From 3103d3e0985d8f9feb531c23a543689283dfcbc5 Mon Sep 17 00:00:00 2001
From: Max Nuding <max.nuding@icloud.com>
Date: Wed, 14 Jun 2023 20:37:31 +0200
Subject: [PATCH] Fix #26: Scale images to the correct size and use more
 efficient image formats

---
 .gitignore                                |   1 +
 src/lib/components/AvatarComponent.svelte |  17 ++-
 src/lib/components/PostComponent.svelte   |  79 ++++++++++++--
 src/lib/mastodon/response.ts              |  13 +++
 src/lib/odesliResponse.ts                 |   3 +
 src/lib/server/db.ts                      | 109 ++++++++++++++++++-
 src/lib/server/timeline.ts                | 122 ++++++++++++++++++++--
 7 files changed, 324 insertions(+), 20 deletions(-)

diff --git a/.gitignore b/.gitignore
index ed79676..c2c3b68 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,7 @@ inventory.yml
 ansible.cfg
 
 avatars/*
+thumbnails/*
 node_modules
 /build
 /.svelte-kit
diff --git a/src/lib/components/AvatarComponent.svelte b/src/lib/components/AvatarComponent.svelte
index c93d977..0575be1 100644
--- a/src/lib/components/AvatarComponent.svelte
+++ b/src/lib/components/AvatarComponent.svelte
@@ -6,8 +6,23 @@
   let sourceSetHtml: string;
   $: avatarDescription = `Avatar for ${account.acct}`;
   $: {
+    // Sort thumbnails by file type. This is important, because the order of the srcset entries matter.
+    // We need the best format to be first
+    const formatPriority = new Map<string, number>([
+      ['avif', 0],
+      ['webp', 1],
+      ['jpg', 99],
+      ['jpeg', 99]
+    ]);
+    const resizedAvatars = (account.resizedAvatars ?? []).sort((a, b) => {
+      const extensionA = a.file.split('.').pop() ?? '';
+      const extensionB = b.file.split('.').pop() ?? '';
+      const prioA = formatPriority.get(extensionA) ?? 3;
+      const prioB = formatPriority.get(extensionB) ?? 3;
+      return prioA - prioB;
+    });
     const m = new Map<string, string[]>();
-    for (const resizedAvatar of account.resizedAvatars ?? []) {
+    for (const resizedAvatar of resizedAvatars) {
       const extension = resizedAvatar.file.split('.').pop();
       const mime = extension ? `image/${extension}` : 'application/octet-stream';
       const sourceSetEntry = `${resizedAvatar.file} ${resizedAvatar.sizeDescriptor}`;
diff --git a/src/lib/components/PostComponent.svelte b/src/lib/components/PostComponent.svelte
index 2434733..a83578d 100644
--- a/src/lib/components/PostComponent.svelte
+++ b/src/lib/components/PostComponent.svelte
@@ -1,5 +1,6 @@
 <script lang="ts">
-  import type { Post } from '$lib/mastodon/response';
+  import { type Post, SongThumbnailImageKind } from '$lib/mastodon/response';
+  import type { SongInfo } from '$lib/odesliResponse';
   import AvatarComponent from '$lib/components/AvatarComponent.svelte';
   import AccountComponent from '$lib/components/AccountComponent.svelte';
   import { secondsSince, relativeTime } from '$lib/relativeTime';
@@ -14,6 +15,62 @@
     dateCreated = relativeTime($timePassed) ?? absoluteDate;
   }
 
+  // Blurred thumbs aren't generated (yet, unclear of they ever will)
+  // So blurred forces using the small one, by skipping the others and removing its media query.
+  // This is technically unnecessary - the blurred one will only show if it matches the small media query,
+  // but this makes it more explicit
+  function getSourceSetHtml(song: SongInfo, isBlurred: boolean = false): string {
+    const small = new Map<string, string[]>();
+    const large = new Map<string, string[]>();
+
+    // Sort thumbnails by file type. This is important, because the order of the srcset entries matter.
+    // We need the best format to be first
+    const formatPriority = new Map<string, number>([
+      ['avif', 0],
+      ['webp', 1],
+      ['jpg', 99],
+      ['jpeg', 99]
+    ]);
+    const thumbs = (song.resizedThumbnails ?? []).sort((a, b) => {
+      const extensionA = a.file.split('.').pop() ?? '';
+      const extensionB = b.file.split('.').pop() ?? '';
+      const prioA = formatPriority.get(extensionA) ?? 3;
+      const prioB = formatPriority.get(extensionB) ?? 3;
+      return prioA - prioB;
+    });
+
+    for (const resizedThumb of thumbs) {
+      if (isBlurred && resizedThumb.kind !== SongThumbnailImageKind.Small) {
+        continue;
+      }
+      const extension = resizedThumb.file.split('.').pop();
+      const mime = extension ? `image/${extension}` : 'application/octet-stream';
+      const sourceSetEntry = `${resizedThumb.file} ${resizedThumb.sizeDescriptor}`;
+      switch (resizedThumb.kind) {
+        case SongThumbnailImageKind.Big:
+          large.set(mime, [...(large.get(mime) || []), sourceSetEntry]);
+          break;
+        case SongThumbnailImageKind.Small:
+          small.set(mime, [...(small.get(mime) || []), sourceSetEntry]);
+          break;
+        case SongThumbnailImageKind.Blurred: // currently not generated
+          break;
+      }
+    }
+    let html = '';
+    const mediaAttribute = isBlurred ? '' : 'media="(max-width: 650px)"';
+    for (const entry of small.entries()) {
+      const srcset = entry[1].join(', ');
+      html += `<source srcset="${srcset}" type="${entry[0]}" ${mediaAttribute} />`;
+    }
+    html += '\n';
+    for (const entry of large.entries()) {
+      const srcset = entry[1].join(', ');
+      html += `<source srcset="${srcset}" type="${entry[0]}" />`;
+    }
+    return html;
+  }
+
   onMount(() => {
     // Display relative time only after mount:
     // When JS is disabled the server-side rendered absolute date will be shown,
@@ -33,15 +90,21 @@
     {#if post.songs}
       {#each post.songs as song (song.pageUrl)}
         <div class="info-wrapper">
-          <img class="bgimage" src={song.thumbnailUrl} loading="lazy" alt="Blurred cover" />
+          <picture>
+            {@html getSourceSetHtml(song)}
+            <img class="bgimage" src={song.thumbnailUrl} loading="lazy" alt="Blurred cover" />
+          </picture>
           <a href={song.pageUrl ?? song.postedUrl} target="_blank">
             <div class="info">
-              <img
-                src={song.thumbnailUrl}
-                class="cover"
-                alt="Cover for {song.artistName} - {song.title}"
-                loading="lazy"
-              />
+              <picture>
+                {@html getSourceSetHtml(song)}
+                <img
+                  src={song.thumbnailUrl}
+                  class="cover"
+                  alt="Cover for {song.artistName} - {song.title}"
+                  loading="lazy"
+                />
+              </picture>
               <span class="text">{song.artistName} - {song.title}</span>
             </div>
           </a>
diff --git a/src/lib/mastodon/response.ts b/src/lib/mastodon/response.ts
index cb3f3f3..a5e9910 100644
--- a/src/lib/mastodon/response.ts
+++ b/src/lib/mastodon/response.ts
@@ -45,3 +45,16 @@ export type AccountAvatar = {
   file: string;
   sizeDescriptor: string;
 };
+
+export enum SongThumbnailImageKind {
+  Big = 1,
+  Small,
+  Blurred
+}
+
+export type SongThumbnailImage = {
+  songThumbnailUrl: string;
+  file: string;
+  sizeDescriptor: string;
+  kind: SongThumbnailImageKind;
+};
diff --git a/src/lib/odesliResponse.ts b/src/lib/odesliResponse.ts
index 736a4c5..c01d5de 100644
--- a/src/lib/odesliResponse.ts
+++ b/src/lib/odesliResponse.ts
@@ -1,3 +1,5 @@
+import type { SongThumbnailImage } from '$lib/mastodon/response';
+
 export type SongInfo = {
   pageUrl: string;
   youtubeUrl?: string;
@@ -6,6 +8,7 @@ export type SongInfo = {
   artistName?: string;
   thumbnailUrl?: string;
   postedUrl: string;
+  resizedThumbnails?: SongThumbnailImage[];
 };
 
 export type SongwhipReponse = {
diff --git a/src/lib/server/db.ts b/src/lib/server/db.ts
index 6bda67d..fdd78a1 100644
--- a/src/lib/server/db.ts
+++ b/src/lib/server/db.ts
@@ -1,6 +1,6 @@
 import { IGNORE_USERS, MASTODON_INSTANCE } from '$env/static/private';
 import { enableVerboseLog, log } from '$lib/log';
-import type { Account, AccountAvatar, Post, Tag } from '$lib/mastodon/response';
+import type { Account, AccountAvatar, Post, SongThumbnailImage, Tag } from '$lib/mastodon/response';
 import type { SongInfo } from '$lib/odesliResponse';
 import { TimelineReader } from '$lib/server/timeline';
 import sqlite3 from 'sqlite3';
@@ -48,6 +48,13 @@ type AccountAvatarRow = {
   sizeDescriptor: string;
 };
 
+type SongThumbnailAvatarRow = {
+  song_thumbnailUrl: string;
+  file: string;
+  sizeDescriptor: string;
+  kind: number;
+};
+
 type Migration = {
   id: number;
   name: string;
@@ -280,11 +287,23 @@ function getMigrations(): Migration[] {
       name: 'resized avatars',
       statement: `
       CREATE TABLE accountsavatars (
-        file TEXT  NOT NULL PRIMARY KEY,
+        file TEXT NOT NULL PRIMARY KEY,
         account_url TEXT NOT NULL,
         sizeDescriptor TEXT NOT NULL,
         FOREIGN KEY (account_url) REFERENCES accounts(url)
       );`
+    },
+    {
+      id: 6,
+      name: 'resized song thumbnails',
+      statement: `
+      CREATE TABLE songsthumbnails (
+        file TEXT NOT NULL PRIMARY KEY,
+        song_thumbnailUrl TEXT NOT NULL,
+        sizeDescriptor TEXT NOT NULL,
+        kind INTEGER NOT NULL,
+        FOREIGN KEY (song_thumbnailUrl) REFERENCES songs(thumbnailUrl)
+      );`
     }
   ];
 }
@@ -487,7 +506,7 @@ function getPostData(filterQuery: string, params: FilterParameter): Promise<Post
   });
 }
 
-function getTagData(postIdsParams: String, postIds: string[]): Promise<Map<string, Tag[]>> {
+function getTagData(postIdsParams: string, postIds: string[]): Promise<Map<string, Tag[]>> {
   return new Promise((resolve, reject) => {
     db.all(
       `SELECT post_id, tags.url, tags.tag
@@ -552,7 +571,7 @@ function getSongData(postIdsParams: String, postIds: string[]): Promise<Map<stri
 }
 
 function getAvatarData(
-  accountUrlsParams: String,
+  accountUrlsParams: string,
   accountUrls: string[]
 ): Promise<Map<string, AccountAvatar[]>> {
   return new Promise((resolve, reject) => {
@@ -585,6 +604,44 @@ function getAvatarData(
   });
 }
 
+function getSongThumbnailData(
+  thumbUrlsParams: string,
+  thumbUrls: string[]
+): Promise<Map<string, SongThumbnailImage[]>> {
+  return new Promise((resolve, reject) => {
+    db.all(
+      `SELECT song_thumbnailUrl, file, sizeDescriptor, kind
+      FROM songsthumbnails
+      WHERE song_thumbnailUrl IN (${thumbUrlsParams});`,
+      thumbUrls,
+      (err, rows: SongThumbnailAvatarRow[]) => {
+        if (err != null) {
+          log.error('Error loading avatars', err);
+          reject(err);
+          return;
+        }
+        const thumbnailMap: Map<string, SongThumbnailImage[]> = rows.reduce(
+          (result: Map<string, SongThumbnailImage[]>, item) => {
+            const info: SongThumbnailImage = {
+              songThumbnailUrl: item.song_thumbnailUrl,
+              file: item.file,
+              sizeDescriptor: item.sizeDescriptor,
+              kind: item.kind
+            };
+            result.set(item.song_thumbnailUrl, [
+              ...(result.get(item.song_thumbnailUrl) || []),
+              info
+            ]);
+            return result;
+          },
+          new Map()
+        );
+        resolve(thumbnailMap);
+      }
+    );
+  });
+}
+
 export async function getPosts(
   since: string | null,
   before: string | null,
@@ -634,6 +691,12 @@ async function getPostsInternal(
   const postIds = rows.map((r: PostRow) => r.url);
   const tagMap = await getTagData(postIdsParams, postIds);
   const songMap = await getSongData(postIdsParams, postIds);
+  for (const entry of songMap) {
+    for (const songInfo of entry[1]) {
+      const thumbs = await getSongThumbnails(songInfo);
+      songInfo.resizedThumbnails = thumbs;
+    }
+  }
 
   const accountUrls = [...new Set(rows.map((r: PostRow) => r.account_url))];
   const accountUrlsParams = accountUrls.map(() => '?').join(', ');
@@ -679,6 +742,36 @@ export async function removeAvatars(accountUrl: string): Promise<void> {
   });
 }
 
+export async function saveSongThumbnail(thumb: SongThumbnailImage): Promise<void> {
+  // Will be null if file already existed
+  if (thumb.file === null) {
+    return;
+  }
+  const params: FilterParameter = {
+    $songId: thumb.songThumbnailUrl,
+    $file: thumb.file,
+    $sizeDescriptor: thumb.sizeDescriptor,
+    $kind: thumb.kind.valueOf()
+  };
+  const sql = `
+  INSERT INTO songsthumbnails
+  (song_thumbnailUrl, file, sizeDescriptor, kind) VALUES ($songId, $file, $sizeDescriptor, $kind)
+  ON CONFLICT(file) DO UPDATE SET
+  song_thumbnailUrl=excluded.song_thumbnailUrl,
+  sizeDescriptor=excluded.sizeDescriptor,
+  kind=excluded.kind;`;
+  await waitReady();
+  return new Promise((resolve, reject) => {
+    db.run(sql, params, (err) => {
+      if (err) {
+        reject(err);
+        return;
+      }
+      resolve();
+    });
+  });
+}
+
 export async function saveAvatar(avatar: AccountAvatar): Promise<void> {
   // Will be null if file already existed
   if (avatar.file === null) {
@@ -745,3 +838,11 @@ export async function getAvatars(
     });
   });
 }
+
+export async function getSongThumbnails(song: SongInfo): Promise<SongThumbnailImage[]> {
+  if (!song.thumbnailUrl) {
+    return [];
+  }
+  const rows = await getSongThumbnailData('?', [song.thumbnailUrl]);
+  return rows.get(song.thumbnailUrl) ?? [];
+}
diff --git a/src/lib/server/timeline.ts b/src/lib/server/timeline.ts
index fc41d75..36191b8 100644
--- a/src/lib/server/timeline.ts
+++ b/src/lib/server/timeline.ts
@@ -1,10 +1,27 @@
 import { HASHTAG_FILTER, MASTODON_INSTANCE, ODESLI_API_KEY } from '$env/static/private';
 import { log } from '$lib/log';
-import type { Account, AccountAvatar, Post, Tag, TimelineEvent } from '$lib/mastodon/response';
+import type {
+  Account,
+  AccountAvatar,
+  Post,
+  SongThumbnailImage,
+  Tag,
+  TimelineEvent
+} from '$lib/mastodon/response';
+import { SongThumbnailImageKind } from '$lib/mastodon/response';
 import type { OdesliResponse, Platform, SongInfo } from '$lib/odesliResponse';
-import { getAvatars, getPosts, removeAvatars, saveAvatar, savePost } from '$lib/server/db';
+import {
+  getAvatars,
+  getPosts,
+  getSongThumbnails,
+  removeAvatars,
+  saveAvatar,
+  savePost,
+  saveSongThumbnail
+} from '$lib/server/db';
 import { createFeed, saveAtomFeed } from '$lib/server/rss';
 import { sleep } from '$lib/sleep';
+import crypto from 'crypto';
 import fs from 'fs/promises';
 import sharp from 'sharp';
 import { WebSocket } from 'ws';
@@ -95,9 +112,10 @@ export class TimelineReader {
     baseName: string,
     size: number,
     suffix: string,
+    folder: string,
     sharpAvatar: sharp.Sharp
   ): Promise<string | null> {
-    const fileName = `avatars/${baseName}_${suffix}`;
+    const fileName = `${folder}/${baseName}_${suffix}`;
     const exists = await fs
       .access(fileName, fs.constants.F_OK)
       .then(() => true)
@@ -124,7 +142,13 @@ export class TimelineReader {
     for (let i = 1; i <= maxPixelDensity; i++) {
       promises.push(
         ...formats.map((f) =>
-          TimelineReader.resizeAvatar(avatarFilenameBase, baseSize * i, `${i}x.${f}`, sharpAvatar)
+          TimelineReader.resizeAvatar(
+            avatarFilenameBase,
+            baseSize * i,
+            `${i}x.${f}`,
+            'avatars',
+            sharpAvatar
+          )
             .then(
               (fn) =>
                 ({
@@ -140,6 +164,43 @@ export class TimelineReader {
     return promises;
   }
 
+  private static resizeThumbnailPromiseMaker(
+    filenameBase: string,
+    baseSize: number,
+    maxPixelDensity: number,
+    songThumbnailUrl: string,
+    formats: string[],
+    image: ArrayBuffer,
+    kind: SongThumbnailImageKind
+  ): Promise<void>[] {
+    const sharpAvatar = sharp(image);
+    const promises: Promise<void>[] = [];
+    for (let i = 1; i <= maxPixelDensity; i++) {
+      promises.push(
+        ...formats.map((f) =>
+          TimelineReader.resizeAvatar(
+            filenameBase,
+            baseSize * i,
+            `${i}x.${f}`,
+            'thumbnails',
+            sharpAvatar
+          )
+            .then(
+              (fn) =>
+                ({
+                  songThumbnailUrl: songThumbnailUrl,
+                  file: fn,
+                  sizeDescriptor: `${i}x`,
+                  kind: kind
+                } as SongThumbnailImage)
+            )
+            .then(saveSongThumbnail)
+        )
+      );
+    }
+    return promises;
+  }
+
   private static async saveAvatar(account: Account) {
     try {
       const existingAvatars = await getAvatars(account.url, 1);
@@ -176,6 +237,52 @@ export class TimelineReader {
     }
   }
 
+  private static async saveSongThumbnails(songs: SongInfo[]) {
+    for (const song of songs) {
+      if (!song.thumbnailUrl) {
+        continue;
+      }
+      try {
+        const existingThumbs = await getSongThumbnails(song);
+        if (existingThumbs.length) {
+          continue;
+        }
+        const fileBaseName = crypto.createHash('sha256').update(song.thumbnailUrl).digest('hex');
+        const imageResponse = await fetch(song.thumbnailUrl);
+        const avatar = await imageResponse.arrayBuffer();
+        await Promise.all(
+          TimelineReader.resizeThumbnailPromiseMaker(
+            fileBaseName + '_large',
+            200,
+            3,
+            song.thumbnailUrl,
+            ['webp', 'avif', 'jpeg'],
+            avatar,
+            SongThumbnailImageKind.Big
+          )
+        );
+        await Promise.all(
+          TimelineReader.resizeThumbnailPromiseMaker(
+            fileBaseName + '_small',
+            60,
+            3,
+            song.thumbnailUrl,
+            ['webp', 'avif', 'jpeg'],
+            avatar,
+            SongThumbnailImageKind.Small
+          )
+        );
+      } catch (e) {
+        console.error(
+          'Could not resize and save song thumbnail for',
+          song.pageUrl,
+          song.thumbnailUrl,
+          e
+        );
+      }
+    }
+  }
+
   private startWebsocket() {
     const socket = new WebSocket(`wss://${MASTODON_INSTANCE}/api/v1/streaming`);
     socket.onopen = () => {
@@ -190,9 +297,6 @@ export class TimelineReader {
         }
         const post: Post = JSON.parse(data.payload);
 
-        // TODO: Move down after post has been confirmed after testing
-        await TimelineReader.saveAvatar(post.account);
-
         const hashttags: string[] = HASHTAG_FILTER.split(',');
         const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
 
@@ -206,6 +310,10 @@ export class TimelineReader {
         }
 
         await savePost(post, songs);
+
+        await TimelineReader.saveAvatar(post.account);
+        await TimelineReader.saveSongThumbnails(songs);
+
         log.debug('Saved post', post.url);
 
         const posts = await getPosts(null, null, 100);