refactor API paths, increase home page caching time

.env.EXAMPLE

@@ -10,8 +10,8 @@ BASE_URL = 'https://moshingmammut.phlaym.net'
 VERBOSE = false
 DEBUG_LOG = false
 IGNORE_USERS = @moshhead@metalhead.club
-WEBSUB_HUB = 'http://pubsubhubbub.superfeedr.com'
 
+PUBLIC_WEBSUB_HUB = 'http://pubsubhubbub.superfeedr.com'
 PUBLIC_REFRESH_INTERVAL = 10000
 PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME = 'Metalhead.club'
 PORT = 3001

.gitignore

@@ -3,6 +3,8 @@ yt_auth_token
 spotify_auth_token
 tidal_auth_token
 *.db
+*.db-shm
+*.db-wal
 feed.xml
 playbook.yml
 inventory.yml

apache2.conf.EXAMPLE

@@ -32,6 +32,10 @@
       Header set Cache-Control "public,max-age=31536000,immutable"
     </Directory>
 
+    Header set X-Content-Type-Options "nosniff"
+    Header set Strict-Transport-Security "max-age=15768000; includeSubDomains"
+    Header set X-Frame-Options "DENY"
+
     ProxyPass / http://localhost:3000/
     ProxyPassReverse / http://localhost:3000/
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "moshing-mammut",
-  "version": "2.0.1",
+  "version": "2.1.1",
   "private": true,
   "license": "LGPL-3.0-or-later",
   "scripts": {
@@ -18,7 +18,7 @@
     "@eslint/js": "^9.30.1",
     "@sveltejs/adapter-node": "^5.2.12",
     "@sveltejs/kit": "^2.22.2",
-    "@sveltejs/vite-plugin-svelte": "^5.1.0",
+    "@sveltejs/vite-plugin-svelte": "^6.1.0",
     "@types/node": "^22.9.0",
     "@types/sqlite3": "^3.0.0",
     "@types/ws": "^8.5.0",
@@ -32,6 +32,7 @@
     "prettier": "^3.1.0",
     "prettier-plugin-svelte": "^3.2.6",
     "svelte": "^5",
+    "svelte-adapter-bun": "^0.5.2",
     "svelte-check": "^4.0.0",
     "tslib": "^2.0.0",
     "typescript": "^5.0.0"
@@ -44,6 +45,11 @@
     "sqlite3": "^5.0.0",
     "ws": "^8.18.0"
   },
+  "overrides": {
+    "@sveltejs/kit": {
+      "cookie": ">=0.7.0"
+    }
+  },
   "engines": {
     "node": ">=20.0.0"
   }

src/app.html

@@ -19,7 +19,7 @@
     <meta name="theme-color" content="#17063b" media="(prefers-color-scheme: dark)" />
     <meta name="theme-color" content="#BCB9B2" media="(prefers-color-scheme: light)" />
     <link rel="alternate" type="application/atom+xml" href="/feed.xml" title="Atom Feed" />
-    <link rel="hub" href="https://pubsubhubbub.superfeedr.com" />
+    <link rel="hub" href="%sveltekit.env.PUBLIC_WEBSUB_HUB%" />
     %sveltekit.head%
     <style>
       body {

src/lib/components/AvatarComponent.svelte

@@ -3,10 +3,12 @@
 
   interface Props {
     account: Account;
+    lazyLoadImages: Boolean;
   }
 
-  let { account }: Props = $props();
+  let { account, lazyLoadImages = true }: Props = $props();
   let avatarDescription: string = $derived(`Avatar for ${account.acct}`);
+  let loadingProp = $derived(lazyLoadImages ? 'lazy' : 'eager');
   let sourceSetHtml: string = $derived.by(() => {
     // Sort thumbnails by file type. This is important, because the order of the srcset entries matter.
     // We need the best format to be first
@@ -41,7 +43,7 @@
 
 <picture>
   {@html sourceSetHtml}
-  <img src={account.avatar} alt={avatarDescription} loading="lazy" width="50" height="50" />
+  <img src={account.avatar} alt={avatarDescription} loading={loadingProp} width="50" height="50" />
 </picture>
 
 <style>

src/lib/components/PostComponent.svelte

@@ -8,9 +8,10 @@
 
   interface Props {
     post: Post;
+    lazyLoadImages: Boolean;
   }
 
-  let { post }: Props = $props();
+  let { post, lazyLoadImages = true }: Props = $props();
   let displayRelativeTime = $state(false);
   const absoluteDate = new Date(post.created_at).toLocaleString();
   const timePassed = secondsSince(new Date(post.created_at));
@@ -20,6 +21,7 @@
     }
     return absoluteDate;
   });
+  let loadingProp = $derived(lazyLoadImages ? 'lazy' : 'eager');
 
   const songs = filterDuplicates(post.songs ?? []);
 
@@ -115,7 +117,7 @@
 </script>
 
 <div class="wrapper">
-  <div class="avatar"><AvatarComponent account={post.account} /></div>
+  <div class="avatar"><AvatarComponent account={post.account} {lazyLoadImages} /></div>
   <div class="account"><AccountComponent account={post.account} /></div>
   <div class="meta">
     <small><a href={post.url} target="_blank" title={absoluteDate}>{dateCreated}</a></small>
@@ -127,7 +129,12 @@
         <div class="info-wrapper">
           <picture>
             {@html getSourceSetHtml(song)}
-            <img class="bgimage" src={song.thumbnailUrl} loading="lazy" alt="Blurred cover" />
+            <img
+              class="bgimage"
+              src={song.thumbnailUrl}
+              loading={loadingProp}
+              alt="Blurred cover"
+            />
           </picture>
           <a href={song.pageUrl ?? song.postedUrl} target="_blank">
             <div class="info">
@@ -136,7 +143,7 @@
                 <img
                   src={song.thumbnailUrl}
                   alt="Cover for {song.artistName} - {song.title}"
-                  loading="lazy"
+                  loading={loadingProp}
                   width={song.thumbnailWidth}
                   height={song.thumbnailHeight}
                 />
@@ -193,6 +200,12 @@
     border-radius: 3px;
     margin-bottom: 3px;
   }
+  .cover img {
+    max-width: 200px;
+    max-height: 200px;
+    object-fit: contain;
+    border-radius: 3px;
+  }
   .bgimage {
     display: none;
     background-color: var(--color-bg);
@@ -231,6 +244,10 @@
       backdrop-filter: blur(10px);
       -webkit-backdrop-filter: blur(10px);
     }
+    .cover img {
+      max-width: 60px;
+      max-height: 60px;
+    }
     .bgimage {
       display: block;
       width: 100%;

src/lib/server/db.ts

@@ -67,8 +67,12 @@ type Migration = {
   name: string;
   statement: string;
 };
-
+const db: sqlite3.Database = new sqlite3.cached.Database('moshingmammut.db');
-const db: sqlite3.Database = new sqlite3.Database('moshingmammut.db');
+const maxThumbnailCacheSize = 100;
+const maxThumbnailCacheAge = 1 * 60 * 60 * 1000; //1h in ms
+const thumbnailCache = new Map<string, { data: SongThumbnailImage[]; ts: number }>();
+let thumbnailCacheHits = 0;
+let thumbnailCacheMisses = 0;
 
 export function close() {
   try {
@@ -93,6 +97,7 @@ const ignoredUsers: string[] =
 let databaseReady = false;
 
 if (enableVerboseLog) {
+  logger.verbose('Enabling verbose DB log');
   sqlite3.verbose();
   db.on('change', (t, d, table, rowid) => {
     logger.verbose('DB change event', t, d, table, rowid);
@@ -102,8 +107,16 @@ if (enableVerboseLog) {
     logger.verbose('Running', sql);
   });
 
-  db.on('profile', (sql) => {
+  db.on('profile', (sql, time) => {
-    logger.verbose('Finished', sql);
+    if (sql.startsWith('EXPLAIN')) {
+      return;
+    }
+    logger.verbose('Finished', sql, 'time', time);
+    if (sql.startsWith('SELECT')) {
+      db.get('EXPLAIN QUERY PLAN ' + sql, (a: sqlite3.RunResult, b: Error) => {
+        logger.verbose(sql, a, b);
+      });
+    }
   });
 }
 
@@ -149,6 +162,9 @@ async function applyMigration(migration: Migration) {
 db.on('open', () => {
   logger.info('Opened database');
   db.serialize();
+  db.run('pragma journal_mode = WAL;');
+  db.run('pragma synchronous = normal;');
+  db.run('pragma journal_size_limit = 6144000;');
   db.run('CREATE TABLE IF NOT EXISTS "migrations" ("id" integer,"name" TEXT, PRIMARY KEY (id))');
   db.all('SELECT id FROM migrations', (err, rows: Migration[]) => {
     if (err !== null) {
@@ -340,6 +356,26 @@ function getMigrations(): Migration[] {
       name: 'song tidal id',
       statement: `
       ALTER TABLE songs ADD COLUMN tidalId TEXT NULL;`
+    },
+    {
+      id: 10,
+      name: 'add indexes',
+      statement: `
+      CREATE INDEX posts_created_at ON posts(created_at);
+      CREATE INDEX accounts_acct ON accounts(acct);
+      CREATE INDEX accounts_username ON accounts(username);`
+    },
+    {
+      id: 11,
+      name: 'add FK indexes',
+      statement: `
+      CREATE INDEX migrations_id ON migrations(id);
+      CREATE INDEX accountsavatars_account_url ON accountsavatars(account_url);
+      CREATE INDEX posts_account_id ON posts(account_id);
+      CREATE INDEX poststags_post_id ON poststags(post_id);
+      CREATE INDEX poststags_tag_url ON poststags(tag_url);
+      CREATE INDEX songs_post_url ON songs(post_url);
+      CREATE INDEX songsthumbnails_song_thumbnailUrl ON songsthumbnails(song_thumbnailUrl);`
     }
   ];
 }
@@ -614,7 +650,6 @@ function getSongData(postIdsParams: string, postIds: string[]): Promise<Map<stri
           },
           new Map()
         );
-        logger.verbose('songMap', songMap);
         resolve(songMap);
       }
     );
@@ -660,12 +695,15 @@ function getSongThumbnailData(
   thumbUrls: string[]
 ): Promise<Map<string, SongThumbnailImage[]>> {
   return new Promise((resolve, reject) => {
+    const start = performance.now();
     db.all(
       `SELECT song_thumbnailUrl, file, sizeDescriptor, kind
       FROM songsthumbnails
       WHERE song_thumbnailUrl IN (${thumbUrlsParams});`,
       thumbUrls,
       (err, rows: SongThumbnailAvatarRow[]) => {
+        const afterQuery = performance.now();
+        logger.verbose('thumbnail query took', afterQuery - start, 'ms');
         if (err != null) {
           logger.error('Error loading avatars', err);
           reject(err);
@@ -687,6 +725,8 @@ function getSongThumbnailData(
           },
           new Map()
         );
+        const afterReduce = performance.now();
+        logger.verbose('thumbnail reduce took', afterReduce - afterQuery, 'ms');
         resolve(thumbnailMap);
       }
     );
@@ -730,10 +770,16 @@ export async function getPosts(
   before: string | null,
   limit: number
 ): Promise<Post[]> {
+  const start = performance.now();
   if (!databaseReady) {
     await waitReady();
   }
-  return await getPostsInternal(since, before, limit);
+  const ready = performance.now();
+  logger.debug('DB ready took', ready - start, 'ms');
+  const posts = await getPostsInternal(since, before, limit);
+  const afterPosts = performance.now();
+  logger.debug('DB posts', afterPosts - ready, 'ms');
+  return posts;
 }
 
 async function getPostsInternal(
@@ -741,6 +787,7 @@ async function getPostsInternal(
   before: string | null,
   limit: number
 ): Promise<Post[]> {
+  const start = performance.now();
   let filterQuery = '';
   const params: FilterParameter = { $limit: limit };
   if (since === null && before === null) {
@@ -763,8 +810,12 @@ async function getPostsInternal(
     params[acctParam] = ignoredUser;
     params[usernameParam] = ignoredUser;
   });
+  const afterFilter = performance.now();
+  logger.debug('filterQuery took', afterFilter - start, 'ms');
 
   const rows = await getPostData(filterQuery, params);
+  const afterRows = performance.now();
+  logger.debug('rows took', afterRows - afterFilter, 'ms');
   if (rows.length === 0) {
     // No need to check for tags and songs
     return [];
@@ -772,19 +823,50 @@ async function getPostsInternal(
 
   const postIdsParams = rows.map(() => '?').join(', ');
   const postIds = rows.map((r: PostRow) => r.url);
+  const afterParams = performance.now();
+  logger.debug('params took', afterParams - afterRows, 'ms');
+
   const tagMap = await getTagData(postIdsParams, postIds);
+  const afterTag = performance.now();
+  logger.debug('rows took', afterTag - afterRows, 'ms');
+
   const songMap = await getSongData(postIdsParams, postIds);
+  const afterSong = performance.now();
+  logger.debug('rows took', afterSong - afterTag, 'ms');
+
+  const turls = songMap
+    .values()
+    .flatMap((x) => x)
+    .map((x) => x.thumbnailUrl)
+    .filter((x) => x !== undefined)
+    .filter((x) => getCachedThumbnail(x) === null)
+    .toArray();
+  if (turls) {
+    const tMap = await getSongThumbnailData('?, '.repeat(turls.length).slice(0, -2), turls);
     for (const entry of songMap) {
       for (const songInfo of entry[1]) {
-      const thumbs = await getSongThumbnails(songInfo);
+        if (songInfo.thumbnailUrl === undefined) {
+          continue;
+        }
+        let thumbs = getCachedThumbnail(songInfo.thumbnailUrl);
+        if (thumbs === null) {
+          thumbs = tMap.get(songInfo.thumbnailUrl) ?? [];
+          cacheThumbnail(songInfo.thumbnailUrl, thumbs);
+        }
         songInfo.resizedThumbnails = thumbs;
       }
     }
+  }
+  const afterThumbs2 = performance.now();
+  logger.debug('thumbs took', afterThumbs2 - afterSong, 'ms');
 
   const accountUrls = [...new Set(rows.map((r: PostRow) => r.account_url))];
   const accountUrlsParams = accountUrls.map(() => '?').join(', ');
 
   const avatars = await getAvatarData(accountUrlsParams, accountUrls);
+  const afterAvatar = performance.now();
+  logger.debug('avatar took', afterAvatar - afterThumbs2, 'ms');
+
   const posts = rows.map((row) => {
     return {
       id: row.id,
@@ -804,6 +886,8 @@ async function getPostsInternal(
       songs: songMap.get(row.url) || []
     } as Post;
   });
+  const afterMap = performance.now();
+  logger.debug('map took', afterMap - afterAvatar, 'ms');
   return posts;
 }
 
@@ -926,6 +1010,79 @@ export async function getSongThumbnails(song: SongInfo): Promise<SongThumbnailIm
   if (!song.thumbnailUrl) {
     return [];
   }
-  const rows = await getSongThumbnailData('?', [song.thumbnailUrl]);
+  const cachedThumbnail = getCachedThumbnail(song.thumbnailUrl);
-  return rows.get(song.thumbnailUrl) ?? [];
+  if (cachedThumbnail !== null) {
+    return cachedThumbnail;
+  }
+  const rows = await getSongThumbnailData('?', [song.thumbnailUrl]);
+  const data = rows.get(song.thumbnailUrl) ?? [];
+  cacheThumbnail(song.thumbnailUrl, data);
+  return data;
+}
+
+function getCachedThumbnail(thumbnailUrl: string): SongThumbnailImage[] | null {
+  const cachedThumbnail = thumbnailCache.get(thumbnailUrl);
+  if (cachedThumbnail == undefined) {
+    thumbnailCacheMisses++;
+    return null;
+  }
+  const age = new Date().getTime() - cachedThumbnail.ts;
+  if (age <= maxThumbnailCacheAge) {
+    thumbnailCacheHits++;
+    return cachedThumbnail.data;
+  }
+  thumbnailCache.delete(thumbnailUrl);
+  thumbnailCacheMisses++;
+  return null;
+}
+
+function cacheThumbnail(thumbnailUrl: string, thumbnails: SongThumbnailImage[]) {
+  if (!thumbnails) {
+    // This usually means, that the data is being saved to cached,
+    // while the thumbnail generation is not finished yet
+    logger.debug('will not cache empty thumbnail list', thumbnailUrl);
+    return;
+  }
+  const now = new Date().getTime();
+  const initialSize = thumbnailCache.size;
+  if (initialSize >= maxThumbnailCacheSize) {
+    logger.debug('Sweeping cache. Current size', initialSize);
+    const deleteTimestampsOlderThan =
+      thumbnailCache
+        .values()
+        .map((x) => x.ts)
+        .toArray()
+        .sort((a, b) => a - b)
+        .slice(0, 20)
+        .pop() ?? 0;
+    const timestampExpired = now - maxThumbnailCacheAge;
+    const threshold = Math.max(timestampExpired, deleteTimestampsOlderThan);
+    thumbnailCache.forEach((v, k) => {
+      if (v.ts <= threshold) {
+        thumbnailCache.delete(k);
+      }
+    });
+    if (Logger.isDebugEnabled()) {
+      logger.debug(
+        'Swept cache. New size',
+        thumbnailCache.size,
+        'deleted',
+        initialSize - thumbnailCache.size,
+        'entries'
+      );
+      logger.debug(
+        'Current hitrate',
+        thumbnailCacheHits,
+        '/',
+        thumbnailCacheHits + thumbnailCacheMisses,
+        '=',
+        ((100.0 * thumbnailCacheHits) / (thumbnailCacheHits + thumbnailCacheMisses)).toFixed(2),
+        '%'
+      );
+    }
+  }
+  thumbnailCache.set(thumbnailUrl, {
+    data: thumbnails,
+    ts: now
+  });
 }

src/lib/server/rss.ts

@@ -1,4 +1,5 @@
-import { BASE_URL, WEBSUB_HUB } from '$env/static/private';
+import { BASE_URL } from '$env/static/private';
+import { PUBLIC_WEBSUB_HUB } from '$env/static/public';
 import { PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME } from '$env/static/public';
 import type { Post } from '$lib//mastodon/response';
 import { Logger } from '$lib/log';
@@ -10,7 +11,7 @@ const logger = new Logger('RSS');
 
 export function createFeed(posts: Post[]): Feed {
   const baseUrl = BASE_URL.endsWith('/') ? BASE_URL : BASE_URL + '/';
-  const hub = WEBSUB_HUB ? WEBSUB_HUB : undefined;
+  const hub = PUBLIC_WEBSUB_HUB ? PUBLIC_WEBSUB_HUB : undefined;
   const feed = new Feed({
     title: `${PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME} music feed`,
     description: `Posts about music on ${PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME}`,
@@ -51,15 +52,15 @@ export function createFeed(posts: Post[]): Feed {
 }
 export async function saveAtomFeed(feed: Feed) {
   await fs.writeFile('feed.xml', feed.atom1(), { encoding: 'utf8' });
-  if (!WEBSUB_HUB || !PROD) {
+  if (!PUBLIC_WEBSUB_HUB || !PROD) {
-    logger.info('Skipping Websub publish. hub configured?', WEBSUB_HUB, 'Production?', PROD);
+    logger.info('Skipping Websub publish. hub configured?', PUBLIC_WEBSUB_HUB, 'Production?', PROD);
     return;
   }
   try {
     const param = new FormData();
     param.append('hub.mode', 'publish');
     param.append('hub.url', `${BASE_URL}/feed.xml`);
-    await fetch(WEBSUB_HUB, {
+    await fetch(PUBLIC_WEBSUB_HUB, {
       method: 'POST',
       headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
       body: param

src/lib/server/timeline.ts

@@ -335,7 +335,7 @@ export class TimelineReader {
           50,
           3,
           account.url,
-          ['webp', 'avif', 'jpeg'],
+          ['avif', 'jpeg'],
           avatar
         )
       );
@@ -363,7 +363,7 @@ export class TimelineReader {
             200,
             3,
             song.thumbnailUrl,
-            ['webp', 'avif', 'jpeg'],
+            ['avif', 'jpeg'],
             avatar,
             SongThumbnailImageKind.Big
           )
@@ -374,7 +374,7 @@ export class TimelineReader {
             60,
             3,
             song.thumbnailUrl,
-            ['webp', 'avif', 'jpeg'],
+            ['avif', 'jpeg'],
             avatar,
             SongThumbnailImageKind.Small
           )

src/routes/+page.svelte

@@ -163,7 +163,7 @@
     {#if posts.length === 0}
       Sorry, no posts recommending music have been found yet
     {/if}
-    {#each posts as post (post.url)}
+    {#each posts as post, index (post.url)}
       <div
         class="post"
         transition:edgeFly|global={{
@@ -173,7 +173,7 @@
           easing: cubicInOut
         }}
       >
-        <PostComponent {post} />
+        <PostComponent {post} lazyLoadImages={index >= 4} />
       </div>
     {/each}
     <LoadMoreComponent

src/routes/+page.ts

@@ -2,11 +2,12 @@ import type { Post } from '$lib/mastodon/response';
 import type { PageLoad } from './$types';
 
 export const load = (async ({ fetch, setHeaders }) => {
-  const p = await fetch('/');
+  const p = await fetch('/api/posts?count=5');
   setHeaders({
-    'cache-control': 'public,max-age=60'
+    'cache-control': 'public,max-age=300'
   });
+  const j: Post[] = await p.json();
   return {
-    posts: (await p.json()) as Post[]
+    posts: j
   };
 }) satisfies PageLoad;

src/routes/+server.ts

@@ -1,8 +0,0 @@
-import type { RequestHandler } from './$types';
-
-export const GET = (async ({ fetch, setHeaders }) => {
-  setHeaders({
-    'cache-control': 'max-age=10'
-  });
-  return await fetch('api/posts');
-}) satisfies RequestHandler;

src/routes/api/posts/+server.ts

@@ -1,16 +1,31 @@
 import { getPosts } from '$lib/server/db';
 import { json } from '@sveltejs/kit';
+import { Logger } from '$lib/log';
 
 import type { RequestHandler } from './$types';
+import { performance } from 'perf_hooks';
 
-export const GET = (async ({ url }) => {
+const logger = new Logger('+server.ts API');
+
+export const GET = (async ({ url, setHeaders }) => {
+  setHeaders({
+    'cache-control': 'max-age=10'
+  });
+  const start = performance.now();
   const since = url.searchParams.get('since');
   const before = url.searchParams.get('before');
   let count = Number.parseInt(url.searchParams.get('count') || '');
   if (isNaN(count)) {
-    count = 20;
+    count = 10;
   }
   count = Math.min(count, 100);
+  const afterCount = performance.now();
+  logger.debug('Count took', afterCount - start, 'ms');
   const posts = await getPosts(since, before, count);
-  return json(posts);
+  const afterFetch = performance.now();
+  logger.debug('DB took', afterFetch - afterCount, 'ms');
+  const resp = json(posts);
+  const afterResponse = performance.now();
+  logger.debug('Response took', afterResponse - afterFetch, 'ms');
+  return resp;
 }) satisfies RequestHandler;

svelte.config.js

@@ -1,3 +1,5 @@
+// Does not seem to work
+//import adapter from 'svelte-adapter-bun';
 import adapter from '@sveltejs/adapter-node';
 import { vitePreprocess } from '@sveltejs/vite-plugin-svelte';
 
@@ -17,7 +19,7 @@ const config = {
     },
     csp: {
       directives: {
-        'script-src': ['self', 'unsafe-inline'],
+        'script-src': ['self'],
         'base-uri': ['self'],
         'object-src': ['none']
       }