Add DB indexes for performance

.gitignore

@@ -3,6 +3,8 @@ yt_auth_token
 spotify_auth_token
 tidal_auth_token
 *.db
+*.db-shm
+*.db-wal
 feed.xml
 playbook.yml
 inventory.yml

package.json

@@ -18,7 +18,7 @@
     "@eslint/js": "^9.30.1",
     "@sveltejs/adapter-node": "^5.2.12",
     "@sveltejs/kit": "^2.22.2",
-    "@sveltejs/vite-plugin-svelte": "^5.1.0",
+    "@sveltejs/vite-plugin-svelte": "^6.1.0",
     "@types/node": "^22.9.0",
     "@types/sqlite3": "^3.0.0",
     "@types/ws": "^8.5.0",
@@ -32,6 +32,7 @@
     "prettier": "^3.1.0",
     "prettier-plugin-svelte": "^3.2.6",
     "svelte": "^5",
+    "svelte-adapter-bun": "^0.5.2",
     "svelte-check": "^4.0.0",
     "tslib": "^2.0.0",
     "typescript": "^5.0.0"
@@ -44,6 +45,11 @@
     "sqlite3": "^5.0.0",
     "ws": "^8.18.0"
   },
+  "overrides": {
+    "@sveltejs/kit": {
+      "cookie": ">=0.7.0"
+    }
+  },
   "engines": {
     "node": ">=20.0.0"
   }

src/lib/server/db.ts

@@ -68,6 +68,11 @@ type Migration = {
   statement: string;
 };
 const db: sqlite3.Database = new sqlite3.cached.Database('moshingmammut.db');
+const maxThumbnailCacheSize = 100;
+const maxThumbnailCacheAge = 1 * 60 * 60 * 1000; //1h in ms
+const thumbnailCache = new Map<string, { data: SongThumbnailImage[]; ts: number }>();
+let thumbnailCacheHits = 0;
+let thumbnailCacheMisses = 0;
 
 export function close() {
   try {
@@ -92,6 +97,7 @@ const ignoredUsers: string[] =
 let databaseReady = false;
 
 if (enableVerboseLog) {
+  logger.verbose('Enabling verbose DB log');
   sqlite3.verbose();
   db.on('change', (t, d, table, rowid) => {
     logger.verbose('DB change event', t, d, table, rowid);
@@ -101,8 +107,16 @@ if (enableVerboseLog) {
     logger.verbose('Running', sql);
   });
 
-  db.on('profile', (sql) => {
+  db.on('profile', (sql, time) => {
-    logger.verbose('Finished', sql);
+    if (sql.startsWith('EXPLAIN')) {
+      return;
+    }
+    logger.verbose('Finished', sql, 'time', time);
+    if (sql.startsWith('SELECT')) {
+      db.get('EXPLAIN QUERY PLAN ' + sql, (a: sqlite3.RunResult, b: Error) => {
+        logger.verbose(sql, a, b);
+      });
+    }
   });
 }
 
@@ -148,6 +162,9 @@ async function applyMigration(migration: Migration) {
 db.on('open', () => {
   logger.info('Opened database');
   db.serialize();
+  db.run('pragma journal_mode = WAL;');
+  db.run('pragma synchronous = normal;');
+  db.run('pragma journal_size_limit = 6144000;');
   db.run('CREATE TABLE IF NOT EXISTS "migrations" ("id" integer,"name" TEXT, PRIMARY KEY (id))');
   db.all('SELECT id FROM migrations', (err, rows: Migration[]) => {
     if (err !== null) {
@@ -339,6 +356,26 @@ function getMigrations(): Migration[] {
       name: 'song tidal id',
       statement: `
       ALTER TABLE songs ADD COLUMN tidalId TEXT NULL;`
+    },
+    {
+      id: 10,
+      name: 'add indexes',
+      statement: `
+      CREATE INDEX posts_created_at ON posts(created_at);
+      CREATE INDEX accounts_acct ON accounts(acct);
+      CREATE INDEX accounts_username ON accounts(username);`
+    },
+    {
+      id: 11,
+      name: 'add FK indexes',
+      statement: `
+      CREATE INDEX migrations_id ON migrations(id);
+      CREATE INDEX accountsavatars_account_url ON accountsavatars(account_url);
+      CREATE INDEX posts_account_id ON posts(account_id);
+      CREATE INDEX poststags_post_id ON poststags(post_id);
+      CREATE INDEX poststags_tag_url ON poststags(tag_url);
+      CREATE INDEX songs_post_url ON songs(post_url);
+      CREATE INDEX songsthumbnails_song_thumbnailUrl ON songsthumbnails(song_thumbnailUrl);`
     }
   ];
 }
@@ -613,7 +650,6 @@ function getSongData(postIdsParams: string, postIds: string[]): Promise<Map<stri
           },
           new Map()
         );
-        logger.verbose('songMap', songMap);
         resolve(songMap);
       }
     );
@@ -803,6 +839,7 @@ async function getPostsInternal(
     .flatMap((x) => x)
     .map((x) => x.thumbnailUrl)
     .filter((x) => x !== undefined)
+    .filter((x) => getCachedThumbnail(x) === null)
     .toArray();
   if (turls) {
     const tMap = await getSongThumbnailData('?, '.repeat(turls.length).slice(0, -2), turls);
@@ -811,7 +848,11 @@ async function getPostsInternal(
         if (songInfo.thumbnailUrl === undefined) {
           continue;
         }
-        const thumbs = tMap.get(songInfo.thumbnailUrl) ?? [];
+        let thumbs = getCachedThumbnail(songInfo.thumbnailUrl);
+        if (thumbs === null) {
+          thumbs = tMap.get(songInfo.thumbnailUrl) ?? [];
+          cacheThumbnail(songInfo.thumbnailUrl, thumbs);
+        }
         songInfo.resizedThumbnails = thumbs;
       }
     }
@@ -969,6 +1010,73 @@ export async function getSongThumbnails(song: SongInfo): Promise<SongThumbnailIm
   if (!song.thumbnailUrl) {
     return [];
   }
+  const cachedThumbnail = getCachedThumbnail(song.thumbnailUrl);
+  if (cachedThumbnail !== null) {
+    return cachedThumbnail;
+  }
   const rows = await getSongThumbnailData('?', [song.thumbnailUrl]);
-  return rows.get(song.thumbnailUrl) ?? [];
+  const data = rows.get(song.thumbnailUrl) ?? [];
+  cacheThumbnail(song.thumbnailUrl, data);
+  return data;
+}
+
+function getCachedThumbnail(thumbnailUrl: string): SongThumbnailImage[] | null {
+  const cachedThumbnail = thumbnailCache.get(thumbnailUrl);
+  if (cachedThumbnail == undefined) {
+    thumbnailCacheMisses++;
+    return null;
+  }
+  const age = new Date().getTime() - cachedThumbnail.ts;
+  if (age <= maxThumbnailCacheAge) {
+    thumbnailCacheHits++;
+    return cachedThumbnail.data;
+  }
+  thumbnailCache.delete(thumbnailUrl);
+  thumbnailCacheMisses++;
+  return null;
+}
+
+function cacheThumbnail(thumbnailUrl: string, thumbnails: SongThumbnailImage[]) {
+  const now = new Date().getTime();
+  const initialSize = thumbnailCache.size;
+  if (initialSize >= maxThumbnailCacheSize) {
+    logger.debug('Sweeping cache. Current size', initialSize);
+    const deleteTimestampsOlderThan =
+      thumbnailCache
+        .values()
+        .map((x) => x.ts)
+        .toArray()
+        .sort((a, b) => a - b)
+        .slice(0, 20)
+        .pop() ?? 0;
+    const timestampExpired = now - maxThumbnailCacheAge;
+    const threshold = Math.max(timestampExpired, deleteTimestampsOlderThan);
+    thumbnailCache.forEach((v, k) => {
+      if (v.ts <= threshold) {
+        thumbnailCache.delete(k);
+      }
+    });
+    if (Logger.isDebugEnabled()) {
+      logger.debug(
+        'Swept cache. New size',
+        thumbnailCache.size,
+        'deleted',
+        initialSize - thumbnailCache.size,
+        'entries'
+      );
+      logger.debug(
+        'Current hitrate',
+        thumbnailCacheHits,
+        '/',
+        thumbnailCacheHits + thumbnailCacheMisses,
+        '=',
+        ((100.0 * thumbnailCacheHits) / (thumbnailCacheHits + thumbnailCacheMisses)).toFixed(2),
+        '%'
+      );
+    }
+  }
+  thumbnailCache.set(thumbnailUrl, {
+    data: thumbnails,
+    ts: now
+  });
 }

src/routes/+page.ts

@@ -2,18 +2,11 @@ import type { Post } from '$lib/mastodon/response';
 import type { PageLoad } from './$types';
 
 export const load = (async ({ fetch, setHeaders }) => {
-  const start = performance.now();
   const p = await fetch('/');
-  const afterFetch = performance.now();
-  console.debug('+page.ts: Fetch took', afterFetch - start, 'ms');
   setHeaders({
     'cache-control': 'public,max-age=60'
   });
-  const afterHeaders = performance.now();
-  console.debug('+page.ts: Headers took', afterHeaders - afterFetch, 'ms');
   const j: Post[] = await p.json();
-  const afterJson = performance.now();
-  console.debug('+page.ts: JSON took', afterJson - afterHeaders, 'ms');
   return {
     posts: j
   };

svelte.config.js

@@ -1,3 +1,5 @@
+// Does not seem to work
+//import adapter from 'svelte-adapter-bun';
 import adapter from '@sveltejs/adapter-node';
 import { vitePreprocess } from '@sveltejs/vite-plugin-svelte';