prepare for being available on multiple domain names

Reviewed-on: #38

.env.EXAMPLE

@@ -1,12 +1,17 @@
 HASHTAG_FILTER = ichlausche,music,musik,nowplaying,tunetuesday,nowlistening
 YOUTUBE_API_KEY = CHANGE_ME
+YOUTUBE_PLAYLIST_ID = CHANGE_ME
+YOUTUBE_CLIENT_ID = CHANGE_ME
+YOUTUBE_CLIENT_SECRET = CHANGE_ME
 ODESLI_API_KEY = CHANGE_ME
 MASTODON_INSTANCE = 'metalhead.club'
 MASTODON_ACCESS_TOKEN = 'YOUR_ACCESS_TOKEN_HERE'
 BASE_URL = 'https://moshingmammut.phlaym.net'
 VERBOSE = false
+DEBUG_LOG = false
 IGNORE_USERS = @moshhead@metalhead.club
 WEBSUB_HUB = 'http://pubsubhubbub.superfeedr.com'
 
 PUBLIC_REFRESH_INTERVAL = 10000
 PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME = 'Metalhead.club'
+PORT = 3001

.gitignore

@@ -1,3 +1,6 @@
+moshing-mammut.pid
+yt_auth_token
+spotify_auth_token
 *.db
 feed.xml
 playbook.yml

.nova/Configuration.json

@@ -1,6 +1,5 @@
 {
-  "apexskier.eslint.config.eslintConfigPath" : ".eslintrc.cjs",
+  "apexskier.eslint.config.eslintConfigPath" : ".eslint.cjs",
-  "apexskier.eslint.config.eslintPath" : "node_modules\/@eslint\/eslintrc\/dist\/eslintrc.cjs",
   "apexskier.eslint.config.fixOnSave" : "Enable",
   "apexskier.typescript.config.formatDocumentOnSave" : "false",
   "apexskier.typescript.config.isEnabledForJavascript" : "Enable",

.nvmrc

@@ -0,0 +1 @@
+lts/*

.prettierrc

@@ -5,6 +5,5 @@
   "trailingComma": "none",
   "printWidth": 100,
   "plugins": ["prettier-plugin-svelte"],
-  "pluginSearchDirs": ["."],
   "overrides": [{ "files": "*.svelte", "options": { "parser": "svelte" } }]
 }

README.md

@@ -65,7 +65,7 @@ Set up NVM:
 
 ```
 $ cd
-$ curl https://raw.github.com/creationix/nvm/master/install.sh | sh
+$ curl https://raw.githubusercontent.com/nvm-sh/nvm/refs/heads/master/install.sh | bash
 $ source ~/.nvm/nvm.sh
 $ nvm install --lts
 ```
@@ -98,7 +98,7 @@ and set your `User`, `Group`, `ExecStart` and `WorkingDirectory` accordingly.
 Copy `.env.EXAMPLE` to `.env` and add your `YOUTUBE_API_KEY` and `ODESLI_API_KEY`.
 To obtain one follow [YouTube's guide](https://developers.google.com/youtube/registering_an_application) to create an
 _API key_.
-If `YOUTUBE_API_KEY` is unset, no playlist will be updated. Also, _all_ YouTube links will be treated as music videos,
+If `YOUTUBE_API_KEY` is unset _all_ YouTube links will be treated as music videos,
 because the API is the only way to check if a YouTube link leads to music or something else.
 
 If `ODESLI_API_KEY` is unset, your rate limit to the song.link API will be lower.
@@ -106,10 +106,15 @@ If `ODESLI_API_KEY` is unset, your rate limit to the song.link API will be lower
 Add `MASTODON_ACCESS_TOKEN` as well, see [Creating our application
 
 ](https://docs.joinmastodon.org/client/token/#app) in the Mastodon documentation.
-`read:statuses` is the only required scope. An access token will be displayed in your settings. Use that!
+`read:statuses` and `read:search` the only required scope. An access token will be displayed in your settings. Use that!
 
 There are currently no plans to implement an actual authentication flow.
 
+If you want the app to save the songs it encounters into a playlist, YouTube requires OAuth 2.0 credentials.
+Once again, follow [YouTube's guide](https://developers.google.com/youtube/registering_an_application) and the OAuth 2.0 described there
+to obtain a clientId and clientSecret. Add the values as `YOUTUBE_CLIENT_ID` and `YOUTUBE_CLIENT_SECRET`.
+Create a playlist and configure its ID as `YOUTUBE_PLAYLIST_ID`.
+
 Run `npm run build` and copy the output folder, usually `build` to `$APP_DIR` on your server.
 
 #### On your server again
@@ -125,6 +130,9 @@ Verify that everything is okay with `service moshing-mammut status`.
 
 The app should now be reachable on http://localhost:3000 or whatever you configured your domain to be!
 
+If you want to add the songs available on YouTube to a playlist and have configured the environment variables to do so,
+you now need to visit `/ytauth`, e.g. `http://localhost:3000/ytauth`. This will obtain the necessary access tokens from Google.
+
 # Icons
 
 Favicon is a combination of [speaker-line by remix icon](https://remixicon.com/icon/speaker-line)

eslint.config.cjs

@@ -0,0 +1,73 @@
+const { defineConfig, globalIgnores } = require('eslint/config');
+
+const tsParser = require('@typescript-eslint/parser');
+const typescriptEslint = require('@typescript-eslint/eslint-plugin');
+const parser = require('svelte-eslint-parser');
+const globals = require('globals');
+const js = require('@eslint/js');
+
+const { FlatCompat } = require('@eslint/eslintrc');
+
+const compat = new FlatCompat({
+  baseDirectory: __dirname,
+  recommendedConfig: js.configs.recommended,
+  allConfig: js.configs.all
+});
+
+module.exports = defineConfig([
+  {
+    languageOptions: {
+      parser: tsParser,
+      sourceType: 'module',
+      ecmaVersion: 2020,
+
+      parserOptions: {
+        extraFileExtensions: ['.svelte']
+      },
+
+      globals: {
+        ...globals.browser,
+        ...globals.node
+      }
+    },
+
+    extends: compat.extends(
+      'plugin:svelte/recommended',
+      'plugin:@typescript-eslint/recommended',
+      'prettier'
+    ),
+
+    plugins: {
+      '@typescript-eslint': typescriptEslint
+    },
+
+    settings: {
+      'svelte3/typescript': () => require('typescript')
+    }
+  },
+  globalIgnores(['**/*.cjs']),
+  {
+    files: ['**/*.svelte'],
+
+    languageOptions: {
+      parser: parser,
+
+      parserOptions: {
+        parser: '@typescript-eslint/parser'
+      }
+    }
+  },
+  globalIgnores([
+    '**/.DS_Store',
+    '**/node_modules',
+    'build',
+    '.svelte-kit',
+    'package',
+    '**/.env',
+    '**/.env.*',
+    '!**/.env.example',
+    '**/pnpm-lock.yaml',
+    '**/package-lock.json',
+    '**/yarn.lock'
+  ])
+]);

moshing-mammut.service.EXAMPLE

@@ -1,14 +1,17 @@
 [Unit]
 Description=Moshing Mammut
+After=network.target
 
 [Service]
 ExecStart=/home/moshing-mammut/app/start.sh
-Restart=always
+Restart=on-failure
 User=moshing-mammut
 Group=moshing-mammut
 Environment=PATH=/usr/bin:/usr/local/bin
 Environment=NODE_ENV=production
 WorkingDirectory=/home/moshing-mammut/app
+Type=forking
+PIDFile=/home/moshing-mammut/app/moshing-mammut.pid
 
 [Install]
 WantedBy=multi-user.target

package.json

@@ -1,6 +1,6 @@
 {
   "name": "moshing-mammut",
-  "version": "1.3.2",
+  "version": "2.0.1",
   "private": true,
   "license": "LGPL-3.0-or-later",
   "scripts": {
@@ -14,31 +14,33 @@
     "format": "prettier --plugin-search-dir . --write ."
   },
   "devDependencies": {
-    "@sveltejs/adapter-node": "^5.0.0",
+    "@eslint/eslintrc": "^3.3.1",
-    "@sveltejs/kit": "^2.7.0",
+    "@eslint/js": "^9.30.1",
-    "@sveltejs/vite-plugin-svelte": "^4.0.0",
+    "@sveltejs/adapter-node": "^5.2.12",
-    "@types/node": "^22.6.1",
+    "@sveltejs/kit": "^2.22.2",
+    "@sveltejs/vite-plugin-svelte": "^5.1.0",
+    "@types/node": "^22.9.0",
     "@types/sqlite3": "^3.0.0",
     "@types/ws": "^8.5.0",
     "@typescript-eslint/eslint-plugin": "^8.0.0",
     "@typescript-eslint/parser": "^8.0.0",
     "@zerodevx/svelte-toast": "^0.9.3",
-    "eslint": "^9.11.1",
+    "eslint": "^9.30.1",
-    "eslint-config-prettier": "^9.0.0",
+    "eslint-config-prettier": "^10.0.0",
-    "eslint-plugin-svelte": "^2.45.1",
+    "eslint-plugin-svelte": "^3.0.0",
+    "globals": "^16.3.0",
     "prettier": "^3.1.0",
     "prettier-plugin-svelte": "^3.2.6",
     "svelte": "^5",
     "svelte-check": "^4.0.0",
     "tslib": "^2.0.0",
-    "typescript": "^5.0.0",
+    "typescript": "^5.0.0"
-    "vite": "^5.0.0"
   },
   "type": "module",
   "dependencies": {
-    "dotenv": "^16.0.3",
+    "dotenv": "^17.0.0",
-    "feed": "^4.2.2",
+    "feed": "^5.1.0",
-    "sharp": "^0.33.0",
+    "sharp": "^0.34.2",
     "sqlite3": "^5.0.0",
     "ws": "^8.18.0"
   },

src/hooks.server.ts

@@ -1,15 +1,34 @@
-import { log } from '$lib/log';
+import { Logger } from '$lib/log';
 import { TimelineReader } from '$lib/server/timeline';
 import type { Handle, HandleServerError } from '@sveltejs/kit';
 import { error } from '@sveltejs/kit';
 import fs from 'fs/promises';
+import { close } from '$lib/server/db';
 
-log.log('App startup');
+const logger = new Logger('App');
+
+if (process?.pid) {
+  try {
+    await fs.writeFile('moshing-mammut.pid', process.pid.toString());
+  } catch (e) {
+    logger.error('Could not write PID to file', e);
+  }
+}
+
+logger.log('App startup, PID', process?.pid);
+
+logger.log('Debug log enabled', Logger.isDebugEnabled());
 TimelineReader.init();
 
-export const handleError = (({ error }) => {
+process.on('sveltekit:shutdown', (reason) => {
-  if (error instanceof Error) {
+  close();
-    log.error('Something went wrong: ', error.name, error.message);
+  logger.log('Shutting down', reason);
+  process.exit(0);
+});
+
+export const handleError = (({ error, status }) => {
+  if (error instanceof Error && status !== 404) {
+    logger.error('Something went wrong:', error.name, error.message);
   }
 
   return {
@@ -18,6 +37,12 @@ export const handleError = (({ error }) => {
 }) satisfies HandleServerError;
 
 export const handle = (async ({ event, resolve }) => {
+  const searchParams = event.url.searchParams;
+  const authCode = searchParams.get('code');
+  if (authCode) {
+    logger.debug('received GET hook', event.url.searchParams);
+  }
+
   // Reeder *insists* on checking /feed instead of /feed.xml
   if (event.url.pathname === '/feed') {
     return new Response('', { status: 301, headers: { Location: '/feed.xml' } });
@@ -39,7 +64,7 @@ export const handle = (async ({ event, resolve }) => {
       const readStream = fd
         .readableWebStream()
         .getReader({ mode: 'byob' }) as ReadableStream<Uint8Array>;
-      log.info('sending. size: ', stat.size);
+      logger.info('sending. size: ', stat.size);
       return new Response(readStream, {
         headers: [
           ['Content-Type', 'image/' + suffix],
@@ -51,7 +76,7 @@ export const handle = (async ({ event, resolve }) => {
       const f = await fs.readFile('avatars/' + fileName);
       return new Response(f, { headers: [['Content-Type', 'image/' + suffix]] });
     } catch (e) {
-      log.error('no stream', e);
+      logger.error('no stream', e);
       error(404);
     }
   }

src/lib/components/AvatarComponent.svelte

@@ -16,7 +16,7 @@
       ['jpg', 99],
       ['jpeg', 99]
     ]);
-    const resizedAvatars = (account.resizedAvatars ?? []).sort((a, b) => {
+    const resizedAvatars = (account.resizedAvatars ?? []).toSorted((a, b) => {
       const extensionA = a.file.split('.').pop() ?? '';
       const extensionB = b.file.split('.').pop() ?? '';
       const prioA = formatPriority.get(extensionA) ?? 3;

src/lib/components/PostComponent.svelte

@@ -65,7 +65,7 @@
       ['jpg', 99],
       ['jpeg', 99]
     ]);
-    const thumbs = (song.resizedThumbnails ?? []).sort((a, b) => {
+    const thumbs = (song.resizedThumbnails ?? []).toSorted((a, b) => {
       const extensionA = a.file.split('.').pop() ?? '';
       const extensionB = b.file.split('.').pop() ?? '';
       const prioA = formatPriority.get(extensionA) ?? 3;

src/lib/log.ts

@@ -1,9 +1,13 @@
+import { DEBUG_LOG } from '$env/static/private';
 import { env } from '$env/dynamic/private';
 import { isTruthy } from '$lib/truthyString';
 const { DEV } = import.meta.env;
 
 export const enableVerboseLog = isTruthy(env.VERBOSE);
 
+/**
+ * @deprecated Use the new {@link Logger} class instead.
+ */
 export const log = {
   verbose: (...params: any[]) => {
     if (!enableVerboseLog) {
@@ -12,7 +16,7 @@ export const log = {
     console.debug(new Date().toISOString(), ...params);
   },
   debug: (...params: any[]) => {
-    if (!DEV) {
+    if (!log.isDebugEnabled()) {
       return;
     }
     console.debug(new Date().toISOString(), ...params);
@@ -28,5 +32,59 @@ export const log = {
   },
   error: (...params: any[]) => {
     console.error(new Date().toISOString(), ...params);
+  },
+  isDebugEnabled: (): boolean => {
+    return DEV;
   }
 };
+
+export class Logger {
+  public constructor(private name: string) {}
+
+  public static isDebugEnabled(): boolean {
+    return !!DEBUG_LOG || DEV || enableVerboseLog;
+  }
+  public verbose(...params: any[]) {
+    if (!enableVerboseLog) {
+      return;
+    }
+    console.debug(new Date().toISOString(), `- ${this.name} -`, '- [VRBSE] -', ...params);
+  }
+  public debug(...params: any[]) {
+    if (!Logger.isDebugEnabled()) {
+      return;
+    }
+    console.debug(new Date().toISOString(), `- ${this.name} -`, '- [DEBUG] -', ...params);
+  }
+  public log(...params: any[]) {
+    console.log(new Date().toISOString(), `- ${this.name} -`, '- [ LOG ] -', ...params);
+  }
+  public info(...params: any[]) {
+    console.info(new Date().toISOString(), `- ${this.name} -`, '- [INFO ] -', ...params);
+  }
+  public warn(...params: any[]) {
+    console.warn(new Date().toISOString(), `- ${this.name} -`, '- [WARN ] -', ...params);
+  }
+  public error(...params: any[]) {
+    console.error(new Date().toISOString(), `- ${this.name} -`, '- [ERROR] -', ...params);
+  }
+
+  public static error(...params: any[]) {
+    console.error(new Date().toISOString(), ...params);
+  }
+  public static debug(...params: any[]) {
+    if (!Logger.isDebugEnabled()) {
+      return;
+    }
+    console.debug(new Date().toISOString(), ...params);
+  }
+  public static log(...params: any[]) {
+    console.log(new Date().toISOString(), ...params);
+  }
+  public static info(...params: any[]) {
+    console.info(new Date().toISOString(), ...params);
+  }
+  public static warn(...params: any[]) {
+    console.warn(new Date().toISOString(), ...params);
+  }
+}

src/lib/mastodon/response.ts

@@ -16,6 +16,17 @@ export interface Post {
   songs?: SongInfo[];
 }
 
+export interface OauthResponse {
+  access_token: string;
+  expires_in: number;
+  expires?: Date;
+  refresh_token?: string;
+  refresh_token_expires_in?: number;
+  scope: string;
+  token_type: string;
+  error?: any;
+}
+
 export interface PreviewCard {
   url: string;
   title: string;

src/lib/odesliResponse.ts

@@ -3,6 +3,8 @@ import type { SongThumbnailImage } from '$lib/mastodon/response';
 export type SongInfo = {
   pageUrl: string;
   youtubeUrl?: string;
+  spotifyUrl?: string;
+  spotifyUri?: string;
   type: 'song' | 'album';
   title?: string;
   artistName?: string;

src/lib/server/db.ts

@@ -1,10 +1,12 @@
 import { IGNORE_USERS, MASTODON_INSTANCE } from '$env/static/private';
-import { enableVerboseLog, log } from '$lib/log';
+import { enableVerboseLog, Logger } from '$lib/log';
 import type { Account, AccountAvatar, Post, SongThumbnailImage, Tag } from '$lib/mastodon/response';
 import type { SongInfo } from '$lib/odesliResponse';
 import { TimelineReader } from '$lib/server/timeline';
 import sqlite3 from 'sqlite3';
 
+const logger = new Logger('Database');
+
 type FilterParameter = {
   $limit?: number | undefined | null;
   $since?: string | undefined | null;
@@ -37,6 +39,8 @@ type SongRow = {
   overviewUrl?: string;
   type: 'album' | 'song';
   youtubeUrl?: string;
+  spotifyUrl?: string;
+  spotifyUri?: string;
   title?: string;
   artistName?: string;
   thumbnailUrl?: string;
@@ -64,6 +68,15 @@ type Migration = {
 };
 
 const db: sqlite3.Database = new sqlite3.Database('moshingmammut.db');
+
+export function close() {
+  try {
+    db.close();
+  } catch (e) {
+    logger.error('Could not close DB');
+  }
+}
+
 // for the local masto instance, the instance name is *not* saved
 // as part of the username or acct, so it needs to be stripped
 const ignoredUsers: string[] =
@@ -81,15 +94,15 @@ let databaseReady = false;
 if (enableVerboseLog) {
   sqlite3.verbose();
   db.on('change', (t, d, table, rowid) => {
-    log.verbose('DB change event', t, d, table, rowid);
+    logger.verbose('DB change event', t, d, table, rowid);
   });
 
   db.on('trace', (sql) => {
-    log.verbose('Running', sql);
+    logger.verbose('Running', sql);
   });
 
   db.on('profile', (sql) => {
-    log.verbose('Finished', sql);
+    logger.verbose('Finished', sql);
   });
 }
 
@@ -97,7 +110,7 @@ function applyDbMigration(migration: Migration): Promise<void> {
   return new Promise((resolve, reject) => {
     db.exec(migration.statement, (err) => {
       if (err !== null) {
-        log.error(`Failed to apply migration ${migration.name}`, err);
+        logger.error(`Failed to apply migration ${migration.name}`, err);
         reject(err);
         return;
       }
@@ -118,31 +131,31 @@ async function applyMigration(migration: Migration) {
       if (post.songs && post.songs.length) {
         continue;
       }
-      log.info(
+      logger.info(
         `Fetching songs for existing post ${current.toString().padStart(4, '0')} of ${total}`,
         post.url
       );
-      const songs = await TimelineReader.getSongInfoInPost(post);
+      const songs = await TimelineReader.instance.getSongInfoInPost(post);
       await saveSongInfoData(post.url, songs);
-      log.debug(`Fetched ${songs.length} songs for existing post`, post.url);
+      logger.debug(`Fetched ${songs.length} songs for existing post`, post.url);
     }
-    log.debug(`Finished fetching songs`);
+    logger.debug(`Finished fetching songs`);
   } else {
     await applyDbMigration(migration);
   }
 }
 
 db.on('open', () => {
-  log.info('Opened database');
+  logger.info('Opened database');
   db.serialize();
   db.run('CREATE TABLE IF NOT EXISTS "migrations" ("id" integer,"name" TEXT, PRIMARY KEY (id))');
   db.all('SELECT id FROM migrations', (err, rows: Migration[]) => {
     if (err !== null) {
-      log.error('Could not fetch existing migrations', err);
+      logger.error('Could not fetch existing migrations', err);
       databaseReady = true;
       return;
     }
-    log.debug('Already applied migrations', rows);
+    logger.debug('Already applied migrations', rows);
     const appliedMigrations: Set<number> = new Set(rows.map((row) => row['id']));
     const toApply = getMigrations().filter((m) => !appliedMigrations.has(m.id));
     let remaining = toApply.length;
@@ -159,7 +172,7 @@ db.on('open', () => {
           databaseReady = true;
         }
         if (err !== null) {
-          log.error(`Failed to apply migration ${migration.name}`, err);
+          logger.error(`Failed to apply migration ${migration.name}`, err);
           return;
         }
         db.run(
@@ -167,10 +180,10 @@ db.on('open', () => {
           [migration.id, migration.name],
           (e: Error) => {
             if (e !== null) {
-              log.error(`Failed to mark migration ${migration.name} as applied`, e);
+              logger.error(`Failed to mark migration ${migration.name} as applied`, e);
               return;
             }
-            log.info(`Applied migration ${migration.name}`);
+            logger.info(`Applied migration ${migration.name}`);
           }
         );
       });
@@ -178,7 +191,7 @@ db.on('open', () => {
   });
 });
 db.on('error', (err) => {
-  log.error('Error opening database', err);
+  logger.error('Error opening database', err);
 });
 
 function getMigrations(): Migration[] {
@@ -313,6 +326,13 @@ function getMigrations(): Migration[] {
       statement: `
       ALTER TABLE songs ADD COLUMN thumbnailWidth INTEGER NULL;
       ALTER TABLE songs ADD COLUMN thumbnailHeight INTEGER NULL;`
+    },
+    {
+      id: 8,
+      name: 'song spotify url/uri',
+      statement: `
+      ALTER TABLE songs ADD COLUMN spotifyUrl TEXT NULL;
+      ALTER TABLE songs ADD COLUMN spotifyUri TEXT NULL;`
     }
   ];
 }
@@ -321,9 +341,9 @@ async function waitReady(): Promise<void> {
   // Simpler than a semaphore and is really only needed on startup
   return new Promise((resolve) => {
     const interval = setInterval(() => {
-      log.verbose('Waiting for database to be ready');
+      logger.verbose('Waiting for database to be ready');
       if (databaseReady) {
-        log.verbose('DB is ready');
+        logger.verbose('DB is ready');
         clearInterval(interval);
         resolve();
       }
@@ -354,7 +374,7 @@ function saveAccountData(account: Account): Promise<void> {
       ],
       (err) => {
         if (err !== null) {
-          log.error(`Could not insert/update account ${account.id}`, err);
+          logger.error(`Could not insert/update account ${account.id}`, err);
           reject(err);
           return;
         }
@@ -377,7 +397,7 @@ function savePostData(post: Post): Promise<void> {
       [post.id, post.content, post.created_at, post.url, post.account.url],
       (postErr) => {
         if (postErr !== null) {
-          log.error(`Could not insert post ${post.url}`, postErr);
+          logger.error(`Could not insert post ${post.url}`, postErr);
           reject(postErr);
           return;
         }
@@ -405,7 +425,7 @@ function savePostTagData(post: Post): Promise<void> {
           [tag.url, tag.name],
           (tagErr) => {
             if (tagErr !== null) {
-              log.error(`Could not insert/update tag ${tag.url}`, tagErr);
+              logger.error(`Could not insert/update tag ${tag.url}`, tagErr);
               reject(tagErr);
               return;
             }
@@ -414,7 +434,7 @@ function savePostTagData(post: Post): Promise<void> {
               [post.url, tag.url],
               (posttagserr) => {
                 if (posttagserr !== null) {
-                  log.error(`Could not insert poststags ${tag.url}, ${post.url}`, posttagserr);
+                  logger.error(`Could not insert poststags ${tag.url}, ${post.url}`, posttagserr);
                   reject(posttagserr);
                   return;
                 }
@@ -444,14 +464,16 @@ function saveSongInfoData(postUrl: string, songs: SongInfo[]): Promise<void> {
       for (const song of songs) {
         db.run(
           `
-          INSERT INTO songs (postedUrl, overviewUrl, type, youtubeUrl, title, artistName, thumbnailUrl, post_url, thumbnailWidth, thumbnailHeight)
+          INSERT INTO songs (postedUrl, overviewUrl, type, youtubeUrl, spotifyUrl, spotifyUri, title, artistName, thumbnailUrl, post_url, thumbnailWidth, thumbnailHeight)
-          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
           `,
           [
             song.postedUrl,
             song.pageUrl,
             song.type,
             song.youtubeUrl,
+            song.spotifyUrl,
+            song.spotifyUri,
             song.title,
             song.artistName,
             song.thumbnailUrl,
@@ -461,7 +483,7 @@ function saveSongInfoData(postUrl: string, songs: SongInfo[]): Promise<void> {
           ],
           (songErr) => {
             if (songErr !== null) {
-              log.error(`Could not insert song ${song.postedUrl}`, songErr);
+              logger.error(`Could not insert song ${song.postedUrl}`, songErr);
               reject(songErr);
               return;
             }
@@ -479,20 +501,20 @@ function saveSongInfoData(postUrl: string, songs: SongInfo[]): Promise<void> {
 }
 
 export async function savePost(post: Post, songs: SongInfo[]) {
-  log.debug(`Saving post ${post.url}`);
+  logger.debug(`Saving post ${post.url}`);
   if (!databaseReady) {
     await waitReady();
   }
 
   const account = post.account;
   await saveAccountData(account);
-  log.debug(`Saved account data ${post.url}`);
+  logger.debug(`Saved account data ${post.url}`);
   await savePostData(post);
-  log.debug(`Saved post data ${post.url}`);
+  logger.debug(`Saved post data ${post.url}`);
   await savePostTagData(post);
-  log.debug(`Saved ${post.tags.length} tag data ${post.url}`);
+  logger.debug(`Saved ${post.tags.length} tag data ${post.url}`);
   await saveSongInfoData(post.url, songs);
-  log.debug(
+  logger.debug(
     `Saved ${songs.length} song info data ${post.url}`,
     songs.map((s) => s.thumbnailHeight)
   );
@@ -511,7 +533,7 @@ function getPostData(filterQuery: string, params: FilterParameter): Promise<Post
   return new Promise((resolve, reject) => {
     db.all(sql, params, (err, rows: PostRow[]) => {
       if (err != null) {
-        log.error('Error loading posts', err);
+        logger.error('Error loading posts', err);
         reject(err);
         return;
       }
@@ -530,7 +552,7 @@ function getTagData(postIdsParams: string, postIds: string[]): Promise<Map<strin
       postIds,
       (tagErr, tagRows: PostTagRow[]) => {
         if (tagErr != null) {
-          log.error('Error loading post tags', tagErr);
+          logger.error('Error loading post tags', tagErr);
           reject(tagErr);
           return;
         }
@@ -551,14 +573,14 @@ function getTagData(postIdsParams: string, postIds: string[]): Promise<Map<strin
 function getSongData(postIdsParams: string, postIds: string[]): Promise<Map<string, SongInfo[]>> {
   return new Promise((resolve, reject) => {
     db.all(
-      `SELECT post_url, songs.postedUrl, songs.overviewUrl, songs.type, songs.youtubeUrl,
+      `SELECT post_url, songs.postedUrl, songs.overviewUrl, songs.type, songs.youtubeUrl, songs.spotifyUri, songs.spotifyUri,
       songs.title, songs.artistName, songs.thumbnailUrl, songs.post_url, songs.thumbnailWidth, songs.thumbnailHeight
       FROM songs
       WHERE post_url IN (${postIdsParams});`,
       postIds,
       (tagErr, tagRows: SongRow[]) => {
         if (tagErr != null) {
-          log.error('Error loading post songs', tagErr);
+          logger.error('Error loading post songs', tagErr);
           reject(tagErr);
           return;
         }
@@ -567,6 +589,8 @@ function getSongData(postIdsParams: string, postIds: string[]): Promise<Map<stri
             const info = {
               pageUrl: item.overviewUrl,
               youtubeUrl: item.youtubeUrl,
+              spotifyUrl: item.spotifyUrl,
+              spotifyUri: item.spotifyUri,
               type: item.type,
               title: item.title,
               artistName: item.artistName,
@@ -580,7 +604,7 @@ function getSongData(postIdsParams: string, postIds: string[]): Promise<Map<stri
           },
           new Map()
         );
-        log.verbose('songMap', songMap);
+        logger.verbose('songMap', songMap);
         resolve(songMap);
       }
     );
@@ -599,7 +623,7 @@ function getAvatarData(
       accountUrls,
       (err, rows: AccountAvatarRow[]) => {
         if (err != null) {
-          log.error('Error loading avatars', err);
+          logger.error('Error loading avatars', err);
           reject(err);
           return;
         }
@@ -633,7 +657,7 @@ function getSongThumbnailData(
       thumbUrls,
       (err, rows: SongThumbnailAvatarRow[]) => {
         if (err != null) {
-          log.error('Error loading avatars', err);
+          logger.error('Error loading avatars', err);
           reject(err);
           return;
         }

src/lib/server/playlist/oauthPlaylistAdder.ts

@@ -0,0 +1,166 @@
+import { BASE_URL } from '$env/static/private';
+import { Logger } from '$lib/log';
+import type { OauthResponse } from '$lib/mastodon/response';
+import fs from 'fs/promises';
+
+export abstract class OauthPlaylistAdder {
+  /// How many minutes before expiry the token will be refreshed
+  protected refresh_time: number = 15;
+  protected logger: Logger = new Logger('OauthPlaylistAdder');
+  protected redirectUri?: URL;
+
+  protected constructor(
+    protected apiBase: string,
+    protected token_file_name: string
+  ) {}
+
+  public async authCodeExists(): Promise<boolean> {
+    try {
+      const token = await this.auth();
+      return token !== null && !token.error;
+    } catch {
+      this.logger.info('No auth token yet, authorizing...');
+      return false;
+    }
+  }
+
+  protected getRedirectUri(suffix: string): URL {
+    const uri = this.redirectUri ?? new URL(`${BASE_URL}/${suffix}`);
+    this.logger.debug('getRedirectUri', uri);
+    return uri;
+  }
+
+  protected constructAuthUrlInternal(
+    endpointUrl: string,
+    clientId: string,
+    scope: string,
+    redirectUri: URL,
+    additionalParameters: Map<string, string> = new Map()
+  ): URL {
+    const authUrl = new URL(endpointUrl);
+    authUrl.searchParams.append('client_id', clientId);
+    authUrl.searchParams.append('redirect_uri', redirectUri.toString());
+    authUrl.searchParams.append('response_type', 'code');
+    authUrl.searchParams.append('scope', scope);
+    for (let p of additionalParameters.entries()) {
+      authUrl.searchParams.append(p[0], p[1]);
+    }
+    return authUrl;
+  }
+
+  public async receivedAuthCodeInternal(
+    tokenUrl: URL,
+    clientId: string,
+    code: string,
+    redirectUri: URL,
+    client_secret?: string,
+    customHeader?: HeadersInit
+  ) {
+    this.logger.debug('received code');
+    const params = new URLSearchParams();
+    params.append('client_id', clientId);
+    params.append('code', code);
+    params.append('grant_type', 'authorization_code');
+    params.append('redirect_uri', redirectUri.toString());
+    if (client_secret) {
+      params.append('client_secret', client_secret);
+    }
+    this.logger.debug('sending token req', params);
+    const resp: OauthResponse = await fetch(tokenUrl, {
+      method: 'POST',
+      body: params,
+      headers: customHeader
+    }).then((r) => r.json());
+    this.logger.debug('received access token', resp);
+    let expiration = new Date();
+    expiration.setTime(expiration.getTime() + resp.expires_in * 1000);
+    expiration.setSeconds(expiration.getSeconds() + resp.expires_in);
+    resp.expires = expiration;
+    await fs.writeFile(this.token_file_name, JSON.stringify(resp));
+  }
+
+  protected async auth(): Promise<OauthResponse | null> {
+    try {
+      const token_file = await fs.readFile(this.token_file_name, { encoding: 'utf8' });
+      let token = JSON.parse(token_file);
+      if (token.expires) {
+        if (typeof token.expires === typeof '') {
+          token.expires = new Date(token.expires);
+        }
+      }
+      return token;
+    } catch (e) {
+      this.logger.error('Could not read access token', e);
+      return null;
+    }
+  }
+
+  protected async shouldRefreshToken(): Promise<{ token: OauthResponse; refresh: boolean } | null> {
+    const token = await this.auth();
+    if (token == null || !token?.expires) {
+      return null;
+    }
+    let refreshAt = new Date();
+    refreshAt.setTime(refreshAt.getTime() - this.refresh_time * 60 * 1000);
+    this.logger.info('token expiry', token.expires, 'vs refresh @', refreshAt);
+    if (token.expires.getTime() > refreshAt.getTime()) {
+      return {
+        token: token,
+        refresh: false
+      };
+    }
+
+    this.logger.info(
+      'Token expires',
+      token.expires,
+      token.expires.getTime(),
+      `which is after the refresh time`,
+      refreshAt,
+      refreshAt.getTime()
+    );
+    return {
+      token: token,
+      refresh: true
+    };
+  }
+
+  protected async requestRefreshToken(
+    tokenUrl: URL,
+    clientId: string,
+    refresh_token: string,
+    redirect_uri?: string,
+    client_secret?: string,
+    customHeader?: HeadersInit
+  ) {
+    const params = new URLSearchParams();
+    params.append('client_id', clientId);
+    params.append('grant_type', 'refresh_token');
+    params.append('refresh_token', refresh_token);
+    if (client_secret) {
+      params.append('client_secret', client_secret);
+    }
+    if (redirect_uri) {
+      params.append('redirect_uri', redirect_uri);
+    }
+    this.logger.debug('sending token req', params);
+    const resp: OauthResponse = await fetch(tokenUrl, {
+      method: 'POST',
+      body: params,
+      headers: customHeader
+    }).then((r) => r.json());
+    this.logger.verbose('received access token', resp);
+    if (resp.error) {
+      this.logger.error('token resp error', resp);
+      return null;
+    }
+    if (!resp.refresh_token) {
+      resp.refresh_token = refresh_token;
+    }
+    let expiration = new Date();
+    expiration.setTime(expiration.getTime() + resp.expires_in * 1000);
+    expiration.setSeconds(expiration.getSeconds() + resp.expires_in);
+    resp.expires = expiration;
+    await fs.writeFile(this.token_file_name, JSON.stringify(resp));
+    return resp;
+  }
+}

src/lib/server/playlist/playlistAdder.ts

@@ -0,0 +1,5 @@
+import type { SongInfo } from '$lib/odesliResponse';
+
+export interface PlaylistAdder {
+  addToPlaylist(song: SongInfo): Promise<void>;
+}

src/lib/server/playlist/spotifyPlaylistAdder.ts

@@ -0,0 +1,124 @@
+import { SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET, SPOTIFY_PLAYLIST_ID } from '$env/static/private';
+import { Logger } from '$lib/log';
+import type { OauthResponse } from '$lib/mastodon/response';
+import type { SongInfo } from '$lib/odesliResponse';
+import { OauthPlaylistAdder } from './oauthPlaylistAdder';
+import type { PlaylistAdder } from './playlistAdder';
+
+export class SpotifyPlaylistAdder extends OauthPlaylistAdder implements PlaylistAdder {
+  public constructor() {
+    super('https://api.spotify.com/v1', 'spotify_auth_token');
+    this.logger = new Logger('SpotifyPlaylistAdder');
+  }
+
+  public constructAuthUrl(redirectUri: URL): URL {
+    const endpoint = 'https://accounts.spotify.com/authorize';
+    return this.constructAuthUrlInternal(
+      endpoint,
+      SPOTIFY_CLIENT_ID,
+      'playlist-modify-private playlist-modify-public',
+      redirectUri
+    );
+  }
+
+  public async receivedAuthCode(code: string, url: URL) {
+    this.logger.debug('received code');
+    const authHeader =
+      'Basic ' + Buffer.from(SPOTIFY_CLIENT_ID + ':' + SPOTIFY_CLIENT_SECRET).toString('base64');
+
+    const tokenUrl = new URL('https://accounts.spotify.com/api/token');
+    await this.receivedAuthCodeInternal(tokenUrl, SPOTIFY_CLIENT_ID, code, url, undefined, {
+      Authorization: authHeader
+    });
+  }
+
+  private async refreshToken(force: boolean = false): Promise<OauthResponse | null> {
+    const tokenInfo = await this.shouldRefreshToken();
+    if (tokenInfo == null) {
+      return null;
+    }
+    let token = tokenInfo.token;
+    if (!tokenInfo.refresh && !force) {
+      return token;
+    }
+
+    if (!token.refresh_token) {
+      this.logger.error('Need to refresh access token, but no refresh token provided');
+      return null;
+    }
+
+    const authHeader =
+      'Basic ' + Buffer.from(SPOTIFY_CLIENT_ID + ':' + SPOTIFY_CLIENT_SECRET).toString('base64');
+    const tokenUrl = new URL('https://accounts.spotify.com/api/token');
+    return await this.requestRefreshToken(
+      tokenUrl,
+      SPOTIFY_CLIENT_ID,
+      token.refresh_token,
+      undefined,
+      undefined,
+      { Authorization: authHeader }
+    );
+  }
+
+  private async addToPlaylistRetry(song: SongInfo, remaning: number = 3) {
+    if (remaning < 0) {
+      this.logger.error('max retries reached, song will not be added to spotify playlist');
+    }
+    this.logger.debug('addToSpotifyPlaylist', remaning);
+    const token = await this.refreshToken();
+    if (token == null) {
+      return;
+    }
+
+    if (!SPOTIFY_PLAYLIST_ID || SPOTIFY_PLAYLIST_ID === 'CHANGE_ME') {
+      this.logger.debug('no spotify playlist ID configured');
+      return;
+    }
+    if (!song.spotifyUri) {
+      this.logger.info('Skip adding song to spotify playlist, no Uri', song);
+      return;
+    }
+
+    // TODO. Spotify's check for "is this song already in the playlist" is... ugh
+    /*
+    const playlistItemsUrl = new URL(`${this.apiBase}/playlists/${SPOTIFY_PLAYLIST_ID}/tracks`);
+    playlistItemsUrl.searchParams.append('videoId', youtubeId);
+    playlistItemsUrl.searchParams.append('playlistId', SPOTIFY_PLAYLIST_ID);
+    playlistItemsUrl.searchParams.append('part', 'id');*/
+    /*const existingPlaylistItem = await fetch(this.apiBase + '/playlistItems', {
+      headers: { Authorization: `${token.token_type} ${token.access_token}` }
+    }).then((r) => r.json());
+    logger.debug('existingPlaylistItem', existingPlaylistItem);
+    if (existingPlaylistItem.pageInfo && existingPlaylistItem.pageInfo.totalResults > 0) {
+      logger.info('Item already in playlist');
+      return;
+    }*/
+
+    //const searchParams = new URLSearchParams([['part', 'snippet']]);
+    const options: RequestInit = {
+      method: 'POST',
+      headers: { Authorization: `${token.token_type} ${token.access_token}` },
+      body: JSON.stringify({
+        uris: [song.spotifyUri]
+      })
+    };
+    const apiUrl = new URL(`${this.apiBase}/playlists/${SPOTIFY_PLAYLIST_ID}/tracks`);
+    const resp = await fetch(apiUrl, options);
+    const respObj = await resp.json();
+    if (respObj.error) {
+      this.logger.debug('Add to playlist failed', song.spotifyUri, respObj.error);
+      if (respObj.error.status === 401) {
+        const token = await this.refreshToken(true);
+        if (token == null) {
+          return;
+        }
+        this.addToPlaylistRetry(song, remaning--);
+      }
+    } else {
+      this.logger.info('Added to playlist', song.spotifyUri, song.title);
+    }
+  }
+  public async addToPlaylist(song: SongInfo) {
+    await this.addToPlaylistRetry(song);
+  }
+}

src/lib/server/playlist/ytPlaylistAdder.ts

@@ -0,0 +1,155 @@
+import { YOUTUBE_CLIENT_ID, YOUTUBE_CLIENT_SECRET, YOUTUBE_PLAYLIST_ID } from '$env/static/private';
+import { Logger } from '$lib/log';
+import type { OauthResponse } from '$lib/mastodon/response';
+import type { SongInfo } from '$lib/odesliResponse';
+import { OauthPlaylistAdder } from './oauthPlaylistAdder';
+import type { PlaylistAdder } from './playlistAdder';
+
+export class YoutubePlaylistAdder extends OauthPlaylistAdder implements PlaylistAdder {
+  public constructor() {
+    super('https://www.googleapis.com/youtube/v3', 'yt_auth_token');
+    this.logger = new Logger('YoutubePlaylistAdder');
+  }
+
+  public constructAuthUrl(redirectUri: URL): URL {
+    this.redirectUri = redirectUri;
+    let additionalParameters = new Map([
+      ['access_type', 'offline'],
+      ['include_granted_scopes', 'false']
+    ]);
+    const endpoint = 'https://accounts.google.com/o/oauth2/v2/auth';
+    return this.constructAuthUrlInternal(
+      endpoint,
+      YOUTUBE_CLIENT_ID,
+      'https://www.googleapis.com/auth/youtube',
+      redirectUri,
+      additionalParameters
+    );
+  }
+
+  public async receivedAuthCode(code: string, url: URL) {
+    this.logger.debug('received code');
+    this.redirectUri = url;
+    const tokenUrl = new URL('https://oauth2.googleapis.com/token');
+    await this.receivedAuthCodeInternal(
+      tokenUrl,
+      YOUTUBE_CLIENT_ID,
+      code,
+      url,
+      YOUTUBE_CLIENT_SECRET
+    );
+    const token = await this.refreshToken();
+    if (token == null) {
+      return;
+    }
+    try {
+      this.logger.debug('Checking authorized token');
+      const res = await fetch(this.apiBase + '/channels?part=id&mine=true', {
+        method: 'GET',
+        headers: { Authorization: `${token.token_type} ${token.access_token}` }
+      }).then((r) => r.json());
+      this.logger.debug('Checked authorized token', res);
+    } catch (e) {
+      this.logger.debug('Error checking authorized token', e);
+    }
+  }
+
+  private async refreshToken(force: boolean = false): Promise<OauthResponse | null> {
+    const tokenInfo = await this.shouldRefreshToken();
+    if (tokenInfo == null) {
+      return null;
+    }
+    let token = tokenInfo.token;
+    if (!tokenInfo.refresh && !force) {
+      return token;
+    }
+    if (!token.refresh_token) {
+      this.logger.error('Need to refresh access token, but no refresh token provided');
+      return null;
+    }
+
+    const tokenUrl = new URL('https://oauth2.googleapis.com/token');
+    return await this.requestRefreshToken(
+      tokenUrl,
+      YOUTUBE_CLIENT_ID,
+      token.refresh_token,
+      this.getRedirectUri('ytauth').toString(),
+      YOUTUBE_CLIENT_SECRET
+    );
+  }
+
+  public async addToPlaylist(song: SongInfo) {}
+
+  private async addToPlaylistRetry(song: SongInfo, remaning: number = 3) {
+    if (remaning < 0) {
+      this.logger.error('max retries reached, song will not be added to spotify playlist');
+    }
+    this.logger.debug('addToYoutubePlaylist');
+    const token = await this.refreshToken();
+    if (token == null) {
+      return;
+    }
+
+    if (!YOUTUBE_PLAYLIST_ID || YOUTUBE_PLAYLIST_ID === 'CHANGE_ME') {
+      this.logger.debug('no playlist ID configured');
+      return;
+    }
+    if (!song.youtubeUrl) {
+      this.logger.info('Skip adding song to YT playlist, no youtube Url', song);
+      return;
+    }
+
+    const songUrl = new URL(song.youtubeUrl);
+    const youtubeId = songUrl.searchParams.get('v');
+    if (!youtubeId) {
+      this.logger.debug(
+        'Skip adding song to YT playlist, could not extract YT id from URL',
+        song.youtubeUrl
+      );
+      return;
+    }
+    this.logger.debug('Found YT id from URL', song.youtubeUrl, youtubeId);
+
+    const playlistItemsUrl = new URL(this.apiBase + '/playlistItems');
+    playlistItemsUrl.searchParams.append('videoId', youtubeId);
+    playlistItemsUrl.searchParams.append('playlistId', YOUTUBE_PLAYLIST_ID);
+    playlistItemsUrl.searchParams.append('part', 'id');
+    const existingPlaylistItem = await fetch(playlistItemsUrl, {
+      headers: { Authorization: `${token.token_type} ${token.access_token}` }
+    }).then((r) => r.json());
+    if (existingPlaylistItem.pageInfo && existingPlaylistItem.pageInfo.totalResults > 0) {
+      this.logger.info('Item already in playlist', existingPlaylistItem);
+      return;
+    }
+
+    const addItemUrl = new URL(this.apiBase + '/playlistItems');
+    addItemUrl.searchParams.append('part', 'snippet');
+    const options: RequestInit = {
+      method: 'POST',
+      headers: { Authorization: `${token.token_type} ${token.access_token}` },
+      body: JSON.stringify({
+        snippet: {
+          playlistId: YOUTUBE_PLAYLIST_ID,
+          resourceId: {
+            videoId: youtubeId,
+            kind: 'youtube#video'
+          }
+        }
+      })
+    };
+    const resp = await fetch(addItemUrl, options);
+    const respObj = await resp.json();
+    this.logger.info('Added to playlist', youtubeId, song.title);
+    if (respObj.error) {
+      this.logger.error('Add to playlist failed', respObj.error.errors);
+      if (respObj.error.errors && respObj.error.errors[0].reason === 'authError') {
+        this.logger.info('Refreshing auth token');
+        const token = await this.refreshToken(true);
+        if (token == null) {
+          return;
+        }
+        this.addToPlaylistRetry(song, remaning--);
+      }
+    }
+  }
+}

src/lib/server/rss.ts

@@ -1,10 +1,12 @@
 import { BASE_URL, WEBSUB_HUB } from '$env/static/private';
 import { PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME } from '$env/static/public';
 import type { Post } from '$lib//mastodon/response';
-import { log } from '$lib/log';
+import { Logger } from '$lib/log';
 import { Feed } from 'feed';
 import fs from 'fs/promises';
 
+const logger = new Logger('RSS');
+
 export function createFeed(posts: Post[]): Feed {
   const baseUrl = BASE_URL.endsWith('/') ? BASE_URL : BASE_URL + '/';
   const hub = WEBSUB_HUB ? WEBSUB_HUB : undefined;
@@ -60,6 +62,6 @@ export async function saveAtomFeed(feed: Feed) {
       body: params
     });
   } catch (e) {
-    log.error('Failed to update WebSub hub', e);
+    logger.error('Failed to update WebSub hub', e);
   }
 }

src/lib/server/timeline.ts

@@ -2,10 +2,11 @@ import {
   HASHTAG_FILTER,
   MASTODON_ACCESS_TOKEN,
   MASTODON_INSTANCE,
+  IGNORE_USERS,
   ODESLI_API_KEY,
   YOUTUBE_API_KEY
 } from '$env/static/private';
-import { log } from '$lib/log';
+import { Logger } from '$lib/log';
 import type {
   Account,
   AccountAvatar,
@@ -25,12 +26,17 @@ import {
   savePost,
   saveSongThumbnail
 } from '$lib/server/db';
+import { SpotifyPlaylistAdder } from '$lib/server/playlist/spotifyPlaylistAdder';
+import { YoutubePlaylistAdder } from '$lib/server/playlist/ytPlaylistAdder';
 import { createFeed, saveAtomFeed } from '$lib/server/rss';
 import { sleep } from '$lib/sleep';
 import crypto from 'crypto';
 import fs from 'fs/promises';
+import { console } from 'inspector/promises';
 import sharp from 'sharp';
+import { URL, URLSearchParams } from 'url';
 import { WebSocket } from 'ws';
+import type { PlaylistAdder } from './playlist/playlistAdder';
 
 const URL_REGEX = new RegExp(/href="(?<postUrl>[^>]+?)" target="_blank"/gm);
 const INVIDIOUS_REGEX = new RegExp(/invidious.*?watch.*?v=(?<videoId>[a-zA-Z_0-9-]+)/gm);
@@ -40,10 +46,15 @@ const YOUTUBE_REGEX = new RegExp(
 
 export class TimelineReader {
   private static _instance: TimelineReader;
+  private lastPosts: string[] = [];
+  private playlistAdders: PlaylistAdder[];
+  private logger: Logger;
+  private ignoredUsers: string[];
 
-  private static async isMusicVideo(videoId: string) {
+  private async isMusicVideo(videoId: string) {
     if (!YOUTUBE_API_KEY || YOUTUBE_API_KEY === 'CHANGE_ME') {
       // Assume that it *is* a music link when no YT API key is provided
+      this.logger.debug('YT API not configured');
       return true;
     }
     const searchParams = new URLSearchParams([
@@ -55,13 +66,13 @@ export class TimelineReader {
     const resp = await fetch(youtubeVideoUrl);
     const respObj = await resp.json();
     if (!respObj.items.length) {
-      console.warn('Could not find video with id', videoId);
+      this.logger.warn('Could not find video with id', videoId);
       return false;
     }
 
     const item = respObj.items[0];
     if (!item.snippet) {
-      console.warn('Could not load snippet for video', videoId, item);
+      this.logger.warn('Could not load snippet for video', videoId, item);
       return false;
     }
     if (item.snippet.tags?.includes('music')) {
@@ -79,15 +90,19 @@ export class TimelineReader {
     const categoryTitle: string = await fetch(youtubeCategoryUrl)
       .then((r) => r.json())
       .then((r) => r.items[0]?.snippet?.title);
+    this.logger.debug('YT category', categoryTitle);
     return categoryTitle === 'Music';
   }
 
-  public static async getSongInfoInPost(post: Post): Promise<SongInfo[]> {
+  public async getSongInfoInPost(post: Post): Promise<SongInfo[]> {
     const urlMatches = post.content.matchAll(URL_REGEX);
     const songs: SongInfo[] = [];
     for (const match of urlMatches) {
       if (match === undefined || match.groups === undefined) {
-        log.warn('Match listed in allMatches, but either it or its groups are undefined', match);
+        this.logger.warn(
+          'Match listed in allMatches, but either it or its groups are undefined',
+          match
+        );
         continue;
       }
       const urlMatch = match.groups.postUrl.toString();
@@ -95,14 +110,14 @@ export class TimelineReader {
       try {
         url = new URL(urlMatch);
       } catch (e) {
-        log.error('URL found via Regex does not seem to be a valud url', urlMatch, e);
+        this.logger.error('URL found via Regex does not seem to be a valud url', urlMatch, e);
         continue;
       }
 
       // Check *all* found url and let odesli determine if it is music or not
-      log.debug(`Checking ${url} if it contains song data`);
+      this.logger.debug(`Checking ${url} if it contains song data`);
-      const info = await TimelineReader.getSongInfo(url);
+      const info = await this.getSongInfo(url);
-      log.debug(`Found song info for ${url}?`, info);
+      //this.logger.debug(`Found song info for ${url}?`, info);
       if (info) {
         songs.push(info);
       }
@@ -110,9 +125,9 @@ export class TimelineReader {
     return songs;
   }
 
-  private static async getSongInfo(url: URL, remainingTries = 6): Promise<SongInfo | null> {
+  private async getSongInfo(url: URL, remainingTries = 6): Promise<SongInfo | null> {
     if (remainingTries === 0) {
-      log.error('No tries remaining. Lookup failed!');
+      this.logger.error('No tries remaining. Lookup failed!');
       return null;
     }
     if (url.hostname === 'songwhip.com') {
@@ -144,6 +159,7 @@ export class TimelineReader {
         return null;
       }
       const info = odesliInfo.entitiesByUniqueId[odesliInfo.entityUniqueId];
+      //this.logger.debug('odesli response', info);
       const platform: Platform = 'youtube';
       if (info.platforms.includes(platform)) {
         const youtubeId =
@@ -151,33 +167,50 @@ export class TimelineReader {
           YOUTUBE_REGEX.exec(url.href)?.groups?.videoId ??
           new URL(odesliInfo.pageUrl).pathname.split('/y/').pop();
         if (youtubeId === undefined) {
-          log.warn('Looks like a youtube video, but could not extract a video id', url, odesliInfo);
+          this.logger.warn(
+            'Looks like a youtube video, but could not extract a video id',
+            url,
+            odesliInfo
+          );
           return null;
         }
-        const isMusic = await TimelineReader.isMusicVideo(youtubeId);
+        const isMusic = await this.isMusicVideo(youtubeId);
         if (!isMusic) {
-          log.debug('Probably not a music video', url);
+          this.logger.debug('Probably not a music video', youtubeId, url);
           return null;
         }
       }
-      return {
+      const spotify: Platform = 'spotify';
+      const songInfo = {
         ...info,
         pageUrl: odesliInfo.pageUrl,
         youtubeUrl: odesliInfo.linksByPlatform[platform]?.url,
+        spotifyUrl: odesliInfo.linksByPlatform[spotify]?.url,
+        spotifyUri: odesliInfo.linksByPlatform[spotify]?.nativeAppUriDesktop,
         postedUrl: url.toString()
       } as SongInfo;
+      if (songInfo.youtubeUrl && !songInfo.spotifyUrl) {
+        this.logger.warn('SongInfo with YT, but no spotify URL', odesliInfo);
+      }
+      return songInfo;
     } catch (e) {
       if (e instanceof Error && e.cause === 429) {
-        log.warn('song.link rate limit reached. Trying again in 10 seconds');
+        this.logger.warn('song.link rate limit reached. Trying again in 10 seconds');
         await sleep(10_000);
         return await this.getSongInfo(url, remainingTries - 1);
       }
-      log.error(`Failed to load ${url} info from song.link`, e);
+      this.logger.error(`Failed to load ${url} info from song.link`, e);
       return null;
     }
   }
 
-  private static async resizeAvatar(
+  private async addToPlaylist(song: SongInfo) {
+    for (let adder of this.playlistAdders) {
+      await adder.addToPlaylist(song);
+    }
+  }
+
+  private async resizeAvatar(
     baseName: string,
     size: number,
     suffix: string,
@@ -190,15 +223,15 @@ export class TimelineReader {
       .then(() => true)
       .catch(() => false);
     if (exists) {
-      log.debug('File already exists', fileName);
+      this.logger.debug('File already exists', fileName);
       return null;
     }
-    log.debug('Saving avatar', fileName);
+    this.logger.debug('Saving avatar', fileName);
     await sharpAvatar.resize(size).toFile(fileName);
     return fileName;
   }
 
-  private static resizeAvatarPromiseMaker(
+  private resizeAvatarPromiseMaker(
     avatarFilenameBase: string,
     baseSize: number,
     maxPixelDensity: number,
@@ -211,13 +244,7 @@ export class TimelineReader {
     for (let i = 1; i <= maxPixelDensity; i++) {
       promises.push(
         ...formats.map((f) =>
-          TimelineReader.resizeAvatar(
+          this.resizeAvatar(avatarFilenameBase, baseSize * i, `${i}x.${f}`, 'avatars', sharpAvatar)
-            avatarFilenameBase,
-            baseSize * i,
-            `${i}x.${f}`,
-            'avatars',
-            sharpAvatar
-          )
             .then(
               (fn) =>
                 ({
@@ -233,7 +260,7 @@ export class TimelineReader {
     return promises;
   }
 
-  private static resizeThumbnailPromiseMaker(
+  private resizeThumbnailPromiseMaker(
     filenameBase: string,
     baseSize: number,
     maxPixelDensity: number,
@@ -247,13 +274,7 @@ export class TimelineReader {
     for (let i = 1; i <= maxPixelDensity; i++) {
       promises.push(
         ...formats.map((f) =>
-          TimelineReader.resizeAvatar(
+          this.resizeAvatar(filenameBase, baseSize * i, `${i}x.${f}`, 'thumbnails', sharpAvatar)
-            filenameBase,
-            baseSize * i,
-            `${i}x.${f}`,
-            'thumbnails',
-            sharpAvatar
-          )
             .then(
               (fn) =>
                 ({
@@ -270,7 +291,7 @@ export class TimelineReader {
     return promises;
   }
 
-  private static async saveAvatar(account: Account) {
+  private async saveAvatar(account: Account) {
     try {
       const existingAvatars = await getAvatars(account.url, 1);
       const existingAvatarBase = existingAvatars.shift()?.file.split('/').pop()?.split('_').shift();
@@ -283,7 +304,7 @@ export class TimelineReader {
         const avatarsToDelete = (await fs.readdir('avatars'))
           .filter((x) => x.startsWith(existingAvatarBase + '_'))
           .map((x) => {
-            log.debug('Removing existing avatar file', x);
+            this.logger.debug('Removing existing avatar file', x);
             return x;
           })
           .map((x) => fs.unlink('avatars/' + x));
@@ -292,7 +313,7 @@ export class TimelineReader {
       const avatarResponse = await fetch(account.avatar);
       const avatar = await avatarResponse.arrayBuffer();
       await Promise.all(
-        TimelineReader.resizeAvatarPromiseMaker(
+        this.resizeAvatarPromiseMaker(
           avatarFilenameBase,
           50,
           3,
@@ -306,7 +327,7 @@ export class TimelineReader {
     }
   }
 
-  private static async saveSongThumbnails(songs: SongInfo[]) {
+  private async saveSongThumbnails(songs: SongInfo[]) {
     for (const song of songs) {
       if (!song.thumbnailUrl) {
         continue;
@@ -320,7 +341,7 @@ export class TimelineReader {
         const imageResponse = await fetch(song.thumbnailUrl);
         const avatar = await imageResponse.arrayBuffer();
         await Promise.all(
-          TimelineReader.resizeThumbnailPromiseMaker(
+          this.resizeThumbnailPromiseMaker(
             fileBaseName + '_large',
             200,
             3,
@@ -331,7 +352,7 @@ export class TimelineReader {
           )
         );
         await Promise.all(
-          TimelineReader.resizeThumbnailPromiseMaker(
+          this.resizeThumbnailPromiseMaker(
             fileBaseName + '_small',
             60,
             3,
@@ -352,71 +373,151 @@ export class TimelineReader {
     }
   }
 
+  private async checkAndSavePost(post: Post) {
+    if (IGNORE_USERS !== undefined && IGNORE_USERS !== '' && IGNORE_USERS !== 'CHANGE_ME') {
+      const ignorelist = IGNORE_USERS.split(',');
+      const isIgnored = ignorelist.includes(post.account.username);
+      const isIgnoredDb = this.ignoredUsers.includes(post.account.username);
+      this.logger.debug(
+        'Check if user',
+        post.account.username,
+        'is ignored',
+        this.ignoredUsers,
+        isIgnored,
+        isIgnoredDb
+      );
+    }
+
+    const hashttags: string[] = HASHTAG_FILTER.split(',');
+    const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
+
+    const songs = await this.getSongInfoInPost(post);
+
+    // If we don't have any tags or non-youtube urls, check youtube
+    // YT is handled separately, because it requires an API call and therefore is slower
+    if (songs.length === 0 && found_tags.length === 0) {
+      this.logger.log('Ignoring post', post.url);
+      return;
+    }
+
+    await savePost(post, songs);
+
+    await this.saveAvatar(post.account);
+    await this.saveSongThumbnails(songs);
+
+    this.logger.debug('Saved post', post.url, 'songs', songs);
+
+    const posts = await getPosts(null, null, 100);
+    await saveAtomFeed(createFeed(posts));
+
+    for (let song of songs) {
+      this.logger.debug('Adding to playlist', song);
+      await this.addToPlaylist(song);
+    }
+  }
+
   private startWebsocket() {
+    const socketLogger = new Logger('Websocket');
     const socket = new WebSocket(
       `wss://${MASTODON_INSTANCE}/api/v1/streaming?type=subscribe&stream=public:local&access_token=${MASTODON_ACCESS_TOKEN}`
     );
     socket.onopen = () => {
-      log.log('Connected to WS');
+      socketLogger.log('Connected to WS');
     };
     socket.onmessage = async (event) => {
       try {
         const data: TimelineEvent = JSON.parse(event.data.toString());
+        socketLogger.debug('ES event', data.event);
         if (data.event !== 'update') {
-          log.log('Ignoring ES event', data.event);
+          socketLogger.log('Ignoring ES event', data.event);
           return;
         }
         const post: Post = JSON.parse(data.payload);
 
-        const hashttags: string[] = HASHTAG_FILTER.split(',');
+        // Sometimes onmessage is called twice for the same post.
-        const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
+        // This looks to be an issue with automatic reloading in the dev environment,
-
+        // but hard to tell
-        const songs = await TimelineReader.getSongInfoInPost(post);
+        if (this.lastPosts.includes(post.id)) {
-
+          socketLogger.log('Skipping post, already handled', post.id);
-        // If we don't have any tags or non-youtube urls, check youtube
-        // YT is handled separately, because it requires an API call and therefore is slower
-        if (songs.length === 0 && found_tags.length === 0) {
-          log.log('Ignoring post', post.url);
           return;
         }
-
+        this.lastPosts.push(post.id);
-        await savePost(post, songs);
+        while (this.lastPosts.length > 10) {
-
+          this.lastPosts.shift();
-        await TimelineReader.saveAvatar(post.account);
+        }
-        await TimelineReader.saveSongThumbnails(songs);
+        await this.checkAndSavePost(post);
-
-        log.debug('Saved post', post.url);
-
-        const posts = await getPosts(null, null, 100);
-        await saveAtomFeed(createFeed(posts));
       } catch (e) {
-        log.error('error message', event, event.data, e);
+        socketLogger.error('error message', event, event.data, e);
       }
     };
     socket.onclose = (event) => {
-      log.warn(
+      socketLogger.warn(
         `Websocket connection to ${MASTODON_INSTANCE} closed. Code: ${event.code}, reason: '${event.reason}'`,
         event
       );
       setTimeout(() => {
-        log.info(`Attempting to reconenct to WS`);
+        socketLogger.info(`Attempting to reconenct to WS`);
         this.startWebsocket();
       }, 10000);
     };
     socket.onerror = (event) => {
-      log.error(
+      socketLogger.error(
         `Websocket connection to ${MASTODON_INSTANCE} failed. ${event.type}: ${event.error}, message: '${event.message}'`
       );
     };
   }
 
+  private async loadPostsSinceLastRun() {
+    const now = new Date().toISOString();
+    let latestPost = await getPosts(null, now, 1);
+    if (latestPost.length > 0) {
+      this.logger.log('Last post in DB since', now, latestPost[0].created_at);
+    } else {
+      this.logger.log('No posts in DB since');
+    }
+    let u = new URL(`https://${MASTODON_INSTANCE}/api/v1/timelines/public?local=true&limit=40`);
+    if (latestPost.length > 0) {
+      u.searchParams.append('since_id', latestPost[0].id);
+    }
+    for (let tag of HASHTAG_FILTER.split(',')) {
+      u.searchParams.append('q', '#' + tag);
+    }
+    const headers = {
+      Authorization: `Bearer ${MASTODON_ACCESS_TOKEN}`
+    };
+    const latestPosts: Post[] = await fetch(u, { headers }).then((r) => r.json());
+    this.logger.info('searched posts', latestPosts.length);
+    for (const post of latestPosts) {
+      await this.checkAndSavePost(post);
+    }
+  }
+
   private constructor() {
-    log.log('Constructing timeline object');
+    this.logger = new Logger('Timeline');
+    this.logger.log('Constructing timeline object');
+    this.playlistAdders = [new YoutubePlaylistAdder(), new SpotifyPlaylistAdder()];
+    this.ignoredUsers =
+      IGNORE_USERS === undefined
+        ? []
+        : IGNORE_USERS.split(',')
+            .map((u) => (u.startsWith('@') ? u.substring(1) : u))
+            .map((u) =>
+              u.endsWith('@' + MASTODON_INSTANCE)
+                ? u.substring(0, u.length - ('@' + MASTODON_INSTANCE).length)
+                : u
+            );
     this.startWebsocket();
+
+    this.loadPostsSinceLastRun()
+      .then((_) => {
+        this.logger.info('loaded posts since last run');
+      })
+      .catch((e) => {
+        this.logger.error('cannot fetch latest posts', e);
+      });
   }
 
   public static init() {
-    log.log('Timeline object init');
     if (this._instance === undefined) {
       this._instance = new TimelineReader();
     }

src/routes/+page.svelte

@@ -17,6 +17,7 @@
   }
 
   let { data = $bindable() }: Props = $props();
+  let posts: Post[] = $state(data.posts);
 
   interface FetchOptions {
     since?: string;
@@ -44,11 +45,11 @@
    */
   function edgeFly(node: Element, opts: EdgeFlyParams) {
     const createdAt = new Date(opts.created_at).getTime();
-    const diffNewest = Math.abs(new Date(data.posts[0].created_at).getTime() - createdAt);
+    const diffNewest = Math.abs(new Date(posts[0].created_at).getTime() - createdAt);
     const oldest =
       oldestBeforeLastFetch !== null
         ? oldestBeforeLastFetch
-        : new Date(data.posts[data.posts.length - 1].created_at).getTime();
+        : new Date(posts[posts.length - 1].created_at).getTime();
     const diffOldest = Math.abs(oldest - createdAt);
     const fromTop = diffNewest <= diffOldest;
 
@@ -83,15 +84,15 @@
 
   function refresh() {
     let filter: FetchOptions = {};
-    if (data.posts.length > 0) {
+    if (posts.length > 0) {
-      filter = { since: data.posts[0].created_at };
+      filter = { since: posts[0].created_at };
     }
     fetchPosts(filter)
       .then((resp) => {
         if (resp.length > 0) {
           // Prepend new posts, filter dupes
           // There shouldn't be any duplicates, but better be safe than sorry
-          data.posts = filterDuplicates(resp.concat(data.posts));
+          posts = filterDuplicates(resp.concat(posts));
         }
       })
       .catch((e: Error) => {
@@ -99,9 +100,10 @@
       });
   }
 
-  onMount(async () => {
+  onMount(() => {
-    if (data.posts.length > 0) {
+    posts = data.posts;
-      oldestBeforeLastFetch = new Date(data.posts[data.posts.length - 1].created_at).getTime();
+    if (posts.length > 0) {
+      oldestBeforeLastFetch = new Date(posts[posts.length - 1].created_at).getTime();
     }
     interval = setInterval(refresh, refreshInterval);
 
@@ -125,8 +127,8 @@
   function loadOlderPosts() {
     loadingOlderPosts = true;
     const filter: FetchOptions = { count: 20 };
-    if (data.posts.length > 0) {
+    if (posts.length > 0) {
-      const before = data.posts[data.posts.length - 1].created_at;
+      const before = posts[posts.length - 1].created_at;
       filter.before = before;
       oldestBeforeLastFetch = new Date(before).getTime();
     }
@@ -136,7 +138,7 @@
         if (resp.length > 0) {
           // Append old posts, filter dupes
           // There shouldn't be any duplicates, but better be safe than sorry
-          data.posts = filterDuplicates(data.posts.concat(resp));
+          posts = filterDuplicates(posts.concat(resp));
           // If we got less than we expected, there are no older posts available
           moreOlderPostsAvailable = resp.length >= (filter.count ?? 20);
         } else {
@@ -158,10 +160,10 @@
 <div class="wrapper">
   <div></div>
   <div class="posts">
-    {#if data.posts.length === 0}
+    {#if posts.length === 0}
       Sorry, no posts recommending music have been found yet
     {/if}
-    {#each data.posts as post (post.url)}
+    {#each posts as post (post.url)}
       <div
         class="post"
         transition:edgeFly|global={{

src/routes/spotifyAuth/+page.server.ts

@@ -0,0 +1,33 @@
+import { BASE_URL } from '$env/static/private';
+import { Logger } from '$lib/log';
+import { SpotifyPlaylistAdder } from '$lib/server/playlist/spotifyPlaylistAdder';
+import { redirect } from '@sveltejs/kit';
+import type { PageServerLoad } from './$types';
+const { DEV } = import.meta.env;
+
+const logger = new Logger('SpotifyAuth');
+
+export const load: PageServerLoad = async ({ url, request }) => {
+  const baseUrl = request.headers.get('X-Forwarded-Host') ?? BASE_URL;
+  const adder = new SpotifyPlaylistAdder();
+  let redirect_uri = new URL(`${new URL(BASE_URL).protocol}//${baseUrl}/spotifyAuth`);
+  if (url.hostname === 'localhost' && DEV) {
+    redirect_uri.hostname = '127.0.0.1';
+  }
+  logger.debug(url.searchParams, url.hostname);
+  if (url.searchParams.has('code')) {
+    await adder.receivedAuthCode(url.searchParams.get('code') || '', redirect_uri);
+    redirect(307, '/');
+  } else if (url.searchParams.has('error')) {
+    logger.error('received error', url.searchParams.get('error'));
+    return;
+  }
+
+  if (await adder.authCodeExists()) {
+    redirect(307, '/');
+  }
+
+  const authUrl = adder.constructAuthUrl(redirect_uri);
+  logger.debug('+page.server.ts', authUrl.toString());
+  redirect(307, authUrl);
+};

src/routes/spotifyAuth/+page.svelte

@@ -0,0 +1 @@
+<h1>Something went wrong</h1>

src/routes/ytauth/+page.server.ts

@@ -0,0 +1,29 @@
+import { BASE_URL } from '$env/static/private';
+import { Logger } from '$lib/log';
+import { YoutubePlaylistAdder } from '$lib/server/playlist/ytPlaylistAdder';
+import { redirect } from '@sveltejs/kit';
+import type { PageServerLoad } from './$types';
+
+const logger = new Logger('YT Auth');
+
+export const load: PageServerLoad = async ({ url, request }) => {
+  const baseUrl = request.headers.get('X-Forwarded-Host') ?? BASE_URL;
+  const adder = new YoutubePlaylistAdder();
+  logger.debug('redirect URL', `${new URL(BASE_URL).protocol}//${baseUrl}/ytauth`);
+  const redirect_uri = new URL(`${new URL(BASE_URL).protocol}//${baseUrl}/ytauth`);
+  if (url.searchParams.has('code')) {
+    logger.debug(url.searchParams);
+    await adder.receivedAuthCode(url.searchParams.get('code') || '', redirect_uri);
+    redirect(307, '/');
+  } else if (url.searchParams.has('error')) {
+    logger.error('received error', url.searchParams.get('error'));
+    return;
+  }
+
+  if (await adder.authCodeExists()) {
+    redirect(307, '/');
+  }
+  const authUrl = adder.constructAuthUrl(redirect_uri);
+  logger.debug('+page.server.ts', authUrl.toString());
+  redirect(307, authUrl);
+};

src/routes/ytauth/+page.svelte

@@ -0,0 +1 @@
+<h1>Something went wrong</h1>

start.sh.EXAMPLE

@@ -1,3 +1,3 @@
 #!/bin/bash
 . /home/moshing-mammut/.nvm/nvm.sh
-node -r dotenv/config build
+node -r dotenv/config build &

tsconfig.json

@@ -9,6 +9,7 @@
     "skipLibCheck": true,
     "sourceMap": true,
     "strict": true
+    //"lib": ["ESNext.Array"]
   }
   // Path aliases are handled by https://kit.svelte.dev/docs/configuration#alias
   //