update dependencies, add songs to youtube playlist

This reverts commit 5b6dbd327d.

.env.EXAMPLE

@@ -1,11 +1,16 @@
 HASHTAG_FILTER = ichlausche,music,musik,nowplaying,tunetuesday,nowlistening
 YOUTUBE_API_KEY = CHANGE_ME
+YOUTUBE_PLAYLIST_ID = CHANGE_ME
+YOUTUBE_CLIENT_ID = CHANGE_ME
+YOUTUBE_CLIENT_SECRET = CHANGE_ME
 ODESLI_API_KEY = CHANGE_ME
 MASTODON_INSTANCE = 'metalhead.club'
+MASTODON_ACCESS_TOKEN = 'YOUR_ACCESS_TOKEN_HERE'
 BASE_URL = 'https://moshingmammut.phlaym.net'
 VERBOSE = false
 IGNORE_USERS = @moshhead@metalhead.club
 WEBSUB_HUB = 'http://pubsubhubbub.superfeedr.com'
 
 PUBLIC_REFRESH_INTERVAL = 10000
-PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME = 'Metalhead.club'
+PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME = 'Metalhead.club'
+PORT = 3001

.eslintrc.cjs

@@ -1,16 +1,25 @@
 module.exports = {
   root: true,
   parser: '@typescript-eslint/parser',
-  extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended', 'prettier'],
+  extends: ['plugin:svelte/recommended', 'plugin:@typescript-eslint/recommended', 'prettier'],
-  plugins: ['svelte3', '@typescript-eslint'],
+  plugins: ['@typescript-eslint'],
   ignorePatterns: ['*.cjs'],
-  overrides: [{ files: ['*.svelte'], processor: 'svelte3/svelte3' }],
+  overrides: [
+    {
+      files: ['*.svelte'],
+      parser: 'svelte-eslint-parser',
+      parserOptions: {
+        parser: '@typescript-eslint/parser'
+      }
+    }
+  ],
   settings: {
     'svelte3/typescript': () => require('typescript')
   },
   parserOptions: {
     sourceType: 'module',
-    ecmaVersion: 2020
+    ecmaVersion: 2020,
+    extraFileExtensions: ['.svelte']
   },
   env: {
     browser: true,

.gitignore

@@ -1,3 +1,4 @@
+yt_auth_token
 *.db
 feed.xml
 playbook.yml

.nvmrc

@@ -0,0 +1 @@
+lts/*

README.md

@@ -65,7 +65,7 @@ Set up NVM:
 
 ```
 $ cd
-$ curl https://raw.github.com/creationix/nvm/master/install.sh | sh
+$ curl https://raw.githubusercontent.com/nvm-sh/nvm/refs/heads/master/install.sh | bash
 $ source ~/.nvm/nvm.sh
 $ nvm install --lts
 ```
@@ -103,6 +103,13 @@ because the API is the only way to check if a YouTube link leads to music or som
 
 If `ODESLI_API_KEY` is unset, your rate limit to the song.link API will be lower.
 
+Add `MASTODON_ACCESS_TOKEN` as well, see [Creating our application
+
+](https://docs.joinmastodon.org/client/token/#app) in the Mastodon documentation.
+`read:statuses` is the only required scope. An access token will be displayed in your settings. Use that!
+
+There are currently no plans to implement an actual authentication flow.
+
 Run `npm run build` and copy the output folder, usually `build` to `$APP_DIR` on your server.
 
 #### On your server again

package.json

@@ -1,6 +1,6 @@
 {
   "name": "moshing-mammut",
-  "version": "1.4.1",
+  "version": "1.4.0",
   "private": true,
   "license": "LGPL-3.0-or-later",
   "scripts": {
@@ -14,34 +14,35 @@
     "format": "prettier --plugin-search-dir . --write ."
   },
   "devDependencies": {
-    "@sveltejs/adapter-node": "^1.2.3",
+    "@sveltejs/adapter-node": "^5.2.12",
-    "@sveltejs/kit": "^1.5.0",
+    "@sveltejs/kit": "^2.22.2",
-    "@types/node": "^18.16.3",
+    "@sveltejs/vite-plugin-svelte": "^5.1.0",
-    "@types/sqlite3": "^3.1.8",
+    "@types/node": "^22.9.0",
-    "@types/ws": "^8.5.4",
+    "@types/sqlite3": "^3.0.0",
-    "@typescript-eslint/eslint-plugin": "^5.45.0",
+    "@types/ws": "^8.5.0",
-    "@typescript-eslint/parser": "^5.45.0",
+    "@typescript-eslint/eslint-plugin": "^8.0.0",
+    "@typescript-eslint/parser": "^8.0.0",
     "@zerodevx/svelte-toast": "^0.9.3",
-    "eslint": "^8.28.0",
+    "eslint": "^9.11.1",
-    "eslint-config-prettier": "^8.5.0",
+    "eslint-config-prettier": "^10.0.0",
-    "eslint-plugin-svelte3": "^4.0.0",
+    "eslint-plugin-svelte": "^3.0.0",
-    "prettier": "^2.8.0",
+    "prettier": "^3.1.0",
-    "prettier-plugin-svelte": "^2.8.1",
+    "prettier-plugin-svelte": "^3.2.6",
-    "svelte": "^3.54.0",
+    "svelte": "^5",
-    "svelte-check": "^3.0.1",
+    "svelte-check": "^4.0.0",
-    "tslib": "^2.4.1",
+    "tslib": "^2.0.0",
-    "typescript": "^4.9.3",
+    "typescript": "^5.0.0",
-    "vite": "^4.0.0"
+    "vite": "^6.0.0"
   },
   "type": "module",
   "dependencies": {
-    "dotenv": "^16.0.3",
+    "dotenv": "^17.0.0",
-    "feed": "^4.2.2",
+    "feed": "^5.1.0",
-    "sharp": "^0.32.0",
+    "sharp": "^0.34.2",
-    "sqlite3": "^5.1.6",
+    "sqlite3": "^5.0.0",
-    "ws": "^8.13.0"
+    "ws": "^8.18.0"
   },
   "engines": {
-    "node": ">=18.0.0"
+    "node": ">=20.0.0"
   }
 }

src/app.html

@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!doctype html>
 <html lang="en">
   <head>
     <meta charset="utf-8" />
@@ -54,9 +54,22 @@
 
         color: var(--color-text);
         background-color: var(--color-bg);
-        font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen-Sans,
+        font-family:
-          Ubuntu, Cantarell, 'Helvetica Neue', Helvetica, Arial, sans-serif, 'Apple Color Emoji',
+          system-ui,
-          'Segoe UI Emoji', 'Segoe UI Symbol';
+          -apple-system,
+          BlinkMacSystemFont,
+          'Segoe UI',
+          Roboto,
+          Oxygen-Sans,
+          Ubuntu,
+          Cantarell,
+          'Helvetica Neue',
+          Helvetica,
+          Arial,
+          sans-serif,
+          'Apple Color Emoji',
+          'Segoe UI Emoji',
+          'Segoe UI Symbol';
       }
 
       a {

src/hooks.server.ts

@@ -1,9 +1,10 @@
 import { log } from '$lib/log';
 import { TimelineReader } from '$lib/server/timeline';
-import type { HandleServerError } from '@sveltejs/kit';
+import type { Handle, HandleServerError } from '@sveltejs/kit';
 import { error } from '@sveltejs/kit';
 import fs from 'fs/promises';
 
+log.log('App startup');
 TimelineReader.init();
 
 export const handleError = (({ error }) => {
@@ -16,9 +17,13 @@ export const handleError = (({ error }) => {
   };
 }) satisfies HandleServerError;
 
-import type { Handle } from '@sveltejs/kit';
-
 export const handle = (async ({ event, resolve }) => {
+  const searchParams = event.url.searchParams;
+  const authCode = searchParams.get('code');
+  if (authCode) {
+    log.debug('received GET hook', event.url.searchParams);
+  }
+
   // Reeder *insists* on checking /feed instead of /feed.xml
   if (event.url.pathname === '/feed') {
     return new Response('', { status: 301, headers: { Location: '/feed.xml' } });
@@ -53,7 +58,7 @@ export const handle = (async ({ event, resolve }) => {
       return new Response(f, { headers: [['Content-Type', 'image/' + suffix]] });
     } catch (e) {
       log.error('no stream', e);
-      throw error(404);
+      error(404);
     }
   }
 

src/lib/components/AccountComponent.svelte

@@ -1,7 +1,11 @@
 <script lang="ts">
   import type { Account } from '$lib/mastodon/response';
 
-  export let account: Account;
+  interface Props {
+    account: Account;
+  }
+
+  let { account }: Props = $props();
 </script>
 
 <a href={account.url} target="_blank">{account.display_name} @{account.acct}</a>

src/lib/components/AvatarComponent.svelte

@@ -1,11 +1,13 @@
 <script lang="ts">
   import type { Account } from '$lib/mastodon/response';
 
-  export let account: Account;
+  interface Props {
-  let avatarDescription: string;
+    account: Account;
-  let sourceSetHtml: string;
+  }
-  $: avatarDescription = `Avatar for ${account.acct}`;
+
-  $: {
+  let { account }: Props = $props();
+  let avatarDescription: string = $derived(`Avatar for ${account.acct}`);
+  let sourceSetHtml: string = $derived.by(() => {
     // Sort thumbnails by file type. This is important, because the order of the srcset entries matter.
     // We need the best format to be first
     const formatPriority = new Map<string, number>([
@@ -14,7 +16,7 @@
       ['jpg', 99],
       ['jpeg', 99]
     ]);
-    const resizedAvatars = (account.resizedAvatars ?? []).sort((a, b) => {
+    const resizedAvatars = (account.resizedAvatars ?? []).toSorted((a, b) => {
       const extensionA = a.file.split('.').pop() ?? '';
       const extensionB = b.file.split('.').pop() ?? '';
       const prioA = formatPriority.get(extensionA) ?? 3;
@@ -33,8 +35,8 @@
       const srcset = entry[1].join(', ');
       html += `<source srcset="${srcset}" type="${entry[0]}" />`;
     }
-    sourceSetHtml = html;
+    return html;
-  }
+  });
 </script>
 
 <picture>

src/lib/components/LoadMoreComponent.svelte

@@ -1,31 +1,34 @@
 <script lang="ts">
-  import { createEventDispatcher } from 'svelte';
   import LoadingSpinnerComponent from '$lib/components/LoadingSpinnerComponent.svelte';
 
-  export let moreAvailable = false;
+  interface Props {
-  export let isLoading = false;
+    moreAvailable?: boolean;
-  let displayText = '';
+    isLoading?: boolean;
-  let title = '';
+    loadOlderPosts: any;
-  let disabled: boolean;
-
-  $: if (isLoading) {
-    displayText = 'Loading...';
-  } else if (!moreAvailable) {
-    displayText = 'You reached the end';
-  } else {
-    displayText = 'Load More';
   }
-  $: disabled = !moreAvailable || isLoading;
-  $: title = moreAvailable ? 'Load More' : 'There be dragons!';
 
-  const dispatch = createEventDispatcher();
+  let { moreAvailable = false, isLoading = false, loadOlderPosts }: Props = $props();
+  let displayText = $derived.by(() => {
+    if (isLoading) {
+      return 'Loading...';
+    } else if (!moreAvailable) {
+      return 'You reached the end';
+    }
+    return 'Load More';
+  });
+  let title = $derived(moreAvailable ? 'Load More' : 'There be dragons!');
+  let disabled: boolean = $derived(!moreAvailable || isLoading);
 
-  function loadOlderPosts() {
+  /*const dispatch = createEventDispatcher<{
-    dispatch('loadOlderPosts');
+  loadOlderPosts: string;
-  }
+}>();
+
+function loadOlderPosts() {
+  dispatch('loadOlderPosts');
+}*/
 </script>
 
-<button on:click={loadOlderPosts} {disabled} {title}>
+<button onclick={() => loadOlderPosts()} {disabled} {title}>
   <div class="loading" class:collapsed={!isLoading}>
     <LoadingSpinnerComponent size="0.5em" thickness="6px" />
   </div>

src/lib/components/LoadingSpinnerComponent.svelte

@@ -1,9 +1,13 @@
 <script lang="ts">
-  export let size = '64px';
+  interface Props {
-  export let thickness = '6px';
+    size?: string;
+    thickness?: string;
+  }
+
+  let { size = '64px', thickness = '6px' }: Props = $props();
 </script>
 
-<div class="lds-dual-ring" style="--size: {size}; --thickness: {thickness}" />
+<div class="lds-dual-ring" style="--size: {size}; --thickness: {thickness}"></div>
 
 <style>
   .lds-dual-ring {

src/lib/components/PostComponent.svelte

@@ -6,15 +6,21 @@
   import { secondsSince, relativeTime } from '$lib/relativeTime';
   import { onMount } from 'svelte';
 
-  export let post: Post;
+  interface Props {
-  let displayRelativeTime = false;
+    post: Post;
-  const absoluteDate = new Date(post.created_at).toLocaleString();
-  let dateCreated = absoluteDate;
-  const timePassed = secondsSince(new Date(post.created_at));
-  $: if (displayRelativeTime) {
-    dateCreated = relativeTime($timePassed) ?? absoluteDate;
   }
 
+  let { post }: Props = $props();
+  let displayRelativeTime = $state(false);
+  const absoluteDate = new Date(post.created_at).toLocaleString();
+  const timePassed = secondsSince(new Date(post.created_at));
+  let dateCreated = $derived.by(() => {
+    if (displayRelativeTime) {
+      return relativeTime($timePassed) ?? absoluteDate;
+    }
+    return absoluteDate;
+  });
+
   const songs = filterDuplicates(post.songs ?? []);
 
   function filterDuplicates(songs: SongInfo[]): SongInfo[] {
@@ -59,7 +65,7 @@
       ['jpg', 99],
       ['jpeg', 99]
     ]);
-    const thumbs = (song.resizedThumbnails ?? []).sort((a, b) => {
+    const thumbs = (song.resizedThumbnails ?? []).toSorted((a, b) => {
       const extensionA = a.file.split('.').pop() ?? '';
       const extensionB = b.file.split('.').pop() ?? '';
       const prioA = formatPriority.get(extensionA) ?? 3;

src/lib/mastodon/response.ts

@@ -16,6 +16,17 @@ export interface Post {
   songs?: SongInfo[];
 }
 
+export interface OauthResponse {
+  access_token: string;
+  expires_in: number;
+  expires?: Date;
+  refresh_token?: string;
+  refresh_token_expires_in?: number;
+  scope: string;
+  token_type: string;
+  error?: any;
+}
+
 export interface PreviewCard {
   url: string;
   title: string;

src/lib/server/db.ts

@@ -479,11 +479,11 @@ function saveSongInfoData(postUrl: string, songs: SongInfo[]): Promise<void> {
 }
 
 export async function savePost(post: Post, songs: SongInfo[]) {
+  log.debug(`Saving post ${post.url}`);
   if (!databaseReady) {
     await waitReady();
   }
 
-  log.debug(`Saving post ${post.url}`);
   const account = post.account;
   await saveAccountData(account);
   log.debug(`Saved account data ${post.url}`);

src/lib/server/timeline.ts

@@ -1,5 +1,6 @@
 import {
   HASHTAG_FILTER,
+  MASTODON_ACCESS_TOKEN,
   MASTODON_INSTANCE,
   ODESLI_API_KEY,
   YOUTUBE_API_KEY
@@ -25,10 +26,13 @@ import {
   saveSongThumbnail
 } from '$lib/server/db';
 import { createFeed, saveAtomFeed } from '$lib/server/rss';
+import { YoutubePlaylistAdder } from '$lib/server/ytPlaylistAdder';
 import { sleep } from '$lib/sleep';
 import crypto from 'crypto';
 import fs from 'fs/promises';
+import { console } from 'inspector/promises';
 import sharp from 'sharp';
+import { URL, URLSearchParams } from 'url';
 import { WebSocket } from 'ws';
 
 const URL_REGEX = new RegExp(/href="(?<postUrl>[^>]+?)" target="_blank"/gm);
@@ -39,10 +43,13 @@ const YOUTUBE_REGEX = new RegExp(
 
 export class TimelineReader {
   private static _instance: TimelineReader;
+  private lastPosts: string[] = [];
+  private youtubePlaylistAdder: YoutubePlaylistAdder;
 
   private static async isMusicVideo(videoId: string) {
     if (!YOUTUBE_API_KEY || YOUTUBE_API_KEY === 'CHANGE_ME') {
       // Assume that it *is* a music link when no YT API key is provided
+      log.debug('YT API not configured');
       return true;
     }
     const searchParams = new URLSearchParams([
@@ -54,13 +61,13 @@ export class TimelineReader {
     const resp = await fetch(youtubeVideoUrl);
     const respObj = await resp.json();
     if (!respObj.items.length) {
-      console.warn('Could not find video with id', videoId);
+      log.warn('Could not find video with id', videoId);
       return false;
     }
 
     const item = respObj.items[0];
     if (!item.snippet) {
-      console.warn('Could not load snippet for video', videoId, item);
+      log.warn('Could not load snippet for video', videoId, item);
       return false;
     }
     if (item.snippet.tags?.includes('music')) {
@@ -78,6 +85,7 @@ export class TimelineReader {
     const categoryTitle: string = await fetch(youtubeCategoryUrl)
       .then((r) => r.json())
       .then((r) => r.items[0]?.snippet?.title);
+    log.debug('YT category', categoryTitle);
     return categoryTitle === 'Music';
   }
 
@@ -101,7 +109,7 @@ export class TimelineReader {
       // Check *all* found url and let odesli determine if it is music or not
       log.debug(`Checking ${url} if it contains song data`);
       const info = await TimelineReader.getSongInfo(url);
-      log.debug(`Found song info for ${url}?`, info);
+      //log.debug(`Found song info for ${url}?`, info);
       if (info) {
         songs.push(info);
       }
@@ -143,6 +151,7 @@ export class TimelineReader {
         return null;
       }
       const info = odesliInfo.entitiesByUniqueId[odesliInfo.entityUniqueId];
+      //log.debug('odesli response', info);
       const platform: Platform = 'youtube';
       if (info.platforms.includes(platform)) {
         const youtubeId =
@@ -155,7 +164,7 @@ export class TimelineReader {
         }
         const isMusic = await TimelineReader.isMusicVideo(youtubeId);
         if (!isMusic) {
-          log.debug('Probably not a music video', url);
+          log.debug('Probably not a music video', youtubeId, url);
           return null;
         }
       }
@@ -176,6 +185,88 @@ export class TimelineReader {
     }
   }
 
+  /*
+  private async addToYoutubePlaylist(song: SongInfo) {
+    log.debug('addToYoutubePlaylist');
+    let token: OauthResponse;
+    try {
+      const youtube_token_file = await fs.readFile('yt_auth_token', { encoding: 'utf8' });
+      token = JSON.parse(youtube_token_file);
+      log.debug('read youtube access token', token);
+    } catch (e) {
+      log.error('Could not read youtube access token', e);
+      return;
+    }
+
+    if (!YOUTUBE_PLAYLIST_ID || YOUTUBE_PLAYLIST_ID === 'CHANGE_ME') {
+      log.debug('no playlist ID configured');
+      return;
+    }
+    if (!song.youtubeUrl) {
+      log.debug('Skip adding song to YT playlist, no youtube Url', song);
+      return;
+    }
+
+    const songUrl = new URL(song.youtubeUrl);
+    const youtubeId = songUrl.searchParams.get('v');
+    if (!youtubeId) {
+      log.debug(
+        'Skip adding song to YT playlist, could not extract YT id from URL',
+        song.youtubeUrl
+      );
+      return;
+    }
+    log.debug('Found YT id from URL', song.youtubeUrl, youtubeId);
+
+    const playlistItemsUrl = new URL('https://www.googleapis.com/youtube/v3/playlistItems');
+    playlistItemsUrl.searchParams.append('videoId', youtubeId);
+    playlistItemsUrl.searchParams.append('playlistId', YOUTUBE_PLAYLIST_ID);
+    playlistItemsUrl.searchParams.append('part', 'id');
+    const existingPlaylistItem = await fetch(
+      'https://www.googleapis.com/youtube/v3/playlistItems',
+      {
+        headers: { Authorization: `${token.token_type} ${token.access_token}` }
+      }
+    ).then((r) => r.json());
+    log.debug('existingPlaylistItem', existingPlaylistItem);
+    if (existingPlaylistItem.pageInfo && existingPlaylistItem.pageInfo.totalResults > 0) {
+      log.info('Item already in playlist');
+      return;
+    }
+
+    const searchParams = new URLSearchParams([
+      ['part', 'snippet']
+      //['key', token.access_token]
+    ]);
+    const options: RequestInit = {
+      method: 'POST',
+      headers: { Authorization: `${token.token_type} ${token.access_token}` },
+      body: JSON.stringify({
+        snippet: {
+          playlistId: YOUTUBE_PLAYLIST_ID,
+          resourceId: {
+            videoId: youtubeId,
+            kind: 'youtube#video'
+          }
+        }
+      })
+    };
+    const youtubeApiUrl = new URL(
+      `https://www.googleapis.com/youtube/v3/playlistItems?${searchParams}`
+    );
+    const resp = await fetch(youtubeApiUrl, options);
+    const respObj = await resp.json();
+    log.debug('Added to playlist', options, respObj);
+    if (respObj.error) {
+      log.debug('Add to playlist failed', respObj.error.errors);
+    }
+  }
+  */
+  private async addToPlaylist(song: SongInfo) {
+    //await this.addToYoutubePlaylist(song);
+    await this.youtubePlaylistAdder.addToPlaylist(song);
+  }
+
   private static async resizeAvatar(
     baseName: string,
     size: number,
@@ -223,7 +314,7 @@ export class TimelineReader {
                   accountUrl: accountUrl,
                   file: fn,
                   sizeDescriptor: `${i}x`
-                } as AccountAvatar)
+                }) as AccountAvatar
             )
             .then(saveAvatar)
         )
@@ -260,7 +351,7 @@ export class TimelineReader {
                   file: fn,
                   sizeDescriptor: `${i}x`,
                   kind: kind
-                } as SongThumbnailImage)
+                }) as SongThumbnailImage
             )
             .then(saveSongThumbnail)
         )
@@ -351,48 +442,123 @@ export class TimelineReader {
     }
   }
 
+  private async checkAndSavePost(post: Post) {
+    const hashttags: string[] = HASHTAG_FILTER.split(',');
+    const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
+
+    const songs = await TimelineReader.getSongInfoInPost(post);
+
+    // If we don't have any tags or non-youtube urls, check youtube
+    // YT is handled separately, because it requires an API call and therefore is slower
+    if (songs.length === 0 && found_tags.length === 0) {
+      log.log('Ignoring post', post.url);
+      return;
+    }
+
+    await savePost(post, songs);
+
+    await TimelineReader.saveAvatar(post.account);
+    await TimelineReader.saveSongThumbnails(songs);
+
+    log.debug('Saved post', post.url, 'songs', songs);
+
+    const posts = await getPosts(null, null, 100);
+    await saveAtomFeed(createFeed(posts));
+
+    for (let song of songs) {
+      log.debug('Adding to playlist', song);
+      await this.addToPlaylist(song);
+    }
+  }
+
   private startWebsocket() {
-    const socket = new WebSocket(`wss://${MASTODON_INSTANCE}/api/v1/streaming`);
+    const socket = new WebSocket(
+      `wss://${MASTODON_INSTANCE}/api/v1/streaming?type=subscribe&stream=public:local&access_token=${MASTODON_ACCESS_TOKEN}`
+    );
     socket.onopen = () => {
       log.log('Connected to WS');
-      socket.send('{ "type": "subscribe", "stream": "public:local"}');
     };
     socket.onmessage = async (event) => {
       try {
+        /*
+        let token: OauthResponse;
+        try {
+          const youtube_token_file = await fs.readFile('yt_auth_token', { encoding: 'utf8' });
+          token = JSON.parse(youtube_token_file);
+          if (token.expires) {
+            if (typeof token.expires === typeof '') {
+              token.expires = new Date(token.expires);
+            }
+            let now = new Date();
+            now.setTime(now.getTime() - 15 * 60 * 1000);
+            log.info('token expiry', token.expires, 'vs refresh @', now);
+            if (token.expires.getTime() <= now.getTime()) {
+              log.info(
+                'YT token expires',
+                token.expires,
+                token.expires.getTime(),
+                'which is less than 15 minutes from now',
+                now,
+                now.getTime()
+              );
+              const tokenUrl = new URL('https://oauth2.googleapis.com/token');
+              const params = new URLSearchParams();
+              params.append('client_id', YOUTUBE_CLIENT_ID);
+              params.append('client_secret', YOUTUBE_CLIENT_SECRET);
+              params.append('refresh_token', token.refresh_token || '');
+              params.append('grant_type', 'refresh_token');
+              params.append('redirect_uri', `${BASE_URL}/ytauth`);
+              if (token.refresh_token) {
+                log.debug('sending token req', params);
+                const resp = await fetch(tokenUrl, {
+                  method: 'POST',
+                  body: params
+                }).then((r) => r.json());
+                if (!resp.error) {
+                  if (!resp.refresh_token) {
+                    resp.refresh_token = token.refresh_token;
+                  }
+                  let expiration = new Date();
+                  expiration.setSeconds(expiration.getSeconds() + resp.expires_in);
+                  resp.expires = expiration;
+                  await fs.writeFile('yt_auth_token', JSON.stringify(resp));
+                } else {
+                  log.error('token resp error', resp);
+                }
+              } else {
+                log.error('no refresg token');
+              }
+            }
+          }
+        } catch (e) {
+          log.error('onmessage Could not read youtube access token', e);
+        }
+        */
+
         const data: TimelineEvent = JSON.parse(event.data.toString());
+        log.debug('ES event', data.event);
         if (data.event !== 'update') {
+          log.log('Ignoring ES event', data.event);
           return;
         }
         const post: Post = JSON.parse(data.payload);
-
+        if (this.lastPosts.includes(post.id)) {
-        const hashttags: string[] = HASHTAG_FILTER.split(',');
+          log.log('Skipping post, already handled', post.id);
-        const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
-
-        const songs = await TimelineReader.getSongInfoInPost(post);
-
-        // If we don't have any tags or non-youtube urls, check youtube
-        // YT is handled separately, because it requires an API call and therefore is slower
-        if (songs.length === 0 && found_tags.length === 0) {
-          log.log('Ignoring post', post.url);
           return;
         }
-
+        this.lastPosts.push(post.id);
-        await savePost(post, songs);
+        while (this.lastPosts.length > 10) {
-
+          this.lastPosts.shift();
-        await TimelineReader.saveAvatar(post.account);
+        }
-        await TimelineReader.saveSongThumbnails(songs);
+        await this.checkAndSavePost(post);
-
-        log.debug('Saved post', post.url);
-
-        const posts = await getPosts(null, null, 100);
-        await saveAtomFeed(createFeed(posts));
       } catch (e) {
         log.error('error message', event, event.data, e);
       }
     };
     socket.onclose = (event) => {
       log.warn(
-        `Websocket connection to ${MASTODON_INSTANCE} closed. Code: ${event.code}, reason: '${event.reason}'`
+        `Websocket connection to ${MASTODON_INSTANCE} closed. Code: ${event.code}, reason: '${event.reason}'`,
+        event
       );
       setTimeout(() => {
         log.info(`Attempting to reconenct to WS`);
@@ -406,11 +572,47 @@ export class TimelineReader {
     };
   }
 
+  private async loadPostsSinceLastRun() {
+    const now = new Date().toISOString();
+    let latestPost = await getPosts(null, now, 1);
+    if (latestPost.length > 0) {
+      log.log('Last post in DB since', now, latestPost[0].created_at);
+    } else {
+      log.log('No posts in DB since');
+    }
+    let u = new URL(`https://${MASTODON_INSTANCE}/api/v1/timelines/public?local=true&limit=40`);
+    if (latestPost.length > 0) {
+      u.searchParams.append('since_id', latestPost[0].id);
+    }
+    for (let tag of HASHTAG_FILTER.split(',')) {
+      u.searchParams.append('q', '#' + tag);
+    }
+    const headers = {
+      Authorization: `Bearer ${MASTODON_ACCESS_TOKEN}`
+    };
+    const latestPosts: Post[] = await fetch(u, { headers }).then((r) => r.json());
+    log.info('searched posts', latestPosts.length);
+    for (const post of latestPosts) {
+      await this.checkAndSavePost(post);
+    }
+  }
+
   private constructor() {
+    log.log('Constructing timeline object');
+    this.youtubePlaylistAdder = new YoutubePlaylistAdder();
     this.startWebsocket();
+
+    this.loadPostsSinceLastRun()
+      .then((_) => {
+        log.info('loaded posts since last run');
+      })
+      .catch((e) => {
+        log.error('cannot fetch latest posts', e);
+      });
   }
 
   public static init() {
+    log.log('Timeline object init');
     if (this._instance === undefined) {
       this._instance = new TimelineReader();
     }

src/lib/server/ytPlaylistAdder.ts

@@ -0,0 +1,194 @@
+import {
+  BASE_URL,
+  YOUTUBE_CLIENT_ID,
+  YOUTUBE_CLIENT_SECRET,
+  YOUTUBE_PLAYLIST_ID
+} from '$env/static/private';
+import { log } from '$lib/log';
+import type { OauthResponse } from '$lib/mastodon/response';
+import type { SongInfo } from '$lib/odesliResponse';
+import fs from 'fs/promises';
+
+export class YoutubePlaylistAdder {
+  private apiBase: string = 'https://www.googleapis.com/youtube/v3';
+  private token_file_name: string = 'yt_auth_token';
+
+  /// How many minutes before expiry the token will be refreshed
+  private refresh_time: number = 15;
+
+  public async authCodeExists(): Promise<boolean> {
+    try {
+      const fileHandle = await fs.open(this.token_file_name);
+      await fileHandle.close();
+      return true;
+    } catch {
+      log.info('No auth token yet, authorizing...');
+      return false;
+    }
+  }
+
+  public constructAuthUrl(redirectUri: URL): URL {
+    const endpoint = 'https://accounts.google.com/o/oauth2/v2/auth';
+    const authUrl = new URL(endpoint);
+    authUrl.searchParams.append('client_id', YOUTUBE_CLIENT_ID);
+    authUrl.searchParams.append('redirect_uri', redirectUri.toString());
+    authUrl.searchParams.append('response_type', 'code');
+    authUrl.searchParams.append('scope', 'https://www.googleapis.com/auth/youtube');
+    authUrl.searchParams.append('access_type', 'offline');
+    authUrl.searchParams.append('include_granted_scopes', 'false');
+    return authUrl;
+  }
+
+  public async receivedAuthCode(code: string, url: URL) {
+    log.debug('received code');
+    const tokenUrl = new URL('https://oauth2.googleapis.com/token');
+    const params = new URLSearchParams();
+    params.append('client_id', YOUTUBE_CLIENT_ID);
+    params.append('client_secret', YOUTUBE_CLIENT_SECRET);
+    params.append('code', code);
+    params.append('grant_type', 'authorization_code');
+    params.append('redirect_uri', `${url.origin}${url.pathname}`);
+    log.debug('sending token req', params);
+    const resp: OauthResponse = await fetch(tokenUrl, {
+      method: 'POST',
+      body: params
+    }).then((r) => r.json());
+    log.debug('received access token', resp);
+    let expiration = new Date();
+    expiration.setTime(expiration.getTime() + resp.expires_in * 1000);
+    expiration.setSeconds(expiration.getSeconds() + resp.expires_in);
+    resp.expires = expiration;
+    await fs.writeFile(this.token_file_name, JSON.stringify(resp));
+  }
+
+  private async auth(): Promise<OauthResponse | null> {
+    try {
+      const youtube_token_file = await fs.readFile(this.token_file_name, { encoding: 'utf8' });
+      let token = JSON.parse(youtube_token_file);
+      log.debug('read youtube access token', token);
+      if (token.expires) {
+        if (typeof token.expires === typeof '') {
+          token.expires = new Date(token.expires);
+        }
+      }
+      return token;
+    } catch (e) {
+      log.error('Could not read youtube access token', e);
+      return null;
+    }
+  }
+
+  private async refreshToken(): Promise<OauthResponse | null> {
+    const token = await this.auth();
+    if (token == null || !token?.expires) {
+      return null;
+    }
+    let now = new Date();
+    now.setTime(now.getTime() - this.refresh_time * 60 * 1000);
+    log.info('token expiry', token.expires, 'vs refresh @', now);
+    if (token.expires.getTime() > now.getTime()) {
+      return token;
+    }
+
+    log.info(
+      'YT token expires',
+      token.expires,
+      token.expires.getTime(),
+      `which is less than ${this.refresh_time} minutes from now`,
+      now,
+      now.getTime()
+    );
+
+    const tokenUrl = new URL('https://oauth2.googleapis.com/token');
+    const params = new URLSearchParams();
+    params.append('client_id', YOUTUBE_CLIENT_ID);
+    params.append('client_secret', YOUTUBE_CLIENT_SECRET);
+    params.append('refresh_token', token.refresh_token || '');
+    params.append('grant_type', 'refresh_token');
+    params.append('redirect_uri', `${BASE_URL}/ytauth`);
+    if (!token.refresh_token) {
+      log.error('Need to refresh access token, but no refresh token provided');
+      return null;
+    }
+    log.debug('sending token req', params);
+    let resp: OauthResponse = await fetch(tokenUrl, {
+      method: 'POST',
+      body: params
+    }).then((r) => r.json());
+    if (resp.error) {
+      log.error('token resp error', resp);
+      return null;
+    }
+    if (!resp.refresh_token) {
+      resp.refresh_token = token.refresh_token;
+    }
+    let expiration = new Date();
+    expiration.setSeconds(expiration.getSeconds() + resp.expires_in);
+    resp.expires = expiration;
+    await fs.writeFile(this.token_file_name, JSON.stringify(resp));
+    return resp;
+  }
+
+  public async addToPlaylist(song: SongInfo) {
+    log.debug('addToYoutubePlaylist');
+    const token = await this.refreshToken();
+    if (token == null) {
+      return;
+    }
+
+    if (!YOUTUBE_PLAYLIST_ID || YOUTUBE_PLAYLIST_ID === 'CHANGE_ME') {
+      log.debug('no playlist ID configured');
+      return;
+    }
+    if (!song.youtubeUrl) {
+      log.debug('Skip adding song to YT playlist, no youtube Url', song);
+      return;
+    }
+
+    const songUrl = new URL(song.youtubeUrl);
+    const youtubeId = songUrl.searchParams.get('v');
+    if (!youtubeId) {
+      log.debug(
+        'Skip adding song to YT playlist, could not extract YT id from URL',
+        song.youtubeUrl
+      );
+      return;
+    }
+    log.debug('Found YT id from URL', song.youtubeUrl, youtubeId);
+
+    const playlistItemsUrl = new URL(this.apiBase + '/playlistItems');
+    playlistItemsUrl.searchParams.append('videoId', youtubeId);
+    playlistItemsUrl.searchParams.append('playlistId', YOUTUBE_PLAYLIST_ID);
+    playlistItemsUrl.searchParams.append('part', 'id');
+    const existingPlaylistItem = await fetch(this.apiBase + '/playlistItems', {
+      headers: { Authorization: `${token.token_type} ${token.access_token}` }
+    }).then((r) => r.json());
+    log.debug('existingPlaylistItem', existingPlaylistItem);
+    if (existingPlaylistItem.pageInfo && existingPlaylistItem.pageInfo.totalResults > 0) {
+      log.info('Item already in playlist');
+      return;
+    }
+
+    const searchParams = new URLSearchParams([['part', 'snippet']]);
+    const options: RequestInit = {
+      method: 'POST',
+      headers: { Authorization: `${token.token_type} ${token.access_token}` },
+      body: JSON.stringify({
+        snippet: {
+          playlistId: YOUTUBE_PLAYLIST_ID,
+          resourceId: {
+            videoId: youtubeId,
+            kind: 'youtube#video'
+          }
+        }
+      })
+    };
+    const youtubeApiUrl = new URL(`${this.apiBase}/playlistItems?${searchParams}`);
+    const resp = await fetch(youtubeApiUrl, options);
+    const respObj = await resp.json();
+    log.debug('Added to playlist', options, respObj);
+    if (respObj.error) {
+      log.debug('Add to playlist failed', respObj.error.errors);
+    }
+  }
+}

src/routes/+layout.svelte

@@ -1,6 +1,11 @@
 <script lang="ts">
   import FooterComponent from '$lib/components/FooterComponent.svelte';
   import { SvelteToast } from '@zerodevx/svelte-toast';
+  interface Props {
+    children?: import('svelte').Snippet;
+  }
+
+  let { children }: Props = $props();
 
   const options = {
     pausable: true,
@@ -8,7 +13,7 @@
   };
 </script>
 
-<slot />
+{@render children?.()}
 <SvelteToast {options} />
 <div class="footer">
   <FooterComponent />

src/routes/+page.svelte

@@ -12,7 +12,12 @@
   import { cubicInOut } from 'svelte/easing';
   import { errorToast } from '$lib/errorToast';
 
-  export let data: PageData;
+  interface Props {
+    data: PageData;
+  }
+
+  let { data = $bindable() }: Props = $props();
+  let posts: Post[] = $state(data.posts);
 
   interface FetchOptions {
     since?: string;
@@ -26,8 +31,8 @@
 
   const refreshInterval = parseInt(PUBLIC_REFRESH_INTERVAL);
   let interval: ReturnType<typeof setTimeout> | null = null;
-  let moreOlderPostsAvailable = true;
+  let moreOlderPostsAvailable = $state(true);
-  let loadingOlderPosts = false;
+  let loadingOlderPosts = $state(false);
 
   // Needed, so that edgeFly() can do its thing:
   // To determine whether a newly loaded post is older than the existing ones, is required to know what the oldest
@@ -40,11 +45,11 @@
    */
   function edgeFly(node: Element, opts: EdgeFlyParams) {
     const createdAt = new Date(opts.created_at).getTime();
-    const diffNewest = Math.abs(new Date(data.posts[0].created_at).getTime() - createdAt);
+    const diffNewest = Math.abs(new Date(posts[0].created_at).getTime() - createdAt);
     const oldest =
       oldestBeforeLastFetch !== null
         ? oldestBeforeLastFetch
-        : new Date(data.posts[data.posts.length - 1].created_at).getTime();
+        : new Date(posts[posts.length - 1].created_at).getTime();
     const diffOldest = Math.abs(oldest - createdAt);
     const fromTop = diffNewest <= diffOldest;
 
@@ -79,15 +84,15 @@
 
   function refresh() {
     let filter: FetchOptions = {};
-    if (data.posts.length > 0) {
+    if (posts.length > 0) {
-      filter = { since: data.posts[0].created_at };
+      filter = { since: posts[0].created_at };
     }
     fetchPosts(filter)
       .then((resp) => {
         if (resp.length > 0) {
           // Prepend new posts, filter dupes
           // There shouldn't be any duplicates, but better be safe than sorry
-          data.posts = filterDuplicates(resp.concat(data.posts));
+          posts = filterDuplicates(resp.concat(posts));
         }
       })
       .catch((e: Error) => {
@@ -96,8 +101,9 @@
   }
 
   onMount(async () => {
-    if (data.posts.length > 0) {
+    posts = data.posts;
-      oldestBeforeLastFetch = new Date(data.posts[data.posts.length - 1].created_at).getTime();
+    if (posts.length > 0) {
+      oldestBeforeLastFetch = new Date(posts[posts.length - 1].created_at).getTime();
     }
     interval = setInterval(refresh, refreshInterval);
 
@@ -121,8 +127,8 @@
   function loadOlderPosts() {
     loadingOlderPosts = true;
     const filter: FetchOptions = { count: 20 };
-    if (data.posts.length > 0) {
+    if (posts.length > 0) {
-      const before = data.posts[data.posts.length - 1].created_at;
+      const before = posts[posts.length - 1].created_at;
       filter.before = before;
       oldestBeforeLastFetch = new Date(before).getTime();
     }
@@ -132,7 +138,7 @@
         if (resp.length > 0) {
           // Append old posts, filter dupes
           // There shouldn't be any duplicates, but better be safe than sorry
-          data.posts = filterDuplicates(data.posts.concat(resp));
+          posts = filterDuplicates(posts.concat(resp));
           // If we got less than we expected, there are no older posts available
           moreOlderPostsAvailable = resp.length >= (filter.count ?? 20);
         } else {
@@ -152,15 +158,15 @@
 </svelte:head>
 <h2>{PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME} music list</h2>
 <div class="wrapper">
-  <div />
+  <div></div>
   <div class="posts">
-    {#if data.posts.length === 0}
+    {#if posts.length === 0}
       Sorry, no posts recommending music have been found yet
     {/if}
-    {#each data.posts as post (post.url)}
+    {#each posts as post (post.url)}
       <div
         class="post"
-        transition:edgeFly={{
+        transition:edgeFly|global={{
           y: 10,
           created_at: post.created_at,
           duration: 300,
@@ -171,12 +177,12 @@
       </div>
     {/each}
     <LoadMoreComponent
-      on:loadOlderPosts={loadOlderPosts}
+      {loadOlderPosts}
       moreAvailable={moreOlderPostsAvailable}
       isLoading={loadingOlderPosts}
     />
   </div>
-  <div />
+  <div></div>
 </div>
 
 <style>

src/routes/ytauth/+page.server.ts

@@ -0,0 +1,24 @@
+import { log } from '$lib/log';
+import { YoutubePlaylistAdder } from '$lib/server/ytPlaylistAdder';
+import { redirect } from '@sveltejs/kit';
+import type { PageServerLoad } from './$types';
+
+export const load: PageServerLoad = async ({ url }) => {
+  const adder = new YoutubePlaylistAdder();
+  if (url.searchParams.has('code')) {
+    log.debug(url.searchParams);
+    await adder.receivedAuthCode(url.searchParams.get('code') || '', url);
+    redirect(307, '/');
+  } else if (url.searchParams.has('error')) {
+    log.error('received error', url.searchParams.get('error'));
+    return;
+  }
+
+  if (await adder.authCodeExists()) {
+    redirect(307, '/');
+  }
+
+  const authUrl = adder.constructAuthUrl(url);
+  log.debug('+page.server.ts', authUrl.toString());
+  redirect(307, authUrl);
+};

src/routes/ytauth/+page.svelte

@@ -0,0 +1,2 @@
+<h1>Hello and welcome to my site!</h1>
+<a href="/about">About my site</a>

svelte.config.js

@@ -1,5 +1,5 @@
 import adapter from '@sveltejs/adapter-node';
-import { vitePreprocess } from '@sveltejs/kit/vite';
+import { vitePreprocess } from '@sveltejs/vite-plugin-svelte';
 
 /** @type {import('@sveltejs/kit').Config} */
 const config = {

tsconfig.json

@@ -9,6 +9,7 @@
     "skipLibCheck": true,
     "sourceMap": true,
     "strict": true
+    //"lib": ["ESNext.Array"]
   }
   // Path aliases are handled by https://kit.svelte.dev/docs/configuration#alias
   //