check for missed posts

README.md

@@ -65,7 +65,7 @@ Set up NVM:
 
 ```
 $ cd
-$ curl https://raw.github.com/creationix/nvm/master/install.sh | sh
+$ curl https://raw.githubusercontent.com/nvm-sh/nvm/refs/heads/master/install.sh | bash
 $ source ~/.nvm/nvm.sh
 $ nvm install --lts
 ```

package.json

@@ -14,9 +14,9 @@
     "format": "prettier --plugin-search-dir . --write ."
   },
   "devDependencies": {
-    "@sveltejs/adapter-node": "^5.0.0",
+    "@sveltejs/adapter-node": "^5.2.12",
-    "@sveltejs/kit": "^2.7.0",
+    "@sveltejs/kit": "^2.21.5",
-    "@sveltejs/vite-plugin-svelte": "^4.0.0",
+    "@sveltejs/vite-plugin-svelte": "^5.0.0",
     "@types/node": "^22.6.1",
     "@types/sqlite3": "^3.0.0",
     "@types/ws": "^8.5.0",
@@ -24,21 +24,21 @@
     "@typescript-eslint/parser": "^8.0.0",
     "@zerodevx/svelte-toast": "^0.9.3",
     "eslint": "^9.11.1",
-    "eslint-config-prettier": "^9.0.0",
+    "eslint-config-prettier": "^10.0.0",
-    "eslint-plugin-svelte": "^2.45.1",
+    "eslint-plugin-svelte": "^3.0.0",
     "prettier": "^3.1.0",
     "prettier-plugin-svelte": "^3.2.6",
     "svelte": "^5",
     "svelte-check": "^4.0.0",
     "tslib": "^2.0.0",
     "typescript": "^5.0.0",
-    "vite": "^5.0.0"
+    "vite": "^6.0.0"
   },
   "type": "module",
   "dependencies": {
     "dotenv": "^16.0.3",
-    "feed": "^4.2.2",
+    "feed": "^5.1.0",
-    "sharp": "^0.33.0",
+    "sharp": "^0.34.2",
     "sqlite3": "^5.0.0",
     "ws": "^8.18.0"
   },

src/lib/components/AvatarComponent.svelte

@@ -16,7 +16,7 @@
       ['jpg', 99],
       ['jpeg', 99]
     ]);
-    const resizedAvatars = (account.resizedAvatars ?? []).sort((a, b) => {
+    const resizedAvatars = (account.resizedAvatars ?? []).toSorted((a, b) => {
       const extensionA = a.file.split('.').pop() ?? '';
       const extensionB = b.file.split('.').pop() ?? '';
       const prioA = formatPriority.get(extensionA) ?? 3;

src/lib/components/PostComponent.svelte

@@ -65,7 +65,7 @@
       ['jpg', 99],
       ['jpeg', 99]
     ]);
-    const thumbs = (song.resizedThumbnails ?? []).sort((a, b) => {
+    const thumbs = (song.resizedThumbnails ?? []).toSorted((a, b) => {
       const extensionA = a.file.split('.').pop() ?? '';
       const extensionB = b.file.split('.').pop() ?? '';
       const prioA = formatPriority.get(extensionA) ?? 3;

src/lib/server/timeline.ts

@@ -352,6 +352,30 @@ export class TimelineReader {
     }
   }
 
+  private async checkAndSavePost(post: Post) {
+    const hashttags: string[] = HASHTAG_FILTER.split(',');
+    const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
+
+    const songs = await TimelineReader.getSongInfoInPost(post);
+
+    // If we don't have any tags or non-youtube urls, check youtube
+    // YT is handled separately, because it requires an API call and therefore is slower
+    if (songs.length === 0 && found_tags.length === 0) {
+      log.log('Ignoring post', post.url);
+      return;
+    }
+
+    await savePost(post, songs);
+
+    await TimelineReader.saveAvatar(post.account);
+    await TimelineReader.saveSongThumbnails(songs);
+
+    log.debug('Saved post', post.url);
+
+    const posts = await getPosts(null, null, 100);
+    await saveAtomFeed(createFeed(posts));
+  }
+
   private startWebsocket() {
     const socket = new WebSocket(
       `wss://${MASTODON_INSTANCE}/api/v1/streaming?type=subscribe&stream=public:local&access_token=${MASTODON_ACCESS_TOKEN}`
@@ -367,28 +391,7 @@ export class TimelineReader {
           return;
         }
         const post: Post = JSON.parse(data.payload);
-
+        await this.checkAndSavePost(post);
-        const hashttags: string[] = HASHTAG_FILTER.split(',');
-        const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
-
-        const songs = await TimelineReader.getSongInfoInPost(post);
-
-        // If we don't have any tags or non-youtube urls, check youtube
-        // YT is handled separately, because it requires an API call and therefore is slower
-        if (songs.length === 0 && found_tags.length === 0) {
-          log.log('Ignoring post', post.url);
-          return;
-        }
-
-        await savePost(post, songs);
-
-        await TimelineReader.saveAvatar(post.account);
-        await TimelineReader.saveSongThumbnails(songs);
-
-        log.debug('Saved post', post.url);
-
-        const posts = await getPosts(null, null, 100);
-        await saveAtomFeed(createFeed(posts));
       } catch (e) {
         log.error('error message', event, event.data, e);
       }
@@ -410,9 +413,38 @@ export class TimelineReader {
     };
   }
 
+  private async loadPostsSinceLastRun() {
+    const now = new Date().toISOString();
+    let latestPost = await getPosts(null, now, 1);
+    log.log('Last post in DB since', now, latestPost);
+    let u = new URL(`https://${MASTODON_INSTANCE}/api/v1/timelines/public?local=true&limit=40`);
+    if (latestPost.length > 0) {
+      u.searchParams.append('since_id', latestPost[0].id);
+    }
+    for (let tag of HASHTAG_FILTER.split(',')) {
+      u.searchParams.append('q', '#' + tag);
+    }
+    const headers = {
+      Authorization: `Bearer ${MASTODON_ACCESS_TOKEN}`
+    };
+    const latestPosts: Post[] = await fetch(u, { headers }).then((r) => r.json());
+    log.info('searched posts', latestPosts);
+    for (const post of latestPosts) {
+      await this.checkAndSavePost(post);
+    }
+  }
+
   private constructor() {
     log.log('Constructing timeline object');
     this.startWebsocket();
+
+    this.loadPostsSinceLastRun()
+      .then((_) => {
+        log.info('loaded posts since last run');
+      })
+      .catch((e) => {
+        log.error('cannot fetch latest posts', e);
+      });
   }
 
   public static init() {

src/routes/+page.svelte

@@ -17,6 +17,7 @@
   }
 
   let { data = $bindable() }: Props = $props();
+  let posts: Post[] = $state(data.posts);
 
   interface FetchOptions {
     since?: string;
@@ -44,11 +45,11 @@
    */
   function edgeFly(node: Element, opts: EdgeFlyParams) {
     const createdAt = new Date(opts.created_at).getTime();
-    const diffNewest = Math.abs(new Date(data.posts[0].created_at).getTime() - createdAt);
+    const diffNewest = Math.abs(new Date(posts[0].created_at).getTime() - createdAt);
     const oldest =
       oldestBeforeLastFetch !== null
         ? oldestBeforeLastFetch
-        : new Date(data.posts[data.posts.length - 1].created_at).getTime();
+        : new Date(posts[posts.length - 1].created_at).getTime();
     const diffOldest = Math.abs(oldest - createdAt);
     const fromTop = diffNewest <= diffOldest;
 
@@ -83,15 +84,15 @@
 
   function refresh() {
     let filter: FetchOptions = {};
-    if (data.posts.length > 0) {
+    if (posts.length > 0) {
-      filter = { since: data.posts[0].created_at };
+      filter = { since: posts[0].created_at };
     }
     fetchPosts(filter)
       .then((resp) => {
         if (resp.length > 0) {
           // Prepend new posts, filter dupes
           // There shouldn't be any duplicates, but better be safe than sorry
-          data.posts = filterDuplicates(resp.concat(data.posts));
+          posts = filterDuplicates(resp.concat(posts));
         }
       })
       .catch((e: Error) => {
@@ -100,8 +101,9 @@
   }
 
   onMount(async () => {
-    if (data.posts.length > 0) {
+    posts = data.posts;
-      oldestBeforeLastFetch = new Date(data.posts[data.posts.length - 1].created_at).getTime();
+    if (posts.length > 0) {
+      oldestBeforeLastFetch = new Date(posts[posts.length - 1].created_at).getTime();
     }
     interval = setInterval(refresh, refreshInterval);
 
@@ -125,8 +127,8 @@
   function loadOlderPosts() {
     loadingOlderPosts = true;
     const filter: FetchOptions = { count: 20 };
-    if (data.posts.length > 0) {
+    if (posts.length > 0) {
-      const before = data.posts[data.posts.length - 1].created_at;
+      const before = posts[posts.length - 1].created_at;
       filter.before = before;
       oldestBeforeLastFetch = new Date(before).getTime();
     }
@@ -136,7 +138,7 @@
         if (resp.length > 0) {
           // Append old posts, filter dupes
           // There shouldn't be any duplicates, but better be safe than sorry
-          data.posts = filterDuplicates(data.posts.concat(resp));
+          posts = filterDuplicates(posts.concat(resp));
           // If we got less than we expected, there are no older posts available
           moreOlderPostsAvailable = resp.length >= (filter.count ?? 20);
         } else {
@@ -158,10 +160,10 @@
 <div class="wrapper">
   <div></div>
   <div class="posts">
-    {#if data.posts.length === 0}
+    {#if posts.length === 0}
       Sorry, no posts recommending music have been found yet
     {/if}
-    {#each data.posts as post (post.url)}
+    {#each posts as post (post.url)}
       <div
         class="post"
         transition:edgeFly|global={{