Auto reconnect to Mastodon WebSocket if it fails

.env.EXAMPLE

@@ -1,9 +1,12 @@
 HASHTAG_FILTER = ichlausche,music,musik,nowplaying,tunetuesday,nowlistening
 URL_FILTER = song.link,album.link,spotify.com,music.apple.com,bandcamp.com
 YOUTUBE_API_KEY = CHANGE_ME
+YOUTUBE_DISABLE = false
 MASTODON_INSTANCE = 'metalhead.club'
 BASE_URL = 'https://moshingmammut.phlaym.net'
 VERBOSE = false
+IGNORE_USERS = @moshhead@metalhead.club
+WEBSUB_HUB = 'http://pubsubhubbub.superfeedr.com'
 
 PUBLIC_REFRESH_INTERVAL = 10000
 PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME = 'Metalhead.club'

README.md

@@ -38,7 +38,7 @@ complexity. I'm willing to make this change if people prefer this though.
 
 Additionally, I ended up adding a few things which turned out to be not needed:
 The `tags` table (tags are included in the post's content and I don't do anything separately with tags) and
-`accounts.username` and `accounts.avatar_static`. I will keep these in until the initial wave of feedback arrives, and
+~~`accounts.username`~~ (s being used for #18) ~~and `accounts.avatar_static`~~ (has been removed). I will keep these in until the initial wave of feedback arrives, and
 remove it if no new features required them.
 
 I'll gladly accept any help in coming up with a good solution which doesn't need to store anything at all!
@@ -95,7 +95,9 @@ and set your `User`, `Group`, `ExecStart` and `WorkingDirectory` accordingly.
 
 Copy `.env.EXAMPLE` to `.env` and add your `YOUTUBE_API_KEY`.
 To obtain one follow [YouTube's guide](https://developers.google.com/youtube/registering_an_application) to create an
-_API key_. As soon as #13 is implemented, this will be optional!
+_API key_.
+If `YOUTUBE_API_KEY` is unset, all YouTube videos will be assumed to contain music links.
+If this is unwanted, set `YOUTUBE_DISABLE` to `true`).
 
 Run `npm run build` and copy the output folder, usually `build` to `$APP_DIR` on your server.
 

package.json

@@ -5,6 +5,7 @@
   "license": "LGPL-3.0-or-later",
   "scripts": {
     "dev": "vite dev",
+    "devn": "vite dev --host",
     "build": "vite build",
     "preview": "vite preview",
     "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",

src/app.html

@@ -10,12 +10,12 @@
     <meta name="apple-mobile-web-app-title" content="Moshing Mammut" />
     <meta name="application-name" content="Moshing Mammut" />
     <meta name="msapplication-TileColor" content="#2e0b78" />
-    <meta name="theme-color" content="#2e0b78" />
     <link rel="stylesheet" href="%sveltekit.assets%/style.css" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover" />
     <meta name="theme-color" content="#17063b" media="(prefers-color-scheme: dark)" />
     <meta name="theme-color" content="#BCB9B2" media="(prefers-color-scheme: light)" />
     <link rel="alternate" type="application/atom+xml" href="/feed.xml" title="Atom Feed" />
+    <link rel="hub" href="https://pubsubhubbub.superfeedr.com" />
     %sveltekit.head%
     <style>
       body {

src/lib/components/FooterComponent.svelte

@@ -42,6 +42,7 @@
     padding: 0.3em 1em;
     margin: 0 -8px;
     border-radius: 3px;
+    padding-bottom: env(safe-area-inset-bottom);
   }
   .icon {
     position: relative;
@@ -57,7 +58,7 @@
       background-color: var(--color-grey-translucent);
     }
   }
-  @media only screen and (max-device-width: 620px) {
+  @media only screen and (max-width: 620px) {
     .mastodonInstance,
     .feedSuffix {
       display: none;
@@ -67,7 +68,7 @@
     }
   }
 
-  @media only screen and (max-device-width: 430px) {
+  @media only screen and (max-width: 430px) {
     .mastodonInstance,
     .feedSuffix,
     .secretIngredient {
@@ -75,7 +76,7 @@
     }
   }
 
-  @media only screen and (max-device-width: 370px) {
+  @media only screen and (max-width: 370px) {
     .label {
       display: none;
     }

src/lib/components/PostComponent.svelte

@@ -41,6 +41,7 @@
     display: flex;
     flex-direction: column;
     flex-grow: 2;
+    word-break: break-word;
   }
   .meta {
     display: flex;

src/lib/mastodon/response.ts

@@ -24,5 +24,4 @@ export interface Account {
   display_name: string;
   url: string;
   avatar: string;
-  avatar_static: string;
 }

src/lib/server/db.ts

@@ -1,11 +1,26 @@
 import { env } from '$env/dynamic/private';
+import { IGNORE_USERS, MASTODON_INSTANCE } from '$env/static/private';
 import type { Account, Post, Tag } from '$lib/mastodon/response';
+import { isTruthy } from '$lib/truthyString';
 import sqlite3 from 'sqlite3';
 const { DEV } = import.meta.env;
 
 const db: sqlite3.Database = new sqlite3.Database('moshingmammut.db');
+// for the local masto instance, the instance name is *not* saved
+// as part of the username or acct, so it needs to be stripped
+const ignoredUsers: string[] =
+  IGNORE_USERS === undefined
+    ? []
+    : IGNORE_USERS.split(',')
+        .map((u) => (u.startsWith('@') ? u.substring(1) : u))
+        .map((u) =>
+          u.endsWith('@' + MASTODON_INSTANCE)
+            ? u.substring(0, u.length - ('@' + MASTODON_INSTANCE).length)
+            : u
+        );
+let databaseReady = false;
 
-if (DEV && env.VERBOSE === 'true') {
+if (DEV && isTruthy(env.VERBOSE)) {
   sqlite3.verbose();
   db.on('change', (t, d, table, rowid) => {
     console.debug('DB change event', t, d, table, rowid);
@@ -33,13 +48,25 @@ db.on('open', () => {
   db.all('SELECT id FROM migrations', (err, rows) => {
     if (err !== null) {
       console.error('Could not fetch existing migrations', err);
+      databaseReady = true;
       return;
     }
     console.debug('Already applied migrations', rows);
     const appliedMigrations: Set<number> = new Set(rows.map((row: any) => row['id']));
     const toApply = getMigrations().filter((m) => !appliedMigrations.has(m.id));
+    let remaining = toApply.length;
+    if (remaining === 0) {
+      databaseReady = true;
+      return;
+    }
     for (const migration of toApply) {
       db.exec(migration.statement, (err) => {
+        remaining--;
+        // This will set databaseReady to true before the migration has been inserted as applies,
+        // but that doesn't matter. It's only important that is has been applied
+        if (remaining === 0) {
+          databaseReady = true;
+        }
         if (err !== null) {
           console.error(`Failed to apply migration ${migration.name}`, err);
           return;
@@ -94,34 +121,102 @@ function getMigrations(): Migration[] {
         FOREIGN KEY (post_id) REFERENCES posts(id),
         FOREIGN KEY (tag_url) REFERENCES tags(url)
       )`
+    },
+    {
+      id: 2,
+      name: 'urls as keys',
+      statement: `
+      CREATE TABLE accounts_new (
+        id TEXT NOT NULL,
+        acct TEXT,
+        username TEXT,
+        display_name TEXT,
+        url TEXT NOT NULL PRIMARY KEY,
+        avatar TEXT
+      );
+      INSERT INTO accounts_new (id, acct, username, display_name, url, avatar)
+         SELECT id, acct, username, display_name, url, avatar
+         FROM accounts;
+      DROP TABLE accounts;
+      ALTER TABLE accounts_new RENAME TO accounts;
+
+      CREATE TABLE posts_new (
+        id TEXT NOT NULL,
+        content TEXT,
+        created_at TEXT,
+        url TEXT NOT NULL PRIMARY KEY,
+        account_id TEXT NOT NULL,
+        FOREIGN KEY (account_id) REFERENCES accounts(url)
+      );
+      INSERT INTO posts_new (id, content, created_at, url, account_id)
+         SELECT p.id, p.content, p.created_at, p.url, accounts.url
+         FROM posts as p
+         JOIN accounts ON accounts.id = p.account_id;
+      DROP TABLE posts;
+      ALTER TABLE posts_new RENAME TO posts;
+
+      CREATE TABLE poststags_new (
+        id integer PRIMARY KEY,
+        post_id TEXT NOT NULL,
+        tag_url TEXT NOT NULL,
+        FOREIGN KEY (post_id) REFERENCES posts(url),
+        FOREIGN KEY (tag_url) REFERENCES tags(url)
+      );
+
+      INSERT INTO poststags_new (id, post_id, tag_url)
+         SELECT pt.id, posts.url, pt.tag_url
+         FROM poststags as pt
+         JOIN posts ON posts.id = pt.post_id;
+      DROP TABLE poststags;
+      ALTER TABLE poststags_new RENAME TO poststags;
+      `
     }
   ];
 }
 
+async function waitReady(): Promise<undefined> {
+  // Simpler than a semaphore and is really only needed on startup
+  return new Promise((resolve) => {
+    const interval = setInterval(() => {
+      if (DEV) {
+        console.debug('Waiting for database to be ready');
+      }
+      if (databaseReady) {
+        if (DEV) {
+          console.debug('DB is ready');
+        }
+        clearInterval(interval);
+        resolve(undefined);
+      }
+    }, 100);
+  });
+}
+
 export async function savePost(post: Post): Promise<undefined> {
-  return new Promise((resolve, reject) => {
+  if (!databaseReady) {
+    await waitReady();
+  }
+  return await new Promise<undefined>((resolve, reject) => {
     console.debug(`Saving post ${post.url}`);
     const account = post.account;
     db.run(
       `
-      INSERT INTO accounts (id, acct, username, display_name, url, avatar, avatar_static)
+      INSERT INTO accounts (id, acct, username, display_name, url, avatar)
-      VALUES(?, ?, ?, ?, ?, ?, ?)
+      VALUES(?, ?, ?, ?, ?, ?)
-      ON CONFLICT(id)
+      ON CONFLICT(url)
       DO UPDATE SET
       acct=excluded.acct,
       username=excluded.username,
       display_name=excluded.display_name,
-      url=excluded.url,
+      id=excluded.id,
-      avatar=excluded.avatar,
+      avatar=excluded.avatar;`,
-      avatar_static=excluded.avatar_static;`,
       [
         account.id,
         account.acct,
         account.username,
         account.display_name,
         account.url,
-        account.avatar,
+        account.avatar
-        account.avatar_static
       ],
       (err) => {
         if (err !== null) {
@@ -132,12 +227,12 @@ export async function savePost(post: Post): Promise<undefined> {
         db.run(
           `
           INSERT INTO posts (id, content, created_at, url, account_id)
-          VALUES (?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET
+          VALUES (?, ?, ?, ?, ?) ON CONFLICT(url) DO UPDATE SET
           content=excluded.content,
           created_at=excluded.created_at,
-          url=excluded.url,
+          id=excluded.id,
           account_id=excluded.account_id;`,
-          [post.id, post.content, post.created_at, post.url, post.account.id],
+          [post.id, post.content, post.created_at, post.url, post.account.url],
           (postErr) => {
             if (postErr !== null) {
               console.error(`Could not insert post ${post.url}`, postErr);
@@ -162,7 +257,7 @@ export async function savePost(post: Post): Promise<undefined> {
                     }
                     db.run(
                       'INSERT INTO poststags (post_id, tag_url) VALUES (?, ?)',
-                      [post.id, tag.url],
+                      [post.url, tag.url],
                       (posttagserr) => {
                         if (posttagserr !== null) {
                           console.error(
@@ -192,8 +287,11 @@ export async function savePost(post: Post): Promise<undefined> {
 }
 
 export async function getPosts(since: string | null, before: string | null, limit: number) {
+  if (!databaseReady) {
+    await waitReady();
+  }
   const promise = await new Promise<Post[]>((resolve, reject) => {
-    let filter_query;
+    let filter_query = '';
     const params: any = { $limit: limit };
     if (since === null && before === null) {
       filter_query = '';
@@ -205,11 +303,22 @@ export async function getPosts(since: string | null, before: string | null, limi
       filter_query = 'WHERE posts.created_at < $before';
       params.$before = before;
     }
+
+    ignoredUsers.forEach((ignoredUser, index) => {
+      const userParam = `$user_${index}`;
+      const acctParam = userParam + 'a';
+      const usernameParam = userParam + 'u';
+      const prefix = filter_query === '' ? ' WHERE' : ' AND';
+      filter_query += `${prefix} acct != ${acctParam} AND username != ${usernameParam} `;
+      params[acctParam] = ignoredUser;
+      params[usernameParam] = ignoredUser;
+    });
+
     const sql = `SELECT posts.id, posts.content, posts.created_at, posts.url,
       accounts.id AS account_id, accounts.acct, accounts.username, accounts.display_name,
       accounts.url AS account_url, accounts.avatar
       FROM posts
-    JOIN accounts ON posts.account_id = accounts.id
+      JOIN accounts ON posts.account_id = accounts.url
       ${filter_query}
       ORDER BY created_at DESC
       LIMIT $limit`;
@@ -230,7 +339,7 @@ export async function getPosts(since: string | null, before: string | null, limi
           FROM poststags
           JOIN tags ON poststags.tag_url = tags.url
           WHERE post_id IN (${postIdsParams});`,
-        rows.map((r: any) => r.id),
+        rows.map((r: any) => r.url),
         (tagErr, tagRows: any[]) => {
           if (tagErr != null) {
             console.error('Error loading post tags', tagErr);
@@ -258,8 +367,7 @@ export async function getPosts(since: string | null, before: string | null, limi
                 username: row.username,
                 display_name: row.display_name,
                 url: row.account_url,
-                avatar: row.avatar,
+                avatar: row.avatar
-                avatar_static: ''
               } as Account
             } as Post;
           });

src/lib/server/rss.ts

@@ -1,4 +1,4 @@
-import { BASE_URL } from '$env/static/private';
+import { BASE_URL, WEBSUB_HUB } from '$env/static/private';
 import { PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME } from '$env/static/public';
 import type { Post } from '$lib//mastodon/response';
 import { Feed } from 'feed';
@@ -6,6 +6,7 @@ import fs from 'fs/promises';
 
 export function createFeed(posts: Post[]): Feed {
   const baseUrl = BASE_URL.endsWith('/') ? BASE_URL : BASE_URL + '/';
+  const hub = WEBSUB_HUB ? WEBSUB_HUB : undefined;
   const feed = new Feed({
     title: `${PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME} music feed`,
     description: `Posts about music on ${PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME}`,
@@ -19,6 +20,7 @@ export function createFeed(posts: Post[]): Feed {
     feedLinks: {
       atom: `${BASE_URL}/feed.xml`
     },
+    hub: hub,
     author: {
       name: '@aymm',
       link: 'https://metalhead.club/@aymm'
@@ -40,8 +42,23 @@ export function createFeed(posts: Post[]): Feed {
     });
   });
   feed.addCategory('Music');
+
   return feed;
 }
 export async function saveAtomFeed(feed: Feed) {
   await fs.writeFile('feed.xml', feed.atom1(), { encoding: 'utf8' });
+  if (!WEBSUB_HUB) {
+    return;
+  }
+  try {
+    const params = new URLSearchParams();
+    params.append('hub.mode', 'publish');
+    params.append('hub.url', `${BASE_URL}/feed.xml`);
+    await fetch(WEBSUB_HUB, {
+      method: 'POST',
+      body: params
+    });
+  } catch (e) {
+    console.error('Failed to update WebSub hub', e);
+  }
 }

src/lib/server/timeline.ts

@@ -2,11 +2,13 @@ import {
   HASHTAG_FILTER,
   MASTODON_INSTANCE,
   URL_FILTER,
-  YOUTUBE_API_KEY
+  YOUTUBE_API_KEY,
+  YOUTUBE_DISABLE
 } from '$env/static/private';
 import type { Post, Tag, TimelineEvent } from '$lib/mastodon/response';
 import { getPosts, savePost } from '$lib/server/db';
 import { createFeed, saveAtomFeed } from '$lib/server/rss';
+import { isTruthy } from '$lib/truthyString';
 import { WebSocket } from 'ws';
 
 const YOUTUBE_REGEX = new RegExp(
@@ -17,6 +19,11 @@ export class TimelineReader {
   private static _instance: TimelineReader;
 
   private static async isMusicVideo(videoId: string) {
+    if (YOUTUBE_API_KEY === undefined) {
+      // Assume that it *is* a music link when no YT API key is provided
+      // If it should assumed to not be YOUTUBE_DISABLE needs to be set to something truthy
+      return true;
+    }
     const searchParams = new URLSearchParams([
       ['part', 'snippet'],
       ['id', videoId],
@@ -50,6 +57,9 @@ export class TimelineReader {
   }
 
   private static async checkYoutubeMatches(postContent: string): Promise<boolean> {
+    if (isTruthy(YOUTUBE_DISABLE)) {
+      return false;
+    }
     const matches = postContent.matchAll(YOUTUBE_REGEX);
     for (const match of matches) {
       if (match === undefined || match.groups === undefined) {
@@ -68,7 +78,7 @@ export class TimelineReader {
     return false;
   }
 
-  private constructor() {
+  private startWebsocket() {
     const socket = new WebSocket(`wss://${MASTODON_INSTANCE}/api/v1/streaming`);
     socket.onopen = () => {
       socket.send('{ "type": "subscribe", "stream": "public:local"}');
@@ -103,13 +113,25 @@ export class TimelineReader {
       }
     };
     socket.onclose = (event) => {
-      console.log('Closed', event, event.code, event.reason);
+      console.warn(
+        `Websocket connection to ${MASTODON_INSTANCE} closed. Code: ${event.code}, reason: '${event.reason}'`
+      );
+      setTimeout(() => {
+        console.info(`Attempting to reconenct to WS`);
+        this.startWebsocket();
+      }, 10000);
     };
     socket.onerror = (event) => {
-      console.log('error', event, event.message, event.error);
+      console.error(
+        `Websocket connection to ${MASTODON_INSTANCE} failed. ${event.type}: ${event.error}, message: '${event.message}'`
+      );
     };
   }
 
+  private constructor() {
+    this.startWebsocket();
+  }
+
   public static init() {
     if (this._instance === undefined) {
       this._instance = new TimelineReader();

src/lib/truthyString.ts

@@ -0,0 +1,7 @@
+export function isTruthy(value: string | number | boolean | null | undefined): boolean {
+  if (typeof value === 'string') {
+    return value.toLowerCase() === 'true' || !!+value; // here we parse to number first
+  }
+
+  return !!value;
+}

src/routes/+layout.svelte

@@ -34,9 +34,9 @@
     align-items: center;
     gap: 10px;
   }
-  @media only screen and (max-device-width: 620px) {
+  @media only screen and (max-width: 620px) {
     .footer {
-      width: calc(100% + 16px);
+      width: 100%;
     }
   }
 </style>

src/routes/+page.svelte

@@ -155,7 +155,7 @@
   <div />
   <div class="posts">
     {#if data.posts.length === 0}
-      Sorry, no posts recommending music aave been found yet
+      Sorry, no posts recommending music have been found yet
     {/if}
     {#each data.posts as post (post.url)}
       <div
@@ -202,9 +202,10 @@
     z-index: 100;
   }
 
-  @media only screen and (max-device-width: 650px) {
+  @media only screen and (max-width: 650px) {
     .post {
-      max-width: 100vw;
+      max-width: calc(100vw - 16px);
+      padding: 1em 0;
     }
   }
 </style>