Improve type safety

Reviewed-on: https://phlaym.net/git/phlaym/moshing-mammut/pulls/20

.env.EXAMPLE

@@ -1,9 +1,13 @@
 HASHTAG_FILTER = ichlausche,music,musik,nowplaying,tunetuesday,nowlistening
-URL_FILTER = song.link,album.link,spotify.com,music.apple.com,bandcamp.com
+URL_FILTER = song.link,album.link,spotify.com,music.apple.com,bandcamp.com,songwhip.com
 YOUTUBE_API_KEY = CHANGE_ME
+YOUTUBE_DISABLE = false
+ODESLI_API_KEY = CHANGE_ME
 MASTODON_INSTANCE = 'metalhead.club'
 BASE_URL = 'https://moshingmammut.phlaym.net'
 VERBOSE = false
+IGNORE_USERS = @moshhead@metalhead.club
+WEBSUB_HUB = 'http://pubsubhubbub.superfeedr.com'
 
 PUBLIC_REFRESH_INTERVAL = 10000
 PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME = 'Metalhead.club'

.nova/Configuration.json

@@ -1,7 +1,8 @@
 {
-  "apexskier.typescript.config.formatDocumentOnSave": "true",
+  "apexskier.eslint.config.eslintPath" : "node_modules\/@eslint\/eslintrc\/dist\/eslintrc.cjs",
-  "apexskier.typescript.config.isEnabledForJavascript": "Enable",
+  "apexskier.typescript.config.formatDocumentOnSave" : "true",
-  "apexskier.typescript.config.organizeImportsOnSave": "true",
+  "apexskier.typescript.config.isEnabledForJavascript" : "Enable",
-  "apexskier.typescript.config.userPreferences.quotePreference": "single",
+  "apexskier.typescript.config.organizeImportsOnSave" : "true",
-  "apexskier.typescript.config.userPreferences.useLabelDetailsInCompletionEntries": true
+  "apexskier.typescript.config.userPreferences.quotePreference" : "single",
+  "apexskier.typescript.config.userPreferences.useLabelDetailsInCompletionEntries" : true
 }

.prettierignore

@@ -6,6 +6,7 @@ node_modules
 .env
 .env.*
 !.env.example
+/.nova
 
 # Ignore files for PNPM, NPM and YARN
 pnpm-lock.yaml

README.md

@@ -33,12 +33,12 @@ I can see that there are plenty of posts using only descriptions and links witho
 be missed. This isn't a great solution.
 
 Another idea was to store only URLs of posts and resolve the content and account information live.
-This would be better, but I'm *still* storing post information while also slowing the app down and introduce more code
+This would be better, but I'm _still_ storing post information while also slowing the app down and introduce more code
 complexity. I'm willing to make this change if people prefer this though.
 
 Additionally, I ended up adding a few things which turned out to be not needed:
 The `tags` table (tags are included in the post's content and I don't do anything separately with tags) and
-`accounts.username` and `accounts.avatar_static`. I will keep these in until the initial wave of feedback arrives, and
+~~`accounts.username`~~ (s being used for #18) ~~and `accounts.avatar_static`~~ (has been removed). I will keep these in until the initial wave of feedback arrives, and
 remove it if no new features required them.
 
 I'll gladly accept any help in coming up with a good solution which doesn't need to store anything at all!
@@ -51,7 +51,6 @@ By default, NVM is used to install NodeJS, but you can install it any way you wa
 
 This is based on [SvelteKit's instructions](https://kit.svelte.dev/docs/adapter-node#deploying) and [How To Deploy Node.js Applications Using Systemd and Nginx](https://www.digitalocean.com/community/tutorials/how-to-deploy-node-js-applications-using-systemd-and-nginx)
 
-
 #### On your server
 
 Install Apache2 if not already installed.
@@ -78,7 +77,6 @@ Enter `$APP_DIR`.
 Place `package-lock.json` and `start.sh.EXAMPLE` in this directory.
 Run `npm ci --omit dev` to install the dependencies.
 
-
 Rename `start.sh.EXAMPLE` to `start.sh` and set the path to your NVM.
 
 If you do not have NVM installed, simply remove the line and make sure your node executable can be found either by
@@ -93,12 +91,13 @@ If you do, add the path to your SSLCertificateFile and SSLCertificateKeyFile.
 Copy `moshing-mammut.service.EXAMPLE` to `/etc/systemd/system/moshing-mammut.service`
 and set your `User`, `Group`, `ExecStart` and `WorkingDirectory` accordingly.
 
-
 #### On your development machine
 
 Copy `.env.EXAMPLE` to `.env` and add your `YOUTUBE_API_KEY`.
 To obtain one follow [YouTube's guide](https://developers.google.com/youtube/registering_an_application) to create an
-*API key*. As soon as #13 is implemented, this will be optional!
+_API key_.
+If `YOUTUBE_API_KEY` is unset, all YouTube videos will be assumed to contain music links.
+If this is unwanted, set `YOUTUBE_DISABLE` to `true`).
 
 Run `npm run build` and copy the output folder, usually `build` to `$APP_DIR` on your server.
 

faviconDescription.json

@@ -1,4 +1,3 @@
-
 {
   "masterPicture": "./icon.svg",
   "iconsPath": "/static",
@@ -61,4 +60,4 @@
     "htmlCodeFile": false,
     "usePathAsIs": false
   }
-}
+}

package.json

@@ -5,6 +5,7 @@
   "license": "LGPL-3.0-or-later",
   "scripts": {
     "dev": "vite dev",
+    "devn": "vite dev --host",
     "build": "vite build",
     "preview": "vite preview",
     "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",

src/app.html

@@ -1,53 +1,53 @@
 <!DOCTYPE html>
 <html lang="en">
-	<head>
+  <head>
-		<meta charset="utf-8" />
+    <meta charset="utf-8" />
-		<link rel="apple-touch-icon" sizes="180x180" href="%sveltekit.assets%/apple-touch-icon.png" />
+    <link rel="apple-touch-icon" sizes="180x180" href="%sveltekit.assets%/apple-touch-icon.png" />
-		<link rel="icon" type="image/png" sizes="32x32" href="%sveltekit.assets%/favicon-32x32.png" />
+    <link rel="icon" type="image/png" sizes="32x32" href="%sveltekit.assets%/favicon-32x32.png" />
-		<link rel="icon" type="image/png" sizes="16x16" href="%sveltekit.assets%/favicon-16x16.png" />
+    <link rel="icon" type="image/png" sizes="16x16" href="%sveltekit.assets%/favicon-16x16.png" />
-		<link rel="manifest" href="%sveltekit.assets%/site.webmanifest" />
+    <link rel="manifest" href="%sveltekit.assets%/site.webmanifest" />
-		<link rel="mask-icon" href="%sveltekit.assets%/safari-pinned-tab.svg" color="#2e0b78" />
+    <link rel="mask-icon" href="%sveltekit.assets%/safari-pinned-tab.svg" color="#2e0b78" />
-		<meta name="apple-mobile-web-app-title" content="Moshing Mammut" />
+    <meta name="apple-mobile-web-app-title" content="Moshing Mammut" />
-		<meta name="application-name" content="Moshing Mammut" />
+    <meta name="application-name" content="Moshing Mammut" />
-		<meta name="msapplication-TileColor" content="#2e0b78" />
+    <meta name="msapplication-TileColor" content="#2e0b78" />
-		<meta name="theme-color" content="#2e0b78" />
+    <link rel="stylesheet" href="%sveltekit.assets%/style.css" />
-		<link rel="stylesheet" href="%sveltekit.assets%/style.css" />
+    <meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover" />
-		<meta name="viewport" content="width=device-width" />
+    <meta name="theme-color" content="#17063b" media="(prefers-color-scheme: dark)" />
-		<meta name="theme-color" content="#2E0B78" media="(prefers-color-scheme: dark)" />
+    <meta name="theme-color" content="#BCB9B2" media="(prefers-color-scheme: light)" />
-		<meta name="theme-color" content="#BCB9B2" media="(prefers-color-scheme: light)" />
+    <link rel="alternate" type="application/atom+xml" href="/feed.xml" title="Atom Feed" />
-		<link rel="alternate" type="application/atom+xml" href="/feed.xml" title="Atom Feed" />
+    <link rel="hub" href="https://pubsubhubbub.superfeedr.com" />
-		%sveltekit.head%
+    %sveltekit.head%
-		<style>
+    <style>
-			body {
+      body {
-				--color-text: #2F0C7A;
+        --color-text: #2f0c7a;
-				--color-bg: white;
+        --color-bg: white;
-				--color-border: #17063B;
+        --color-border: #17063b;
-				--color-link: #563ACC;
+        --color-link: #563acc;
-				--color-link-visited: #858AFA;
+        --color-link-visited: #858afa;
 
-				color: var(--color-text);
+        color: var(--color-text);
-				background-color: var(--color-bg);
+        background-color: var(--color-bg);
-			}
+      }
 
-			a {
+      a {
-				color: var(--color-link);
+        color: var(--color-link);
-			}
+      }
-			a:visited {
+      a:visited {
-				color: var(--color-link-visited);
+        color: var(--color-link-visited);
-			}
+      }
 
-			@media (prefers-color-scheme: dark) {
+      @media (prefers-color-scheme: dark) {
-				body {
+        body {
-					--color-text: white;
+          --color-text: white;
-					--color-bg: #17063B;
+          --color-bg: #17063b;
-					--color-border: white;
+          --color-border: white;
-					--color-link: #8A9BF0;
+          --color-link: #8a9bf0;
-					--color-link-visited: #C384FB;
+          --color-link-visited: #c384fb;
-				}
+        }
-			}
+      }
-		</style>
+    </style>
-	</head>
+  </head>
-	<body data-sveltekit-preload-data="hover">
+  <body data-sveltekit-preload-data="hover">
-		<div style="display: contents">%sveltekit.body%</div>
+    <div style="display: contents">%sveltekit.body%</div>
-	</body>
+  </body>
 </html>

src/hooks.server.ts

@@ -10,8 +10,7 @@ export const handleError = (({ error }) => {
   }
 
   return {
-    message: 'Whoops!',
+    message: `Something went wrong! ${error}`
-    code: (error as any)?.code ?? 'UNKNOWN'
   };
 }) satisfies HandleServerError;
 
@@ -24,12 +23,9 @@ export const handle = (async ({ event, resolve }) => {
   }
   if (event.url.pathname === '/feed.xml') {
     const f = await fs.readFile('feed.xml', { encoding: 'utf8' });
-    return new Response(
+    return new Response(f, { headers: [['Content-Type', 'application/atom+xml']] });
-      f,
-      { headers: [['Content-Type', 'application/atom+xml']] }
-    );
   }
 
   const response = await resolve(event);
   return response;
-}) satisfies Handle;
+}) satisfies Handle;

src/lib/components/AccountComponent.svelte

@@ -4,4 +4,4 @@
   export let account: Account;
 </script>
 
-<a href="{account.url}" target="_blank">{account.display_name} @{account.acct}</a>
+<a href={account.url} target="_blank">{account.display_name} @{account.acct}</a>

src/lib/components/AvatarComponent.svelte

@@ -3,18 +3,18 @@
 
   export let account: Account;
   let avatarDescription: string;
-  $: avatarDescription = `Avatar for ${account.acct}`
+  $: avatarDescription = `Avatar for ${account.acct}`;
 </script>
 
-<img src="{account.avatar}" alt={avatarDescription}/>
+<img src={account.avatar} alt={avatarDescription} />
 
 <style>
-img {
+  img {
-  max-width: 50px;
+    max-width: 50px;
-  max-height: 50px;
+    max-height: 50px;
-  width: auto;
+    width: auto;
-  height: auto;
+    height: auto;
-  object-fit: contain;
+    object-fit: contain;
-  border-radius: 3px;;
+    border-radius: 3px;
-}
+  }
-</style>
+</style>

src/lib/components/FooterComponent.svelte

@@ -4,24 +4,28 @@
 </script>
 
 <div class="footer">
-    <div>
+  <div>
-        <span>Made with &#x1F918; by&nbsp;</span>
+    <span class="label"
-        <a href="https://metalhead.club/@aymm" rel="me">@aymm@metalhead.club</a>
+      >Made<span class="secretIngredient">&nbsp;with &#x1F918;</span>&nbsp;by&nbsp;</span
-    </div>
+    >
-    |
+    <a href="https://metalhead.club/@aymm" rel="me"
-    <div>
+      >@aymm<span class="mastodonInstance">@metalhead.club</span></a
-        <a href="https://phlaym.net/git/phlaym/moshing-mammut">
+    >
-            <img alt="Git branch" src={git} class="icon" />
+  </div>
-            Source Code
+  |
-        </a>
+  <div>
-    </div>
+    <a href="https://phlaym.net/git/phlaym/moshing-mammut">
-    |
+      <img alt="Git branch" src={git} class="icon" />
-    <div>
+      <span class="label">Source Code</span>
-        <a href="/feed.xml">
+    </a>
-            <img alt="RSS" src={rss} class="icon" />
+  </div>
-            RSS Feed
+  |
-        </a>
+  <div>
-    </div>
+    <a href="/feed.xml">
+      <img alt="RSS" src={rss} class="icon" />
+      <span class="label">RSS<span class="feedSuffix">&nbsp;Feed</span></span>
+    </a>
+  </div>
 </div>
 
 <style>
@@ -38,6 +42,7 @@
     padding: 0.3em 1em;
     margin: 0 -8px;
     border-radius: 3px;
+    padding-bottom: env(safe-area-inset-bottom);
   }
   .icon {
     position: relative;
@@ -53,4 +58,27 @@
       background-color: var(--color-grey-translucent);
     }
   }
-</style>
+  @media only screen and (max-width: 620px) {
+    .mastodonInstance,
+    .feedSuffix {
+      display: none;
+    }
+    .footer {
+      justify-content: center;
+    }
+  }
+
+  @media only screen and (max-width: 430px) {
+    .mastodonInstance,
+    .feedSuffix,
+    .secretIngredient {
+      display: none;
+    }
+  }
+
+  @media only screen and (max-width: 370px) {
+    .label {
+      display: none;
+    }
+  }
+</style>

src/lib/components/LoadMoreComponent.svelte

@@ -2,19 +2,19 @@
   import { createEventDispatcher } from 'svelte';
   import LoadingSpinnerComponent from '$lib/components/LoadingSpinnerComponent.svelte';
 
-  export let moreAvailable: boolean = false;
+  export let moreAvailable = false;
-  export let isLoading: boolean = false;
+  export let isLoading = false;
   let displayText = '';
   let title = '';
   let disabled: boolean;
 
   $: if (isLoading) {
-      displayText = 'Loading...';
+    displayText = 'Loading...';
   } else if (!moreAvailable) {
-      displayText = 'You reached the end';
+    displayText = 'You reached the end';
   } else {
-      displayText = 'Load More';
+    displayText = 'Load More';
-  };
+  }
   $: disabled = !moreAvailable || isLoading;
   $: title = moreAvailable ? 'Load More' : 'There be dragons!';
 
@@ -26,10 +26,10 @@
 </script>
 
 <button on:click={loadOlderPosts} {disabled} {title}>
-    <div class="loading" class:collapsed={!isLoading}>
+  <div class="loading" class:collapsed={!isLoading}>
-      <LoadingSpinnerComponent size='0.5em' thickness='6px' />
+    <LoadingSpinnerComponent size="0.5em" thickness="6px" />
-    </div>
+  </div>
-    <span>{displayText}</span>
+  <span>{displayText}</span>
 </button>
 
 <style>
@@ -78,4 +78,4 @@
     max-width: 0;
     margin-right: 0;
   }
-</style>
+</style>

src/lib/components/LoadingSpinnerComponent.svelte

@@ -1,9 +1,10 @@
 <script lang="ts">
-  export let size: string = '64px';
+  export let size = '64px';
-  export let thickness: string = '6px';
+  export let thickness = '6px';
 </script>
 
-<div class="lds-dual-ring" style="--size: {size}; --thickness: {thickness}"></div>
+<div class="lds-dual-ring" style="--size: {size}; --thickness: {thickness}" />
+
 <style>
   .lds-dual-ring {
     display: inline-block;
@@ -11,7 +12,7 @@
     height: 100%;
   }
   .lds-dual-ring:after {
-    content: " ";
+    content: ' ';
     display: block;
     width: var(--size);
     height: var(--size);
@@ -34,5 +35,4 @@
       transform: rotate(360deg);
     }
   }
-
+</style>
-</style>

src/lib/components/PostComponent.svelte

@@ -3,14 +3,14 @@
   import AvatarComponent from '$lib/components/AvatarComponent.svelte';
   import AccountComponent from '$lib/components/AccountComponent.svelte';
   import { secondsSince, relativeTime } from '$lib/relativeTime';
-  import { onMount } from "svelte";
+  import { onMount } from 'svelte';
 
   export let post: Post;
   let displayRelativeTime = false;
   const absoluteDate = new Date(post.created_at).toLocaleString();
   let dateCreated = absoluteDate;
   const timePassed = secondsSince(new Date(post.created_at));
-  $: if(displayRelativeTime) {
+  $: if (displayRelativeTime) {
     dateCreated = relativeTime($timePassed) ?? absoluteDate;
   }
 
@@ -19,16 +19,15 @@
     // When JS is disabled the server-side rendered absolute date will be shown,
     // otherwise the relative date would be outdated very quickly
     displayRelativeTime = true;
-  })
+  });
 </script>
 
-
 <div class="wrapper">
   <div class="avatar"><AvatarComponent account={post.account} /></div>
   <div class="post">
     <div class="meta">
       <AccountComponent account={post.account} />
-      <small><a href={post.url} target="_blank" title="{absoluteDate}">{dateCreated}</a></small>
+      <small><a href={post.url} target="_blank" title={absoluteDate}>{dateCreated}</a></small>
     </div>
     <div class="content">{@html post.content}</div>
   </div>
@@ -42,6 +41,7 @@
     display: flex;
     flex-direction: column;
     flex-grow: 2;
+    word-break: break-word;
   }
   .meta {
     display: flex;
@@ -54,4 +54,4 @@
     max-width: calc(600px - 1em - 50px);
     overflow-x: auto;
   }
-</style>
+</style>

src/lib/errorToast.ts

@@ -5,4 +5,4 @@ export function errorToast(message: string): number {
   return toast.push(`<img src="${errorIcon}" />${message}`, {
     classes: ['error']
   });
-}
+}

src/lib/mastodon/response.ts

@@ -1,29 +1,37 @@
 export interface TimelineEvent {
-  event: string,
+  event: string;
-  payload: string
+  payload: string;
 }
 
 export interface Post {
-  id: string,
+  id: string;
-  created_at: string,
+  created_at: string;
-  tags: Tag[],
+  tags: Tag[];
-  url: string,
+  url: string;
-  content: string,
+  content: string;
-  account: Account
+  account: Account;
+  card?: PreviewCard;
+}
 
+export interface PreviewCard {
+  url: string;
+  title: string;
+  image?: string;
+  blurhash?: string;
+  width: number;
+  height: number;
 }
 
 export interface Tag {
-  name: string,
+  name: string;
-  url: string
+  url: string;
 }
 
 export interface Account {
-  id: string,
+  id: string;
-  acct: string,
+  acct: string;
-  username: string,
+  username: string;
-  display_name: string,
+  display_name: string;
-  url: string,
+  url: string;
-  avatar: string,
+  avatar: string;
-  avatar_static: string
+}
-}

src/lib/odesliResponse.ts

@@ -0,0 +1,143 @@
+export type SongInfo = {
+  pageUrl: string;
+  youtubeUrl?: string;
+  type: 'song' | 'album';
+  title?: string;
+  artistName?: string;
+  thumbnailUrl?: string;
+};
+
+export type SongwhipReponse = {
+  type: 'track' | 'album';
+  name: string;
+  image?: string;
+  url: string;
+};
+
+export type OdesliResponse = {
+  /**
+   * The unique ID for the input entity that was supplied in the request. The
+   * data for this entity, such as title, artistName, etc. will be found in
+   * an object at `nodesByUniqueId[entityUniqueId]`
+   */
+  entityUniqueId: string;
+
+  /**
+   * The userCountry query param that was supplied in the request. It signals
+   * the country/availability we use to query the streaming platforms. Defaults
+   * to 'US' if no userCountry supplied in the request.
+   *
+   * NOTE: As a fallback, our service may respond with matches that were found
+   * in a locale other than the userCountry supplied
+   */
+  userCountry: string;
+
+  /**
+   * A URL that will render the Songlink page for this entity
+   */
+  pageUrl: string;
+
+  /**
+   * A collection of objects. Each key is a platform, and each value is an
+   * object that contains data for linking to the match
+   */
+  linksByPlatform: {
+    /**
+     * Each key in `linksByPlatform` is a Platform. A Platform will exist here
+     * only if there is a match found. E.g. if there is no YouTube match found,
+     * then neither `youtube` or `youtubeMusic` properties will exist here
+     */
+    [k in Platform]: {
+      /**
+       * The unique ID for this entity. Use it to look up data about this entity
+       * at `entitiesByUniqueId[entityUniqueId]`
+       */
+      entityUniqueId: string;
+
+      /**
+       * The URL for this match
+       */
+      url: string;
+
+      /**
+       * The native app URI that can be used on mobile devices to open this
+       * entity directly in the native app
+       */
+      nativeAppUriMobile?: string;
+
+      /**
+       * The native app URI that can be used on desktop devices to open this
+       * entity directly in the native app
+       */
+      nativeAppUriDesktop?: string;
+    };
+  };
+
+  // A collection of objects. Each key is a unique identifier for a streaming
+  // entity, and each value is an object that contains data for that entity,
+  // such as `title`, `artistName`, `thumbnailUrl`, etc.
+  entitiesByUniqueId: {
+    [entityUniqueId: string]: {
+      // This is the unique identifier on the streaming platform/API provider
+      id: string;
+
+      type: 'song' | 'album';
+
+      title?: string;
+      artistName?: string;
+      thumbnailUrl?: string;
+      thumbnailWidth?: number;
+      thumbnailHeight?: number;
+
+      // The API provider that powered this match. Useful if you'd like to use
+      // this entity's data to query the API directly
+      apiProvider: APIProvider;
+
+      // An array of platforms that are "powered" by this entity. E.g. an entity
+      // from Apple Music will generally have a `platforms` array of
+      // `["appleMusic", "itunes"]` since both those platforms/links are derived
+      // from this single entity
+      platforms: Platform[];
+    };
+  };
+};
+
+export type Platform =
+  | 'spotify'
+  | 'itunes'
+  | 'appleMusic'
+  | 'youtube'
+  | 'youtubeMusic'
+  | 'google'
+  | 'googleStore'
+  | 'pandora'
+  | 'deezer'
+  | 'tidal'
+  | 'amazonStore'
+  | 'amazonMusic'
+  | 'soundcloud'
+  | 'napster'
+  | 'yandex'
+  | 'spinrilla'
+  | 'audius'
+  | 'audiomack'
+  | 'anghami'
+  | 'boomplay';
+
+export type APIProvider =
+  | 'spotify'
+  | 'itunes'
+  | 'youtube'
+  | 'google'
+  | 'pandora'
+  | 'deezer'
+  | 'tidal'
+  | 'amazon'
+  | 'soundcloud'
+  | 'napster'
+  | 'yandex'
+  | 'spinrilla'
+  | 'audius'
+  | 'audiomack'
+  | 'anghami'
+  | 'boomplay';

src/lib/relativeTime.ts

@@ -11,10 +11,7 @@ export const time = readable(new Date(), function start(set) {
 });
 
 export function secondsSince(date: Date): Readable<number> {
-  return derived(
+  return derived(time, ($time) => Math.round(($time.getTime() - date.getTime()) / 1000));
-    time,
-    $time => Math.round(($time.getTime() - date.getTime()) / 1000)
-  );
 }
 
 export function relativeTime(seconds: number): string | null {
@@ -30,11 +27,11 @@ export function relativeTime(seconds: number): string | null {
 
   const day = hour * 24;
   if (seconds < day) {
-    return `${(Math.floor(seconds / hour))}h`;
+    return `${Math.floor(seconds / hour)}h`;
   }
   const maxRelative = day * 31;
   if (seconds < maxRelative) {
     return `${Math.floor(seconds / day)}d`;
   }
   return null;
-}
+}

src/lib/server/db.ts

@@ -1,15 +1,30 @@
 import { env } from '$env/dynamic/private';
+import { IGNORE_USERS, MASTODON_INSTANCE } from '$env/static/private';
 import type { Account, Post, Tag } from '$lib/mastodon/response';
+import { isTruthy } from '$lib/truthyString';
 import sqlite3 from 'sqlite3';
 const { DEV } = import.meta.env;
 
 const db: sqlite3.Database = new sqlite3.Database('moshingmammut.db');
+// for the local masto instance, the instance name is *not* saved
+// as part of the username or acct, so it needs to be stripped
+const ignoredUsers: string[] =
+  IGNORE_USERS === undefined
+    ? []
+    : IGNORE_USERS.split(',')
+        .map((u) => (u.startsWith('@') ? u.substring(1) : u))
+        .map((u) =>
+          u.endsWith('@' + MASTODON_INSTANCE)
+            ? u.substring(0, u.length - ('@' + MASTODON_INSTANCE).length)
+            : u
+        );
+let databaseReady = false;
 
-if (DEV && env.VERBOSE === 'true') {
+if (DEV && isTruthy(env.VERBOSE)) {
   sqlite3.verbose();
   db.on('change', (t, d, table, rowid) => {
     console.debug('DB change event', t, d, table, rowid);
-  })
+  });
 
   db.on('trace', (sql) => {
     console.debug('Running', sql);
@@ -21,36 +36,52 @@ if (DEV && env.VERBOSE === 'true') {
 }
 
 interface Migration {
-  id: number,
+  id: number;
-  name: string,
+  name: string;
-  statement: string
+  statement: string;
 }
 
 db.on('open', () => {
   console.log('Opened database');
   db.serialize();
   db.run('CREATE TABLE IF NOT EXISTS "migrations" ("id" integer,"name" TEXT, PRIMARY KEY (id))');
-  db.all('SELECT id FROM migrations', (err, rows) => {
+  db.all('SELECT id FROM migrations', (err, rows: Migration[]) => {
     if (err !== null) {
       console.error('Could not fetch existing migrations', err);
+      databaseReady = true;
       return;
     }
     console.debug('Already applied migrations', rows);
-    const appliedMigrations: Set<number> = new Set(rows.map((row: any) => row['id']));
+    const appliedMigrations: Set<number> = new Set(rows.map((row) => row['id']));
-    const toApply = getMigrations().filter(m => !appliedMigrations.has(m.id));
+    const toApply = getMigrations().filter((m) => !appliedMigrations.has(m.id));
-    for (let migration of toApply) {
+    let remaining = toApply.length;
+    if (remaining === 0) {
+      databaseReady = true;
+      return;
+    }
+    for (const migration of toApply) {
       db.exec(migration.statement, (err) => {
+        remaining--;
+        // This will set databaseReady to true before the migration has been inserted as applies,
+        // but that doesn't matter. It's only important that is has been applied
+        if (remaining === 0) {
+          databaseReady = true;
+        }
         if (err !== null) {
           console.error(`Failed to apply migration ${migration.name}`, err);
           return;
         }
-        db.run('INSERT INTO migrations (id, name) VALUES(?, ?)', [migration.id, migration.name], (e: Error) => {
+        db.run(
-          if (e !== null) {
+          'INSERT INTO migrations (id, name) VALUES(?, ?)',
-            console.error(`Failed to mark migration ${migration.name} as applied`, e);
+          [migration.id, migration.name],
-            return;
+          (e: Error) => {
+            if (e !== null) {
+              console.error(`Failed to mark migration ${migration.name} as applied`, e);
+              return;
+            }
+            console.info(`Applied migration ${migration.name}`);
           }
-          console.info(`Applied migration ${migration.name}`);
+        );
-        });
       });
     }
   });
@@ -60,10 +91,11 @@ db.on('error', (err) => {
 });
 
 function getMigrations(): Migration[] {
-  return [{
+  return [
-    id: 1,
+    {
-    name: 'initial',
+      id: 1,
-    statement: `
+      name: 'initial',
+      statement: `
       CREATE TABLE accounts (
         id TEXT NOT NULL PRIMARY KEY,
         acct TEXT,
@@ -89,32 +121,102 @@ function getMigrations(): Migration[] {
         FOREIGN KEY (post_id) REFERENCES posts(id),
         FOREIGN KEY (tag_url) REFERENCES tags(url)
       )`
-  }];
+    },
+    {
+      id: 2,
+      name: 'urls as keys',
+      statement: `
+      CREATE TABLE accounts_new (
+        id TEXT NOT NULL,
+        acct TEXT,
+        username TEXT,
+        display_name TEXT,
+        url TEXT NOT NULL PRIMARY KEY,
+        avatar TEXT
+      );
+      INSERT INTO accounts_new (id, acct, username, display_name, url, avatar)
+         SELECT id, acct, username, display_name, url, avatar
+         FROM accounts;
+      DROP TABLE accounts;
+      ALTER TABLE accounts_new RENAME TO accounts;
+
+      CREATE TABLE posts_new (
+        id TEXT NOT NULL,
+        content TEXT,
+        created_at TEXT,
+        url TEXT NOT NULL PRIMARY KEY,
+        account_id TEXT NOT NULL,
+        FOREIGN KEY (account_id) REFERENCES accounts(url)
+      );
+      INSERT INTO posts_new (id, content, created_at, url, account_id)
+         SELECT p.id, p.content, p.created_at, p.url, accounts.url
+         FROM posts as p
+         JOIN accounts ON accounts.id = p.account_id;
+      DROP TABLE posts;
+      ALTER TABLE posts_new RENAME TO posts;
+
+      CREATE TABLE poststags_new (
+        id integer PRIMARY KEY,
+        post_id TEXT NOT NULL,
+        tag_url TEXT NOT NULL,
+        FOREIGN KEY (post_id) REFERENCES posts(url),
+        FOREIGN KEY (tag_url) REFERENCES tags(url)
+      );
+
+      INSERT INTO poststags_new (id, post_id, tag_url)
+         SELECT pt.id, posts.url, pt.tag_url
+         FROM poststags as pt
+         JOIN posts ON posts.id = pt.post_id;
+      DROP TABLE poststags;
+      ALTER TABLE poststags_new RENAME TO poststags;
+      `
+    }
+  ];
+}
+
+async function waitReady(): Promise<undefined> {
+  // Simpler than a semaphore and is really only needed on startup
+  return new Promise((resolve) => {
+    const interval = setInterval(() => {
+      if (DEV) {
+        console.debug('Waiting for database to be ready');
+      }
+      if (databaseReady) {
+        if (DEV) {
+          console.debug('DB is ready');
+        }
+        clearInterval(interval);
+        resolve(undefined);
+      }
+    }, 100);
+  });
 }
 
 export async function savePost(post: Post): Promise<undefined> {
-  return new Promise((resolve, reject) => {
+  if (!databaseReady) {
+    await waitReady();
+  }
+  return await new Promise<undefined>((resolve, reject) => {
     console.debug(`Saving post ${post.url}`);
     const account = post.account;
-    db.run(`
+    db.run(
-      INSERT INTO accounts (id, acct, username, display_name, url, avatar, avatar_static)
+      `
-      VALUES(?, ?, ?, ?, ?, ?, ?)
+      INSERT INTO accounts (id, acct, username, display_name, url, avatar)
-      ON CONFLICT(id)
+      VALUES(?, ?, ?, ?, ?, ?)
+      ON CONFLICT(url)
       DO UPDATE SET
       acct=excluded.acct,
       username=excluded.username,
       display_name=excluded.display_name,
-      url=excluded.url,
+      id=excluded.id,
-      avatar=excluded.avatar,
+      avatar=excluded.avatar;`,
-      avatar_static=excluded.avatar_static;`,
       [
         account.id,
         account.acct,
         account.username,
         account.display_name,
         account.url,
-        account.avatar,
+        account.avatar
-        account.avatar_static
       ],
       (err) => {
         if (err !== null) {
@@ -122,20 +224,15 @@ export async function savePost(post: Post): Promise<undefined> {
           reject(err);
           return;
         }
-        db.run(`
+        db.run(
+          `
           INSERT INTO posts (id, content, created_at, url, account_id)
-          VALUES (?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET
+          VALUES (?, ?, ?, ?, ?) ON CONFLICT(url) DO UPDATE SET
           content=excluded.content,
           created_at=excluded.created_at,
-          url=excluded.url,
+          id=excluded.id,
           account_id=excluded.account_id;`,
-          [
+          [post.id, post.content, post.created_at, post.url, post.account.url],
-            post.id,
-            post.content,
-            post.created_at,
-            post.url,
-            post.account.id
-          ],
           (postErr) => {
             if (postErr !== null) {
               console.error(`Could not insert post ${post.url}`, postErr);
@@ -143,28 +240,35 @@ export async function savePost(post: Post): Promise<undefined> {
               return;
             }
 
+            if (!post.tags.length) {
+              resolve(undefined);
+              return;
+            }
+
             db.parallelize(() => {
               let remaining = post.tags.length;
-              for (let tag of post.tags) {
+              for (const tag of post.tags) {
-                db.run(`
+                db.run(
+                  `
                   INSERT INTO tags (url, tag) VALUES (?, ?)
                   ON CONFLICT(url) DO UPDATE SET
                   tag=excluded.tag;`,
-                  [
+                  [tag.url, tag.name],
-                    tag.url,
-                    tag.name
-                  ],
                   (tagErr) => {
                     if (tagErr !== null) {
                       console.error(`Could not insert/update tag ${tag.url}`, tagErr);
                       reject(tagErr);
                       return;
                     }
-                    db.run('INSERT INTO poststags (post_id, tag_url) VALUES (?, ?)',
+                    db.run(
-                      [post.id, tag.url],
+                      'INSERT INTO poststags (post_id, tag_url) VALUES (?, ?)',
+                      [post.url, tag.url],
                       (posttagserr) => {
                         if (posttagserr !== null) {
-                          console.error(`Could not insert poststags ${tag.url}, ${post.url}`, posttagserr);
+                          console.error(
+                            `Could not insert poststags ${tag.url}, ${post.url}`,
+                            posttagserr
+                          );
                           reject(posttagserr);
                           return;
                         }
@@ -180,15 +284,27 @@ export async function savePost(post: Post): Promise<undefined> {
                 );
               }
             });
-          });
+          }
-      });
+        );
+      }
+    );
   });
 }
 
+type FilterParameter = {
+  $limit: number | undefined | null;
+  $since?: string | undefined | null;
+  $before?: string | undefined | null;
+  [x: string]: string | number | undefined | null;
+};
+
 export async function getPosts(since: string | null, before: string | null, limit: number) {
-  let promise = await new Promise<Post[]>((resolve, reject) => {
+  if (!databaseReady) {
-    let filter_query;
+    await waitReady();
-    let params: any = { $limit: limit };
+  }
+  const promise = await new Promise<Post[]>((resolve, reject) => {
+    let filter_query = '';
+    const params: FilterParameter = { $limit: limit };
     if (since === null && before === null) {
       filter_query = '';
     } else if (since !== null) {
@@ -199,74 +315,97 @@ export async function getPosts(since: string | null, before: string | null, limi
       filter_query = 'WHERE posts.created_at < $before';
       params.$before = before;
     }
+
+    ignoredUsers.forEach((ignoredUser, index) => {
+      const userParam = `$user_${index}`;
+      const acctParam = userParam + 'a';
+      const usernameParam = userParam + 'u';
+      const prefix = filter_query === '' ? ' WHERE' : ' AND';
+      filter_query += `${prefix} acct != ${acctParam} AND username != ${usernameParam} `;
+      params[acctParam] = ignoredUser;
+      params[usernameParam] = ignoredUser;
+    });
+
+    type PostResult = {
+      id: string;
+      content: string;
+      created_at: string;
+      url: string;
+      account_id: string;
+      acct: string;
+      username: string;
+      display_name: string;
+      account_url: string;
+      avatar: string;
+    };
+
+    type PostTagResult = {
+      post_id: string;
+      tag: string;
+      url: string;
+    };
+
     const sql = `SELECT posts.id, posts.content, posts.created_at, posts.url,
-    accounts.id AS account_id, accounts.acct, accounts.username, accounts.display_name,
+      accounts.id AS account_id, accounts.acct, accounts.username, accounts.display_name,
-    accounts.url AS account_url, accounts.avatar
+      accounts.url AS account_url, accounts.avatar
-    FROM posts
+      FROM posts
-    JOIN accounts ON posts.account_id = accounts.id
+      JOIN accounts ON posts.account_id = accounts.url
-    ${filter_query}
+      ${filter_query}
-    ORDER BY created_at DESC
+      ORDER BY created_at DESC
-    LIMIT $limit`;
+      LIMIT $limit`;
-    db.all(
+    db.all(sql, params, (err, rows: PostResult[]) => {
-      sql,
+      if (err != null) {
-      params,
+        console.error('Error loading posts', err);
-      (err, rows: any[]) => {
+        reject(err);
-        if (err != null) {
+        return;
-          console.error('Error loading posts', err);
+      }
-          reject(err);
+      if (rows.length === 0) {
-          return;
+        // No need to check for tags
-        }
+        resolve([]);
-        if (rows.length === 0) {
+        return;
-          // No need to check for tags
+      }
-          resolve([]);
+      const postIdsParams = rows.map(() => '?').join(', ');
-          return;
+      db.all(
-        }
+        `SELECT post_id, tags.url, tags.tag
-        const postIdsParams = rows.map(() => '?').join(', ');
-        db.all(
-          `SELECT post_id, tags.url, tags.tag
           FROM poststags
           JOIN tags ON poststags.tag_url = tags.url
           WHERE post_id IN (${postIdsParams});`,
-          rows.map((r: any) => r.id),
+        rows.map((r: PostResult) => r.url),
-          (tagErr, tagRows: any[]) => {
+        (tagErr, tagRows: PostTagResult[]) => {
-            if (tagErr != null) {
+          if (tagErr != null) {
-              console.error('Error loading post tags', tagErr);
+            console.error('Error loading post tags', tagErr);
-              reject(tagErr);
+            reject(tagErr);
-              return;
+            return;
-            }
-            const tagMap: Map<string, Tag[]> = tagRows.reduce(
-              (result: Map<string, Tag[]>, item) => {
-                const tag: Tag = {
-                  url: item.url,
-                  name: item.tag
-                };
-                result.set(item.post_id, [...result.get(item.post_id) || [], tag]);
-                return result;
-              }, new Map());
-            const posts = rows.map(row => {
-              return {
-                id: row.id,
-                content: row.content,
-                created_at: row.created_at,
-                url: row.url,
-                tags: tagMap.get(row.id) || [],
-                account: {
-                  id: row.account_id,
-                  acct: row.acct,
-                  username: row.username,
-                  display_name: row.display_name,
-                  url: row.account_url,
-                  avatar: row.avatar,
-                  avatar_static: ''
-                } as Account
-              } as Post
-            });
-            resolve(posts);
           }
-        );
+          const tagMap: Map<string, Tag[]> = tagRows.reduce((result: Map<string, Tag[]>, item) => {
-      }
+            const tag: Tag = {
-    );
+              url: item.url,
+              name: item.tag
+            };
+            result.set(item.post_id, [...(result.get(item.post_id) || []), tag]);
+            return result;
+          }, new Map());
+          const posts = rows.map((row) => {
+            return {
+              id: row.id,
+              content: row.content,
+              created_at: row.created_at,
+              url: row.url,
+              tags: tagMap.get(row.id) || [],
+              account: {
+                id: row.account_id,
+                acct: row.acct,
+                username: row.username,
+                display_name: row.display_name,
+                url: row.account_url,
+                avatar: row.avatar
+              } as Account
+            } as Post;
+          });
+          resolve(posts);
+        }
+      );
+    });
   });
   return promise;
 }
-

src/lib/server/rss.ts

@@ -1,4 +1,4 @@
-import { BASE_URL } from '$env/static/private';
+import { BASE_URL, WEBSUB_HUB } from '$env/static/private';
 import { PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME } from '$env/static/public';
 import type { Post } from '$lib//mastodon/response';
 import { Feed } from 'feed';
@@ -6,6 +6,7 @@ import fs from 'fs/promises';
 
 export function createFeed(posts: Post[]): Feed {
   const baseUrl = BASE_URL.endsWith('/') ? BASE_URL : BASE_URL + '/';
+  const hub = WEBSUB_HUB ? WEBSUB_HUB : undefined;
   const feed = new Feed({
     title: `${PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME} music feed`,
     description: `Posts about music on ${PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME}`,
@@ -19,27 +20,45 @@ export function createFeed(posts: Post[]): Feed {
     feedLinks: {
       atom: `${BASE_URL}/feed.xml`
     },
+    hub: hub,
     author: {
       name: '@aymm',
       link: 'https://metalhead.club/@aymm'
-    },
+    }
   });
-  posts.forEach(p => {
+  posts.forEach((p) => {
     feed.addItem({
       title: p.content,
       id: p.url,
       link: p.url,
       content: p.content,
-      author: [{
+      author: [
-        name: p.account.acct,
+        {
-        link: p.account.url
+          name: p.account.acct,
-      }],
+          link: p.account.url
+        }
+      ],
       date: new Date(p.created_at)
-    })
+    });
   });
   feed.addCategory('Music');
+
   return feed;
 }
 export async function saveAtomFeed(feed: Feed) {
   await fs.writeFile('feed.xml', feed.atom1(), { encoding: 'utf8' });
-}
+  if (!WEBSUB_HUB) {
+    return;
+  }
+  try {
+    const params = new URLSearchParams();
+    params.append('hub.mode', 'publish');
+    params.append('hub.url', `${BASE_URL}/feed.xml`);
+    await fetch(WEBSUB_HUB, {
+      method: 'POST',
+      body: params
+    });
+  } catch (e) {
+    console.error('Failed to update WebSub hub', e);
+  }
+}

src/lib/server/timeline.ts

@@ -1,19 +1,39 @@
-import { HASHTAG_FILTER, MASTODON_INSTANCE, URL_FILTER, YOUTUBE_API_KEY } from '$env/static/private';
+import {
+  HASHTAG_FILTER,
+  MASTODON_INSTANCE,
+  ODESLI_API_KEY,
+  URL_FILTER,
+  YOUTUBE_API_KEY,
+  YOUTUBE_DISABLE
+} from '$env/static/private';
 import type { Post, Tag, TimelineEvent } from '$lib/mastodon/response';
+import type { OdesliResponse, Platform, SongInfo } from '$lib/odesliResponse';
 import { getPosts, savePost } from '$lib/server/db';
 import { createFeed, saveAtomFeed } from '$lib/server/rss';
-import { WebSocket } from "ws";
+import { sleep } from '$lib/sleep';
+import { isTruthy } from '$lib/truthyString';
+import { WebSocket } from 'ws';
 
-const YOUTUBE_REGEX = new RegExp(/https?:\/\/(www\.)?youtu((be.com\/.*?v=)|(\.be\/))(?<videoId>[a-zA-Z_0-9-]+)/gm);
+const YOUTUBE_REGEX = new RegExp(
+  /https?:\/\/(www\.)?youtu((be.com\/.*?v=)|(\.be\/))(?<videoId>[a-zA-Z_0-9-]+)/gm
+);
+
+const URL_REGEX = new RegExp(/href="(?<postUrl>[^>]+?)" target="_blank"/gm);
 
 export class TimelineReader {
   private static _instance: TimelineReader;
 
   private static async isMusicVideo(videoId: string) {
+    if (!YOUTUBE_API_KEY || YOUTUBE_API_KEY === 'CHANGE_ME') {
+      // Assume that it *is* a music link when no YT API key is provided
+      // If it should assumed to not be YOUTUBE_DISABLE needs to be set to something truthy
+      return true;
+    }
     const searchParams = new URLSearchParams([
       ['part', 'snippet'],
       ['id', videoId],
-      ['key', YOUTUBE_API_KEY]]);
+      ['key', YOUTUBE_API_KEY]
+    ]);
     const youtubeVideoUrl = new URL(`https://www.googleapis.com/youtube/v3/videos?${searchParams}`);
     const resp = await fetch(youtubeVideoUrl);
     const respObj = await resp.json();
@@ -30,15 +50,23 @@ export class TimelineReader {
     const categorySearchParams = new URLSearchParams([
       ['part', 'snippet'],
       ['id', item.categoryId],
-      ['key', YOUTUBE_API_KEY]]);
+      ['key', YOUTUBE_API_KEY]
-    const youtubeCategoryUrl = new URL(`https://www.googleapis.com/youtube/v3/videoCategories?${categorySearchParams}`);
+    ]);
-    const categoryTitle: string = await fetch(youtubeCategoryUrl).then(r => r.json()).then(r => r.items[0]?.title);
+    const youtubeCategoryUrl = new URL(
+      `https://www.googleapis.com/youtube/v3/videoCategories?${categorySearchParams}`
+    );
+    const categoryTitle: string = await fetch(youtubeCategoryUrl)
+      .then((r) => r.json())
+      .then((r) => r.items[0]?.title);
     return categoryTitle === 'Music';
   }
 
-  private static async checkYoutubeMatches(postContent: string): Promise<boolean> {
+  private static async checkYoutubeMatches(postContent: string): Promise<string | null> {
+    if (isTruthy(YOUTUBE_DISABLE)) {
+      return null;
+    }
     const matches = postContent.matchAll(YOUTUBE_REGEX);
-    for (let match of matches) {
+    for (const match of matches) {
       if (match === undefined || match.groups === undefined) {
         continue;
       }
@@ -46,21 +74,93 @@ export class TimelineReader {
       try {
         const isMusic = await TimelineReader.isMusicVideo(videoId);
         if (isMusic) {
-          return true;
+          return match[0];
         }
       } catch (e) {
         console.error('Could not check if', videoId, 'is a music video', e);
       }
     }
-    return false;
+    return null;
   }
 
-  private constructor() {
+  private static async getSongInfo(url: string, remainingTries = 6): Promise<SongInfo | null> {
+    if (remainingTries === 0) {
+      console.error('No tries remaining. Lookup failed!');
+      return null;
+    }
+    let hostname: string;
+    try {
+      hostname = new URL(url).hostname;
+    } catch (e) {
+      console.error(`Could not construct URL ${url}`, e);
+      return null;
+    }
+    if (hostname === 'songwhip.com') {
+      // song.link doesn't support songwhip links and songwhip themselves will provide metadata if you pass in a
+      // Apple Music/Spotify/etc link, but won't when provided with their own link, so no way to extract song info
+      // except maybe scraping their HTML
+      return null;
+    }
+
+    const odesliParams = new URLSearchParams();
+    odesliParams.append('url', url);
+    odesliParams.append('userCountry', 'DE');
+    odesliParams.append('songIfSingle', 'true');
+    if (ODESLI_API_KEY && ODESLI_API_KEY !== 'CHANGE_ME') {
+      odesliParams.append('key', ODESLI_API_KEY);
+    }
+    const odesliApiUrl = `https://api.song.link/v1-alpha.1/links?${odesliParams}`;
+    try {
+      return fetch(odesliApiUrl).then(async (response) => {
+        if (response.status === 429) {
+          throw new Error('Rate limit reached', { cause: 429 });
+        }
+        return response.json().then((odesliInfo: OdesliResponse) => {
+          const info = odesliInfo.entitiesByUniqueId[odesliInfo.entityUniqueId];
+          const platform: Platform = 'youtube';
+          return {
+            ...info,
+            pageUrl: odesliInfo.pageUrl,
+            youtubeUrl: odesliInfo.linksByPlatform[platform]?.url
+          } as SongInfo;
+        });
+      });
+    } catch (e) {
+      if (e instanceof Error && e.cause === 429) {
+        console.warn('song.link rate limit reached. Trying again in 10 seconds');
+        await sleep(10_000);
+        return await this.getSongInfo(url, remainingTries - 1);
+      }
+      console.error(`Failed to load ${url} info from song.link`, e);
+      return null;
+    }
+  }
+
+  private static async getUrlFromPreviewCard(post: Post): Promise<string | undefined> {
+    return undefined;
+    // Currently disabled, because it seems to always be null, even after re-fetching the post from Mastodon
+    /*
+    if (post.card) {
+      return post.card?.url;
+    }
+    try {
+      const status: Post = await (
+        await fetch(`https://${MASTODON_INSTANCE}/api/v1/statuses/${post.id}`)
+      ).json();
+      return status.card?.url;
+    } catch (e) {
+      console.error(`Could not fetch status ${post.url}`, e);
+    }
+    */
+  }
+
+  private startWebsocket() {
     const socket = new WebSocket(`wss://${MASTODON_INSTANCE}/api/v1/streaming`);
-    socket.onopen = (_event) => {
+    socket.onopen = () => {
+      console.log('Connected to WS');
       socket.send('{ "type": "subscribe", "stream": "public:local"}');
     };
-    socket.onmessage = (async (event) => {
+    socket.onmessage = async (event) => {
       try {
         const data: TimelineEvent = JSON.parse(event.data.toString());
         if (data.event !== 'update') {
@@ -71,31 +171,94 @@ export class TimelineReader {
         const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
 
         const urls: string[] = URL_FILTER.split(',');
-        const found_urls = urls.filter(t => post.content.includes(t));
+        const found_urls = urls.filter((t) => post.content.includes(t));
-
+        const urlsToCheck: string[] = [];
         // If we don't have any tags or non-youtube urls, check youtube
         // YT is handled separately, because it requires an API call and therefore is slower
-        if (found_urls.length === 0 &&
+        if (found_urls.length === 0 && found_tags.length === 0) {
-          found_tags.length === 0 &&
+          const youtubeUrl = await TimelineReader.checkYoutubeMatches(post.content);
-          !await TimelineReader.checkYoutubeMatches(post.content)) {
+          if (youtubeUrl === null) {
-          return;
+            console.log('Ignoring post', post.url);
+            return;
+          }
+          urlsToCheck.push(youtubeUrl);
+          console.log('Found YT URL', youtubeUrl, found_urls, found_urls.length);
         }
+
+        // TODO: Change URL detection above to use this regex.
+        // Looks like we're stuck with regex for now instead of using preview cards.
+        // Might as well use it to find URLs. Could also use this for YouTube: If Odesli finds something, it's a song,
+        // if not, ignore it. No need to consult the YT API and give those links a special handling
+        const musicUrls: string[] = [];
+        const musicUrl = await TimelineReader.getUrlFromPreviewCard(post);
+        if (musicUrl) {
+          musicUrls.push(musicUrl);
+        } else {
+          const urlMatches = post.content.matchAll(URL_REGEX);
+          for (const match of urlMatches) {
+            if (match === undefined || match.groups === undefined) {
+              continue;
+            }
+            const urlMatch = match.groups.postUrl.toString();
+            const musicUrl = urls.find((u) => urlMatch.includes(u));
+            if (musicUrl) {
+              musicUrls.push(urlMatch);
+            }
+          }
+        }
+
+        for (const url of musicUrls) {
+          let hostname: string | null = null;
+          try {
+            hostname = new URL(url).hostname;
+          } catch (e) {
+            console.error(`Could not check hostname for URL ${url}`, e);
+          }
+          if (hostname === 'songwhip.com') {
+            // TODO: Implement checking the songwhip API
+            continue;
+          }
+          const info = await TimelineReader.getSongInfo(url);
+          if (info) {
+            console.info(
+              'Got song info for',
+              post.url,
+              url,
+              info.artistName,
+              info.title,
+              info.thumbnailUrl,
+              info.pageUrl,
+              info.youtubeUrl
+            );
+          }
+        }
+
         await savePost(post);
+
         const posts = await getPosts(null, null, 100);
         await saveAtomFeed(createFeed(posts));
-
       } catch (e) {
-        console.error("error message", event, event.data, e)
+        console.error('error message', event, event.data, e);
       }
-
+    };
-    });
     socket.onclose = (event) => {
-      console.log("Closed", event, event.code, event.reason)
+      console.warn(
+        `Websocket connection to ${MASTODON_INSTANCE} closed. Code: ${event.code}, reason: '${event.reason}'`
+      );
+      setTimeout(() => {
+        console.info(`Attempting to reconenct to WS`);
+        this.startWebsocket();
+      }, 10000);
     };
     socket.onerror = (event) => {
-      console.log("error", event, event.message, event.error)
+      console.error(
+        `Websocket connection to ${MASTODON_INSTANCE} failed. ${event.type}: ${event.error}, message: '${event.message}'`
+      );
     };
+  }
 
+  private constructor() {
+    this.startWebsocket();
   }
 
   public static init() {
@@ -108,4 +271,4 @@ export class TimelineReader {
     TimelineReader.init();
     return this._instance;
   }
-}
+}

src/lib/sleep.ts

@@ -0,0 +1,5 @@
+export function sleep(timeInMs: number): Promise<undefined> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, timeInMs);
+  });
+}

src/lib/truthyString.ts

@@ -0,0 +1,7 @@
+export function isTruthy(value: string | number | boolean | null | undefined): boolean {
+  if (typeof value === 'string') {
+    return value.toLowerCase() === 'true' || !!+value; // here we parse to number first
+  }
+
+  return !!value;
+}

src/routes/+layout.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-  import FooterComponent from '$lib/components/FooterComponent.svelte'
+  import FooterComponent from '$lib/components/FooterComponent.svelte';
   import { SvelteToast } from '@zerodevx/svelte-toast';
 
   const options = {
@@ -7,6 +7,7 @@
     classes: ['toast']
   };
 </script>
+
 <slot />
 <SvelteToast {options} />
 <div class="footer">
@@ -33,4 +34,9 @@
     align-items: center;
     gap: 10px;
   }
-</style>
+  @media only screen and (max-width: 620px) {
+    .footer {
+      width: 100%;
+    }
+  }
+</style>

src/routes/+page.svelte

@@ -1,174 +1,184 @@
 <script lang="ts">
-import { onMount } from "svelte";
+  import { onMount } from 'svelte';
-import type { PageData } from './$types';
+  import type { PageData } from './$types';
-import type { Post } from '$lib/mastodon/response';
+  import type { Post } from '$lib/mastodon/response';
-import { PUBLIC_REFRESH_INTERVAL, PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME } from '$env/static/public';
+  import {
-import PostComponent from '$lib/components/PostComponent.svelte';
+    PUBLIC_REFRESH_INTERVAL,
-import LoadMoreComponent from '$lib/components/LoadMoreComponent.svelte';
+    PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME
-import { fly, type FlyParams } from 'svelte/transition';
+  } from '$env/static/public';
-import { cubicInOut } from 'svelte/easing';
+  import PostComponent from '$lib/components/PostComponent.svelte';
-import { errorToast } from '$lib/errorToast'
+  import LoadMoreComponent from '$lib/components/LoadMoreComponent.svelte';
+  import { fly, type FlyParams } from 'svelte/transition';
+  import { cubicInOut } from 'svelte/easing';
+  import { errorToast } from '$lib/errorToast';
 
+  export let data: PageData;
 
-export let data: PageData;
+  interface FetchOptions {
-
+    since?: string;
-interface FetchOptions {
+    before?: string;
-  since?: string,
+    count?: number;
-  before?: string,
-  count?: number
-}
-
-interface EdgeFlyParams extends FlyParams {
-  created_at: string
-}
-
-const refreshInterval = parseInt(PUBLIC_REFRESH_INTERVAL);
-let interval: NodeJS.Timer | null = null;
-let moreOlderPostsAvailable = true;
-let loadingOlderPosts = false;
-
-// Needed, so that edgeFly() can do its thing:
-// To determine whether a newly loaded post is older than the existing ones, is required to know what the oldest
-// post was, before the fetch happened.
-let oldestBeforeLastFetch: number | null = null;
-
-/**
- * Animate either from the top, or the bottom of the window, depending if the post is
- * newer than the existing ones or older.
- */
-function edgeFly(node: Element, opts: EdgeFlyParams) {
-  const createdAt =  new Date(opts.created_at).getTime();
-  const diffNewest = Math.abs(new Date(data.posts[0].created_at).getTime() - createdAt);
-  const oldest = oldestBeforeLastFetch !== null
-    ? oldestBeforeLastFetch
-    : new Date(data.posts[data.posts.length - 1].created_at).getTime();
-  const diffOldest = Math.abs(oldest - createdAt);
-  const fromTop = diffNewest <= diffOldest;
-
-  const rect = node.getBoundingClientRect();
-  const paramY = +`${opts.y}`;
-  let offset = isNaN(paramY) ? 0 : paramY + rect.height;
-  opts.y = fromTop ? -offset : window.innerHeight + offset;
-  return fly(node, opts);
-}
-
-async function fetchPosts(options: FetchOptions): Promise<Post[]> {
-  const params = new URLSearchParams();
-  if (options?.since !== undefined) {
-    params.set('since', options.since);
-  }
-  if (options?.before !== undefined) {
-    params.set('before', options.before);
-  }
-  if (options?.count !== undefined) {
-    params.set('count', options.count.toFixed(0));
   }
 
-  const response = await fetch(`/api/posts?${params}`);
+  interface EdgeFlyParams extends FlyParams {
-  return await response.json();
+    created_at: string;
-}
-
-function filterDuplicates(posts: Post[]): Post[] {
-  return posts.filter((obj, index, arr) => {
-    return arr.map(mapObj => mapObj.url).indexOf(obj.url) === index;
-  });
-}
-
-function refresh() {
-  let filter: FetchOptions = {};
-  if (data.posts.length > 0) {
-    filter = { since: data.posts[0].created_at };
   }
-  fetchPosts(filter).then(resp => {
+
-    if (resp.length > 0) {
+  const refreshInterval = parseInt(PUBLIC_REFRESH_INTERVAL);
-      // Prepend new posts, filter dupes
+  let interval: ReturnType<typeof setTimeout> | null = null;
-      // There shouldn't be any duplicates, but better be safe than sorry
+  let moreOlderPostsAvailable = true;
-      data.posts = filterDuplicates(resp.concat(data.posts));
+  let loadingOlderPosts = false;
+
+  // Needed, so that edgeFly() can do its thing:
+  // To determine whether a newly loaded post is older than the existing ones, is required to know what the oldest
+  // post was, before the fetch happened.
+  let oldestBeforeLastFetch: number | null = null;
+
+  /**
+   * Animate either from the top, or the bottom of the window, depending if the post is
+   * newer than the existing ones or older.
+   */
+  function edgeFly(node: Element, opts: EdgeFlyParams) {
+    const createdAt = new Date(opts.created_at).getTime();
+    const diffNewest = Math.abs(new Date(data.posts[0].created_at).getTime() - createdAt);
+    const oldest =
+      oldestBeforeLastFetch !== null
+        ? oldestBeforeLastFetch
+        : new Date(data.posts[data.posts.length - 1].created_at).getTime();
+    const diffOldest = Math.abs(oldest - createdAt);
+    const fromTop = diffNewest <= diffOldest;
+
+    const rect = node.getBoundingClientRect();
+    const paramY = +`${opts.y}`;
+    let offset = isNaN(paramY) ? 0 : paramY + rect.height;
+    opts.y = fromTop ? -offset : window.innerHeight + offset;
+    return fly(node, opts);
+  }
+
+  async function fetchPosts(options: FetchOptions): Promise<Post[]> {
+    const params = new URLSearchParams();
+    if (options?.since !== undefined) {
+      params.set('since', options.since);
     }
-  })
+    if (options?.before !== undefined) {
-  .catch((e: Error) => {
+      params.set('before', options.before);
-    errorToast('Error loading newest posts: ' + e.message);
-  });
-}
-
-onMount(async () => {
-  if (data.posts.length > 0) {
-    oldestBeforeLastFetch = new Date(data.posts[data.posts.length - 1].created_at).getTime();
-  }
-  interval = setInterval(refresh, refreshInterval);
-
-  // - If the page is hidden, slow down refresh rate
-  // - If the page is shown, bump up refresh rate
-  document.addEventListener('visibilitychange', () => {
-    const delay = document.hidden ? refreshInterval * 10 : refreshInterval;
-    if (interval) {
-      clearInterval(interval);
     }
-    interval = setInterval(refresh, delay);
+    if (options?.count !== undefined) {
+      params.set('count', options.count.toFixed(0));
+    }
+
+    const response = await fetch(`/api/posts?${params}`);
+    return await response.json();
+  }
+
+  function filterDuplicates(posts: Post[]): Post[] {
+    return posts.filter((obj, index, arr) => {
+      return arr.map((mapObj) => mapObj.url).indexOf(obj.url) === index;
+    });
+  }
+
+  function refresh() {
+    let filter: FetchOptions = {};
+    if (data.posts.length > 0) {
+      filter = { since: data.posts[0].created_at };
+    }
+    fetchPosts(filter)
+      .then((resp) => {
+        if (resp.length > 0) {
+          // Prepend new posts, filter dupes
+          // There shouldn't be any duplicates, but better be safe than sorry
+          data.posts = filterDuplicates(resp.concat(data.posts));
+        }
+      })
+      .catch((e: Error) => {
+        errorToast('Error loading newest posts: ' + e.message);
+      });
+  }
+
+  onMount(async () => {
+    if (data.posts.length > 0) {
+      oldestBeforeLastFetch = new Date(data.posts[data.posts.length - 1].created_at).getTime();
+    }
+    interval = setInterval(refresh, refreshInterval);
+
+    // - If the page is hidden, slow down refresh rate
+    // - If the page is shown, bump up refresh rate
+    document.addEventListener('visibilitychange', () => {
+      const delay = document.hidden ? refreshInterval * 10 : refreshInterval;
+      if (interval) {
+        clearInterval(interval);
+      }
+      interval = setInterval(refresh, delay);
+    });
+
+    return () => {
+      if (interval !== null) {
+        clearInterval(interval);
+      }
+    };
   });
 
-  return () => {
+  function loadOlderPosts() {
-    if (interval !== null) {
+    loadingOlderPosts = true;
-      clearInterval(interval)
+    const filter: FetchOptions = { count: 20 };
+    if (data.posts.length > 0) {
+      const before = data.posts[data.posts.length - 1].created_at;
+      filter.before = before;
+      oldestBeforeLastFetch = new Date(before).getTime();
     }
+
+    fetchPosts(filter)
+      .then((resp) => {
+        if (resp.length > 0) {
+          // Append old posts, filter dupes
+          // There shouldn't be any duplicates, but better be safe than sorry
+          data.posts = filterDuplicates(data.posts.concat(resp));
+          // If we got less than we expected, there are no older posts available
+          moreOlderPostsAvailable = resp.length >= (filter.count ?? 20);
+        } else {
+          moreOlderPostsAvailable = false;
+        }
+        loadingOlderPosts = false;
+      })
+      .catch((e) => {
+        loadingOlderPosts = false;
+        errorToast('Error loading older posts: ' + e.message);
+      });
   }
-});
-
-function loadOlderPosts() {
-  loadingOlderPosts = true;
-  const filter: FetchOptions = { count: 20 };
-  if (data.posts.length > 0) {
-    const before = data.posts[data.posts.length - 1].created_at;
-    filter.before = before;
-    oldestBeforeLastFetch = new Date(before).getTime();
-  }
-
-
-  fetchPosts(filter).then(resp => {
-    if (resp.length > 0) {
-      // Append old posts, filter dupes
-      // There shouldn't be any duplicates, but better be safe than sorry
-      data.posts = filterDuplicates(data.posts.concat(resp));
-      // If we got less than we expected, there are no older posts available
-      moreOlderPostsAvailable = resp.length >= (filter.count ?? 20);
-    } else {
-      moreOlderPostsAvailable = false;
-    }
-    loadingOlderPosts = false;
-  })
-  .catch(e => {
-    loadingOlderPosts = false;
-    errorToast('Error loading older posts: ' + e.message);
-  });
-
-}
-
 </script>
+
 <svelte:head>
   <title>{PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME} music list</title>
 </svelte:head>
 <h2>{PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME} music list</h2>
 <div class="wrapper">
-  <div></div>
+  <div />
   <div class="posts">
-  {#if data.posts.length === 0}
+    {#if data.posts.length === 0}
-    Sorry, no posts recommending music aave been found yet
+      Sorry, no posts recommending music have been found yet
-  {/if}
+    {/if}
-  {#each data.posts as post (post.url)}
+    {#each data.posts as post (post.url)}
       <div
         class="post"
-        transition:edgeFly="{{ y: 10, created_at: post.created_at, duration: 300, easing: cubicInOut }}"
+        transition:edgeFly={{
+          y: 10,
+          created_at: post.created_at,
+          duration: 300,
+          easing: cubicInOut
+        }}
       >
         <PostComponent {post} />
       </div>
-  {/each}
+    {/each}
-  <LoadMoreComponent
+    <LoadMoreComponent
-    on:loadOlderPosts={loadOlderPosts}
+      on:loadOlderPosts={loadOlderPosts}
-    moreAvailable={moreOlderPostsAvailable}
+      moreAvailable={moreOlderPostsAvailable}
-    isLoading={loadingOlderPosts}/>
+      isLoading={loadingOlderPosts}
-  </div>
+    />
-  <div></div>
   </div>
+  <div />
+</div>
+
 <style>
   .posts {
     display: flex;
@@ -191,4 +201,11 @@ function loadOlderPosts() {
     text-align: center;
     z-index: 100;
   }
-</style>
+
+  @media only screen and (max-width: 650px) {
+    .post {
+      max-width: calc(100vw - 16px);
+      padding: 1em 0;
+    }
+  }
+</style>

src/routes/+page.ts

@@ -2,8 +2,8 @@ import type { Post } from '$lib/mastodon/response';
 import type { PageLoad } from './$types';
 
 export const load = (async ({ fetch }) => {
-    const p = await fetch('/');
+  const p = await fetch('/');
-    return {
+  return {
-        posts: await p.json() as Post[]
+    posts: (await p.json()) as Post[]
-    };
+  };
-}) satisfies PageLoad;
+}) satisfies PageLoad;

src/routes/+server.ts

@@ -2,4 +2,4 @@ import type { RequestHandler } from './$types';
 
 export const GET = (async ({ fetch }) => {
   return await fetch('api/posts');
-}) satisfies RequestHandler;
+}) satisfies RequestHandler;

src/routes/api/posts/+server.ts

@@ -13,4 +13,4 @@ export const GET = (async ({ url }) => {
   count = Math.min(count, 100);
   const posts = await getPosts(since, before, count);
   return json(posts);
-}) satisfies RequestHandler;
+}) satisfies RequestHandler;

static/site.webmanifest

@@ -1,19 +1,19 @@
 {
-    "name": "Moshing Mammut",
+  "name": "Moshing Mammut",
-    "short_name": "Moshing Mammut",
+  "short_name": "Moshing Mammut",
-    "icons": [
+  "icons": [
-        {
+    {
-            "src": "/android-chrome-192x192.png",
+      "src": "/android-chrome-192x192.png",
-            "sizes": "192x192",
+      "sizes": "192x192",
-            "type": "image/png"
+      "type": "image/png"
-        },
+    },
-        {
+    {
-            "src": "/android-chrome-512x512.png",
+      "src": "/android-chrome-512x512.png",
-            "sizes": "512x512",
+      "sizes": "512x512",
-            "type": "image/png"
+      "type": "image/png"
-        }
+    }
-    ],
+  ],
-    "theme_color": "#2e0b78",
+  "theme_color": "#2e0b78",
-    "background_color": "#2e0b78",
+  "background_color": "#2e0b78",
-    "display": "standalone"
+  "display": "standalone"
 }

static/style.css

@@ -16,7 +16,6 @@ body {
   --color-red-desat-dark: hsl(7, 20%, 30%);
   --color-red-desat-desat: hsl(7, 8%, 56%);
 
-
   --color-text: var(--color-blue);
   --color-border: var(--color-grey);
   --color-link: var(--color-mauve);
@@ -30,9 +29,9 @@ body {
 
   color: var(--color-text);
   background-color: var(--color-bg);
-  font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans,
+  font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen-Sans, Ubuntu,
-  Ubuntu, Cantarell, "Helvetica Neue", Helvetica, Arial, sans-serif,
+    Cantarell, 'Helvetica Neue', Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji',
-  "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
+    'Segoe UI Symbol';
 }
 
 a {
@@ -55,4 +54,4 @@ a:visited {
     --color-button-deactivated: var(--color-red-desat-desat);
     --color-button-text: var(--color-blue-dark);
   }
-}
+}

svelte.config.js

@@ -11,8 +11,8 @@ const config = {
     // adapter-auto only supports some environments, see https://kit.svelte.dev/docs/adapter-auto for a list.
     // If your environment is not supported or you settled on a specific environment, switch out the adapter.
     // See https://kit.svelte.dev/docs/adapters for more information about adapters.
-    adapter: adapter(),
+    adapter: adapter()
-  },
+  }
 };
 
 export default config;