Basic post display

.gitignore

@@ -1,3 +1,5 @@
+*.db
+
 node_modules
 /build
 /.svelte-kit

package.json

@@ -27,5 +27,11 @@
     "typescript": "^4.9.3",
     "vite": "^4.0.0"
   },
-  "type": "module"
+  "type": "module",
+  "dependencies": {
+    "@types/sqlite3": "^3.1.8",
+    "@types/ws": "^8.5.4",
+    "sqlite3": "^5.1.6",
+    "ws": "^8.13.0"
+  }
 }

src/lib/components/AccountComponent.svelte

@@ -0,0 +1,7 @@
+<script lang="ts">
+  import type { Account } from '$lib/mastodon/response';
+
+  export let account: Account;
+</script>
+
+<a href="{account.url}" target="_blank">{account.display_name} @{account.acct}</a>

src/lib/components/AvatarComponent.svelte

@@ -0,0 +1,20 @@
+<script lang="ts">
+  import type { Account } from '$lib/mastodon/response';
+
+  export let account: Account;
+  let avatarDescription: string;
+  $: avatarDescription = `Avatar for ${account.acct}`
+</script>
+
+<img src="{account.avatar}" alt={avatarDescription}/>
+
+<style>
+img {
+  max-width: 50px;
+  max-height: 50px;
+  width: auto;
+  height: auto;
+  object-fit: contain;
+  border-radius: 3px;;
+}
+</style>

src/lib/components/PostComponent.svelte

@@ -0,0 +1,31 @@
+<script lang="ts">
+  import type { Post } from '$lib/mastodon/response';
+  import AvatarComponent from '$lib/components/AvatarComponent.svelte';
+  import AccountComponent from '$lib/components/AccountComponent.svelte';
+
+  export let post: Post;
+  let dateCreated: string;
+  $: dateCreated = new Date(post.created_at).toLocaleString();
+</script>
+
+<div class="wrapper">
+  <div class="avatar"><AvatarComponent account={post.account} /></div>
+  <div class="post">
+    <AccountComponent account={post.account} />
+    <small><a href={post.url} target="_blank">{dateCreated}</a></small>
+    {@html post.content}
+  </div>
+</div>
+
+<style>
+  .wrapper {
+    display: flex;
+  }
+  .post {
+    display: flex;
+    flex-direction: column;
+  }
+  .avatar {
+    margin-right: 1em;
+  }
+</style>

src/lib/mastodon/response.ts

@@ -0,0 +1,29 @@
+export interface TimelineEvent {
+  event: string,
+  payload: string
+}
+
+export interface Post {
+  id: string,
+  created_at: string,
+  tags: Tag[],
+  url: string,
+  content: string,
+  account: Account
+
+}
+
+export interface Tag {
+  name: string,
+  url: string
+}
+
+export interface Account {
+  id: string,
+  acct: string,
+  username: string,
+  display_name: string,
+  url: string,
+  avatar: string,
+  avatar_static: string
+}

src/lib/server/db.ts

@@ -0,0 +1,250 @@
+import type { Account, Post, Tag } from '$lib/mastodon/response';
+import sqlite3 from 'sqlite3';
+const { MODE } = import.meta.env;
+
+const db: sqlite3.Database = new sqlite3.Database('moshingmammut.db');
+
+if (MODE === 'development') {
+  sqlite3.verbose();
+  db.on('change', (t, d, table, rowid) => {
+    console.debug('DB change event', t, d, table, rowid);
+  })
+
+  db.on('trace', (sql) => {
+    console.debug('Running', sql);
+  });
+
+  db.on('profile', (sql) => {
+    console.debug('Finished', sql);
+  });
+}
+
+interface Migration {
+  id: number,
+  name: string,
+  statement: string
+}
+
+db.on('open', () => {
+  console.log('Opened database');
+  db.run('CREATE TABLE IF NOT EXISTS "migrations" ("id" integer,"name" TEXT, PRIMARY KEY (id))');
+  db.serialize();
+  db.all('SELECT id FROM migrations', (err, rows) => {
+    if (err !== null) {
+      console.error('Could not fetch existing migrations', err);
+      return;
+    }
+    console.debug('Already applied migrations', rows);
+    const appliedMigrations: Set<number> = new Set(rows.map((row: any) => row['id']));
+    const toApply = getMigrations().filter(m => !appliedMigrations.has(m.id));
+    for (let migration of toApply) {
+      db.exec(migration.statement, (err) => {
+        if (err !== null) {
+          console.error(`Failed to apply migration ${migration.name}`, err);
+          return;
+        }
+        db.run('INSERT INTO migrations (id, name) VALUES(?, ?)', [migration.id, migration.name], (e: Error) => {
+          if (e !== null) {
+            console.error(`Failed to mark migration ${migration.name} as applied`, e);
+            return;
+          }
+          console.info(`Applied migration ${migration.name}`);
+        });
+      });
+    }
+  });
+});
+db.on('error', (err) => {
+  console.error('Error opening database', err);
+});
+
+function getMigrations(): Migration[] {
+  return [{
+    id: 1,
+    name: 'initial',
+    statement: `
+      CREATE TABLE accounts (
+        id TEXT NOT NULL PRIMARY KEY,
+        acct TEXT,
+        username TEXT,
+        display_name TEXT,
+        url TEXT,
+        avatar TEXT,
+        avatar_static TEXT
+      );
+      CREATE TABLE tags (url TEXT NOT NULL PRIMARY KEY, tag TEXT NOT NULL);
+      CREATE TABLE posts (
+        id TEXT NOT NULL PRIMARY KEY,
+        content TEXT,
+        created_at TEXT,
+        url TEXT NOT NULL,
+        account_id TEXT NOT NULL,
+        FOREIGN KEY (account_id) REFERENCES accounts(id)
+      );
+      CREATE TABLE poststags (
+        id integer PRIMARY KEY,
+        post_id TEXT NOT NULL,
+        tag_url TEXT NOT NULL,
+        FOREIGN KEY (post_id) REFERENCES posts(id),
+        FOREIGN KEY (tag_url) REFERENCES tags(url)
+      )`
+  }];
+}
+
+export function savePost(post: Post): void {
+  console.debug(`Saving post ${post.url}`);
+  const account = post.account;
+  db.run(`
+    INSERT INTO accounts (id, acct, username, display_name, url, avatar, avatar_static)
+    VALUES(?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(id)
+    DO UPDATE SET
+    acct=excluded.acct,
+    username=excluded.username,
+    display_name=excluded.display_name,
+    url=excluded.url,
+    avatar=excluded.avatar,
+    avatar_static=excluded.avatar_static;`,
+    [
+      account.id,
+      account.acct,
+      account.username,
+      account.display_name,
+      account.url,
+      account.avatar,
+      account.avatar_static
+    ],
+    (err) => {
+      if (err !== null) {
+        console.error(`Could not insert/update account ${account.id}`, err);
+        return;
+      }
+      db.run(`
+        INSERT INTO posts (id, content, created_at, url, account_id)
+        VALUES (?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET
+        content=excluded.content,
+        created_at=excluded.created_at,
+        url=excluded.url,
+        account_id=excluded.account_id;`,
+        [
+          post.id,
+          post.content,
+          post.created_at,
+          post.url,
+          post.account.id
+        ],
+        (postErr) => {
+          if (postErr !== null) {
+            console.error(`Could not insert post ${post.url}`, postErr);
+            return;
+          }
+
+          db.parallelize(() => {
+            for (let tag of post.tags) {
+              db.run(`
+                INSERT INTO tags (url, tag) VALUES (?, ?)
+                ON CONFLICT(url) DO UPDATE SET
+                tag=excluded.tag;`,
+                [
+                  tag.url,
+                  tag.name
+                ],
+                (tagErr) => {
+                  if (tagErr !== null) {
+                    console.error(`Could not insert/update tag ${tag.url}`, tagErr);
+                    return;
+                  }
+                  db.run('INSERT INTO poststags (post_id, tag_url) VALUES (?, ?)',
+                    [post.id, tag.url],
+                    (posttagserr) => {
+                      if (posttagserr !== null) {
+                        console.error(`Could not insert poststags ${tag.url}, ${post.url}`, posttagserr);
+                        return;
+                      }
+                    }
+                  );
+                }
+              );
+            }
+          });
+        });
+    });
+}
+
+export async function getPosts(since: string | null, limit: number) {
+  let promise = await new Promise<Post[]>((resolve, reject) => {
+    let filter_query;
+    let params: any = { $limit: limit };
+    if (since === null) {
+      filter_query = '';
+    } else {
+      filter_query = 'WHERE posts.created_at > $since';
+      params.$since = since;
+      console.debug('Filtering by created_at > ', since);
+    }
+    const sql = `SELECT posts.id, posts.content, posts.created_at, posts.url,
+    accounts.id AS account_id, accounts.acct, accounts.username, accounts.display_name,
+    accounts.url AS account_url, accounts.avatar
+    FROM posts
+    JOIN accounts ON posts.account_id = accounts.id
+    ${filter_query}
+    ORDER BY created_at DESC
+    LIMIT $limit`;
+    db.all(
+      sql,
+      params,
+      (err, rows: any[]) => {
+        if (err != null) {
+          console.error('Error loading posts', err);
+          reject(err);
+          return;
+        }
+        const postIdsParams = rows.map(() => '?').join(', ');
+        db.all(
+          `SELECT post_id, tags.url, tags.tag
+          FROM poststags
+          JOIN tags ON poststags.tag_url = tags.url
+          WHERE post_id IN (${postIdsParams});`,
+          rows.map((r: any) => r.id),
+          (tagErr, tagRows: any[]) => {
+            if (tagErr != null) {
+              console.error('Error loading post tags', tagErr);
+              reject(tagErr);
+              return;
+            }
+            const tagMap: Map<string, Tag[]> = tagRows.reduce(
+              (result: Map<string, Tag[]>, item) => {
+                const tag: Tag = {
+                  url: item.url,
+                  name: item.tag
+                };
+                result.set(item.post_id, [...result.get(item.post_id) || [], tag]);
+                return result;
+              }, new Map());
+            const posts = rows.map(row => {
+              return {
+                id: row.id,
+                content: row.content,
+                created_at: row.created_at,
+                url: row.url,
+                tags: tagMap.get(row.id) || [],
+                account: {
+                  id: row.account_id,
+                  acct: row.acct,
+                  username: row.username,
+                  display_name: row.display_name,
+                  url: row.account_url,
+                  avatar: row.avatar,
+                  avatar_static: ''
+                } as Account
+              } as Post
+            });
+            resolve(posts);
+          }
+        );
+      }
+    );
+  });
+  return promise;
+}
+

src/lib/server/timeline.ts

@@ -0,0 +1,74 @@
+import { HASHTAG_FILTER, URL_FILTER } from '$env/static/private';
+import type { Post, Tag, TimelineEvent } from '$lib/mastodon/response';
+import { savePost } from '$lib/server/db';
+import { WebSocket } from "ws";
+/*
+Filter youtube: /v3/videos (part: snippet), /v3/videoCategories (part: snippet). Category 10 is Music
+*/
+
+const YOUTUBE_REGEX = new RegExp('https?:\/\/(www\.)?youtu((be.com\/.*v=)|(\.be\/))(?<videoId>[a-zA-Z_0-9-]+)');
+
+export class TimelineReader {
+  private static _instance: TimelineReader;
+  private constructor() {
+    //const socket = new WebSocket("wss://metalhead.club/api/v1/streaming/public/local")
+    const socket = new WebSocket("wss://metalhead.club/api/v1/streaming")
+    socket.onopen = (_event) => {
+      socket.send('{ "type": "subscribe", "stream": "public:local"}');
+    };
+    socket.onmessage = (event) => {
+      try {
+        const data: TimelineEvent = JSON.parse(event.data.toString());
+        if (data.event !== 'update') {
+          return;
+        }
+        const post: Post = JSON.parse(data.payload);
+        const hashttags: string[] = HASHTAG_FILTER.split(',');
+        const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
+
+        const urls: string[] = URL_FILTER.split(',');
+        const found_urls = urls.filter(t => post.content.includes(t));
+
+        if (found_urls.length === 0 && found_tags.length === 0) {
+          //return;
+        }
+
+        const youtubeMatches = found_urls.map(u => u.match(YOUTUBE_REGEX)).filter(i => i !== null);
+        if (found_urls.length > 0 && youtubeMatches.length === found_urls.length) {
+          // TODO: Check with YouTube API if it is in YT music or labeled as music
+          console.info('Found youtube urls', youtubeMatches);
+        }
+        console.log(
+          "message",
+          `Update by @${post.account?.username}: ${post.content}`,
+          'tags',
+          post.tags,
+          found_tags,
+          found_urls);
+        savePost(post);
+
+      } catch (e) {
+        console.error("error message", event, event.data, e)
+      }
+
+    };
+    socket.onclose = (event) => {
+      console.log("Closed", event, event.code, event.reason)
+    };
+    socket.onerror = (event) => {
+      console.log("error", event, event.message, event.error)
+    };
+
+  }
+
+  public static init() {
+    if (this._instance === undefined) {
+      this._instance = new TimelineReader();
+    }
+  }
+
+  public static get instance(): TimelineReader {
+    TimelineReader.init();
+    return this._instance;
+  }
+}

src/routes/+page.svelte

@@ -1,2 +1,41 @@
-<h1>Welcome to SvelteKit</h1>
+<script lang="ts">
-<p>Visit <a href="https://kit.svelte.dev">kit.svelte.dev</a> to read the documentation</p>
+import { onMount } from "svelte";
+import type { PageData } from './$types';
+import type { Post } from '$lib/mastodon/response';
+import { PUBLIC_REFRESH_INTERVAL } from '$env/static/public';
+import PostComponent from '$lib/components/PostComponent.svelte';
+
+export let data: PageData;
+
+let interval: NodeJS.Timer | null = null;
+
+onMount(async () => {
+  interval = setInterval(async () => {
+    const params = new URLSearchParams();
+    if (data.posts.length > 0) {
+      params.set('since', data.posts[0].created_at);
+    }
+    await fetch(`/api/posts?${params}`)
+    .then(r => r.json())
+    .then((resp: Post[]) => {
+      if (resp.length > 0) {
+        data.posts = resp.concat(data.posts);
+        console.log('updated data', resp, data.posts);
+      }
+    })
+    .catch(e => {
+      // TODO: Show error in UI
+      console.error('Error loading newest posts', e);
+    });
+  }, parseInt(PUBLIC_REFRESH_INTERVAL));
+  return () => {
+    if (interval !== null) {
+      clearInterval(interval)
+    }
+  }
+})
+</script>
+
+{#each data.posts as post (post.id)}
+  <PostComponent {post}></PostComponent>
+{/each}

src/routes/+page.ts

@@ -0,0 +1,10 @@
+import type { Post } from '$lib/mastodon/response';
+import type { PageLoad } from './$types';
+
+export const load = (async ({ fetch }) => {
+    const p = await fetch('/');
+    //const posts: Post[] = await p.json();
+    return {
+        posts: await p.json() as Post[]
+    };
+}) satisfies PageLoad;

src/routes/+server.ts

@@ -0,0 +1,8 @@
+import { TimelineReader } from '$lib/server/timeline';
+import type { RequestHandler } from './$types';
+
+TimelineReader.init();
+
+export const GET = (async ({ fetch }) => {
+  return await fetch('api/posts');
+}) satisfies RequestHandler;

src/routes/api/posts/+server.ts

@@ -0,0 +1,15 @@
+import { getPosts } from '$lib/server/db';
+import { json } from '@sveltejs/kit';
+
+import type { RequestHandler } from './$types';
+
+export const GET = (async ({ url }) => {
+    const since = url.searchParams.get('since');
+    let count = Number.parseInt(url.searchParams.get('count') || '');
+    if (isNaN(count)) {
+        count = 20;
+    }
+    count = Math.min(count, 100);
+    const posts = await getPosts(since, count);
+    return json(posts);
+}) satisfies RequestHandler;