Saving song infos to DB, refactor logging

.nova/Configuration.json

@@ -1,8 +1,11 @@
 {
+  "apexskier.eslint.config.eslintConfigPath" : ".eslintrc.cjs",
   "apexskier.eslint.config.eslintPath" : "node_modules\/@eslint\/eslintrc\/dist\/eslintrc.cjs",
-  "apexskier.typescript.config.formatDocumentOnSave" : "true",
+  "apexskier.eslint.config.fixOnSave" : "Enable",
+  "apexskier.typescript.config.formatDocumentOnSave" : "false",
   "apexskier.typescript.config.isEnabledForJavascript" : "Enable",
   "apexskier.typescript.config.organizeImportsOnSave" : "true",
   "apexskier.typescript.config.userPreferences.quotePreference" : "single",
-  "apexskier.typescript.config.userPreferences.useLabelDetailsInCompletionEntries" : true
+  "apexskier.typescript.config.userPreferences.useLabelDetailsInCompletionEntries" : true,
+  "prettier.format-on-save" : "Global Default"
 }

src/hooks.server.ts

@@ -1,3 +1,4 @@
+import { log } from '$lib/log';
 import { TimelineReader } from '$lib/server/timeline';
 import type { HandleServerError } from '@sveltejs/kit';
 import fs from 'fs/promises';
@@ -6,7 +7,7 @@ TimelineReader.init();
 
 export const handleError = (({ error }) => {
   if (error instanceof Error) {
-    console.error('Something went wrong: ', error.name, error.message);
+    log.error('Something went wrong: ', error.name, error.message);
   }
 
   return {

src/lib/log.ts

@@ -0,0 +1,32 @@
+import { env } from '$env/dynamic/private';
+import { isTruthy } from '$lib/truthyString';
+const { DEV } = import.meta.env;
+
+export const enableVerboseLog = isTruthy(env.VERBOSE);
+
+export const log = {
+  verbose: (...params: any[]) => {
+    if (!enableVerboseLog) {
+      return;
+    }
+    console.debug(new Date().toISOString(), ...params);
+  },
+  debug: (...params: any[]) => {
+    if (!DEV) {
+      return;
+    }
+    console.debug(new Date().toISOString(), ...params);
+  },
+  log: (...params: any[]) => {
+    console.log(new Date().toISOString(), ...params);
+  },
+  info: (...params: any[]) => {
+    console.info(new Date().toISOString(), ...params);
+  },
+  warn: (...params: any[]) => {
+    console.warn(new Date().toISOString(), ...params);
+  },
+  error: (...params: any[]) => {
+    console.error(new Date().toISOString(), ...params);
+  }
+};

src/lib/mastodon/response.ts

@@ -1,3 +1,5 @@
+import type { SongInfo } from '$lib/odesliResponse';
+
 export interface TimelineEvent {
   event: string;
   payload: string;
@@ -11,6 +13,7 @@ export interface Post {
   content: string;
   account: Account;
   card?: PreviewCard;
+  songs?: SongInfo[];
 }
 
 export interface PreviewCard {

src/lib/odesliResponse.ts

@@ -5,6 +5,7 @@ export type SongInfo = {
   title?: string;
   artistName?: string;
   thumbnailUrl?: string;
+  postedUrl: string;
 };
 
 export type SongwhipReponse = {

src/lib/server/db.ts

@@ -1,10 +1,47 @@
-import { env } from '$env/dynamic/private';
 import { IGNORE_USERS, MASTODON_INSTANCE } from '$env/static/private';
+import { enableVerboseLog, log } from '$lib/log';
 import type { Account, Post, Tag } from '$lib/mastodon/response';
-import { isTruthy } from '$lib/truthyString';
+import type { SongInfo } from '$lib/odesliResponse';
 import sqlite3 from 'sqlite3';
+
 const { DEV } = import.meta.env;
 
+type FilterParameter = {
+  $limit: number | undefined | null;
+  $since?: string | undefined | null;
+  $before?: string | undefined | null;
+  [x: string]: string | number | undefined | null;
+};
+
+type PostRow = {
+  id: string;
+  content: string;
+  created_at: string;
+  url: string;
+  account_id: string;
+  acct: string;
+  username: string;
+  display_name: string;
+  account_url: string;
+  avatar: string;
+};
+
+type PostTagRow = {
+  post_id: string;
+  tag: string;
+  url: string;
+};
+
+type SongRow = SongInfo & {
+  post_url: string;
+};
+
+type Migration = {
+  id: number;
+  name: string;
+  statement: string;
+};
+
 const db: sqlite3.Database = new sqlite3.Database('moshingmammut.db');
 // for the local masto instance, the instance name is *not* saved
 // as part of the username or acct, so it needs to be stripped
@@ -20,38 +57,32 @@ const ignoredUsers: string[] =
         );
 let databaseReady = false;
 
-if (DEV && isTruthy(env.VERBOSE)) {
+if (enableVerboseLog) {
   sqlite3.verbose();
   db.on('change', (t, d, table, rowid) => {
-    console.debug('DB change event', t, d, table, rowid);
+    log.verbose('DB change event', t, d, table, rowid);
   });
 
   db.on('trace', (sql) => {
-    console.debug('Running', sql);
+    log.verbose('Running', sql);
   });
 
   db.on('profile', (sql) => {
-    console.debug('Finished', sql);
+    log.verbose('Finished', sql);
   });
 }
 
-interface Migration {
-  id: number;
-  name: string;
-  statement: string;
-}
-
 db.on('open', () => {
-  console.log('Opened database');
+  log.info('Opened database');
   db.serialize();
   db.run('CREATE TABLE IF NOT EXISTS "migrations" ("id" integer,"name" TEXT, PRIMARY KEY (id))');
   db.all('SELECT id FROM migrations', (err, rows: Migration[]) => {
     if (err !== null) {
-      console.error('Could not fetch existing migrations', err);
+      log.error('Could not fetch existing migrations', err);
       databaseReady = true;
       return;
     }
-    console.debug('Already applied migrations', rows);
+    log.debug('Already applied migrations', rows);
     const appliedMigrations: Set<number> = new Set(rows.map((row) => row['id']));
     const toApply = getMigrations().filter((m) => !appliedMigrations.has(m.id));
     let remaining = toApply.length;
@@ -68,7 +99,7 @@ db.on('open', () => {
           databaseReady = true;
         }
         if (err !== null) {
-          console.error(`Failed to apply migration ${migration.name}`, err);
+          log.error(`Failed to apply migration ${migration.name}`, err);
           return;
         }
         db.run(
@@ -76,10 +107,10 @@ db.on('open', () => {
           [migration.id, migration.name],
           (e: Error) => {
             if (e !== null) {
-              console.error(`Failed to mark migration ${migration.name} as applied`, e);
+              log.error(`Failed to mark migration ${migration.name} as applied`, e);
               return;
             }
-            console.info(`Applied migration ${migration.name}`);
+            log.info(`Applied migration ${migration.name}`);
           }
         );
       });
@@ -87,7 +118,7 @@ db.on('open', () => {
   });
 });
 db.on('error', (err) => {
-  console.error('Error opening database', err);
+  log.error('Error opening database', err);
 });
 
 function getMigrations(): Migration[] {
@@ -170,6 +201,23 @@ function getMigrations(): Migration[] {
       DROP TABLE poststags;
       ALTER TABLE poststags_new RENAME TO poststags;
       `
+    },
+    {
+      id: 3,
+      name: 'song info for posts',
+      statement: `
+      CREATE TABLE songs (
+        id integer PRIMARY KEY,
+        postedUrl TEXT NOT NULL,
+        overviewUrl TEXT,
+        type TEXT CHECK ( type in ('album', 'song') ),
+        youtubeUrl TEXT,
+        title TEXT,
+        artistName TEXT,
+        thumbnailUrl TEXT,
+        post_url TEXT,
+        FOREIGN KEY (post_url) REFERENCES posts(url)
+      );`
     }
   ];
 }
@@ -179,11 +227,11 @@ async function waitReady(): Promise<undefined> {
   return new Promise((resolve) => {
     const interval = setInterval(() => {
       if (DEV) {
-        console.debug('Waiting for database to be ready');
+        log.debug('Waiting for database to be ready');
       }
       if (databaseReady) {
         if (DEV) {
-          console.debug('DB is ready');
+          log.debug('DB is ready');
         }
         clearInterval(interval);
         resolve(undefined);
@@ -192,13 +240,8 @@ async function waitReady(): Promise<undefined> {
   });
 }
 
-export async function savePost(post: Post): Promise<undefined> {
+function saveAccountData(account: Account): Promise<undefined> {
-  if (!databaseReady) {
+  return new Promise<undefined>((resolve, reject) => {
-    await waitReady();
-  }
-  return await new Promise<undefined>((resolve, reject) => {
-    console.debug(`Saving post ${post.url}`);
-    const account = post.account;
     db.run(
       `
       INSERT INTO accounts (id, acct, username, display_name, url, avatar)
@@ -220,192 +263,281 @@ export async function savePost(post: Post): Promise<undefined> {
       ],
       (err) => {
         if (err !== null) {
-          console.error(`Could not insert/update account ${account.id}`, err);
+          log.error(`Could not insert/update account ${account.id}`, err);
           reject(err);
           return;
         }
-        db.run(
+        resolve(undefined);
-          `
-          INSERT INTO posts (id, content, created_at, url, account_id)
-          VALUES (?, ?, ?, ?, ?) ON CONFLICT(url) DO UPDATE SET
-          content=excluded.content,
-          created_at=excluded.created_at,
-          id=excluded.id,
-          account_id=excluded.account_id;`,
-          [post.id, post.content, post.created_at, post.url, post.account.url],
-          (postErr) => {
-            if (postErr !== null) {
-              console.error(`Could not insert post ${post.url}`, postErr);
-              reject(postErr);
-              return;
-            }
-
-            if (!post.tags.length) {
-              resolve(undefined);
-              return;
-            }
-
-            db.parallelize(() => {
-              let remaining = post.tags.length;
-              for (const tag of post.tags) {
-                db.run(
-                  `
-                  INSERT INTO tags (url, tag) VALUES (?, ?)
-                  ON CONFLICT(url) DO UPDATE SET
-                  tag=excluded.tag;`,
-                  [tag.url, tag.name],
-                  (tagErr) => {
-                    if (tagErr !== null) {
-                      console.error(`Could not insert/update tag ${tag.url}`, tagErr);
-                      reject(tagErr);
-                      return;
-                    }
-                    db.run(
-                      'INSERT INTO poststags (post_id, tag_url) VALUES (?, ?)',
-                      [post.url, tag.url],
-                      (posttagserr) => {
-                        if (posttagserr !== null) {
-                          console.error(
-                            `Could not insert poststags ${tag.url}, ${post.url}`,
-                            posttagserr
-                          );
-                          reject(posttagserr);
-                          return;
-                        }
-                        // Don't decrease on fail
-                        remaining--;
-                        // Only resolve after all have been inserted
-                        if (remaining === 0) {
-                          resolve(undefined);
-                        }
-                      }
-                    );
-                  }
-                );
-              }
-            });
-          }
-        );
       }
     );
   });
 }
 
-type FilterParameter = {
+function savePostData(post: Post): Promise<undefined> {
-  $limit: number | undefined | null;
+  return new Promise<undefined>((resolve, reject) => {
-  $since?: string | undefined | null;
+    db.run(
-  $before?: string | undefined | null;
+      `
-  [x: string]: string | number | undefined | null;
+    INSERT INTO posts (id, content, created_at, url, account_id)
-};
+    VALUES (?, ?, ?, ?, ?) ON CONFLICT(url) DO UPDATE SET
+    content=excluded.content,
+    created_at=excluded.created_at,
+    id=excluded.id,
+    account_id=excluded.account_id;`,
+      [post.id, post.content, post.created_at, post.url, post.account.url],
+      (postErr) => {
+        if (postErr !== null) {
+          log.error(`Could not insert post ${post.url}`, postErr);
+          reject(postErr);
+          return;
+        }
+        resolve(undefined);
+      }
+    );
+  });
+}
 
-export async function getPosts(since: string | null, before: string | null, limit: number) {
+function savePostTagData(post: Post): Promise<undefined> {
+  return new Promise<undefined>((resolve, reject) => {
+    if (!post.tags.length) {
+      resolve(undefined);
+      return;
+    }
+
+    db.parallelize(() => {
+      let remaining = post.tags.length;
+      for (const tag of post.tags) {
+        db.run(
+          `
+          INSERT INTO tags (url, tag) VALUES (?, ?)
+          ON CONFLICT(url) DO UPDATE SET
+          tag=excluded.tag;`,
+          [tag.url, tag.name],
+          (tagErr) => {
+            if (tagErr !== null) {
+              log.error(`Could not insert/update tag ${tag.url}`, tagErr);
+              reject(tagErr);
+              return;
+            }
+            db.run(
+              'INSERT INTO poststags (post_id, tag_url) VALUES (?, ?)',
+              [post.url, tag.url],
+              (posttagserr) => {
+                if (posttagserr !== null) {
+                  log.error(`Could not insert poststags ${tag.url}, ${post.url}`, posttagserr);
+                  reject(posttagserr);
+                  return;
+                }
+                // Don't decrease on fail
+                remaining--;
+                // Only resolve after all have been inserted
+                if (remaining === 0) {
+                  resolve(undefined);
+                }
+              }
+            );
+          }
+        );
+      }
+    });
+  });
+}
+
+function saveSongInfoData(postUrl: string, songs: SongInfo[]): Promise<undefined> {
+  return new Promise<undefined>((resolve, reject) => {
+    if (songs.length === 0) {
+      resolve(undefined);
+      return;
+    }
+    db.parallelize(() => {
+      let remaining = songs.length;
+      for (const song of songs) {
+        db.run(
+          `
+          INSERT INTO songs (postedUrl, overviewUrl, type, youtubeUrl, title, artistName, thumbnailUrl, post_url)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+          `,
+          [
+            song.postedUrl,
+            song.pageUrl,
+            song.type,
+            song.youtubeUrl,
+            song.title,
+            song.artistName,
+            song.thumbnailUrl,
+            postUrl
+          ],
+          (songErr) => {
+            if (songErr !== null) {
+              log.error(`Could not insert song ${song.postedUrl}`, songErr);
+              reject(songErr);
+              return;
+            }
+            // Don't decrease on fail
+            remaining--;
+            // Only resolve after all have been inserted
+            if (remaining === 0) {
+              resolve(undefined);
+            }
+          }
+        );
+      }
+    });
+  });
+}
+
+export async function savePost(post: Post, songs: SongInfo[]) {
   if (!databaseReady) {
     await waitReady();
   }
-  const promise = await new Promise<Post[]>((resolve, reject) => {
-    let filter_query = '';
-    const params: FilterParameter = { $limit: limit };
-    if (since === null && before === null) {
-      filter_query = '';
-    } else if (since !== null) {
-      filter_query = 'WHERE posts.created_at > $since';
-      params.$since = since;
-    } else if (before !== null) {
-      // Setting both, before and since doesn't make sense, so this case is not explicitly handled
-      filter_query = 'WHERE posts.created_at < $before';
-      params.$before = before;
-    }
 
-    ignoredUsers.forEach((ignoredUser, index) => {
+  log.debug(`Saving post ${post.url}`);
-      const userParam = `$user_${index}`;
+  const account = post.account;
-      const acctParam = userParam + 'a';
+  await saveAccountData(account);
-      const usernameParam = userParam + 'u';
+  log.debug(`Saved account data ${post.url}`);
-      const prefix = filter_query === '' ? ' WHERE' : ' AND';
+  await savePostData(post);
-      filter_query += `${prefix} acct != ${acctParam} AND username != ${usernameParam} `;
+  log.debug(`Saved post data ${post.url}`);
-      params[acctParam] = ignoredUser;
+  await savePostTagData(post);
-      params[usernameParam] = ignoredUser;
+  log.debug(`Saved ${post.tags.length} tag data ${post.url}`);
-    });
+  await saveSongInfoData(post.url, songs);
+  log.debug(`Saved ${songs.length} song info data ${post.url}`);
+}
 
-    type PostResult = {
+function getPostData(filterQuery: string, params: FilterParameter): Promise<PostRow[]> {
-      id: string;
+  const sql = `SELECT posts.id, posts.content, posts.created_at, posts.url,
-      content: string;
+  accounts.id AS account_id, accounts.acct, accounts.username, accounts.display_name,
-      created_at: string;
+  accounts.url AS account_url, accounts.avatar
-      url: string;
+  FROM posts
-      account_id: string;
+  JOIN accounts ON posts.account_id = accounts.url
-      acct: string;
+  ${filterQuery}
-      username: string;
+  ORDER BY created_at DESC
-      display_name: string;
+  LIMIT $limit`;
-      account_url: string;
-      avatar: string;
-    };
 
-    type PostTagResult = {
+  return new Promise((resolve, reject) => {
-      post_id: string;
+    db.all(sql, params, (err, rows: PostRow[]) => {
-      tag: string;
-      url: string;
-    };
-
-    const sql = `SELECT posts.id, posts.content, posts.created_at, posts.url,
-      accounts.id AS account_id, accounts.acct, accounts.username, accounts.display_name,
-      accounts.url AS account_url, accounts.avatar
-      FROM posts
-      JOIN accounts ON posts.account_id = accounts.url
-      ${filter_query}
-      ORDER BY created_at DESC
-      LIMIT $limit`;
-    db.all(sql, params, (err, rows: PostResult[]) => {
       if (err != null) {
-        console.error('Error loading posts', err);
+        log.error('Error loading posts', err);
         reject(err);
         return;
       }
-      if (rows.length === 0) {
+      resolve(rows);
-        // No need to check for tags
-        resolve([]);
-        return;
-      }
-      const postIdsParams = rows.map(() => '?').join(', ');
-      db.all(
-        `SELECT post_id, tags.url, tags.tag
-          FROM poststags
-          JOIN tags ON poststags.tag_url = tags.url
-          WHERE post_id IN (${postIdsParams});`,
-        rows.map((r: PostResult) => r.url),
-        (tagErr, tagRows: PostTagResult[]) => {
-          if (tagErr != null) {
-            console.error('Error loading post tags', tagErr);
-            reject(tagErr);
-            return;
-          }
-          const tagMap: Map<string, Tag[]> = tagRows.reduce((result: Map<string, Tag[]>, item) => {
-            const tag: Tag = {
-              url: item.url,
-              name: item.tag
-            };
-            result.set(item.post_id, [...(result.get(item.post_id) || []), tag]);
-            return result;
-          }, new Map());
-          const posts = rows.map((row) => {
-            return {
-              id: row.id,
-              content: row.content,
-              created_at: row.created_at,
-              url: row.url,
-              tags: tagMap.get(row.id) || [],
-              account: {
-                id: row.account_id,
-                acct: row.acct,
-                username: row.username,
-                display_name: row.display_name,
-                url: row.account_url,
-                avatar: row.avatar
-              } as Account
-            } as Post;
-          });
-          resolve(posts);
-        }
-      );
     });
   });
-  return promise;
+}
+
+function getTagData(postIdsParams: String, postIds: string[]): Promise<Map<string, Tag[]>> {
+  return new Promise((resolve, reject) => {
+    db.all(
+      `SELECT post_id, tags.url, tags.tag
+      FROM poststags
+      JOIN tags ON poststags.tag_url = tags.url
+      WHERE post_id IN (${postIdsParams});`,
+      postIds,
+      (tagErr, tagRows: PostTagRow[]) => {
+        if (tagErr != null) {
+          log.error('Error loading post tags', tagErr);
+          reject(tagErr);
+          return;
+        }
+        const tagMap: Map<string, Tag[]> = tagRows.reduce((result: Map<string, Tag[]>, item) => {
+          const tag: Tag = {
+            url: item.url,
+            name: item.tag
+          };
+          result.set(item.post_id, [...(result.get(item.post_id) || []), tag]);
+          return result;
+        }, new Map());
+        resolve(tagMap);
+      }
+    );
+  });
+}
+
+function getSongData(postIdsParams: String, postIds: string[]): Promise<Map<string, SongInfo[]>> {
+  return new Promise((resolve, reject) => {
+    db.all(
+      `SELECT post_url, songs.postedUrl, songs.overviewUrl, songs.type, songs.youtubeUrl,
+      songs.title, songs.artistName, songs.thumbnailUrl, songs.post_url
+      FROM songs
+      WHERE post_url IN (${postIdsParams});`,
+      postIds,
+      (tagErr, tagRows: SongRow[]) => {
+        if (tagErr != null) {
+          log.error('Error loading post tags', tagErr);
+          reject(tagErr);
+          return;
+        }
+        const songMap: Map<string, SongInfo[]> = tagRows.reduce(
+          (result: Map<string, SongInfo[]>, item) => {
+            result.set(item.post_url, [...(result.get(item.post_url) || []), item]);
+            return result;
+          },
+          new Map()
+        );
+        resolve(songMap);
+      }
+    );
+  });
+}
+
+export async function getPosts(
+  since: string | null,
+  before: string | null,
+  limit: number
+): Promise<Post[]> {
+  if (!databaseReady) {
+    await waitReady();
+  }
+
+  let filterQuery = '';
+  const params: FilterParameter = { $limit: limit };
+  if (since === null && before === null) {
+    filterQuery = '';
+  } else if (since !== null) {
+    filterQuery = 'WHERE posts.created_at > $since';
+    params.$since = since;
+  } else if (before !== null) {
+    // Setting both, before and since doesn't make sense, so this case is not explicitly handled
+    filterQuery = 'WHERE posts.created_at < $before';
+    params.$before = before;
+  }
+
+  ignoredUsers.forEach((ignoredUser, index) => {
+    const userParam = `$user_${index}`;
+    const acctParam = userParam + 'a';
+    const usernameParam = userParam + 'u';
+    const prefix = filterQuery === '' ? ' WHERE' : ' AND';
+    filterQuery += `${prefix} acct != ${acctParam} AND username != ${usernameParam} `;
+    params[acctParam] = ignoredUser;
+    params[usernameParam] = ignoredUser;
+  });
+
+  const rows = await getPostData(filterQuery, params);
+  if (rows.length === 0) {
+    // No need to check for tags and songs
+    return [];
+  }
+
+  const postIdsParams = rows.map(() => '?').join(', ');
+  const postIds = rows.map((r: PostRow) => r.url);
+  const tagMap = await getTagData(postIdsParams, postIds);
+  const songMap = await getSongData(postIdsParams, postIds);
+  const posts = rows.map((row) => {
+    return {
+      id: row.id,
+      content: row.content,
+      created_at: row.created_at,
+      url: row.url,
+      tags: tagMap.get(row.url) || [],
+      account: {
+        id: row.account_id,
+        acct: row.acct,
+        username: row.username,
+        display_name: row.display_name,
+        url: row.account_url,
+        avatar: row.avatar
+      } as Account,
+      songs: songMap.get(row.url) || []
+    } as Post;
+  });
+  return posts;
 }

src/lib/server/rss.ts

@@ -1,6 +1,7 @@
 import { BASE_URL, WEBSUB_HUB } from '$env/static/private';
 import { PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME } from '$env/static/public';
 import type { Post } from '$lib//mastodon/response';
+import { log } from '$lib/log';
 import { Feed } from 'feed';
 import fs from 'fs/promises';
 
@@ -59,6 +60,6 @@ export async function saveAtomFeed(feed: Feed) {
       body: params
     });
   } catch (e) {
-    console.error('Failed to update WebSub hub', e);
+    log.error('Failed to update WebSub hub', e);
   }
 }

src/lib/server/timeline.ts

@@ -6,6 +6,7 @@ import {
   YOUTUBE_API_KEY,
   YOUTUBE_DISABLE
 } from '$env/static/private';
+import { log } from '$lib/log';
 import type { Post, Tag, TimelineEvent } from '$lib/mastodon/response';
 import type { OdesliResponse, Platform, SongInfo } from '$lib/odesliResponse';
 import { getPosts, savePost } from '$lib/server/db';
@@ -38,7 +39,7 @@ export class TimelineReader {
     const resp = await fetch(youtubeVideoUrl);
     const respObj = await resp.json();
     if (!respObj.items.length) {
-      console.warn('Could not find video with id', videoId);
+      log.warn('Could not find video with id', videoId);
       return false;
     }
 
@@ -77,7 +78,7 @@ export class TimelineReader {
           return match[0];
         }
       } catch (e) {
-        console.error('Could not check if', videoId, 'is a music video', e);
+        log.error('Could not check if', videoId, 'is a music video', e);
       }
     }
     return null;
@@ -85,14 +86,14 @@ export class TimelineReader {
 
   private static async getSongInfo(url: string, remainingTries = 6): Promise<SongInfo | null> {
     if (remainingTries === 0) {
-      console.error('No tries remaining. Lookup failed!');
+      log.error('No tries remaining. Lookup failed!');
       return null;
     }
     let hostname: string;
     try {
       hostname = new URL(url).hostname;
     } catch (e) {
-      console.error(`Could not construct URL ${url}`, e);
+      log.error(`Could not construct URL ${url}`, e);
       return null;
     }
     if (hostname === 'songwhip.com') {
@@ -121,17 +122,18 @@ export class TimelineReader {
           return {
             ...info,
             pageUrl: odesliInfo.pageUrl,
-            youtubeUrl: odesliInfo.linksByPlatform[platform]?.url
+            youtubeUrl: odesliInfo.linksByPlatform[platform]?.url,
+            postedUrl: url
           } as SongInfo;
         });
       });
     } catch (e) {
       if (e instanceof Error && e.cause === 429) {
-        console.warn('song.link rate limit reached. Trying again in 10 seconds');
+        log.warn('song.link rate limit reached. Trying again in 10 seconds');
         await sleep(10_000);
         return await this.getSongInfo(url, remainingTries - 1);
       }
-      console.error(`Failed to load ${url} info from song.link`, e);
+      log.error(`Failed to load ${url} info from song.link`, e);
       return null;
     }
   }
@@ -149,7 +151,7 @@ export class TimelineReader {
       ).json();
       return status.card?.url;
     } catch (e) {
-      console.error(`Could not fetch status ${post.url}`, e);
+      log.error(`Could not fetch status ${post.url}`, e);
     }
     */
   }
@@ -157,7 +159,7 @@ export class TimelineReader {
   private startWebsocket() {
     const socket = new WebSocket(`wss://${MASTODON_INSTANCE}/api/v1/streaming`);
     socket.onopen = () => {
-      console.log('Connected to WS');
+      log.log('Connected to WS');
       socket.send('{ "type": "subscribe", "stream": "public:local"}');
     };
     socket.onmessage = async (event) => {
@@ -172,17 +174,15 @@ export class TimelineReader {
 
         const urls: string[] = URL_FILTER.split(',');
         const found_urls = urls.filter((t) => post.content.includes(t));
-        const urlsToCheck: string[] = [];
         // If we don't have any tags or non-youtube urls, check youtube
         // YT is handled separately, because it requires an API call and therefore is slower
         if (found_urls.length === 0 && found_tags.length === 0) {
           const youtubeUrl = await TimelineReader.checkYoutubeMatches(post.content);
           if (youtubeUrl === null) {
-            console.log('Ignoring post', post.url);
+            log.log('Ignoring post', post.url);
             return;
           }
-          urlsToCheck.push(youtubeUrl);
+          log.debug('Found YT URL', youtubeUrl, found_urls, found_urls.length);
-          console.log('Found YT URL', youtubeUrl, found_urls, found_urls.length);
         }
 
         // TODO: Change URL detection above to use this regex.
@@ -207,51 +207,46 @@ export class TimelineReader {
           }
         }
 
+        const songs: SongInfo[] = [];
+        log.debug(`Checking ${musicUrls.length} URLs if they contain song data`);
         for (const url of musicUrls) {
           let hostname: string | null = null;
           try {
             hostname = new URL(url).hostname;
           } catch (e) {
-            console.error(`Could not check hostname for URL ${url}`, e);
+            log.error(`Could not check hostname for URL ${url}`, e);
           }
           if (hostname === 'songwhip.com') {
             // TODO: Implement checking the songwhip API
             continue;
           }
           const info = await TimelineReader.getSongInfo(url);
+          log.debug(`Found song info for ${url}?`, info);
           if (info) {
-            console.info(
+            songs.push(info);
-              'Got song info for',
-              post.url,
-              url,
-              info.artistName,
-              info.title,
-              info.thumbnailUrl,
-              info.pageUrl,
-              info.youtubeUrl
-            );
           }
         }
 
-        await savePost(post);
+        await savePost(post, songs);
+        log.debug('Saved post', post.url);
 
         const posts = await getPosts(null, null, 100);
         await saveAtomFeed(createFeed(posts));
       } catch (e) {
-        console.error('error message', event, event.data, e);
+        log.error('error message', event, event.data, e);
       }
     };
     socket.onclose = (event) => {
-      console.warn(
+      log.warn(
         `Websocket connection to ${MASTODON_INSTANCE} closed. Code: ${event.code}, reason: '${event.reason}'`
       );
       setTimeout(() => {
-        console.info(`Attempting to reconenct to WS`);
+        log.info(`Attempting to reconenct to WS`);
         this.startWebsocket();
       }, 10000);
     };
     socket.onerror = (event) => {
-      console.error(
+      log.error(
         `Websocket connection to ${MASTODON_INSTANCE} failed. ${event.type}: ${event.error}, message: '${event.message}'`
       );
     };