From 971c846dd1d5e4b6efd1d6a9336a8c452d8f593e Mon Sep 17 00:00:00 2001 From: Max Nuding Date: Sun, 23 Apr 2023 12:46:14 +0200 Subject: [PATCH] Saving song infos to DB, refactor logging --- .nova/Configuration.json | 7 +- src/hooks.server.ts | 3 +- src/lib/log.ts | 32 +++ src/lib/mastodon/response.ts | 3 + src/lib/odesliResponse.ts | 1 + src/lib/server/db.ts | 522 ++++++++++++++++++++++------------- src/lib/server/rss.ts | 3 +- src/lib/server/timeline.ts | 53 ++-- 8 files changed, 396 insertions(+), 228 deletions(-) create mode 100644 src/lib/log.ts diff --git a/.nova/Configuration.json b/.nova/Configuration.json index 1cc2406..3d7f496 100644 --- a/.nova/Configuration.json +++ b/.nova/Configuration.json @@ -1,8 +1,11 @@ { + "apexskier.eslint.config.eslintConfigPath" : ".eslintrc.cjs", "apexskier.eslint.config.eslintPath" : "node_modules\/@eslint\/eslintrc\/dist\/eslintrc.cjs", - "apexskier.typescript.config.formatDocumentOnSave" : "true", + "apexskier.eslint.config.fixOnSave" : "Enable", + "apexskier.typescript.config.formatDocumentOnSave" : "false", "apexskier.typescript.config.isEnabledForJavascript" : "Enable", "apexskier.typescript.config.organizeImportsOnSave" : "true", "apexskier.typescript.config.userPreferences.quotePreference" : "single", - "apexskier.typescript.config.userPreferences.useLabelDetailsInCompletionEntries" : true + "apexskier.typescript.config.userPreferences.useLabelDetailsInCompletionEntries" : true, + "prettier.format-on-save" : "Global Default" } diff --git a/src/hooks.server.ts b/src/hooks.server.ts index ca9e60f..ac5c8e3 100644 --- a/src/hooks.server.ts +++ b/src/hooks.server.ts @@ -1,3 +1,4 @@ +import { log } from '$lib/log'; import { TimelineReader } from '$lib/server/timeline'; import type { HandleServerError } from '@sveltejs/kit'; import fs from 'fs/promises'; @@ -6,7 +7,7 @@ TimelineReader.init(); export const handleError = (({ error }) => { if (error instanceof Error) { - console.error('Something went wrong: ', error.name, error.message); + log.error('Something went wrong: ', error.name, error.message); } return { diff --git a/src/lib/log.ts b/src/lib/log.ts new file mode 100644 index 0000000..0c448d4 --- /dev/null +++ b/src/lib/log.ts @@ -0,0 +1,32 @@ +import { env } from '$env/dynamic/private'; +import { isTruthy } from '$lib/truthyString'; +const { DEV } = import.meta.env; + +export const enableVerboseLog = isTruthy(env.VERBOSE); + +export const log = { + verbose: (...params: any[]) => { + if (!enableVerboseLog) { + return; + } + console.debug(new Date().toISOString(), ...params); + }, + debug: (...params: any[]) => { + if (!DEV) { + return; + } + console.debug(new Date().toISOString(), ...params); + }, + log: (...params: any[]) => { + console.log(new Date().toISOString(), ...params); + }, + info: (...params: any[]) => { + console.info(new Date().toISOString(), ...params); + }, + warn: (...params: any[]) => { + console.warn(new Date().toISOString(), ...params); + }, + error: (...params: any[]) => { + console.error(new Date().toISOString(), ...params); + } +}; diff --git a/src/lib/mastodon/response.ts b/src/lib/mastodon/response.ts index c73dd53..2fb08bb 100644 --- a/src/lib/mastodon/response.ts +++ b/src/lib/mastodon/response.ts @@ -1,3 +1,5 @@ +import type { SongInfo } from '$lib/odesliResponse'; + export interface TimelineEvent { event: string; payload: string; @@ -11,6 +13,7 @@ export interface Post { content: string; account: Account; card?: PreviewCard; + songs?: SongInfo[]; } export interface PreviewCard { diff --git a/src/lib/odesliResponse.ts b/src/lib/odesliResponse.ts index e500adf..736a4c5 100644 --- a/src/lib/odesliResponse.ts +++ b/src/lib/odesliResponse.ts @@ -5,6 +5,7 @@ export type SongInfo = { title?: string; artistName?: string; thumbnailUrl?: string; + postedUrl: string; }; export type SongwhipReponse = { diff --git a/src/lib/server/db.ts b/src/lib/server/db.ts index b159973..5fb7ed7 100644 --- a/src/lib/server/db.ts +++ b/src/lib/server/db.ts @@ -1,10 +1,47 @@ -import { env } from '$env/dynamic/private'; import { IGNORE_USERS, MASTODON_INSTANCE } from '$env/static/private'; +import { enableVerboseLog, log } from '$lib/log'; import type { Account, Post, Tag } from '$lib/mastodon/response'; -import { isTruthy } from '$lib/truthyString'; +import type { SongInfo } from '$lib/odesliResponse'; import sqlite3 from 'sqlite3'; + const { DEV } = import.meta.env; +type FilterParameter = { + $limit: number | undefined | null; + $since?: string | undefined | null; + $before?: string | undefined | null; + [x: string]: string | number | undefined | null; +}; + +type PostRow = { + id: string; + content: string; + created_at: string; + url: string; + account_id: string; + acct: string; + username: string; + display_name: string; + account_url: string; + avatar: string; +}; + +type PostTagRow = { + post_id: string; + tag: string; + url: string; +}; + +type SongRow = SongInfo & { + post_url: string; +}; + +type Migration = { + id: number; + name: string; + statement: string; +}; + const db: sqlite3.Database = new sqlite3.Database('moshingmammut.db'); // for the local masto instance, the instance name is *not* saved // as part of the username or acct, so it needs to be stripped @@ -20,38 +57,32 @@ const ignoredUsers: string[] = ); let databaseReady = false; -if (DEV && isTruthy(env.VERBOSE)) { +if (enableVerboseLog) { sqlite3.verbose(); db.on('change', (t, d, table, rowid) => { - console.debug('DB change event', t, d, table, rowid); + log.verbose('DB change event', t, d, table, rowid); }); db.on('trace', (sql) => { - console.debug('Running', sql); + log.verbose('Running', sql); }); db.on('profile', (sql) => { - console.debug('Finished', sql); + log.verbose('Finished', sql); }); } -interface Migration { - id: number; - name: string; - statement: string; -} - db.on('open', () => { - console.log('Opened database'); + log.info('Opened database'); db.serialize(); db.run('CREATE TABLE IF NOT EXISTS "migrations" ("id" integer,"name" TEXT, PRIMARY KEY (id))'); db.all('SELECT id FROM migrations', (err, rows: Migration[]) => { if (err !== null) { - console.error('Could not fetch existing migrations', err); + log.error('Could not fetch existing migrations', err); databaseReady = true; return; } - console.debug('Already applied migrations', rows); + log.debug('Already applied migrations', rows); const appliedMigrations: Set = new Set(rows.map((row) => row['id'])); const toApply = getMigrations().filter((m) => !appliedMigrations.has(m.id)); let remaining = toApply.length; @@ -68,7 +99,7 @@ db.on('open', () => { databaseReady = true; } if (err !== null) { - console.error(`Failed to apply migration ${migration.name}`, err); + log.error(`Failed to apply migration ${migration.name}`, err); return; } db.run( @@ -76,10 +107,10 @@ db.on('open', () => { [migration.id, migration.name], (e: Error) => { if (e !== null) { - console.error(`Failed to mark migration ${migration.name} as applied`, e); + log.error(`Failed to mark migration ${migration.name} as applied`, e); return; } - console.info(`Applied migration ${migration.name}`); + log.info(`Applied migration ${migration.name}`); } ); }); @@ -87,7 +118,7 @@ db.on('open', () => { }); }); db.on('error', (err) => { - console.error('Error opening database', err); + log.error('Error opening database', err); }); function getMigrations(): Migration[] { @@ -170,6 +201,23 @@ function getMigrations(): Migration[] { DROP TABLE poststags; ALTER TABLE poststags_new RENAME TO poststags; ` + }, + { + id: 3, + name: 'song info for posts', + statement: ` + CREATE TABLE songs ( + id integer PRIMARY KEY, + postedUrl TEXT NOT NULL, + overviewUrl TEXT, + type TEXT CHECK ( type in ('album', 'song') ), + youtubeUrl TEXT, + title TEXT, + artistName TEXT, + thumbnailUrl TEXT, + post_url TEXT, + FOREIGN KEY (post_url) REFERENCES posts(url) + );` } ]; } @@ -179,11 +227,11 @@ async function waitReady(): Promise { return new Promise((resolve) => { const interval = setInterval(() => { if (DEV) { - console.debug('Waiting for database to be ready'); + log.debug('Waiting for database to be ready'); } if (databaseReady) { if (DEV) { - console.debug('DB is ready'); + log.debug('DB is ready'); } clearInterval(interval); resolve(undefined); @@ -192,13 +240,8 @@ async function waitReady(): Promise { }); } -export async function savePost(post: Post): Promise { - if (!databaseReady) { - await waitReady(); - } - return await new Promise((resolve, reject) => { - console.debug(`Saving post ${post.url}`); - const account = post.account; +function saveAccountData(account: Account): Promise { + return new Promise((resolve, reject) => { db.run( ` INSERT INTO accounts (id, acct, username, display_name, url, avatar) @@ -220,192 +263,281 @@ export async function savePost(post: Post): Promise { ], (err) => { if (err !== null) { - console.error(`Could not insert/update account ${account.id}`, err); + log.error(`Could not insert/update account ${account.id}`, err); reject(err); return; } - db.run( - ` - INSERT INTO posts (id, content, created_at, url, account_id) - VALUES (?, ?, ?, ?, ?) ON CONFLICT(url) DO UPDATE SET - content=excluded.content, - created_at=excluded.created_at, - id=excluded.id, - account_id=excluded.account_id;`, - [post.id, post.content, post.created_at, post.url, post.account.url], - (postErr) => { - if (postErr !== null) { - console.error(`Could not insert post ${post.url}`, postErr); - reject(postErr); - return; - } - - if (!post.tags.length) { - resolve(undefined); - return; - } - - db.parallelize(() => { - let remaining = post.tags.length; - for (const tag of post.tags) { - db.run( - ` - INSERT INTO tags (url, tag) VALUES (?, ?) - ON CONFLICT(url) DO UPDATE SET - tag=excluded.tag;`, - [tag.url, tag.name], - (tagErr) => { - if (tagErr !== null) { - console.error(`Could not insert/update tag ${tag.url}`, tagErr); - reject(tagErr); - return; - } - db.run( - 'INSERT INTO poststags (post_id, tag_url) VALUES (?, ?)', - [post.url, tag.url], - (posttagserr) => { - if (posttagserr !== null) { - console.error( - `Could not insert poststags ${tag.url}, ${post.url}`, - posttagserr - ); - reject(posttagserr); - return; - } - // Don't decrease on fail - remaining--; - // Only resolve after all have been inserted - if (remaining === 0) { - resolve(undefined); - } - } - ); - } - ); - } - }); - } - ); + resolve(undefined); } ); }); } -type FilterParameter = { - $limit: number | undefined | null; - $since?: string | undefined | null; - $before?: string | undefined | null; - [x: string]: string | number | undefined | null; -}; +function savePostData(post: Post): Promise { + return new Promise((resolve, reject) => { + db.run( + ` + INSERT INTO posts (id, content, created_at, url, account_id) + VALUES (?, ?, ?, ?, ?) ON CONFLICT(url) DO UPDATE SET + content=excluded.content, + created_at=excluded.created_at, + id=excluded.id, + account_id=excluded.account_id;`, + [post.id, post.content, post.created_at, post.url, post.account.url], + (postErr) => { + if (postErr !== null) { + log.error(`Could not insert post ${post.url}`, postErr); + reject(postErr); + return; + } + resolve(undefined); + } + ); + }); +} -export async function getPosts(since: string | null, before: string | null, limit: number) { +function savePostTagData(post: Post): Promise { + return new Promise((resolve, reject) => { + if (!post.tags.length) { + resolve(undefined); + return; + } + + db.parallelize(() => { + let remaining = post.tags.length; + for (const tag of post.tags) { + db.run( + ` + INSERT INTO tags (url, tag) VALUES (?, ?) + ON CONFLICT(url) DO UPDATE SET + tag=excluded.tag;`, + [tag.url, tag.name], + (tagErr) => { + if (tagErr !== null) { + log.error(`Could not insert/update tag ${tag.url}`, tagErr); + reject(tagErr); + return; + } + db.run( + 'INSERT INTO poststags (post_id, tag_url) VALUES (?, ?)', + [post.url, tag.url], + (posttagserr) => { + if (posttagserr !== null) { + log.error(`Could not insert poststags ${tag.url}, ${post.url}`, posttagserr); + reject(posttagserr); + return; + } + // Don't decrease on fail + remaining--; + // Only resolve after all have been inserted + if (remaining === 0) { + resolve(undefined); + } + } + ); + } + ); + } + }); + }); +} + +function saveSongInfoData(postUrl: string, songs: SongInfo[]): Promise { + return new Promise((resolve, reject) => { + if (songs.length === 0) { + resolve(undefined); + return; + } + db.parallelize(() => { + let remaining = songs.length; + for (const song of songs) { + db.run( + ` + INSERT INTO songs (postedUrl, overviewUrl, type, youtubeUrl, title, artistName, thumbnailUrl, post_url) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, + [ + song.postedUrl, + song.pageUrl, + song.type, + song.youtubeUrl, + song.title, + song.artistName, + song.thumbnailUrl, + postUrl + ], + (songErr) => { + if (songErr !== null) { + log.error(`Could not insert song ${song.postedUrl}`, songErr); + reject(songErr); + return; + } + // Don't decrease on fail + remaining--; + // Only resolve after all have been inserted + if (remaining === 0) { + resolve(undefined); + } + } + ); + } + }); + }); +} + +export async function savePost(post: Post, songs: SongInfo[]) { if (!databaseReady) { await waitReady(); } - const promise = await new Promise((resolve, reject) => { - let filter_query = ''; - const params: FilterParameter = { $limit: limit }; - if (since === null && before === null) { - filter_query = ''; - } else if (since !== null) { - filter_query = 'WHERE posts.created_at > $since'; - params.$since = since; - } else if (before !== null) { - // Setting both, before and since doesn't make sense, so this case is not explicitly handled - filter_query = 'WHERE posts.created_at < $before'; - params.$before = before; - } - ignoredUsers.forEach((ignoredUser, index) => { - const userParam = `$user_${index}`; - const acctParam = userParam + 'a'; - const usernameParam = userParam + 'u'; - const prefix = filter_query === '' ? ' WHERE' : ' AND'; - filter_query += `${prefix} acct != ${acctParam} AND username != ${usernameParam} `; - params[acctParam] = ignoredUser; - params[usernameParam] = ignoredUser; - }); + log.debug(`Saving post ${post.url}`); + const account = post.account; + await saveAccountData(account); + log.debug(`Saved account data ${post.url}`); + await savePostData(post); + log.debug(`Saved post data ${post.url}`); + await savePostTagData(post); + log.debug(`Saved ${post.tags.length} tag data ${post.url}`); + await saveSongInfoData(post.url, songs); + log.debug(`Saved ${songs.length} song info data ${post.url}`); +} - type PostResult = { - id: string; - content: string; - created_at: string; - url: string; - account_id: string; - acct: string; - username: string; - display_name: string; - account_url: string; - avatar: string; - }; +function getPostData(filterQuery: string, params: FilterParameter): Promise { + const sql = `SELECT posts.id, posts.content, posts.created_at, posts.url, + accounts.id AS account_id, accounts.acct, accounts.username, accounts.display_name, + accounts.url AS account_url, accounts.avatar + FROM posts + JOIN accounts ON posts.account_id = accounts.url + ${filterQuery} + ORDER BY created_at DESC + LIMIT $limit`; - type PostTagResult = { - post_id: string; - tag: string; - url: string; - }; - - const sql = `SELECT posts.id, posts.content, posts.created_at, posts.url, - accounts.id AS account_id, accounts.acct, accounts.username, accounts.display_name, - accounts.url AS account_url, accounts.avatar - FROM posts - JOIN accounts ON posts.account_id = accounts.url - ${filter_query} - ORDER BY created_at DESC - LIMIT $limit`; - db.all(sql, params, (err, rows: PostResult[]) => { + return new Promise((resolve, reject) => { + db.all(sql, params, (err, rows: PostRow[]) => { if (err != null) { - console.error('Error loading posts', err); + log.error('Error loading posts', err); reject(err); return; } - if (rows.length === 0) { - // No need to check for tags - resolve([]); - return; - } - const postIdsParams = rows.map(() => '?').join(', '); - db.all( - `SELECT post_id, tags.url, tags.tag - FROM poststags - JOIN tags ON poststags.tag_url = tags.url - WHERE post_id IN (${postIdsParams});`, - rows.map((r: PostResult) => r.url), - (tagErr, tagRows: PostTagResult[]) => { - if (tagErr != null) { - console.error('Error loading post tags', tagErr); - reject(tagErr); - return; - } - const tagMap: Map = tagRows.reduce((result: Map, item) => { - const tag: Tag = { - url: item.url, - name: item.tag - }; - result.set(item.post_id, [...(result.get(item.post_id) || []), tag]); - return result; - }, new Map()); - const posts = rows.map((row) => { - return { - id: row.id, - content: row.content, - created_at: row.created_at, - url: row.url, - tags: tagMap.get(row.id) || [], - account: { - id: row.account_id, - acct: row.acct, - username: row.username, - display_name: row.display_name, - url: row.account_url, - avatar: row.avatar - } as Account - } as Post; - }); - resolve(posts); - } - ); + resolve(rows); }); }); - return promise; +} + +function getTagData(postIdsParams: String, postIds: string[]): Promise> { + return new Promise((resolve, reject) => { + db.all( + `SELECT post_id, tags.url, tags.tag + FROM poststags + JOIN tags ON poststags.tag_url = tags.url + WHERE post_id IN (${postIdsParams});`, + postIds, + (tagErr, tagRows: PostTagRow[]) => { + if (tagErr != null) { + log.error('Error loading post tags', tagErr); + reject(tagErr); + return; + } + const tagMap: Map = tagRows.reduce((result: Map, item) => { + const tag: Tag = { + url: item.url, + name: item.tag + }; + result.set(item.post_id, [...(result.get(item.post_id) || []), tag]); + return result; + }, new Map()); + resolve(tagMap); + } + ); + }); +} + +function getSongData(postIdsParams: String, postIds: string[]): Promise> { + return new Promise((resolve, reject) => { + db.all( + `SELECT post_url, songs.postedUrl, songs.overviewUrl, songs.type, songs.youtubeUrl, + songs.title, songs.artistName, songs.thumbnailUrl, songs.post_url + FROM songs + WHERE post_url IN (${postIdsParams});`, + postIds, + (tagErr, tagRows: SongRow[]) => { + if (tagErr != null) { + log.error('Error loading post tags', tagErr); + reject(tagErr); + return; + } + const songMap: Map = tagRows.reduce( + (result: Map, item) => { + result.set(item.post_url, [...(result.get(item.post_url) || []), item]); + return result; + }, + new Map() + ); + resolve(songMap); + } + ); + }); +} + +export async function getPosts( + since: string | null, + before: string | null, + limit: number +): Promise { + if (!databaseReady) { + await waitReady(); + } + + let filterQuery = ''; + const params: FilterParameter = { $limit: limit }; + if (since === null && before === null) { + filterQuery = ''; + } else if (since !== null) { + filterQuery = 'WHERE posts.created_at > $since'; + params.$since = since; + } else if (before !== null) { + // Setting both, before and since doesn't make sense, so this case is not explicitly handled + filterQuery = 'WHERE posts.created_at < $before'; + params.$before = before; + } + + ignoredUsers.forEach((ignoredUser, index) => { + const userParam = `$user_${index}`; + const acctParam = userParam + 'a'; + const usernameParam = userParam + 'u'; + const prefix = filterQuery === '' ? ' WHERE' : ' AND'; + filterQuery += `${prefix} acct != ${acctParam} AND username != ${usernameParam} `; + params[acctParam] = ignoredUser; + params[usernameParam] = ignoredUser; + }); + + const rows = await getPostData(filterQuery, params); + if (rows.length === 0) { + // No need to check for tags and songs + return []; + } + + const postIdsParams = rows.map(() => '?').join(', '); + const postIds = rows.map((r: PostRow) => r.url); + const tagMap = await getTagData(postIdsParams, postIds); + const songMap = await getSongData(postIdsParams, postIds); + const posts = rows.map((row) => { + return { + id: row.id, + content: row.content, + created_at: row.created_at, + url: row.url, + tags: tagMap.get(row.url) || [], + account: { + id: row.account_id, + acct: row.acct, + username: row.username, + display_name: row.display_name, + url: row.account_url, + avatar: row.avatar + } as Account, + songs: songMap.get(row.url) || [] + } as Post; + }); + return posts; } diff --git a/src/lib/server/rss.ts b/src/lib/server/rss.ts index c712b15..31c440c 100644 --- a/src/lib/server/rss.ts +++ b/src/lib/server/rss.ts @@ -1,6 +1,7 @@ import { BASE_URL, WEBSUB_HUB } from '$env/static/private'; import { PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME } from '$env/static/public'; import type { Post } from '$lib//mastodon/response'; +import { log } from '$lib/log'; import { Feed } from 'feed'; import fs from 'fs/promises'; @@ -59,6 +60,6 @@ export async function saveAtomFeed(feed: Feed) { body: params }); } catch (e) { - console.error('Failed to update WebSub hub', e); + log.error('Failed to update WebSub hub', e); } } diff --git a/src/lib/server/timeline.ts b/src/lib/server/timeline.ts index 7489e7c..daa3a0d 100644 --- a/src/lib/server/timeline.ts +++ b/src/lib/server/timeline.ts @@ -6,6 +6,7 @@ import { YOUTUBE_API_KEY, YOUTUBE_DISABLE } from '$env/static/private'; +import { log } from '$lib/log'; import type { Post, Tag, TimelineEvent } from '$lib/mastodon/response'; import type { OdesliResponse, Platform, SongInfo } from '$lib/odesliResponse'; import { getPosts, savePost } from '$lib/server/db'; @@ -38,7 +39,7 @@ export class TimelineReader { const resp = await fetch(youtubeVideoUrl); const respObj = await resp.json(); if (!respObj.items.length) { - console.warn('Could not find video with id', videoId); + log.warn('Could not find video with id', videoId); return false; } @@ -77,7 +78,7 @@ export class TimelineReader { return match[0]; } } catch (e) { - console.error('Could not check if', videoId, 'is a music video', e); + log.error('Could not check if', videoId, 'is a music video', e); } } return null; @@ -85,14 +86,14 @@ export class TimelineReader { private static async getSongInfo(url: string, remainingTries = 6): Promise { if (remainingTries === 0) { - console.error('No tries remaining. Lookup failed!'); + log.error('No tries remaining. Lookup failed!'); return null; } let hostname: string; try { hostname = new URL(url).hostname; } catch (e) { - console.error(`Could not construct URL ${url}`, e); + log.error(`Could not construct URL ${url}`, e); return null; } if (hostname === 'songwhip.com') { @@ -121,17 +122,18 @@ export class TimelineReader { return { ...info, pageUrl: odesliInfo.pageUrl, - youtubeUrl: odesliInfo.linksByPlatform[platform]?.url + youtubeUrl: odesliInfo.linksByPlatform[platform]?.url, + postedUrl: url } as SongInfo; }); }); } catch (e) { if (e instanceof Error && e.cause === 429) { - console.warn('song.link rate limit reached. Trying again in 10 seconds'); + log.warn('song.link rate limit reached. Trying again in 10 seconds'); await sleep(10_000); return await this.getSongInfo(url, remainingTries - 1); } - console.error(`Failed to load ${url} info from song.link`, e); + log.error(`Failed to load ${url} info from song.link`, e); return null; } } @@ -149,7 +151,7 @@ export class TimelineReader { ).json(); return status.card?.url; } catch (e) { - console.error(`Could not fetch status ${post.url}`, e); + log.error(`Could not fetch status ${post.url}`, e); } */ } @@ -157,7 +159,7 @@ export class TimelineReader { private startWebsocket() { const socket = new WebSocket(`wss://${MASTODON_INSTANCE}/api/v1/streaming`); socket.onopen = () => { - console.log('Connected to WS'); + log.log('Connected to WS'); socket.send('{ "type": "subscribe", "stream": "public:local"}'); }; socket.onmessage = async (event) => { @@ -172,17 +174,15 @@ export class TimelineReader { const urls: string[] = URL_FILTER.split(','); const found_urls = urls.filter((t) => post.content.includes(t)); - const urlsToCheck: string[] = []; // If we don't have any tags or non-youtube urls, check youtube // YT is handled separately, because it requires an API call and therefore is slower if (found_urls.length === 0 && found_tags.length === 0) { const youtubeUrl = await TimelineReader.checkYoutubeMatches(post.content); if (youtubeUrl === null) { - console.log('Ignoring post', post.url); + log.log('Ignoring post', post.url); return; } - urlsToCheck.push(youtubeUrl); - console.log('Found YT URL', youtubeUrl, found_urls, found_urls.length); + log.debug('Found YT URL', youtubeUrl, found_urls, found_urls.length); } // TODO: Change URL detection above to use this regex. @@ -207,51 +207,46 @@ export class TimelineReader { } } + const songs: SongInfo[] = []; + log.debug(`Checking ${musicUrls.length} URLs if they contain song data`); for (const url of musicUrls) { let hostname: string | null = null; try { hostname = new URL(url).hostname; } catch (e) { - console.error(`Could not check hostname for URL ${url}`, e); + log.error(`Could not check hostname for URL ${url}`, e); } if (hostname === 'songwhip.com') { // TODO: Implement checking the songwhip API continue; } const info = await TimelineReader.getSongInfo(url); + log.debug(`Found song info for ${url}?`, info); if (info) { - console.info( - 'Got song info for', - post.url, - url, - info.artistName, - info.title, - info.thumbnailUrl, - info.pageUrl, - info.youtubeUrl - ); + songs.push(info); } } - await savePost(post); + await savePost(post, songs); + log.debug('Saved post', post.url); const posts = await getPosts(null, null, 100); await saveAtomFeed(createFeed(posts)); } catch (e) { - console.error('error message', event, event.data, e); + log.error('error message', event, event.data, e); } }; socket.onclose = (event) => { - console.warn( + log.warn( `Websocket connection to ${MASTODON_INSTANCE} closed. Code: ${event.code}, reason: '${event.reason}'` ); setTimeout(() => { - console.info(`Attempting to reconenct to WS`); + log.info(`Attempting to reconenct to WS`); this.startWebsocket(); }, 10000); }; socket.onerror = (event) => { - console.error( + log.error( `Websocket connection to ${MASTODON_INSTANCE} failed. ${event.type}: ${event.error}, message: '${event.message}'` ); };