diff --git a/package.json b/package.json index 323e9d8..27374e1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "moshing-mammut", - "version": "1.3.0", + "version": "1.3.1", "private": true, "license": "LGPL-3.0-or-later", "scripts": { diff --git a/src/lib/server/db.ts b/src/lib/server/db.ts index e7935fb..986f53f 100644 --- a/src/lib/server/db.ts +++ b/src/lib/server/db.ts @@ -2,6 +2,7 @@ import { IGNORE_USERS, MASTODON_INSTANCE } from '$env/static/private'; import { enableVerboseLog, log } from '$lib/log'; import type { Account, Post, Tag } from '$lib/mastodon/response'; import type { SongInfo } from '$lib/odesliResponse'; +import { TimelineReader } from '$lib/server/timeline'; import sqlite3 from 'sqlite3'; const { DEV } = import.meta.env; @@ -79,6 +80,45 @@ if (enableVerboseLog) { }); } +async function applyDbMigration(migration: Migration): Promise { + return new Promise(async (resolve, reject) => { + db.exec(migration.statement, (err) => { + if (err !== null) { + log.error(`Failed to apply migration ${migration.name}`, err); + reject(err); + return; + } + resolve(); + }); + }); +} + +async function applyMigration(migration: Migration) { + if (migration.id === 4) { + // When this is run, no posts will have added song data, + // so filtering won't help + const posts = await getPostsInternal(null, null, 10000); + let current = 0; + let total = posts.length.toString().padStart(4, '0'); + for (const post of posts) { + current++; + if (post.songs && post.songs.length) { + continue; + } + log.debug( + `Fetching songs for existing post ${current.toString().padStart(4, '0')} of ${total}`, + post.url + ); + const songs = await TimelineReader.getSongInfoInPost(post); + await saveSongInfoData(post.url, songs); + log.debug(`Fetched ${songs.length} songs for existing post`, post.url); + } + log.debug(`Finished fetching songs`); + } else { + await applyDbMigration(migration); + } +} + db.on('open', () => { log.info('Opened database'); db.serialize(); @@ -98,7 +138,7 @@ db.on('open', () => { return; } for (const migration of toApply) { - db.exec(migration.statement, (err) => { + applyMigration(migration).then(() => { remaining--; // This will set databaseReady to true before the migration has been inserted as applies, // but that doesn't matter. It's only important that is has been applied @@ -225,11 +265,16 @@ function getMigrations(): Migration[] { post_url TEXT, FOREIGN KEY (post_url) REFERENCES posts(url) );` + }, + { + id: 4, + name: 'song info for existing posts', + statement: `` } ]; } -async function waitReady(): Promise { +async function waitReady(): Promise { // Simpler than a semaphore and is really only needed on startup return new Promise((resolve) => { const interval = setInterval(() => { @@ -241,14 +286,14 @@ async function waitReady(): Promise { log.debug('DB is ready'); } clearInterval(interval); - resolve(undefined); + resolve(); } }, 100); }); } -function saveAccountData(account: Account): Promise { - return new Promise((resolve, reject) => { +function saveAccountData(account: Account): Promise { + return new Promise((resolve, reject) => { db.run( ` INSERT INTO accounts (id, acct, username, display_name, url, avatar) @@ -274,14 +319,14 @@ function saveAccountData(account: Account): Promise { reject(err); return; } - resolve(undefined); + resolve(); } ); }); } -function savePostData(post: Post): Promise { - return new Promise((resolve, reject) => { +function savePostData(post: Post): Promise { + return new Promise((resolve, reject) => { db.run( ` INSERT INTO posts (id, content, created_at, url, account_id) @@ -297,16 +342,16 @@ function savePostData(post: Post): Promise { reject(postErr); return; } - resolve(undefined); + resolve(); } ); }); } -function savePostTagData(post: Post): Promise { - return new Promise((resolve, reject) => { +function savePostTagData(post: Post): Promise { + return new Promise((resolve, reject) => { if (!post.tags.length) { - resolve(undefined); + resolve(); return; } @@ -338,7 +383,7 @@ function savePostTagData(post: Post): Promise { remaining--; // Only resolve after all have been inserted if (remaining === 0) { - resolve(undefined); + resolve(); } } ); @@ -349,10 +394,10 @@ function savePostTagData(post: Post): Promise { }); } -function saveSongInfoData(postUrl: string, songs: SongInfo[]): Promise { - return new Promise((resolve, reject) => { +function saveSongInfoData(postUrl: string, songs: SongInfo[]): Promise { + return new Promise((resolve, reject) => { if (songs.length === 0) { - resolve(undefined); + resolve(); return; } db.parallelize(() => { @@ -383,7 +428,7 @@ function saveSongInfoData(postUrl: string, songs: SongInfo[]): Promise { let filterQuery = ''; const params: FilterParameter = { $limit: limit }; if (since === null && before === null) { diff --git a/src/lib/server/timeline.ts b/src/lib/server/timeline.ts index 69d82c4..e648a64 100644 --- a/src/lib/server/timeline.ts +++ b/src/lib/server/timeline.ts @@ -12,6 +12,34 @@ const URL_REGEX = new RegExp(/href="(?[^>]+?)" target="_blank"/gm); export class TimelineReader { private static _instance: TimelineReader; + public static async getSongInfoInPost(post: Post): Promise { + const urlMatches = post.content.matchAll(URL_REGEX); + const songs: SongInfo[] = []; + for (const match of urlMatches) { + if (match === undefined || match.groups === undefined) { + log.warn('Match listed in allMatches, but either it or its groups are undefined', match); + continue; + } + const urlMatch = match.groups.postUrl.toString(); + let url: URL; + try { + url = new URL(urlMatch); + } catch (e) { + log.error('URL found via Regex does not seem to be a valud url', urlMatch, e); + continue; + } + + // Check *all* found url and let odesli determine if it is music or not + log.debug(`Checking ${url} if it contains song data`); + const info = await TimelineReader.getSongInfo(url); + log.debug(`Found song info for ${url}?`, info); + if (info) { + songs.push(info); + } + } + return songs; + } + private static async getSongInfo(url: URL, remainingTries = 6): Promise { if (remainingTries === 0) { log.error('No tries remaining. Lookup failed!'); @@ -77,33 +105,7 @@ export class TimelineReader { const hashttags: string[] = HASHTAG_FILTER.split(','); const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name)); - const urlMatches = post.content.matchAll(URL_REGEX); - const songs: SongInfo[] = []; - for (const match of urlMatches) { - if (match === undefined || match.groups === undefined) { - log.warn( - 'Match listed in allMatches, but either it or its groups are undefined', - match - ); - continue; - } - const urlMatch = match.groups.postUrl.toString(); - let url: URL; - try { - url = new URL(urlMatch); - } catch (e) { - log.error('URL found via Regex does not seem to be a valud url', urlMatch, e); - continue; - } - - // Check *all* found url and let odesli determine if it is music or not - log.debug(`Checking ${url} if it contains song data`); - const info = await TimelineReader.getSongInfo(url); - log.debug(`Found song info for ${url}?`, info); - if (info) { - songs.push(info); - } - } + const songs = await TimelineReader.getSongInfoInPost(post); // If we don't have any tags or non-youtube urls, check youtube // YT is handled separately, because it requires an API call and therefore is slower diff --git a/src/lib/sleep.ts b/src/lib/sleep.ts index 3c701f9..ee33e38 100644 --- a/src/lib/sleep.ts +++ b/src/lib/sleep.ts @@ -1,4 +1,4 @@ -export function sleep(timeInMs: number): Promise { +export function sleep(timeInMs: number): Promise { return new Promise((resolve) => { setTimeout(resolve, timeInMs); });