import { IGNORE_USERS, MASTODON_INSTANCE } from '$env/static/private'; import { enableVerboseLog, log } from '$lib/log'; import type { Account, AccountAvatar, Post, SongThumbnailImage, Tag } from '$lib/mastodon/response'; import type { SongInfo } from '$lib/odesliResponse'; import { TimelineReader } from '$lib/server/timeline'; import sqlite3 from 'sqlite3'; type FilterParameter = { $limit?: number | undefined | null; $since?: string | undefined | null; $before?: string | undefined | null; [x: string]: string | number | undefined | null; }; type PostRow = { id: string; content: string; created_at: string; url: string; account_id: string; acct: string; username: string; display_name: string; account_url: string; avatar: string; }; type PostTagRow = { post_id: string; tag: string; url: string; }; type SongRow = { post_url: string; postedUrl: string; overviewUrl?: string; type: 'album' | 'song'; youtubeUrl?: string; title?: string; artistName?: string; thumbnailUrl?: string; thumbnailWidth?: number; thumbnailHeight?: number; }; type AccountAvatarRow = { account_url: string; file: string; sizeDescriptor: string; }; type SongThumbnailAvatarRow = { song_thumbnailUrl: string; file: string; sizeDescriptor: string; kind: number; }; type Migration = { id: number; name: string; statement: string; }; const db: sqlite3.Database = new sqlite3.Database('moshingmammut.db'); // for the local masto instance, the instance name is *not* saved // as part of the username or acct, so it needs to be stripped const ignoredUsers: string[] = IGNORE_USERS === undefined ? [] : IGNORE_USERS.split(',') .map((u) => (u.startsWith('@') ? u.substring(1) : u)) .map((u) => u.endsWith('@' + MASTODON_INSTANCE) ? u.substring(0, u.length - ('@' + MASTODON_INSTANCE).length) : u ); let databaseReady = false; if (enableVerboseLog) { sqlite3.verbose(); db.on('change', (t, d, table, rowid) => { log.verbose('DB change event', t, d, table, rowid); }); db.on('trace', (sql) => { log.verbose('Running', sql); }); db.on('profile', (sql) => { log.verbose('Finished', sql); }); } function applyDbMigration(migration: Migration): Promise { return new Promise((resolve, reject) => { db.exec(migration.statement, (err) => { if (err !== null) { log.error(`Failed to apply migration ${migration.name}`, err); reject(err); return; } resolve(); }); }); } async function applyMigration(migration: Migration) { if (migration.id === 4) { // When this is run, no posts will have added song data, // so filtering won't help const posts = await getPostsInternal(null, null, 10000); let current = 0; const total = posts.length.toString().padStart(4, '0'); for (const post of posts) { current++; if (post.songs && post.songs.length) { continue; } log.info( `Fetching songs for existing post ${current.toString().padStart(4, '0')} of ${total}`, post.url ); const songs = await TimelineReader.getSongInfoInPost(post); await saveSongInfoData(post.url, songs); log.debug(`Fetched ${songs.length} songs for existing post`, post.url); } log.debug(`Finished fetching songs`); } else { await applyDbMigration(migration); } } db.on('open', () => { log.info('Opened database'); db.serialize(); db.run('CREATE TABLE IF NOT EXISTS "migrations" ("id" integer,"name" TEXT, PRIMARY KEY (id))'); db.all('SELECT id FROM migrations', (err, rows: Migration[]) => { if (err !== null) { log.error('Could not fetch existing migrations', err); databaseReady = true; return; } log.debug('Already applied migrations', rows); const appliedMigrations: Set = new Set(rows.map((row) => row['id'])); const toApply = getMigrations().filter((m) => !appliedMigrations.has(m.id)); let remaining = toApply.length; if (remaining === 0) { databaseReady = true; return; } for (const migration of toApply) { applyMigration(migration).then(() => { remaining--; // This will set databaseReady to true before the migration has been inserted as applies, // but that doesn't matter. It's only important that is has been applied if (remaining === 0) { databaseReady = true; } if (err !== null) { log.error(`Failed to apply migration ${migration.name}`, err); return; } db.run( 'INSERT INTO migrations (id, name) VALUES(?, ?)', [migration.id, migration.name], (e: Error) => { if (e !== null) { log.error(`Failed to mark migration ${migration.name} as applied`, e); return; } log.info(`Applied migration ${migration.name}`); } ); }); } }); }); db.on('error', (err) => { log.error('Error opening database', err); }); function getMigrations(): Migration[] { return [ { id: 1, name: 'initial', statement: ` CREATE TABLE accounts ( id TEXT NOT NULL PRIMARY KEY, acct TEXT, username TEXT, display_name TEXT, url TEXT, avatar TEXT, avatar_static TEXT ); CREATE TABLE tags (url TEXT NOT NULL PRIMARY KEY, tag TEXT NOT NULL); CREATE TABLE posts ( id TEXT NOT NULL PRIMARY KEY, content TEXT, created_at TEXT, url TEXT NOT NULL, account_id TEXT NOT NULL, FOREIGN KEY (account_id) REFERENCES accounts(id) ); CREATE TABLE poststags ( id integer PRIMARY KEY, post_id TEXT NOT NULL, tag_url TEXT NOT NULL, FOREIGN KEY (post_id) REFERENCES posts(id), FOREIGN KEY (tag_url) REFERENCES tags(url) )` }, { id: 2, name: 'urls as keys', statement: ` CREATE TABLE accounts_new ( id TEXT NOT NULL, acct TEXT, username TEXT, display_name TEXT, url TEXT NOT NULL PRIMARY KEY, avatar TEXT ); INSERT INTO accounts_new (id, acct, username, display_name, url, avatar) SELECT id, acct, username, display_name, url, avatar FROM accounts; DROP TABLE accounts; ALTER TABLE accounts_new RENAME TO accounts; CREATE TABLE posts_new ( id TEXT NOT NULL, content TEXT, created_at TEXT, url TEXT NOT NULL PRIMARY KEY, account_id TEXT NOT NULL, FOREIGN KEY (account_id) REFERENCES accounts(url) ); INSERT INTO posts_new (id, content, created_at, url, account_id) SELECT p.id, p.content, p.created_at, p.url, accounts.url FROM posts as p JOIN accounts ON accounts.id = p.account_id; DROP TABLE posts; ALTER TABLE posts_new RENAME TO posts; CREATE TABLE poststags_new ( id integer PRIMARY KEY, post_id TEXT NOT NULL, tag_url TEXT NOT NULL, FOREIGN KEY (post_id) REFERENCES posts(url), FOREIGN KEY (tag_url) REFERENCES tags(url) ); INSERT INTO poststags_new (id, post_id, tag_url) SELECT pt.id, posts.url, pt.tag_url FROM poststags as pt JOIN posts ON posts.id = pt.post_id; DROP TABLE poststags; ALTER TABLE poststags_new RENAME TO poststags; ` }, { id: 3, name: 'song info for posts', statement: ` CREATE TABLE songs ( id integer PRIMARY KEY, postedUrl TEXT NOT NULL, overviewUrl TEXT, type TEXT CHECK ( type in ('album', 'song') ), youtubeUrl TEXT, title TEXT, artistName TEXT, thumbnailUrl TEXT, post_url TEXT, FOREIGN KEY (post_url) REFERENCES posts(url) );` }, { id: 4, name: 'song info for existing posts', statement: `` }, { id: 5, name: 'resized avatars', statement: ` CREATE TABLE accountsavatars ( file TEXT NOT NULL PRIMARY KEY, account_url TEXT NOT NULL, sizeDescriptor TEXT NOT NULL, FOREIGN KEY (account_url) REFERENCES accounts(url) );` }, { id: 6, name: 'resized song thumbnails', statement: ` CREATE TABLE songsthumbnails ( file TEXT NOT NULL PRIMARY KEY, song_thumbnailUrl TEXT NOT NULL, sizeDescriptor TEXT NOT NULL, kind INTEGER NOT NULL, FOREIGN KEY (song_thumbnailUrl) REFERENCES songs(thumbnailUrl) );` }, { id: 7, name: 'song thumbnail size', statement: ` ALTER TABLE songs ADD COLUMN thumbnailWidth INTEGER NULL; ALTER TABLE songs ADD COLUMN thumbnailHeight INTEGER NULL;` } ]; } async function waitReady(): Promise { // Simpler than a semaphore and is really only needed on startup return new Promise((resolve) => { const interval = setInterval(() => { log.verbose('Waiting for database to be ready'); if (databaseReady) { log.verbose('DB is ready'); clearInterval(interval); resolve(); } }, 100); }); } function saveAccountData(account: Account): Promise { return new Promise((resolve, reject) => { db.run( ` INSERT INTO accounts (id, acct, username, display_name, url, avatar) VALUES(?, ?, ?, ?, ?, ?) ON CONFLICT(url) DO UPDATE SET acct=excluded.acct, username=excluded.username, display_name=excluded.display_name, id=excluded.id, avatar=excluded.avatar;`, [ account.id, account.acct, account.username, account.display_name, account.url, account.avatar ], (err) => { if (err !== null) { log.error(`Could not insert/update account ${account.id}`, err); reject(err); return; } resolve(); } ); }); } function savePostData(post: Post): Promise { return new Promise((resolve, reject) => { db.run( ` INSERT INTO posts (id, content, created_at, url, account_id) VALUES (?, ?, ?, ?, ?) ON CONFLICT(url) DO UPDATE SET content=excluded.content, created_at=excluded.created_at, id=excluded.id, account_id=excluded.account_id;`, [post.id, post.content, post.created_at, post.url, post.account.url], (postErr) => { if (postErr !== null) { log.error(`Could not insert post ${post.url}`, postErr); reject(postErr); return; } resolve(); } ); }); } function savePostTagData(post: Post): Promise { return new Promise((resolve, reject) => { if (!post.tags.length) { resolve(); return; } db.parallelize(() => { let remaining = post.tags.length; for (const tag of post.tags) { db.run( ` INSERT INTO tags (url, tag) VALUES (?, ?) ON CONFLICT(url) DO UPDATE SET tag=excluded.tag;`, [tag.url, tag.name], (tagErr) => { if (tagErr !== null) { log.error(`Could not insert/update tag ${tag.url}`, tagErr); reject(tagErr); return; } db.run( 'INSERT INTO poststags (post_id, tag_url) VALUES (?, ?)', [post.url, tag.url], (posttagserr) => { if (posttagserr !== null) { log.error(`Could not insert poststags ${tag.url}, ${post.url}`, posttagserr); reject(posttagserr); return; } // Don't decrease on fail remaining--; // Only resolve after all have been inserted if (remaining === 0) { resolve(); } } ); } ); } }); }); } function saveSongInfoData(postUrl: string, songs: SongInfo[]): Promise { return new Promise((resolve, reject) => { if (songs.length === 0) { resolve(); return; } db.parallelize(() => { let remaining = songs.length; for (const song of songs) { db.run( ` INSERT INTO songs (postedUrl, overviewUrl, type, youtubeUrl, title, artistName, thumbnailUrl, post_url, thumbnailWidth, thumbnailHeight) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, [ song.postedUrl, song.pageUrl, song.type, song.youtubeUrl, song.title, song.artistName, song.thumbnailUrl, postUrl, song.thumbnailWidth, song.thumbnailHeight ], (songErr) => { if (songErr !== null) { log.error(`Could not insert song ${song.postedUrl}`, songErr); reject(songErr); return; } // Don't decrease on fail remaining--; // Only resolve after all have been inserted if (remaining === 0) { resolve(); } } ); } }); }); } export async function savePost(post: Post, songs: SongInfo[]) { log.debug(`Saving post ${post.url}`); if (!databaseReady) { await waitReady(); } const account = post.account; await saveAccountData(account); log.debug(`Saved account data ${post.url}`); await savePostData(post); log.debug(`Saved post data ${post.url}`); await savePostTagData(post); log.debug(`Saved ${post.tags.length} tag data ${post.url}`); await saveSongInfoData(post.url, songs); log.debug( `Saved ${songs.length} song info data ${post.url}`, songs.map((s) => s.thumbnailHeight) ); } function getPostData(filterQuery: string, params: FilterParameter): Promise { const sql = `SELECT posts.id, posts.content, posts.created_at, posts.url, accounts.id AS account_id, accounts.acct, accounts.username, accounts.display_name, accounts.url AS account_url, accounts.avatar FROM posts JOIN accounts ON posts.account_id = accounts.url ${filterQuery} ORDER BY created_at DESC LIMIT $limit`; return new Promise((resolve, reject) => { db.all(sql, params, (err, rows: PostRow[]) => { if (err != null) { log.error('Error loading posts', err); reject(err); return; } resolve(rows); }); }); } function getTagData(postIdsParams: string, postIds: string[]): Promise> { return new Promise((resolve, reject) => { db.all( `SELECT post_id, tags.url, tags.tag FROM poststags JOIN tags ON poststags.tag_url = tags.url WHERE post_id IN (${postIdsParams});`, postIds, (tagErr, tagRows: PostTagRow[]) => { if (tagErr != null) { log.error('Error loading post tags', tagErr); reject(tagErr); return; } const tagMap: Map = tagRows.reduce((result: Map, item) => { const tag: Tag = { url: item.url, name: item.tag }; result.set(item.post_id, [...(result.get(item.post_id) || []), tag]); return result; }, new Map()); resolve(tagMap); } ); }); } function getSongData(postIdsParams: string, postIds: string[]): Promise> { return new Promise((resolve, reject) => { db.all( `SELECT post_url, songs.postedUrl, songs.overviewUrl, songs.type, songs.youtubeUrl, songs.title, songs.artistName, songs.thumbnailUrl, songs.post_url, songs.thumbnailWidth, songs.thumbnailHeight FROM songs WHERE post_url IN (${postIdsParams});`, postIds, (tagErr, tagRows: SongRow[]) => { if (tagErr != null) { log.error('Error loading post songs', tagErr); reject(tagErr); return; } const songMap: Map = tagRows.reduce( (result: Map, item) => { const info = { pageUrl: item.overviewUrl, youtubeUrl: item.youtubeUrl, type: item.type, title: item.title, artistName: item.artistName, thumbnailUrl: item.thumbnailUrl, postedUrl: item.postedUrl, thumbnailHeight: item.thumbnailHeight, thumbnailWidth: item.thumbnailWidth } as SongInfo; result.set(item.post_url, [...(result.get(item.post_url) || []), info]); return result; }, new Map() ); log.verbose('songMap', songMap); resolve(songMap); } ); }); } function getAvatarData( accountUrlsParams: string, accountUrls: string[] ): Promise> { return new Promise((resolve, reject) => { db.all( `SELECT account_url, file, sizeDescriptor FROM accountsavatars WHERE account_url IN (${accountUrlsParams});`, accountUrls, (err, rows: AccountAvatarRow[]) => { if (err != null) { log.error('Error loading avatars', err); reject(err); return; } const avatarMap: Map = rows.reduce( (result: Map, item) => { const info: AccountAvatar = { accountUrl: item.account_url, file: item.file, sizeDescriptor: item.sizeDescriptor }; result.set(item.account_url, [...(result.get(item.account_url) || []), info]); return result; }, new Map() ); resolve(avatarMap); } ); }); } function getSongThumbnailData( thumbUrlsParams: string, thumbUrls: string[] ): Promise> { return new Promise((resolve, reject) => { db.all( `SELECT song_thumbnailUrl, file, sizeDescriptor, kind FROM songsthumbnails WHERE song_thumbnailUrl IN (${thumbUrlsParams});`, thumbUrls, (err, rows: SongThumbnailAvatarRow[]) => { if (err != null) { log.error('Error loading avatars', err); reject(err); return; } const thumbnailMap: Map = rows.reduce( (result: Map, item) => { const info: SongThumbnailImage = { songThumbnailUrl: item.song_thumbnailUrl, file: item.file, sizeDescriptor: item.sizeDescriptor, kind: item.kind }; result.set(item.song_thumbnailUrl, [ ...(result.get(item.song_thumbnailUrl) || []), info ]); return result; }, new Map() ); resolve(thumbnailMap); } ); }); } export async function getPosts( since: string | null, before: string | null, limit: number ): Promise { if (!databaseReady) { await waitReady(); } return await getPostsInternal(since, before, limit); } async function getPostsInternal( since: string | null, before: string | null, limit: number ): Promise { let filterQuery = ''; const params: FilterParameter = { $limit: limit }; if (since === null && before === null) { filterQuery = ''; } else if (since !== null) { filterQuery = 'WHERE posts.created_at > $since'; params.$since = since; } else if (before !== null) { // Setting both, before and since doesn't make sense, so this case is not explicitly handled filterQuery = 'WHERE posts.created_at < $before'; params.$before = before; } ignoredUsers.forEach((ignoredUser, index) => { const userParam = `$user_${index}`; const acctParam = userParam + 'a'; const usernameParam = userParam + 'u'; const prefix = filterQuery === '' ? ' WHERE' : ' AND'; filterQuery += `${prefix} acct != ${acctParam} AND username != ${usernameParam} `; params[acctParam] = ignoredUser; params[usernameParam] = ignoredUser; }); const rows = await getPostData(filterQuery, params); if (rows.length === 0) { // No need to check for tags and songs return []; } const postIdsParams = rows.map(() => '?').join(', '); const postIds = rows.map((r: PostRow) => r.url); const tagMap = await getTagData(postIdsParams, postIds); const songMap = await getSongData(postIdsParams, postIds); for (const entry of songMap) { for (const songInfo of entry[1]) { const thumbs = await getSongThumbnails(songInfo); songInfo.resizedThumbnails = thumbs; } } const accountUrls = [...new Set(rows.map((r: PostRow) => r.account_url))]; const accountUrlsParams = accountUrls.map(() => '?').join(', '); const avatars = await getAvatarData(accountUrlsParams, accountUrls); const posts = rows.map((row) => { return { id: row.id, content: row.content, created_at: row.created_at, url: row.url, tags: tagMap.get(row.url) || [], account: { id: row.account_id, acct: row.acct, username: row.username, display_name: row.display_name, url: row.account_url, avatar: row.avatar, resizedAvatars: avatars.get(row.account_url) || [] } as Account, songs: songMap.get(row.url) || [] } as Post; }); return posts; } export async function removeAvatars(accountUrl: string): Promise { const params: FilterParameter = { $account: accountUrl }; const sql = ` DELETE FROM accountsavatars WHERE account_url = $account`; await waitReady(); return new Promise((resolve, reject) => { db.run(sql, params, (err) => { if (err) { reject(err); return; } resolve(); }); }); } export async function saveSongThumbnail(thumb: SongThumbnailImage): Promise { // Will be null if file already existed if (thumb.file === null) { return; } const params: FilterParameter = { $songId: thumb.songThumbnailUrl, $file: thumb.file, $sizeDescriptor: thumb.sizeDescriptor, $kind: thumb.kind.valueOf() }; const sql = ` INSERT INTO songsthumbnails (song_thumbnailUrl, file, sizeDescriptor, kind) VALUES ($songId, $file, $sizeDescriptor, $kind) ON CONFLICT(file) DO UPDATE SET song_thumbnailUrl=excluded.song_thumbnailUrl, sizeDescriptor=excluded.sizeDescriptor, kind=excluded.kind;`; await waitReady(); return new Promise((resolve, reject) => { db.run(sql, params, (err) => { if (err) { reject(err); return; } resolve(); }); }); } export async function saveAvatar(avatar: AccountAvatar): Promise { // Will be null if file already existed if (avatar.file === null) { return; } const params: FilterParameter = { $accountUrl: avatar.accountUrl, $file: avatar.file, $sizeDescriptor: avatar.sizeDescriptor }; const sql = ` INSERT INTO accountsavatars (account_url, file, sizeDescriptor) VALUES ($accountUrl, $file, $sizeDescriptor) ON CONFLICT(file) DO UPDATE SET account_url=excluded.account_url, sizeDescriptor=excluded.sizeDescriptor;`; await waitReady(); return new Promise((resolve, reject) => { db.run(sql, params, (err) => { if (err) { reject(err); return; } resolve(); }); }); } export async function getAvatars( accountUrl: string, limit: number | undefined ): Promise { // TODO: Refactor to use `getAvatarData` await waitReady(); let limitFilter = ''; const params: FilterParameter = { $account: accountUrl, $limit: 100 }; if (limit !== undefined) { limitFilter = 'LIMIT $limit'; params.$limit = limit; } const sql = ` SELECT account_url, file, sizeDescriptor FROM accountsavatars WHERE account_url = $account ${limitFilter};`; return new Promise((resolve, reject) => { db.all(sql, params, (err, rows: AccountAvatarRow[]) => { if (err) { reject(err); return; } resolve( rows.map((r) => { return { accountUrl: r.account_url, file: r.file, sizeDescriptor: r.sizeDescriptor } as AccountAvatar; }) ); }); }); } export async function getSongThumbnails(song: SongInfo): Promise { if (!song.thumbnailUrl) { return []; } const rows = await getSongThumbnailData('?', [song.thumbnailUrl]); return rows.get(song.thumbnailUrl) ?? []; }