phlaym/moshing-mammut
1
0
Fork 0
Code Issues 5 Pull Requests Projects Releases 8 Activity

Saving song infos to DB, refactor logging

Browse Source
This commit is contained in:
Max Nuding 2023-04-23 12:46:14 +02:00
parent 1cd9d83910
commit 971c846dd1
Signed by: phlaym
GPG Key ID: A06651BAB6777237
8 changed files with 396 additions and 228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.nova/Configuration.json
View File

@ -1,8 +1,11 @@
{
"apexskier.eslint.config.eslintConfigPath" : ".eslintrc.cjs",
"apexskier.eslint.config.eslintPath" : "node_modules\/@eslint\/eslintrc\/dist\/eslintrc.cjs",
"apexskier.typescript.config.formatDocumentOnSave" : "true",
"apexskier.eslint.config.fixOnSave" : "Enable",
"apexskier.typescript.config.formatDocumentOnSave" : "false",
"apexskier.typescript.config.isEnabledForJavascript" : "Enable",
"apexskier.typescript.config.organizeImportsOnSave" : "true",
"apexskier.typescript.config.userPreferences.quotePreference" : "single",
"apexskier.typescript.config.userPreferences.useLabelDetailsInCompletionEntries" : true
"apexskier.typescript.config.userPreferences.useLabelDetailsInCompletionEntries" : true,
"prettier.format-on-save" : "Global Default"
}

3
src/hooks.server.ts
View File

@ -1,3 +1,4 @@
import { log } from '$lib/log';
import { TimelineReader } from '$lib/server/timeline';
import type { HandleServerError } from '@sveltejs/kit';
import fs from 'fs/promises';
@ -6,7 +7,7 @@ TimelineReader.init();
export const handleError = (({ error }) => {
if (error instanceof Error) {
console.error('Something went wrong: ', error.name, error.message);
log.error('Something went wrong: ', error.name, error.message);
}
return {

32
src/lib/log.ts Normal file
View File

@ -0,0 +1,32 @@
import { env } from '$env/dynamic/private';
import { isTruthy } from '$lib/truthyString';
const { DEV } = import.meta.env;
export const enableVerboseLog = isTruthy(env.VERBOSE);
export const log = {
verbose: (...params: any[]) => {
if (!enableVerboseLog) {
return;
}
console.debug(new Date().toISOString(), ...params);
},
debug: (...params: any[]) => {
if (!DEV) {
return;
}
console.debug(new Date().toISOString(), ...params);
},
log: (...params: any[]) => {
console.log(new Date().toISOString(), ...params);
},
info: (...params: any[]) => {
console.info(new Date().toISOString(), ...params);
},
warn: (...params: any[]) => {
console.warn(new Date().toISOString(), ...params);
},
error: (...params: any[]) => {
console.error(new Date().toISOString(), ...params);
}
};

3
src/lib/mastodon/response.ts
View File

@ -1,3 +1,5 @@
import type { SongInfo } from '$lib/odesliResponse';
export interface TimelineEvent {
event: string;
payload: string;
@ -11,6 +13,7 @@ export interface Post {
content: string;
account: Account;
card?: PreviewCard;
songs?: SongInfo[];
}
export interface PreviewCard {

1
src/lib/odesliResponse.ts
View File

@ -5,6 +5,7 @@ export type SongInfo = {
title?: string;
artistName?: string;
thumbnailUrl?: string;
postedUrl: string;
};
export type SongwhipReponse = {

346
src/lib/server/db.ts
View File

@ -1,10 +1,47 @@
import { env } from '$env/dynamic/private';
import { IGNORE_USERS, MASTODON_INSTANCE } from '$env/static/private';
import { enableVerboseLog, log } from '$lib/log';
import type { Account, Post, Tag } from '$lib/mastodon/response';
import { isTruthy } from '$lib/truthyString';
import type { SongInfo } from '$lib/odesliResponse';
import sqlite3 from 'sqlite3';
const { DEV } = import.meta.env;
type FilterParameter = {
$limit: number | undefined | null;
$since?: string | undefined | null;
$before?: string | undefined | null;
[x: string]: string | number | undefined | null;
};
type PostRow = {
id: string;
content: string;
created_at: string;
url: string;
account_id: string;
acct: string;
username: string;
display_name: string;
account_url: string;
avatar: string;
};
type PostTagRow = {
post_id: string;
tag: string;
url: string;
};
type SongRow = SongInfo & {
post_url: string;
};
type Migration = {
id: number;
name: string;
statement: string;
};
const db: sqlite3.Database = new sqlite3.Database('moshingmammut.db');
// for the local masto instance, the instance name is *not* saved
// as part of the username or acct, so it needs to be stripped
@ -20,38 +57,32 @@ const ignoredUsers: string[] =
);
let databaseReady = false;
if (DEV && isTruthy(env.VERBOSE)) {
if (enableVerboseLog) {
sqlite3.verbose();
db.on('change', (t, d, table, rowid) => {
console.debug('DB change event', t, d, table, rowid);
log.verbose('DB change event', t, d, table, rowid);
});
db.on('trace', (sql) => {
console.debug('Running', sql);
log.verbose('Running', sql);
});
db.on('profile', (sql) => {
console.debug('Finished', sql);
log.verbose('Finished', sql);
});
}
interface Migration {
id: number;
name: string;
statement: string;
}
db.on('open', () => {
console.log('Opened database');
log.info('Opened database');
db.serialize();
db.run('CREATE TABLE IF NOT EXISTS "migrations" ("id" integer,"name" TEXT, PRIMARY KEY (id))');
db.all('SELECT id FROM migrations', (err, rows: Migration[]) => {
if (err !== null) {
console.error('Could not fetch existing migrations', err);
log.error('Could not fetch existing migrations', err);
databaseReady = true;
return;
}
console.debug('Already applied migrations', rows);
log.debug('Already applied migrations', rows);
const appliedMigrations: Set<number> = new Set(rows.map((row) => row['id']));
const toApply = getMigrations().filter((m) => !appliedMigrations.has(m.id));
let remaining = toApply.length;
@ -68,7 +99,7 @@ db.on('open', () => {
databaseReady = true;
}
if (err !== null) {
console.error(`Failed to apply migration ${migration.name}`, err);
log.error(`Failed to apply migration ${migration.name}`, err);
return;
}
db.run(
@ -76,10 +107,10 @@ db.on('open', () => {
[migration.id, migration.name],
(e: Error) => {
if (e !== null) {
console.error(`Failed to mark migration ${migration.name} as applied`, e);
log.error(`Failed to mark migration ${migration.name} as applied`, e);
return;
}
console.info(`Applied migration ${migration.name}`);
log.info(`Applied migration ${migration.name}`);
}
);
});
@ -87,7 +118,7 @@ db.on('open', () => {
});
});
db.on('error', (err) => {
console.error('Error opening database', err);
log.error('Error opening database', err);
});
function getMigrations(): Migration[] {
@ -170,6 +201,23 @@ function getMigrations(): Migration[] {
DROP TABLE poststags;
ALTER TABLE poststags_new RENAME TO poststags;
`
},
{
id: 3,
name: 'song info for posts',
statement: `
CREATE TABLE songs (
id integer PRIMARY KEY,
postedUrl TEXT NOT NULL,
overviewUrl TEXT,
type TEXT CHECK ( type in ('album', 'song') ),
youtubeUrl TEXT,
title TEXT,
artistName TEXT,
thumbnailUrl TEXT,
post_url TEXT,
FOREIGN KEY (post_url) REFERENCES posts(url)
);`
}
];
}
@ -179,11 +227,11 @@ async function waitReady(): Promise<undefined> {
return new Promise((resolve) => {
const interval = setInterval(() => {
if (DEV) {
console.debug('Waiting for database to be ready');
log.debug('Waiting for database to be ready');
}
if (databaseReady) {
if (DEV) {
console.debug('DB is ready');
log.debug('DB is ready');
}
clearInterval(interval);
resolve(undefined);
@ -192,13 +240,8 @@ async function waitReady(): Promise<undefined> {
});
}
export async function savePost(post: Post): Promise<undefined> {
if (!databaseReady) {
await waitReady();
}
return await new Promise<undefined>((resolve, reject) => {
console.debug(`Saving post ${post.url}`);
const account = post.account;
function saveAccountData(account: Account): Promise<undefined> {
return new Promise<undefined>((resolve, reject) => {
db.run(
`
INSERT INTO accounts (id, acct, username, display_name, url, avatar)
@ -220,10 +263,18 @@ export async function savePost(post: Post): Promise<undefined> {
],
(err) => {
if (err !== null) {
console.error(`Could not insert/update account ${account.id}`, err);
log.error(`Could not insert/update account ${account.id}`, err);
reject(err);
return;
}
resolve(undefined);
}
);
});
}
function savePostData(post: Post): Promise<undefined> {
return new Promise<undefined>((resolve, reject) => {
db.run(
`
INSERT INTO posts (id, content, created_at, url, account_id)
@ -235,11 +286,18 @@ export async function savePost(post: Post): Promise<undefined> {
[post.id, post.content, post.created_at, post.url, post.account.url],
(postErr) => {
if (postErr !== null) {
console.error(`Could not insert post ${post.url}`, postErr);
log.error(`Could not insert post ${post.url}`, postErr);
reject(postErr);
return;
}
resolve(undefined);
}
);
});
}
function savePostTagData(post: Post): Promise<undefined> {
return new Promise<undefined>((resolve, reject) => {
if (!post.tags.length) {
resolve(undefined);
return;
@ -256,7 +314,7 @@ export async function savePost(post: Post): Promise<undefined> {
[tag.url, tag.name],
(tagErr) => {
if (tagErr !== null) {
console.error(`Could not insert/update tag ${tag.url}`, tagErr);
log.error(`Could not insert/update tag ${tag.url}`, tagErr);
reject(tagErr);
return;
}
@ -265,10 +323,7 @@ export async function savePost(post: Post): Promise<undefined> {
[post.url, tag.url],
(posttagserr) => {
if (posttagserr !== null) {
console.error(
`Could not insert poststags ${tag.url}, ${post.url}`,
posttagserr
);
log.error(`Could not insert poststags ${tag.url}, ${post.url}`, posttagserr);
reject(posttagserr);
return;
}
@ -284,96 +339,102 @@ export async function savePost(post: Post): Promise<undefined> {
);
}
});
}
);
}
);
});
}
type FilterParameter = {
$limit: number | undefined | null;
$since?: string | undefined | null;
$before?: string | undefined | null;
[x: string]: string | number | undefined | null;
};
function saveSongInfoData(postUrl: string, songs: SongInfo[]): Promise<undefined> {
return new Promise<undefined>((resolve, reject) => {
if (songs.length === 0) {
resolve(undefined);
return;
}
db.parallelize(() => {
let remaining = songs.length;
for (const song of songs) {
db.run(
`
INSERT INTO songs (postedUrl, overviewUrl, type, youtubeUrl, title, artistName, thumbnailUrl, post_url)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`,
[
song.postedUrl,
song.pageUrl,
song.type,
song.youtubeUrl,
song.title,
song.artistName,
song.thumbnailUrl,
postUrl
],
(songErr) => {
if (songErr !== null) {
log.error(`Could not insert song ${song.postedUrl}`, songErr);
reject(songErr);
return;
}
// Don't decrease on fail
remaining--;
// Only resolve after all have been inserted
if (remaining === 0) {
resolve(undefined);
}
}
);
}
});
});
}
export async function getPosts(since: string | null, before: string | null, limit: number) {
export async function savePost(post: Post, songs: SongInfo[]) {
if (!databaseReady) {
await waitReady();
}
const promise = await new Promise<Post[]>((resolve, reject) => {
let filter_query = '';
const params: FilterParameter = { $limit: limit };
if (since === null && before === null) {
filter_query = '';
} else if (since !== null) {
filter_query = 'WHERE posts.created_at > $since';
params.$since = since;
} else if (before !== null) {
// Setting both, before and since doesn't make sense, so this case is not explicitly handled
filter_query = 'WHERE posts.created_at < $before';
params.$before = before;
}
ignoredUsers.forEach((ignoredUser, index) => {
const userParam = `$user_${index}`;
const acctParam = userParam + 'a';
const usernameParam = userParam + 'u';
const prefix = filter_query === '' ? ' WHERE' : ' AND';
filter_query += `${prefix} acct != ${acctParam} AND username != ${usernameParam} `;
params[acctParam] = ignoredUser;
params[usernameParam] = ignoredUser;
});
type PostResult = {
id: string;
content: string;
created_at: string;
url: string;
account_id: string;
acct: string;
username: string;
display_name: string;
account_url: string;
avatar: string;
};
type PostTagResult = {
post_id: string;
tag: string;
url: string;
};
log.debug(`Saving post ${post.url}`);
const account = post.account;
await saveAccountData(account);
log.debug(`Saved account data ${post.url}`);
await savePostData(post);
log.debug(`Saved post data ${post.url}`);
await savePostTagData(post);
log.debug(`Saved ${post.tags.length} tag data ${post.url}`);
await saveSongInfoData(post.url, songs);
log.debug(`Saved ${songs.length} song info data ${post.url}`);
}
function getPostData(filterQuery: string, params: FilterParameter): Promise<PostRow[]> {
const sql = `SELECT posts.id, posts.content, posts.created_at, posts.url,
accounts.id AS account_id, accounts.acct, accounts.username, accounts.display_name,
accounts.url AS account_url, accounts.avatar
FROM posts
JOIN accounts ON posts.account_id = accounts.url
${filter_query}
${filterQuery}
ORDER BY created_at DESC
LIMIT $limit`;
db.all(sql, params, (err, rows: PostResult[]) => {
return new Promise((resolve, reject) => {
db.all(sql, params, (err, rows: PostRow[]) => {
if (err != null) {
console.error('Error loading posts', err);
log.error('Error loading posts', err);
reject(err);
return;
}
if (rows.length === 0) {
// No need to check for tags
resolve([]);
return;
}
const postIdsParams = rows.map(() => '?').join(', ');
resolve(rows);
});
});
}
function getTagData(postIdsParams: String, postIds: string[]): Promise<Map<string, Tag[]>> {
return new Promise((resolve, reject) => {
db.all(
`SELECT post_id, tags.url, tags.tag
FROM poststags
JOIN tags ON poststags.tag_url = tags.url
WHERE post_id IN (${postIdsParams});`,
rows.map((r: PostResult) => r.url),
(tagErr, tagRows: PostTagResult[]) => {
postIds,
(tagErr, tagRows: PostTagRow[]) => {
if (tagErr != null) {
console.error('Error loading post tags', tagErr);
log.error('Error loading post tags', tagErr);
reject(tagErr);
return;
}
@ -385,13 +446,88 @@ export async function getPosts(since: string | null, before: string | null, limi
result.set(item.post_id, [...(result.get(item.post_id) || []), tag]);
return result;
}, new Map());
resolve(tagMap);
}
);
});
}
function getSongData(postIdsParams: String, postIds: string[]): Promise<Map<string, SongInfo[]>> {
return new Promise((resolve, reject) => {
db.all(
`SELECT post_url, songs.postedUrl, songs.overviewUrl, songs.type, songs.youtubeUrl,
songs.title, songs.artistName, songs.thumbnailUrl, songs.post_url
FROM songs
WHERE post_url IN (${postIdsParams});`,
postIds,
(tagErr, tagRows: SongRow[]) => {
if (tagErr != null) {
log.error('Error loading post tags', tagErr);
reject(tagErr);
return;
}
const songMap: Map<string, SongInfo[]> = tagRows.reduce(
(result: Map<string, SongInfo[]>, item) => {
result.set(item.post_url, [...(result.get(item.post_url) || []), item]);
return result;
},
new Map()
);
resolve(songMap);
}
);
});
}
export async function getPosts(
since: string | null,
before: string | null,
limit: number
): Promise<Post[]> {
if (!databaseReady) {
await waitReady();
}
let filterQuery = '';
const params: FilterParameter = { $limit: limit };
if (since === null && before === null) {
filterQuery = '';
} else if (since !== null) {
filterQuery = 'WHERE posts.created_at > $since';
params.$since = since;
} else if (before !== null) {
// Setting both, before and since doesn't make sense, so this case is not explicitly handled
filterQuery = 'WHERE posts.created_at < $before';
params.$before = before;
}
ignoredUsers.forEach((ignoredUser, index) => {
const userParam = `$user_${index}`;
const acctParam = userParam + 'a';
const usernameParam = userParam + 'u';
const prefix = filterQuery === '' ? ' WHERE' : ' AND';
filterQuery += `${prefix} acct != ${acctParam} AND username != ${usernameParam} `;
params[acctParam] = ignoredUser;
params[usernameParam] = ignoredUser;
});
const rows = await getPostData(filterQuery, params);
if (rows.length === 0) {
// No need to check for tags and songs
return [];
}
const postIdsParams = rows.map(() => '?').join(', ');
const postIds = rows.map((r: PostRow) => r.url);
const tagMap = await getTagData(postIdsParams, postIds);
const songMap = await getSongData(postIdsParams, postIds);
const posts = rows.map((row) => {
return {
id: row.id,
content: row.content,
created_at: row.created_at,
url: row.url,
tags: tagMap.get(row.id) || [],
tags: tagMap.get(row.url) || [],
account: {
id: row.account_id,
acct: row.acct,
@ -399,13 +535,9 @@ export async function getPosts(since: string | null, before: string | null, limi
display_name: row.display_name,
url: row.account_url,
avatar: row.avatar
} as Account
} as Account,
songs: songMap.get(row.url) || []
} as Post;
});
resolve(posts);
}
);
});
});
return promise;
return posts;
}

3
src/lib/server/rss.ts
View File

@ -1,6 +1,7 @@
import { BASE_URL, WEBSUB_HUB } from '$env/static/private';
import { PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME } from '$env/static/public';
import type { Post } from '$lib//mastodon/response';
import { log } from '$lib/log';
import { Feed } from 'feed';
import fs from 'fs/promises';
@ -59,6 +60,6 @@ export async function saveAtomFeed(feed: Feed) {
body: params
});
} catch (e) {
console.error('Failed to update WebSub hub', e);
log.error('Failed to update WebSub hub', e);
}
}

53
src/lib/server/timeline.ts
View File

@ -6,6 +6,7 @@ import {
YOUTUBE_API_KEY,
YOUTUBE_DISABLE
} from '$env/static/private';
import { log } from '$lib/log';
import type { Post, Tag, TimelineEvent } from '$lib/mastodon/response';
import type { OdesliResponse, Platform, SongInfo } from '$lib/odesliResponse';
import { getPosts, savePost } from '$lib/server/db';
@ -38,7 +39,7 @@ export class TimelineReader {
const resp = await fetch(youtubeVideoUrl);
const respObj = await resp.json();
if (!respObj.items.length) {
console.warn('Could not find video with id', videoId);
log.warn('Could not find video with id', videoId);
return false;
}
@ -77,7 +78,7 @@ export class TimelineReader {
return match[0];
}
} catch (e) {
console.error('Could not check if', videoId, 'is a music video', e);
log.error('Could not check if', videoId, 'is a music video', e);
}
}
return null;
@ -85,14 +86,14 @@ export class TimelineReader {
private static async getSongInfo(url: string, remainingTries = 6): Promise<SongInfo | null> {
if (remainingTries === 0) {
console.error('No tries remaining. Lookup failed!');
log.error('No tries remaining. Lookup failed!');
return null;
}
let hostname: string;
try {
hostname = new URL(url).hostname;
} catch (e) {
console.error(`Could not construct URL ${url}`, e);
log.error(`Could not construct URL ${url}`, e);
return null;
}
if (hostname === 'songwhip.com') {
@ -121,17 +122,18 @@ export class TimelineReader {
return {
...info,
pageUrl: odesliInfo.pageUrl,
youtubeUrl: odesliInfo.linksByPlatform[platform]?.url
youtubeUrl: odesliInfo.linksByPlatform[platform]?.url,
postedUrl: url
} as SongInfo;
});
});
} catch (e) {
if (e instanceof Error && e.cause === 429) {
console.warn('song.link rate limit reached. Trying again in 10 seconds');
log.warn('song.link rate limit reached. Trying again in 10 seconds');
await sleep(10_000);
return await this.getSongInfo(url, remainingTries - 1);
}
console.error(`Failed to load ${url} info from song.link`, e);
log.error(`Failed to load ${url} info from song.link`, e);
return null;
}
}
@ -149,7 +151,7 @@ export class TimelineReader {
).json();
return status.card?.url;
} catch (e) {
console.error(`Could not fetch status ${post.url}`, e);
log.error(`Could not fetch status ${post.url}`, e);
}
*/
}
@ -157,7 +159,7 @@ export class TimelineReader {
private startWebsocket() {
const socket = new WebSocket(`wss://${MASTODON_INSTANCE}/api/v1/streaming`);
socket.onopen = () => {
console.log('Connected to WS');
log.log('Connected to WS');
socket.send('{ "type": "subscribe", "stream": "public:local"}');
};
socket.onmessage = async (event) => {
@ -172,17 +174,15 @@ export class TimelineReader {
const urls: string[] = URL_FILTER.split(',');
const found_urls = urls.filter((t) => post.content.includes(t));
const urlsToCheck: string[] = [];
// If we don't have any tags or non-youtube urls, check youtube
// YT is handled separately, because it requires an API call and therefore is slower
if (found_urls.length === 0 && found_tags.length === 0) {
const youtubeUrl = await TimelineReader.checkYoutubeMatches(post.content);
if (youtubeUrl === null) {
console.log('Ignoring post', post.url);
log.log('Ignoring post', post.url);
return;
}
urlsToCheck.push(youtubeUrl);
console.log('Found YT URL', youtubeUrl, found_urls, found_urls.length);
log.debug('Found YT URL', youtubeUrl, found_urls, found_urls.length);
}
// TODO: Change URL detection above to use this regex.
@ -207,51 +207,46 @@ export class TimelineReader {
}
}
const songs: SongInfo[] = [];
log.debug(`Checking ${musicUrls.length} URLs if they contain song data`);
for (const url of musicUrls) {
let hostname: string | null = null;
try {
hostname = new URL(url).hostname;
} catch (e) {
console.error(`Could not check hostname for URL ${url}`, e);
log.error(`Could not check hostname for URL ${url}`, e);
}
if (hostname === 'songwhip.com') {
// TODO: Implement checking the songwhip API
continue;
}
const info = await TimelineReader.getSongInfo(url);
log.debug(`Found song info for ${url}?`, info);
if (info) {
console.info(
'Got song info for',
post.url,
url,
info.artistName,
info.title,
info.thumbnailUrl,
info.pageUrl,
info.youtubeUrl
);
songs.push(info);
}
}
await savePost(post);
await savePost(post, songs);
log.debug('Saved post', post.url);
const posts = await getPosts(null, null, 100);
await saveAtomFeed(createFeed(posts));
} catch (e) {
console.error('error message', event, event.data, e);
log.error('error message', event, event.data, e);
}
};
socket.onclose = (event) => {
console.warn(
log.warn(
`Websocket connection to ${MASTODON_INSTANCE} closed. Code: ${event.code}, reason: '${event.reason}'`
);
setTimeout(() => {
console.info(`Attempting to reconenct to WS`);
log.info(`Attempting to reconenct to WS`);
this.startWebsocket();
}, 10000);
};
socket.onerror = (event) => {
console.error(
log.error(
`Websocket connection to ${MASTODON_INSTANCE} failed. ${event.type}: ${event.error}, message: '${event.message}'`
);
};