Compare commits

...

2 Commits

8 changed files with 421 additions and 242 deletions

View File

@ -1,8 +1,11 @@
{ {
"apexskier.eslint.config.eslintConfigPath" : ".eslintrc.cjs",
"apexskier.eslint.config.eslintPath" : "node_modules\/@eslint\/eslintrc\/dist\/eslintrc.cjs", "apexskier.eslint.config.eslintPath" : "node_modules\/@eslint\/eslintrc\/dist\/eslintrc.cjs",
"apexskier.typescript.config.formatDocumentOnSave" : "true", "apexskier.eslint.config.fixOnSave" : "Enable",
"apexskier.typescript.config.formatDocumentOnSave" : "false",
"apexskier.typescript.config.isEnabledForJavascript" : "Enable", "apexskier.typescript.config.isEnabledForJavascript" : "Enable",
"apexskier.typescript.config.organizeImportsOnSave" : "true", "apexskier.typescript.config.organizeImportsOnSave" : "true",
"apexskier.typescript.config.userPreferences.quotePreference" : "single", "apexskier.typescript.config.userPreferences.quotePreference" : "single",
"apexskier.typescript.config.userPreferences.useLabelDetailsInCompletionEntries" : true "apexskier.typescript.config.userPreferences.useLabelDetailsInCompletionEntries" : true,
"prettier.format-on-save" : "Global Default"
} }

View File

@ -1,3 +1,4 @@
import { log } from '$lib/log';
import { TimelineReader } from '$lib/server/timeline'; import { TimelineReader } from '$lib/server/timeline';
import type { HandleServerError } from '@sveltejs/kit'; import type { HandleServerError } from '@sveltejs/kit';
import fs from 'fs/promises'; import fs from 'fs/promises';
@ -6,7 +7,7 @@ TimelineReader.init();
export const handleError = (({ error }) => { export const handleError = (({ error }) => {
if (error instanceof Error) { if (error instanceof Error) {
console.error('Something went wrong: ', error.name, error.message); log.error('Something went wrong: ', error.name, error.message);
} }
return { return {

32
src/lib/log.ts Normal file
View File

@ -0,0 +1,32 @@
import { env } from '$env/dynamic/private';
import { isTruthy } from '$lib/truthyString';
const { DEV } = import.meta.env;
export const enableVerboseLog = isTruthy(env.VERBOSE);
export const log = {
verbose: (...params: any[]) => {
if (!enableVerboseLog) {
return;
}
console.debug(new Date().toISOString(), ...params);
},
debug: (...params: any[]) => {
if (!DEV) {
return;
}
console.debug(new Date().toISOString(), ...params);
},
log: (...params: any[]) => {
console.log(new Date().toISOString(), ...params);
},
info: (...params: any[]) => {
console.info(new Date().toISOString(), ...params);
},
warn: (...params: any[]) => {
console.warn(new Date().toISOString(), ...params);
},
error: (...params: any[]) => {
console.error(new Date().toISOString(), ...params);
}
};

View File

@ -1,3 +1,5 @@
import type { SongInfo } from '$lib/odesliResponse';
export interface TimelineEvent { export interface TimelineEvent {
event: string; event: string;
payload: string; payload: string;
@ -11,6 +13,7 @@ export interface Post {
content: string; content: string;
account: Account; account: Account;
card?: PreviewCard; card?: PreviewCard;
songs?: SongInfo[];
} }
export interface PreviewCard { export interface PreviewCard {

View File

@ -5,6 +5,7 @@ export type SongInfo = {
title?: string; title?: string;
artistName?: string; artistName?: string;
thumbnailUrl?: string; thumbnailUrl?: string;
postedUrl: string;
}; };
export type SongwhipReponse = { export type SongwhipReponse = {

View File

@ -1,10 +1,47 @@
import { env } from '$env/dynamic/private';
import { IGNORE_USERS, MASTODON_INSTANCE } from '$env/static/private'; import { IGNORE_USERS, MASTODON_INSTANCE } from '$env/static/private';
import { enableVerboseLog, log } from '$lib/log';
import type { Account, Post, Tag } from '$lib/mastodon/response'; import type { Account, Post, Tag } from '$lib/mastodon/response';
import { isTruthy } from '$lib/truthyString'; import type { SongInfo } from '$lib/odesliResponse';
import sqlite3 from 'sqlite3'; import sqlite3 from 'sqlite3';
const { DEV } = import.meta.env; const { DEV } = import.meta.env;
type FilterParameter = {
$limit: number | undefined | null;
$since?: string | undefined | null;
$before?: string | undefined | null;
[x: string]: string | number | undefined | null;
};
type PostRow = {
id: string;
content: string;
created_at: string;
url: string;
account_id: string;
acct: string;
username: string;
display_name: string;
account_url: string;
avatar: string;
};
type PostTagRow = {
post_id: string;
tag: string;
url: string;
};
type SongRow = SongInfo & {
post_url: string;
};
type Migration = {
id: number;
name: string;
statement: string;
};
const db: sqlite3.Database = new sqlite3.Database('moshingmammut.db'); const db: sqlite3.Database = new sqlite3.Database('moshingmammut.db');
// for the local masto instance, the instance name is *not* saved // for the local masto instance, the instance name is *not* saved
// as part of the username or acct, so it needs to be stripped // as part of the username or acct, so it needs to be stripped
@ -20,38 +57,32 @@ const ignoredUsers: string[] =
); );
let databaseReady = false; let databaseReady = false;
if (DEV && isTruthy(env.VERBOSE)) { if (enableVerboseLog) {
sqlite3.verbose(); sqlite3.verbose();
db.on('change', (t, d, table, rowid) => { db.on('change', (t, d, table, rowid) => {
console.debug('DB change event', t, d, table, rowid); log.verbose('DB change event', t, d, table, rowid);
}); });
db.on('trace', (sql) => { db.on('trace', (sql) => {
console.debug('Running', sql); log.verbose('Running', sql);
}); });
db.on('profile', (sql) => { db.on('profile', (sql) => {
console.debug('Finished', sql); log.verbose('Finished', sql);
}); });
} }
interface Migration {
id: number;
name: string;
statement: string;
}
db.on('open', () => { db.on('open', () => {
console.log('Opened database'); log.info('Opened database');
db.serialize(); db.serialize();
db.run('CREATE TABLE IF NOT EXISTS "migrations" ("id" integer,"name" TEXT, PRIMARY KEY (id))'); db.run('CREATE TABLE IF NOT EXISTS "migrations" ("id" integer,"name" TEXT, PRIMARY KEY (id))');
db.all('SELECT id FROM migrations', (err, rows: Migration[]) => { db.all('SELECT id FROM migrations', (err, rows: Migration[]) => {
if (err !== null) { if (err !== null) {
console.error('Could not fetch existing migrations', err); log.error('Could not fetch existing migrations', err);
databaseReady = true; databaseReady = true;
return; return;
} }
console.debug('Already applied migrations', rows); log.debug('Already applied migrations', rows);
const appliedMigrations: Set<number> = new Set(rows.map((row) => row['id'])); const appliedMigrations: Set<number> = new Set(rows.map((row) => row['id']));
const toApply = getMigrations().filter((m) => !appliedMigrations.has(m.id)); const toApply = getMigrations().filter((m) => !appliedMigrations.has(m.id));
let remaining = toApply.length; let remaining = toApply.length;
@ -68,7 +99,7 @@ db.on('open', () => {
databaseReady = true; databaseReady = true;
} }
if (err !== null) { if (err !== null) {
console.error(`Failed to apply migration ${migration.name}`, err); log.error(`Failed to apply migration ${migration.name}`, err);
return; return;
} }
db.run( db.run(
@ -76,10 +107,10 @@ db.on('open', () => {
[migration.id, migration.name], [migration.id, migration.name],
(e: Error) => { (e: Error) => {
if (e !== null) { if (e !== null) {
console.error(`Failed to mark migration ${migration.name} as applied`, e); log.error(`Failed to mark migration ${migration.name} as applied`, e);
return; return;
} }
console.info(`Applied migration ${migration.name}`); log.info(`Applied migration ${migration.name}`);
} }
); );
}); });
@ -87,7 +118,7 @@ db.on('open', () => {
}); });
}); });
db.on('error', (err) => { db.on('error', (err) => {
console.error('Error opening database', err); log.error('Error opening database', err);
}); });
function getMigrations(): Migration[] { function getMigrations(): Migration[] {
@ -170,6 +201,23 @@ function getMigrations(): Migration[] {
DROP TABLE poststags; DROP TABLE poststags;
ALTER TABLE poststags_new RENAME TO poststags; ALTER TABLE poststags_new RENAME TO poststags;
` `
},
{
id: 3,
name: 'song info for posts',
statement: `
CREATE TABLE songs (
id integer PRIMARY KEY,
postedUrl TEXT NOT NULL,
overviewUrl TEXT,
type TEXT CHECK ( type in ('album', 'song') ),
youtubeUrl TEXT,
title TEXT,
artistName TEXT,
thumbnailUrl TEXT,
post_url TEXT,
FOREIGN KEY (post_url) REFERENCES posts(url)
);`
} }
]; ];
} }
@ -179,11 +227,11 @@ async function waitReady(): Promise<undefined> {
return new Promise((resolve) => { return new Promise((resolve) => {
const interval = setInterval(() => { const interval = setInterval(() => {
if (DEV) { if (DEV) {
console.debug('Waiting for database to be ready'); log.debug('Waiting for database to be ready');
} }
if (databaseReady) { if (databaseReady) {
if (DEV) { if (DEV) {
console.debug('DB is ready'); log.debug('DB is ready');
} }
clearInterval(interval); clearInterval(interval);
resolve(undefined); resolve(undefined);
@ -192,13 +240,8 @@ async function waitReady(): Promise<undefined> {
}); });
} }
export async function savePost(post: Post): Promise<undefined> { function saveAccountData(account: Account): Promise<undefined> {
if (!databaseReady) { return new Promise<undefined>((resolve, reject) => {
await waitReady();
}
return await new Promise<undefined>((resolve, reject) => {
console.debug(`Saving post ${post.url}`);
const account = post.account;
db.run( db.run(
` `
INSERT INTO accounts (id, acct, username, display_name, url, avatar) INSERT INTO accounts (id, acct, username, display_name, url, avatar)
@ -220,10 +263,18 @@ export async function savePost(post: Post): Promise<undefined> {
], ],
(err) => { (err) => {
if (err !== null) { if (err !== null) {
console.error(`Could not insert/update account ${account.id}`, err); log.error(`Could not insert/update account ${account.id}`, err);
reject(err); reject(err);
return; return;
} }
resolve(undefined);
}
);
});
}
function savePostData(post: Post): Promise<undefined> {
return new Promise<undefined>((resolve, reject) => {
db.run( db.run(
` `
INSERT INTO posts (id, content, created_at, url, account_id) INSERT INTO posts (id, content, created_at, url, account_id)
@ -235,11 +286,18 @@ export async function savePost(post: Post): Promise<undefined> {
[post.id, post.content, post.created_at, post.url, post.account.url], [post.id, post.content, post.created_at, post.url, post.account.url],
(postErr) => { (postErr) => {
if (postErr !== null) { if (postErr !== null) {
console.error(`Could not insert post ${post.url}`, postErr); log.error(`Could not insert post ${post.url}`, postErr);
reject(postErr); reject(postErr);
return; return;
} }
resolve(undefined);
}
);
});
}
function savePostTagData(post: Post): Promise<undefined> {
return new Promise<undefined>((resolve, reject) => {
if (!post.tags.length) { if (!post.tags.length) {
resolve(undefined); resolve(undefined);
return; return;
@ -256,7 +314,7 @@ export async function savePost(post: Post): Promise<undefined> {
[tag.url, tag.name], [tag.url, tag.name],
(tagErr) => { (tagErr) => {
if (tagErr !== null) { if (tagErr !== null) {
console.error(`Could not insert/update tag ${tag.url}`, tagErr); log.error(`Could not insert/update tag ${tag.url}`, tagErr);
reject(tagErr); reject(tagErr);
return; return;
} }
@ -265,10 +323,7 @@ export async function savePost(post: Post): Promise<undefined> {
[post.url, tag.url], [post.url, tag.url],
(posttagserr) => { (posttagserr) => {
if (posttagserr !== null) { if (posttagserr !== null) {
console.error( log.error(`Could not insert poststags ${tag.url}, ${post.url}`, posttagserr);
`Could not insert poststags ${tag.url}, ${post.url}`,
posttagserr
);
reject(posttagserr); reject(posttagserr);
return; return;
} }
@ -284,96 +339,102 @@ export async function savePost(post: Post): Promise<undefined> {
); );
} }
}); });
}
);
}
);
}); });
} }
type FilterParameter = { function saveSongInfoData(postUrl: string, songs: SongInfo[]): Promise<undefined> {
$limit: number | undefined | null; return new Promise<undefined>((resolve, reject) => {
$since?: string | undefined | null; if (songs.length === 0) {
$before?: string | undefined | null; resolve(undefined);
[x: string]: string | number | undefined | null; return;
}; }
db.parallelize(() => {
let remaining = songs.length;
for (const song of songs) {
db.run(
`
INSERT INTO songs (postedUrl, overviewUrl, type, youtubeUrl, title, artistName, thumbnailUrl, post_url)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`,
[
song.postedUrl,
song.pageUrl,
song.type,
song.youtubeUrl,
song.title,
song.artistName,
song.thumbnailUrl,
postUrl
],
(songErr) => {
if (songErr !== null) {
log.error(`Could not insert song ${song.postedUrl}`, songErr);
reject(songErr);
return;
}
// Don't decrease on fail
remaining--;
// Only resolve after all have been inserted
if (remaining === 0) {
resolve(undefined);
}
}
);
}
});
});
}
export async function getPosts(since: string | null, before: string | null, limit: number) { export async function savePost(post: Post, songs: SongInfo[]) {
if (!databaseReady) { if (!databaseReady) {
await waitReady(); await waitReady();
} }
const promise = await new Promise<Post[]>((resolve, reject) => {
let filter_query = ''; log.debug(`Saving post ${post.url}`);
const params: FilterParameter = { $limit: limit }; const account = post.account;
if (since === null && before === null) { await saveAccountData(account);
filter_query = ''; log.debug(`Saved account data ${post.url}`);
} else if (since !== null) { await savePostData(post);
filter_query = 'WHERE posts.created_at > $since'; log.debug(`Saved post data ${post.url}`);
params.$since = since; await savePostTagData(post);
} else if (before !== null) { log.debug(`Saved ${post.tags.length} tag data ${post.url}`);
// Setting both, before and since doesn't make sense, so this case is not explicitly handled await saveSongInfoData(post.url, songs);
filter_query = 'WHERE posts.created_at < $before'; log.debug(`Saved ${songs.length} song info data ${post.url}`);
params.$before = before;
} }
ignoredUsers.forEach((ignoredUser, index) => { function getPostData(filterQuery: string, params: FilterParameter): Promise<PostRow[]> {
const userParam = `$user_${index}`;
const acctParam = userParam + 'a';
const usernameParam = userParam + 'u';
const prefix = filter_query === '' ? ' WHERE' : ' AND';
filter_query += `${prefix} acct != ${acctParam} AND username != ${usernameParam} `;
params[acctParam] = ignoredUser;
params[usernameParam] = ignoredUser;
});
type PostResult = {
id: string;
content: string;
created_at: string;
url: string;
account_id: string;
acct: string;
username: string;
display_name: string;
account_url: string;
avatar: string;
};
type PostTagResult = {
post_id: string;
tag: string;
url: string;
};
const sql = `SELECT posts.id, posts.content, posts.created_at, posts.url, const sql = `SELECT posts.id, posts.content, posts.created_at, posts.url,
accounts.id AS account_id, accounts.acct, accounts.username, accounts.display_name, accounts.id AS account_id, accounts.acct, accounts.username, accounts.display_name,
accounts.url AS account_url, accounts.avatar accounts.url AS account_url, accounts.avatar
FROM posts FROM posts
JOIN accounts ON posts.account_id = accounts.url JOIN accounts ON posts.account_id = accounts.url
${filter_query} ${filterQuery}
ORDER BY created_at DESC ORDER BY created_at DESC
LIMIT $limit`; LIMIT $limit`;
db.all(sql, params, (err, rows: PostResult[]) => {
return new Promise((resolve, reject) => {
db.all(sql, params, (err, rows: PostRow[]) => {
if (err != null) { if (err != null) {
console.error('Error loading posts', err); log.error('Error loading posts', err);
reject(err); reject(err);
return; return;
} }
if (rows.length === 0) { resolve(rows);
// No need to check for tags });
resolve([]); });
return;
} }
const postIdsParams = rows.map(() => '?').join(', ');
function getTagData(postIdsParams: String, postIds: string[]): Promise<Map<string, Tag[]>> {
return new Promise((resolve, reject) => {
db.all( db.all(
`SELECT post_id, tags.url, tags.tag `SELECT post_id, tags.url, tags.tag
FROM poststags FROM poststags
JOIN tags ON poststags.tag_url = tags.url JOIN tags ON poststags.tag_url = tags.url
WHERE post_id IN (${postIdsParams});`, WHERE post_id IN (${postIdsParams});`,
rows.map((r: PostResult) => r.url), postIds,
(tagErr, tagRows: PostTagResult[]) => { (tagErr, tagRows: PostTagRow[]) => {
if (tagErr != null) { if (tagErr != null) {
console.error('Error loading post tags', tagErr); log.error('Error loading post tags', tagErr);
reject(tagErr); reject(tagErr);
return; return;
} }
@ -385,13 +446,88 @@ export async function getPosts(since: string | null, before: string | null, limi
result.set(item.post_id, [...(result.get(item.post_id) || []), tag]); result.set(item.post_id, [...(result.get(item.post_id) || []), tag]);
return result; return result;
}, new Map()); }, new Map());
resolve(tagMap);
}
);
});
}
function getSongData(postIdsParams: String, postIds: string[]): Promise<Map<string, SongInfo[]>> {
return new Promise((resolve, reject) => {
db.all(
`SELECT post_url, songs.postedUrl, songs.overviewUrl, songs.type, songs.youtubeUrl,
songs.title, songs.artistName, songs.thumbnailUrl, songs.post_url
FROM songs
WHERE post_url IN (${postIdsParams});`,
postIds,
(tagErr, tagRows: SongRow[]) => {
if (tagErr != null) {
log.error('Error loading post tags', tagErr);
reject(tagErr);
return;
}
const songMap: Map<string, SongInfo[]> = tagRows.reduce(
(result: Map<string, SongInfo[]>, item) => {
result.set(item.post_url, [...(result.get(item.post_url) || []), item]);
return result;
},
new Map()
);
resolve(songMap);
}
);
});
}
export async function getPosts(
since: string | null,
before: string | null,
limit: number
): Promise<Post[]> {
if (!databaseReady) {
await waitReady();
}
let filterQuery = '';
const params: FilterParameter = { $limit: limit };
if (since === null && before === null) {
filterQuery = '';
} else if (since !== null) {
filterQuery = 'WHERE posts.created_at > $since';
params.$since = since;
} else if (before !== null) {
// Setting both, before and since doesn't make sense, so this case is not explicitly handled
filterQuery = 'WHERE posts.created_at < $before';
params.$before = before;
}
ignoredUsers.forEach((ignoredUser, index) => {
const userParam = `$user_${index}`;
const acctParam = userParam + 'a';
const usernameParam = userParam + 'u';
const prefix = filterQuery === '' ? ' WHERE' : ' AND';
filterQuery += `${prefix} acct != ${acctParam} AND username != ${usernameParam} `;
params[acctParam] = ignoredUser;
params[usernameParam] = ignoredUser;
});
const rows = await getPostData(filterQuery, params);
if (rows.length === 0) {
// No need to check for tags and songs
return [];
}
const postIdsParams = rows.map(() => '?').join(', ');
const postIds = rows.map((r: PostRow) => r.url);
const tagMap = await getTagData(postIdsParams, postIds);
const songMap = await getSongData(postIdsParams, postIds);
const posts = rows.map((row) => { const posts = rows.map((row) => {
return { return {
id: row.id, id: row.id,
content: row.content, content: row.content,
created_at: row.created_at, created_at: row.created_at,
url: row.url, url: row.url,
tags: tagMap.get(row.id) || [], tags: tagMap.get(row.url) || [],
account: { account: {
id: row.account_id, id: row.account_id,
acct: row.acct, acct: row.acct,
@ -399,13 +535,9 @@ export async function getPosts(since: string | null, before: string | null, limi
display_name: row.display_name, display_name: row.display_name,
url: row.account_url, url: row.account_url,
avatar: row.avatar avatar: row.avatar
} as Account } as Account,
songs: songMap.get(row.url) || []
} as Post; } as Post;
}); });
resolve(posts); return posts;
}
);
});
});
return promise;
} }

View File

@ -1,6 +1,7 @@
import { BASE_URL, WEBSUB_HUB } from '$env/static/private'; import { BASE_URL, WEBSUB_HUB } from '$env/static/private';
import { PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME } from '$env/static/public'; import { PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME } from '$env/static/public';
import type { Post } from '$lib//mastodon/response'; import type { Post } from '$lib//mastodon/response';
import { log } from '$lib/log';
import { Feed } from 'feed'; import { Feed } from 'feed';
import fs from 'fs/promises'; import fs from 'fs/promises';
@ -59,6 +60,6 @@ export async function saveAtomFeed(feed: Feed) {
body: params body: params
}); });
} catch (e) { } catch (e) {
console.error('Failed to update WebSub hub', e); log.error('Failed to update WebSub hub', e);
} }
} }

View File

@ -6,6 +6,7 @@ import {
YOUTUBE_API_KEY, YOUTUBE_API_KEY,
YOUTUBE_DISABLE YOUTUBE_DISABLE
} from '$env/static/private'; } from '$env/static/private';
import { log } from '$lib/log';
import type { Post, Tag, TimelineEvent } from '$lib/mastodon/response'; import type { Post, Tag, TimelineEvent } from '$lib/mastodon/response';
import type { OdesliResponse, Platform, SongInfo } from '$lib/odesliResponse'; import type { OdesliResponse, Platform, SongInfo } from '$lib/odesliResponse';
import { getPosts, savePost } from '$lib/server/db'; import { getPosts, savePost } from '$lib/server/db';
@ -38,7 +39,7 @@ export class TimelineReader {
const resp = await fetch(youtubeVideoUrl); const resp = await fetch(youtubeVideoUrl);
const respObj = await resp.json(); const respObj = await resp.json();
if (!respObj.items.length) { if (!respObj.items.length) {
console.warn('Could not find video with id', videoId); log.warn('Could not find video with id', videoId);
return false; return false;
} }
@ -77,25 +78,18 @@ export class TimelineReader {
return match[0]; return match[0];
} }
} catch (e) { } catch (e) {
console.error('Could not check if', videoId, 'is a music video', e); log.error('Could not check if', videoId, 'is a music video', e);
} }
} }
return null; return null;
} }
private static async getSongInfo(url: string, remainingTries = 6): Promise<SongInfo | null> { private static async getSongInfo(url: URL, remainingTries = 6): Promise<SongInfo | null> {
if (remainingTries === 0) { if (remainingTries === 0) {
console.error('No tries remaining. Lookup failed!'); log.error('No tries remaining. Lookup failed!');
return null; return null;
} }
let hostname: string; if (url.hostname === 'songwhip.com') {
try {
hostname = new URL(url).hostname;
} catch (e) {
console.error(`Could not construct URL ${url}`, e);
return null;
}
if (hostname === 'songwhip.com') {
// song.link doesn't support songwhip links and songwhip themselves will provide metadata if you pass in a // song.link doesn't support songwhip links and songwhip themselves will provide metadata if you pass in a
// Apple Music/Spotify/etc link, but won't when provided with their own link, so no way to extract song info // Apple Music/Spotify/etc link, but won't when provided with their own link, so no way to extract song info
// except maybe scraping their HTML // except maybe scraping their HTML
@ -103,7 +97,7 @@ export class TimelineReader {
} }
const odesliParams = new URLSearchParams(); const odesliParams = new URLSearchParams();
odesliParams.append('url', url); odesliParams.append('url', url.toString());
odesliParams.append('userCountry', 'DE'); odesliParams.append('userCountry', 'DE');
odesliParams.append('songIfSingle', 'true'); odesliParams.append('songIfSingle', 'true');
if (ODESLI_API_KEY && ODESLI_API_KEY !== 'CHANGE_ME') { if (ODESLI_API_KEY && ODESLI_API_KEY !== 'CHANGE_ME') {
@ -121,17 +115,18 @@ export class TimelineReader {
return { return {
...info, ...info,
pageUrl: odesliInfo.pageUrl, pageUrl: odesliInfo.pageUrl,
youtubeUrl: odesliInfo.linksByPlatform[platform]?.url youtubeUrl: odesliInfo.linksByPlatform[platform]?.url,
postedUrl: url.toString()
} as SongInfo; } as SongInfo;
}); });
}); });
} catch (e) { } catch (e) {
if (e instanceof Error && e.cause === 429) { if (e instanceof Error && e.cause === 429) {
console.warn('song.link rate limit reached. Trying again in 10 seconds'); log.warn('song.link rate limit reached. Trying again in 10 seconds');
await sleep(10_000); await sleep(10_000);
return await this.getSongInfo(url, remainingTries - 1); return await this.getSongInfo(url, remainingTries - 1);
} }
console.error(`Failed to load ${url} info from song.link`, e); log.error(`Failed to load ${url} info from song.link`, e);
return null; return null;
} }
} }
@ -149,7 +144,7 @@ export class TimelineReader {
).json(); ).json();
return status.card?.url; return status.card?.url;
} catch (e) { } catch (e) {
console.error(`Could not fetch status ${post.url}`, e); log.error(`Could not fetch status ${post.url}`, e);
} }
*/ */
} }
@ -157,7 +152,7 @@ export class TimelineReader {
private startWebsocket() { private startWebsocket() {
const socket = new WebSocket(`wss://${MASTODON_INSTANCE}/api/v1/streaming`); const socket = new WebSocket(`wss://${MASTODON_INSTANCE}/api/v1/streaming`);
socket.onopen = () => { socket.onopen = () => {
console.log('Connected to WS'); log.log('Connected to WS');
socket.send('{ "type": "subscribe", "stream": "public:local"}'); socket.send('{ "type": "subscribe", "stream": "public:local"}');
}; };
socket.onmessage = async (event) => { socket.onmessage = async (event) => {
@ -172,86 +167,97 @@ export class TimelineReader {
const urls: string[] = URL_FILTER.split(','); const urls: string[] = URL_FILTER.split(',');
const found_urls = urls.filter((t) => post.content.includes(t)); const found_urls = urls.filter((t) => post.content.includes(t));
const urlsToCheck: string[] = [];
// If we don't have any tags or non-youtube urls, check youtube // If we don't have any tags or non-youtube urls, check youtube
// YT is handled separately, because it requires an API call and therefore is slower // YT is handled separately, because it requires an API call and therefore is slower
if (found_urls.length === 0 && found_tags.length === 0) { if (found_urls.length === 0 && found_tags.length === 0) {
const youtubeUrl = await TimelineReader.checkYoutubeMatches(post.content); const youtubeUrl = await TimelineReader.checkYoutubeMatches(post.content);
if (youtubeUrl === null) { if (youtubeUrl === null) {
console.log('Ignoring post', post.url); log.log('Ignoring post', post.url);
return; return;
} }
urlsToCheck.push(youtubeUrl); log.debug('Found YT URL', youtubeUrl, found_urls, found_urls.length);
console.log('Found YT URL', youtubeUrl, found_urls, found_urls.length);
} }
// TODO: Change URL detection above to use this regex. // TODO: Change URL detection above to use this regex.
// Looks like we're stuck with regex for now instead of using preview cards. // Looks like we're stuck with regex for now instead of using preview cards.
// Might as well use it to find URLs. Could also use this for YouTube: If Odesli finds something, it's a song, // Might as well use it to find URLs. Could also use this for YouTube: If Odesli finds something, it's a song,
// if not, ignore it. No need to consult the YT API and give those links a special handling // if not, ignore it. No need to consult the YT API and give those links a special handling
const musicUrls: string[] = []; const musicUrls: URL[] = [];
const musicUrl = await TimelineReader.getUrlFromPreviewCard(post); const musicUrl = await TimelineReader.getUrlFromPreviewCard(post);
if (musicUrl) { if (musicUrl) {
musicUrls.push(musicUrl); try {
musicUrls.push(new URL(musicUrl));
} catch (e) {
log.error(
'URL received from preview card does not seem to be a valid URL',
musicUrl,
e
);
}
} else { } else {
const urlMatches = post.content.matchAll(URL_REGEX); const urlMatches = post.content.matchAll(URL_REGEX);
for (const match of urlMatches) { for (const match of urlMatches) {
if (match === undefined || match.groups === undefined) { if (match === undefined || match.groups === undefined) {
console.warn(
'Match listed in allMatches, but either it or its groups are undefined',
match
);
continue; continue;
} }
const urlMatch = match.groups.postUrl.toString(); const urlMatch = match.groups.postUrl.toString();
const musicUrl = urls.find((u) => urlMatch.includes(u)); let url: URL;
if (musicUrl) { try {
musicUrls.push(urlMatch); url = new URL(urlMatch);
} catch (e) {
log.error('URL found via Regex does not seem to be a valud url', urlMatch, e);
continue;
} }
// Check *all* found url and let odesli determine if it is music or not
musicUrls.push(url);
} }
} }
const songs: SongInfo[] = [];
log.debug(`Checking ${musicUrls.length} URLs if they contain song data`);
for (const url of musicUrls) { for (const url of musicUrls) {
let hostname: string | null = null; let hostname: string | null = null;
try { try {
hostname = new URL(url).hostname; hostname = new URL(url).hostname;
} catch (e) { } catch (e) {
console.error(`Could not check hostname for URL ${url}`, e); log.error(`Could not check hostname for URL ${url}`, e);
} }
if (hostname === 'songwhip.com') { if (hostname === 'songwhip.com') {
// TODO: Implement checking the songwhip API // TODO: Implement checking the songwhip API
continue; continue;
} }
const info = await TimelineReader.getSongInfo(url); const info = await TimelineReader.getSongInfo(url);
log.debug(`Found song info for ${url}?`, info);
if (info) { if (info) {
console.info( songs.push(info);
'Got song info for',
post.url,
url,
info.artistName,
info.title,
info.thumbnailUrl,
info.pageUrl,
info.youtubeUrl
);
} }
} }
await savePost(post); await savePost(post, songs);
log.debug('Saved post', post.url);
const posts = await getPosts(null, null, 100); const posts = await getPosts(null, null, 100);
await saveAtomFeed(createFeed(posts)); await saveAtomFeed(createFeed(posts));
} catch (e) { } catch (e) {
console.error('error message', event, event.data, e); log.error('error message', event, event.data, e);
} }
}; };
socket.onclose = (event) => { socket.onclose = (event) => {
console.warn( log.warn(
`Websocket connection to ${MASTODON_INSTANCE} closed. Code: ${event.code}, reason: '${event.reason}'` `Websocket connection to ${MASTODON_INSTANCE} closed. Code: ${event.code}, reason: '${event.reason}'`
); );
setTimeout(() => { setTimeout(() => {
console.info(`Attempting to reconenct to WS`); log.info(`Attempting to reconenct to WS`);
this.startWebsocket(); this.startWebsocket();
}, 10000); }, 10000);
}; };
socket.onerror = (event) => { socket.onerror = (event) => {
console.error( log.error(
`Websocket connection to ${MASTODON_INSTANCE} failed. ${event.type}: ${event.error}, message: '${event.message}'` `Websocket connection to ${MASTODON_INSTANCE} failed. ${event.type}: ${event.error}, message: '${event.message}'`
); );
}; };