phlaym/moshing-mammut
1
0
Fork 0
Code Issues 4 Pull Requests Projects Releases 8 Activity

Saving song infos to DB, refactor logging

Browse Source
This commit is contained in:
Max Nuding
2023-04-23 12:46:14 +02:00
parent 1cd9d83910
commit 971c846dd1
8 changed files with 396 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
src/lib/server/timeline.ts
View File

@ -6,6 +6,7 @@ import {
YOUTUBE_API_KEY,
YOUTUBE_DISABLE
} from '$env/static/private';
import { log } from '$lib/log';
import type { Post, Tag, TimelineEvent } from '$lib/mastodon/response';
import type { OdesliResponse, Platform, SongInfo } from '$lib/odesliResponse';
import { getPosts, savePost } from '$lib/server/db';
@ -38,7 +39,7 @@ export class TimelineReader {
const resp = await fetch(youtubeVideoUrl);
const respObj = await resp.json();
if (!respObj.items.length) {
console.warn('Could not find video with id', videoId);
log.warn('Could not find video with id', videoId);
return false;
}
@ -77,7 +78,7 @@ export class TimelineReader {
return match[0];
}
} catch (e) {
console.error('Could not check if', videoId, 'is a music video', e);
log.error('Could not check if', videoId, 'is a music video', e);
}
}
return null;
@ -85,14 +86,14 @@ export class TimelineReader {
private static async getSongInfo(url: string, remainingTries = 6): Promise<SongInfo | null> {
if (remainingTries === 0) {
console.error('No tries remaining. Lookup failed!');
log.error('No tries remaining. Lookup failed!');
return null;
}
let hostname: string;
try {
hostname = new URL(url).hostname;
} catch (e) {
console.error(`Could not construct URL ${url}`, e);
log.error(`Could not construct URL ${url}`, e);
return null;
}
if (hostname === 'songwhip.com') {
@ -121,17 +122,18 @@ export class TimelineReader {
return {
...info,
pageUrl: odesliInfo.pageUrl,
youtubeUrl: odesliInfo.linksByPlatform[platform]?.url
youtubeUrl: odesliInfo.linksByPlatform[platform]?.url,
postedUrl: url
} as SongInfo;
});
});
} catch (e) {
if (e instanceof Error && e.cause === 429) {
console.warn('song.link rate limit reached. Trying again in 10 seconds');
log.warn('song.link rate limit reached. Trying again in 10 seconds');
await sleep(10_000);
return await this.getSongInfo(url, remainingTries - 1);
}
console.error(`Failed to load ${url} info from song.link`, e);
log.error(`Failed to load ${url} info from song.link`, e);
return null;
}
}
@ -149,7 +151,7 @@ export class TimelineReader {
).json();
return status.card?.url;
} catch (e) {
console.error(`Could not fetch status ${post.url}`, e);
log.error(`Could not fetch status ${post.url}`, e);
}
*/
}
@ -157,7 +159,7 @@ export class TimelineReader {
private startWebsocket() {
const socket = new WebSocket(`wss://${MASTODON_INSTANCE}/api/v1/streaming`);
socket.onopen = () => {
console.log('Connected to WS');
log.log('Connected to WS');
socket.send('{ "type": "subscribe", "stream": "public:local"}');
};
socket.onmessage = async (event) => {
@ -172,17 +174,15 @@ export class TimelineReader {
const urls: string[] = URL_FILTER.split(',');
const found_urls = urls.filter((t) => post.content.includes(t));
const urlsToCheck: string[] = [];
// If we don't have any tags or non-youtube urls, check youtube
// YT is handled separately, because it requires an API call and therefore is slower
if (found_urls.length === 0 && found_tags.length === 0) {
const youtubeUrl = await TimelineReader.checkYoutubeMatches(post.content);
if (youtubeUrl === null) {
console.log('Ignoring post', post.url);
log.log('Ignoring post', post.url);
return;
}
urlsToCheck.push(youtubeUrl);
console.log('Found YT URL', youtubeUrl, found_urls, found_urls.length);
log.debug('Found YT URL', youtubeUrl, found_urls, found_urls.length);
}
// TODO: Change URL detection above to use this regex.
@ -207,51 +207,46 @@ export class TimelineReader {
}
}
const songs: SongInfo[] = [];
log.debug(`Checking ${musicUrls.length} URLs if they contain song data`);
for (const url of musicUrls) {
let hostname: string | null = null;
try {
hostname = new URL(url).hostname;
} catch (e) {
console.error(`Could not check hostname for URL ${url}`, e);
log.error(`Could not check hostname for URL ${url}`, e);
}
if (hostname === 'songwhip.com') {
// TODO: Implement checking the songwhip API
continue;
}
const info = await TimelineReader.getSongInfo(url);
log.debug(`Found song info for ${url}?`, info);
if (info) {
console.info(
'Got song info for',
post.url,
url,
info.artistName,
info.title,
info.thumbnailUrl,
info.pageUrl,
info.youtubeUrl
);
songs.push(info);
}
}
await savePost(post);
await savePost(post, songs);
log.debug('Saved post', post.url);
const posts = await getPosts(null, null, 100);
await saveAtomFeed(createFeed(posts));
} catch (e) {
console.error('error message', event, event.data, e);
log.error('error message', event, event.data, e);
}
};
socket.onclose = (event) => {
console.warn(
log.warn(
`Websocket connection to ${MASTODON_INSTANCE} closed. Code: ${event.code}, reason: '${event.reason}'`
);
setTimeout(() => {
console.info(`Attempting to reconenct to WS`);
log.info(`Attempting to reconenct to WS`);
this.startWebsocket();
}, 10000);
};
socket.onerror = (event) => {
console.error(
log.error(
`Websocket connection to ${MASTODON_INSTANCE} failed. ${event.type}: ${event.error}, message: '${event.message}'`
);
};