phlaym/moshing-mammut
1
0
Fork 0
Code Issues 7 Pull Requests Projects Releases 8 Activity

update logging

Browse Source
This commit is contained in:
Max Nuding
2025-07-04 08:46:41 +02:00
parent b0465a020d
commit 77e483d637
8 changed files with 96 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
src/lib/server/timeline.ts
View File

@ -5,7 +5,7 @@ import {
ODESLI_API_KEY,
YOUTUBE_API_KEY
} from '$env/static/private';
import { log, Logger } from '$lib/log';
import { Logger } from '$lib/log';
import type {
Account,
AccountAvatar,
@ -47,11 +47,12 @@ export class TimelineReader {
private static _instance: TimelineReader;
private lastPosts: string[] = [];
private playlistAdders: PlaylistAdder[];
private logger: Logger;
private static async isMusicVideo(videoId: string) {
private async isMusicVideo(videoId: string) {
if (!YOUTUBE_API_KEY || YOUTUBE_API_KEY === 'CHANGE_ME') {
// Assume that it *is* a music link when no YT API key is provided
log.debug('YT API not configured');
this.logger.debug('YT API not configured');
return true;
}
const searchParams = new URLSearchParams([
@ -63,13 +64,13 @@ export class TimelineReader {
const resp = await fetch(youtubeVideoUrl);
const respObj = await resp.json();
if (!respObj.items.length) {
log.warn('Could not find video with id', videoId);
this.logger.warn('Could not find video with id', videoId);
return false;
}
const item = respObj.items[0];
if (!item.snippet) {
log.warn('Could not load snippet for video', videoId, item);
this.logger.warn('Could not load snippet for video', videoId, item);
return false;
}
if (item.snippet.tags?.includes('music')) {
@ -87,16 +88,19 @@ export class TimelineReader {
const categoryTitle: string = await fetch(youtubeCategoryUrl)
.then((r) => r.json())
.then((r) => r.items[0]?.snippet?.title);
log.debug('YT category', categoryTitle);
this.logger.debug('YT category', categoryTitle);
return categoryTitle === 'Music';
}
public static async getSongInfoInPost(post: Post): Promise<SongInfo[]> {
public async getSongInfoInPost(post: Post): Promise<SongInfo[]> {
const urlMatches = post.content.matchAll(URL_REGEX);
const songs: SongInfo[] = [];
for (const match of urlMatches) {
if (match === undefined || match.groups === undefined) {
log.warn('Match listed in allMatches, but either it or its groups are undefined', match);
this.logger.warn(
'Match listed in allMatches, but either it or its groups are undefined',
match
);
continue;
}
const urlMatch = match.groups.postUrl.toString();
@ -104,14 +108,14 @@ export class TimelineReader {
try {
url = new URL(urlMatch);
} catch (e) {
log.error('URL found via Regex does not seem to be a valud url', urlMatch, e);
this.logger.error('URL found via Regex does not seem to be a valud url', urlMatch, e);
continue;
}
// Check *all* found url and let odesli determine if it is music or not
log.debug(`Checking ${url} if it contains song data`);
const info = await TimelineReader.getSongInfo(url);
//log.debug(`Found song info for ${url}?`, info);
this.logger.debug(`Checking ${url} if it contains song data`);
const info = await this.getSongInfo(url);
//this.logger.debug(`Found song info for ${url}?`, info);
if (info) {
songs.push(info);
}
@ -119,9 +123,9 @@ export class TimelineReader {
return songs;
}
private static async getSongInfo(url: URL, remainingTries = 6): Promise<SongInfo | null> {
private async getSongInfo(url: URL, remainingTries = 6): Promise<SongInfo | null> {
if (remainingTries === 0) {
log.error('No tries remaining. Lookup failed!');
this.logger.error('No tries remaining. Lookup failed!');
return null;
}
if (url.hostname === 'songwhip.com') {
@ -153,7 +157,7 @@ export class TimelineReader {
return null;
}
const info = odesliInfo.entitiesByUniqueId[odesliInfo.entityUniqueId];
//log.debug('odesli response', info);
//this.logger.debug('odesli response', info);
const platform: Platform = 'youtube';
if (info.platforms.includes(platform)) {
const youtubeId =
@ -161,12 +165,16 @@ export class TimelineReader {
YOUTUBE_REGEX.exec(url.href)?.groups?.videoId ??
new URL(odesliInfo.pageUrl).pathname.split('/y/').pop();
if (youtubeId === undefined) {
log.warn('Looks like a youtube video, but could not extract a video id', url, odesliInfo);
this.logger.warn(
'Looks like a youtube video, but could not extract a video id',
url,
odesliInfo
);
return null;
}
const isMusic = await TimelineReader.isMusicVideo(youtubeId);
const isMusic = await this.isMusicVideo(youtubeId);
if (!isMusic) {
log.debug('Probably not a music video', youtubeId, url);
this.logger.debug('Probably not a music video', youtubeId, url);
return null;
}
}
@ -181,11 +189,11 @@ export class TimelineReader {
} as SongInfo;
} catch (e) {
if (e instanceof Error && e.cause === 429) {
log.warn('song.link rate limit reached. Trying again in 10 seconds');
this.logger.warn('song.link rate limit reached. Trying again in 10 seconds');
await sleep(10_000);
return await this.getSongInfo(url, remainingTries - 1);
}
log.error(`Failed to load ${url} info from song.link`, e);
this.logger.error(`Failed to load ${url} info from song.link`, e);
return null;
}
}
@ -196,7 +204,7 @@ export class TimelineReader {
}
}
private static async resizeAvatar(
private async resizeAvatar(
baseName: string,
size: number,
suffix: string,
@ -209,15 +217,15 @@ export class TimelineReader {
.then(() => true)
.catch(() => false);
if (exists) {
log.debug('File already exists', fileName);
this.logger.debug('File already exists', fileName);
return null;
}
log.debug('Saving avatar', fileName);
this.logger.debug('Saving avatar', fileName);
await sharpAvatar.resize(size).toFile(fileName);
return fileName;
}
private static resizeAvatarPromiseMaker(
private resizeAvatarPromiseMaker(
avatarFilenameBase: string,
baseSize: number,
maxPixelDensity: number,
@ -230,13 +238,7 @@ export class TimelineReader {
for (let i = 1; i <= maxPixelDensity; i++) {
promises.push(
...formats.map((f) =>
TimelineReader.resizeAvatar(
avatarFilenameBase,
baseSize * i,
`${i}x.${f}`,
'avatars',
sharpAvatar
)
this.resizeAvatar(avatarFilenameBase, baseSize * i, `${i}x.${f}`, 'avatars', sharpAvatar)
.then(
(fn) =>
({
@ -252,7 +254,7 @@ export class TimelineReader {
return promises;
}
private static resizeThumbnailPromiseMaker(
private resizeThumbnailPromiseMaker(
filenameBase: string,
baseSize: number,
maxPixelDensity: number,
@ -266,13 +268,7 @@ export class TimelineReader {
for (let i = 1; i <= maxPixelDensity; i++) {
promises.push(
...formats.map((f) =>
TimelineReader.resizeAvatar(
filenameBase,
baseSize * i,
`${i}x.${f}`,
'thumbnails',
sharpAvatar
)
this.resizeAvatar(filenameBase, baseSize * i, `${i}x.${f}`, 'thumbnails', sharpAvatar)
.then(
(fn) =>
({
@ -289,7 +285,7 @@ export class TimelineReader {
return promises;
}
private static async saveAvatar(account: Account) {
private async saveAvatar(account: Account) {
try {
const existingAvatars = await getAvatars(account.url, 1);
const existingAvatarBase = existingAvatars.shift()?.file.split('/').pop()?.split('_').shift();
@ -302,7 +298,7 @@ export class TimelineReader {
const avatarsToDelete = (await fs.readdir('avatars'))
.filter((x) => x.startsWith(existingAvatarBase + '_'))
.map((x) => {
log.debug('Removing existing avatar file', x);
this.logger.debug('Removing existing avatar file', x);
return x;
})
.map((x) => fs.unlink('avatars/' + x));
@ -311,7 +307,7 @@ export class TimelineReader {
const avatarResponse = await fetch(account.avatar);
const avatar = await avatarResponse.arrayBuffer();
await Promise.all(
TimelineReader.resizeAvatarPromiseMaker(
this.resizeAvatarPromiseMaker(
avatarFilenameBase,
50,
3,
@ -325,7 +321,7 @@ export class TimelineReader {
}
}
private static async saveSongThumbnails(songs: SongInfo[]) {
private async saveSongThumbnails(songs: SongInfo[]) {
for (const song of songs) {
if (!song.thumbnailUrl) {
continue;
@ -339,7 +335,7 @@ export class TimelineReader {
const imageResponse = await fetch(song.thumbnailUrl);
const avatar = await imageResponse.arrayBuffer();
await Promise.all(
TimelineReader.resizeThumbnailPromiseMaker(
this.resizeThumbnailPromiseMaker(
fileBaseName + '_large',
200,
3,
@ -350,7 +346,7 @@ export class TimelineReader {
)
);
await Promise.all(
TimelineReader.resizeThumbnailPromiseMaker(
this.resizeThumbnailPromiseMaker(
fileBaseName + '_small',
60,
3,
@ -375,27 +371,27 @@ export class TimelineReader {
const hashttags: string[] = HASHTAG_FILTER.split(',');
const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
const songs = await TimelineReader.getSongInfoInPost(post);
const songs = await this.getSongInfoInPost(post);
// If we don't have any tags or non-youtube urls, check youtube
// YT is handled separately, because it requires an API call and therefore is slower
if (songs.length === 0 && found_tags.length === 0) {
log.log('Ignoring post', post.url);
this.logger.log('Ignoring post', post.url);
return;
}
await savePost(post, songs);
await TimelineReader.saveAvatar(post.account);
await TimelineReader.saveSongThumbnails(songs);
await this.saveAvatar(post.account);
await this.saveSongThumbnails(songs);
log.debug('Saved post', post.url, 'songs', songs);
this.logger.debug('Saved post', post.url, 'songs', songs);
const posts = await getPosts(null, null, 100);
await saveAtomFeed(createFeed(posts));
for (let song of songs) {
log.debug('Adding to playlist', song);
this.logger.debug('Adding to playlist', song);
await this.addToPlaylist(song);
}
}
@ -455,9 +451,9 @@ export class TimelineReader {
const now = new Date().toISOString();
let latestPost = await getPosts(null, now, 1);
if (latestPost.length > 0) {
log.log('Last post in DB since', now, latestPost[0].created_at);
this.logger.log('Last post in DB since', now, latestPost[0].created_at);
} else {
log.log('No posts in DB since');
this.logger.log('No posts in DB since');
}
let u = new URL(`https://${MASTODON_INSTANCE}/api/v1/timelines/public?local=true&limit=40`);
if (latestPost.length > 0) {
@ -470,28 +466,28 @@ export class TimelineReader {
Authorization: `Bearer ${MASTODON_ACCESS_TOKEN}`
};
const latestPosts: Post[] = await fetch(u, { headers }).then((r) => r.json());
log.info('searched posts', latestPosts.length);
this.logger.info('searched posts', latestPosts.length);
for (const post of latestPosts) {
await this.checkAndSavePost(post);
}
}
private constructor() {
log.log('Constructing timeline object');
this.logger = new Logger('Timeline');
this.logger.log('Constructing timeline object');
this.playlistAdders = [new YoutubePlaylistAdder(), new SpotifyPlaylistAdder()];
this.startWebsocket();
this.loadPostsSinceLastRun()
.then((_) => {
log.info('loaded posts since last run');
this.logger.info('loaded posts since last run');
})
.catch((e) => {
log.error('cannot fetch latest posts', e);
this.logger.error('cannot fetch latest posts', e);
});
}
public static init() {
log.log('Timeline object init');
if (this._instance === undefined) {
this._instance = new TimelineReader();
}