phlaym/moshing-mammut
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases 8 Activity

update logging

Browse Source
This commit is contained in:
Max Nuding
2025-07-04 08:46:41 +02:00
parent b0465a020d
commit 77e483d637
8 changed files with 96 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/hooks.server.ts
View File

@ -1,15 +1,17 @@
import { log } from '$lib/log';
import { Logger } from '$lib/log';
import { TimelineReader } from '$lib/server/timeline';
import type { Handle, HandleServerError } from '@sveltejs/kit';
import { error } from '@sveltejs/kit';
import fs from 'fs/promises';
log.log('App startup');
const logger = new Logger('App');
logger.log('App startup');
TimelineReader.init();
export const handleError = (({ error }) => {
if (error instanceof Error) {
log.error('Something went wrong: ', error.name, error.message);
logger.error('Something went wrong: ', error.name, error.message);
}
return {
@ -21,7 +23,7 @@ export const handle = (async ({ event, resolve }) => {
const searchParams = event.url.searchParams;
const authCode = searchParams.get('code');
if (authCode) {
log.debug('received GET hook', event.url.searchParams);
logger.debug('received GET hook', event.url.searchParams);
}
// Reeder *insists* on checking /feed instead of /feed.xml
@ -45,7 +47,7 @@ export const handle = (async ({ event, resolve }) => {
const readStream = fd
.readableWebStream()
.getReader({ mode: 'byob' }) as ReadableStream<Uint8Array>;
log.info('sending. size: ', stat.size);
logger.info('sending. size: ', stat.size);
return new Response(readStream, {
headers: [
['Content-Type', 'image/' + suffix],
@ -57,7 +59,7 @@ export const handle = (async ({ event, resolve }) => {
const f = await fs.readFile('avatars/' + fileName);
return new Response(f, { headers: [['Content-Type', 'image/' + suffix]] });
} catch (e) {
log.error('no stream', e);
logger.error('no stream', e);
error(404);
}
}

3
src/lib/log.ts
View File

@ -4,6 +4,9 @@ const { DEV } = import.meta.env;
export const enableVerboseLog = isTruthy(env.VERBOSE);
/**
* @deprecated Use the new {@link Logger} class instead.
*/
export const log = {
verbose: (...params: any[]) => {
if (!enableVerboseLog) {

5
src/lib/server/playlist/spotifyPlaylistAdder.ts
View File

@ -79,6 +79,7 @@ export class SpotifyPlaylistAdder extends OauthPlaylistAdder implements Playlist
return;
}
// TODO. Spotify's check for "is this song already in the playlist" is... ugh
/*
const playlistItemsUrl = new URL(`${this.apiBase}/playlists/${SPOTIFY_PLAYLIST_ID}/tracks`);
playlistItemsUrl.searchParams.append('videoId', youtubeId);
@ -87,9 +88,9 @@ export class SpotifyPlaylistAdder extends OauthPlaylistAdder implements Playlist
/*const existingPlaylistItem = await fetch(this.apiBase + '/playlistItems', {
headers: { Authorization: `${token.token_type} ${token.access_token}` }
}).then((r) => r.json());
log.debug('existingPlaylistItem', existingPlaylistItem);
logger.debug('existingPlaylistItem', existingPlaylistItem);
if (existingPlaylistItem.pageInfo && existingPlaylistItem.pageInfo.totalResults > 0) {
log.info('Item already in playlist');
logger.info('Item already in playlist');
return;
}*/

23
src/lib/server/playlist/ytPlaylistAdder.ts
View File

@ -4,7 +4,7 @@ import {
YOUTUBE_CLIENT_SECRET,
YOUTUBE_PLAYLIST_ID
} from '$env/static/private';
import { log } from '$lib/log';
import { Logger } from '$lib/log';
import type { OauthResponse } from '$lib/mastodon/response';
import type { SongInfo } from '$lib/odesliResponse';
import { OauthPlaylistAdder } from './oauthPlaylistAdder';
@ -13,6 +13,7 @@ import type { PlaylistAdder } from './playlistAdder';
export class YoutubePlaylistAdder extends OauthPlaylistAdder implements PlaylistAdder {
public constructor() {
super('https://www.googleapis.com/youtube/v3', 'yt_auth_token');
this.logger = new Logger('YoutubePlaylistAdder');
}
public constructAuthUrl(redirectUri: URL): URL {
@ -31,7 +32,7 @@ export class YoutubePlaylistAdder extends OauthPlaylistAdder implements Playlist
}
public async receivedAuthCode(code: string, url: URL) {
log.debug('received code');
this.logger.debug('received code');
const tokenUrl = new URL('https://oauth2.googleapis.com/token');
await this.receivedAuthCodeInternal(
tokenUrl,
@ -52,7 +53,7 @@ export class YoutubePlaylistAdder extends OauthPlaylistAdder implements Playlist
return token;
}
if (!token.refresh_token) {
log.error('Need to refresh access token, but no refresh token provided');
this.logger.error('Need to refresh access token, but no refresh token provided');
return null;
}
@ -67,31 +68,31 @@ export class YoutubePlaylistAdder extends OauthPlaylistAdder implements Playlist
}
public async addToPlaylist(song: SongInfo) {
log.debug('addToYoutubePlaylist');
this.logger.debug('addToYoutubePlaylist');
const token = await this.refreshToken();
if (token == null) {
return;
}
if (!YOUTUBE_PLAYLIST_ID || YOUTUBE_PLAYLIST_ID === 'CHANGE_ME') {
log.debug('no playlist ID configured');
this.logger.debug('no playlist ID configured');
return;
}
if (!song.youtubeUrl) {
log.info('Skip adding song to YT playlist, no youtube Url', song);
this.logger.info('Skip adding song to YT playlist, no youtube Url', song);
return;
}
const songUrl = new URL(song.youtubeUrl);
const youtubeId = songUrl.searchParams.get('v');
if (!youtubeId) {
log.debug(
this.logger.debug(
'Skip adding song to YT playlist, could not extract YT id from URL',
song.youtubeUrl
);
return;
}
log.debug('Found YT id from URL', song.youtubeUrl, youtubeId);
this.logger.debug('Found YT id from URL', song.youtubeUrl, youtubeId);
const playlistItemsUrl = new URL(this.apiBase + '/playlistItems');
playlistItemsUrl.searchParams.append('videoId', youtubeId);
@ -101,7 +102,7 @@ export class YoutubePlaylistAdder extends OauthPlaylistAdder implements Playlist
headers: { Authorization: `${token.token_type} ${token.access_token}` }
}).then((r) => r.json());
if (existingPlaylistItem.pageInfo && existingPlaylistItem.pageInfo.totalResults > 0) {
log.info('Item already in playlist', existingPlaylistItem);
this.logger.info('Item already in playlist', existingPlaylistItem);
return;
}
@ -122,9 +123,9 @@ export class YoutubePlaylistAdder extends OauthPlaylistAdder implements Playlist
};
const resp = await fetch(addItemUrl, options);
const respObj = await resp.json();
log.info('Added to playlist', youtubeId, song.title);
this.logger.info('Added to playlist', youtubeId, song.title);
if (respObj.error) {
log.debug('Add to playlist failed', respObj.error.errors);
this.logger.debug('Add to playlist failed', respObj.error.errors);
}
}
}

6
src/lib/server/rss.ts
View File

@ -1,10 +1,12 @@
import { BASE_URL, WEBSUB_HUB } from '$env/static/private';
import { PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME } from '$env/static/public';
import type { Post } from '$lib//mastodon/response';
import { log } from '$lib/log';
import { Logger } from '$lib/log';
import { Feed } from 'feed';
import fs from 'fs/promises';
const logger = new Logger('RSS');
export function createFeed(posts: Post[]): Feed {
const baseUrl = BASE_URL.endsWith('/') ? BASE_URL : BASE_URL + '/';
const hub = WEBSUB_HUB ? WEBSUB_HUB : undefined;
@ -60,6 +62,6 @@ export async function saveAtomFeed(feed: Feed) {
body: params
});
} catch (e) {
log.error('Failed to update WebSub hub', e);
logger.error('Failed to update WebSub hub', e);
}
}

112
src/lib/server/timeline.ts
View File

@ -5,7 +5,7 @@ import {
ODESLI_API_KEY,
YOUTUBE_API_KEY
} from '$env/static/private';
import { log, Logger } from '$lib/log';
import { Logger } from '$lib/log';
import type {
Account,
AccountAvatar,
@ -47,11 +47,12 @@ export class TimelineReader {
private static _instance: TimelineReader;
private lastPosts: string[] = [];
private playlistAdders: PlaylistAdder[];
private logger: Logger;
private static async isMusicVideo(videoId: string) {
private async isMusicVideo(videoId: string) {
if (!YOUTUBE_API_KEY || YOUTUBE_API_KEY === 'CHANGE_ME') {
// Assume that it *is* a music link when no YT API key is provided
log.debug('YT API not configured');
this.logger.debug('YT API not configured');
return true;
}
const searchParams = new URLSearchParams([
@ -63,13 +64,13 @@ export class TimelineReader {
const resp = await fetch(youtubeVideoUrl);
const respObj = await resp.json();
if (!respObj.items.length) {
log.warn('Could not find video with id', videoId);
this.logger.warn('Could not find video with id', videoId);
return false;
}
const item = respObj.items[0];
if (!item.snippet) {
log.warn('Could not load snippet for video', videoId, item);
this.logger.warn('Could not load snippet for video', videoId, item);
return false;
}
if (item.snippet.tags?.includes('music')) {
@ -87,16 +88,19 @@ export class TimelineReader {
const categoryTitle: string = await fetch(youtubeCategoryUrl)
.then((r) => r.json())
.then((r) => r.items[0]?.snippet?.title);
log.debug('YT category', categoryTitle);
this.logger.debug('YT category', categoryTitle);
return categoryTitle === 'Music';
}
public static async getSongInfoInPost(post: Post): Promise<SongInfo[]> {
public async getSongInfoInPost(post: Post): Promise<SongInfo[]> {
const urlMatches = post.content.matchAll(URL_REGEX);
const songs: SongInfo[] = [];
for (const match of urlMatches) {
if (match === undefined || match.groups === undefined) {
log.warn('Match listed in allMatches, but either it or its groups are undefined', match);
this.logger.warn(
'Match listed in allMatches, but either it or its groups are undefined',
match
);
continue;
}
const urlMatch = match.groups.postUrl.toString();
@ -104,14 +108,14 @@ export class TimelineReader {
try {
url = new URL(urlMatch);
} catch (e) {
log.error('URL found via Regex does not seem to be a valud url', urlMatch, e);
this.logger.error('URL found via Regex does not seem to be a valud url', urlMatch, e);
continue;
}
// Check *all* found url and let odesli determine if it is music or not
log.debug(`Checking ${url} if it contains song data`);
const info = await TimelineReader.getSongInfo(url);
//log.debug(`Found song info for ${url}?`, info);
this.logger.debug(`Checking ${url} if it contains song data`);
const info = await this.getSongInfo(url);
//this.logger.debug(`Found song info for ${url}?`, info);
if (info) {
songs.push(info);
}
@ -119,9 +123,9 @@ export class TimelineReader {
return songs;
}
private static async getSongInfo(url: URL, remainingTries = 6): Promise<SongInfo | null> {
private async getSongInfo(url: URL, remainingTries = 6): Promise<SongInfo | null> {
if (remainingTries === 0) {
log.error('No tries remaining. Lookup failed!');
this.logger.error('No tries remaining. Lookup failed!');
return null;
}
if (url.hostname === 'songwhip.com') {
@ -153,7 +157,7 @@ export class TimelineReader {
return null;
}
const info = odesliInfo.entitiesByUniqueId[odesliInfo.entityUniqueId];
//log.debug('odesli response', info);
//this.logger.debug('odesli response', info);
const platform: Platform = 'youtube';
if (info.platforms.includes(platform)) {
const youtubeId =
@ -161,12 +165,16 @@ export class TimelineReader {
YOUTUBE_REGEX.exec(url.href)?.groups?.videoId ??
new URL(odesliInfo.pageUrl).pathname.split('/y/').pop();
if (youtubeId === undefined) {
log.warn('Looks like a youtube video, but could not extract a video id', url, odesliInfo);
this.logger.warn(
'Looks like a youtube video, but could not extract a video id',
url,
odesliInfo
);
return null;
}
const isMusic = await TimelineReader.isMusicVideo(youtubeId);
const isMusic = await this.isMusicVideo(youtubeId);
if (!isMusic) {
log.debug('Probably not a music video', youtubeId, url);
this.logger.debug('Probably not a music video', youtubeId, url);
return null;
}
}
@ -181,11 +189,11 @@ export class TimelineReader {
} as SongInfo;
} catch (e) {
if (e instanceof Error && e.cause === 429) {
log.warn('song.link rate limit reached. Trying again in 10 seconds');
this.logger.warn('song.link rate limit reached. Trying again in 10 seconds');
await sleep(10_000);
return await this.getSongInfo(url, remainingTries - 1);
}
log.error(`Failed to load ${url} info from song.link`, e);
this.logger.error(`Failed to load ${url} info from song.link`, e);
return null;
}
}
@ -196,7 +204,7 @@ export class TimelineReader {
}
}
private static async resizeAvatar(
private async resizeAvatar(
baseName: string,
size: number,
suffix: string,
@ -209,15 +217,15 @@ export class TimelineReader {
.then(() => true)
.catch(() => false);
if (exists) {
log.debug('File already exists', fileName);
this.logger.debug('File already exists', fileName);
return null;
}
log.debug('Saving avatar', fileName);
this.logger.debug('Saving avatar', fileName);
await sharpAvatar.resize(size).toFile(fileName);
return fileName;
}
private static resizeAvatarPromiseMaker(
private resizeAvatarPromiseMaker(
avatarFilenameBase: string,
baseSize: number,
maxPixelDensity: number,
@ -230,13 +238,7 @@ export class TimelineReader {
for (let i = 1; i <= maxPixelDensity; i++) {
promises.push(
...formats.map((f) =>
TimelineReader.resizeAvatar(
avatarFilenameBase,
baseSize * i,
`${i}x.${f}`,
'avatars',
sharpAvatar
)
this.resizeAvatar(avatarFilenameBase, baseSize * i, `${i}x.${f}`, 'avatars', sharpAvatar)
.then(
(fn) =>
({
@ -252,7 +254,7 @@ export class TimelineReader {
return promises;
}
private static resizeThumbnailPromiseMaker(
private resizeThumbnailPromiseMaker(
filenameBase: string,
baseSize: number,
maxPixelDensity: number,
@ -266,13 +268,7 @@ export class TimelineReader {
for (let i = 1; i <= maxPixelDensity; i++) {
promises.push(
...formats.map((f) =>
TimelineReader.resizeAvatar(
filenameBase,
baseSize * i,
`${i}x.${f}`,
'thumbnails',
sharpAvatar
)
this.resizeAvatar(filenameBase, baseSize * i, `${i}x.${f}`, 'thumbnails', sharpAvatar)
.then(
(fn) =>
({
@ -289,7 +285,7 @@ export class TimelineReader {
return promises;
}
private static async saveAvatar(account: Account) {
private async saveAvatar(account: Account) {
try {
const existingAvatars = await getAvatars(account.url, 1);
const existingAvatarBase = existingAvatars.shift()?.file.split('/').pop()?.split('_').shift();
@ -302,7 +298,7 @@ export class TimelineReader {
const avatarsToDelete = (await fs.readdir('avatars'))
.filter((x) => x.startsWith(existingAvatarBase + '_'))
.map((x) => {
log.debug('Removing existing avatar file', x);
this.logger.debug('Removing existing avatar file', x);
return x;
})
.map((x) => fs.unlink('avatars/' + x));
@ -311,7 +307,7 @@ export class TimelineReader {
const avatarResponse = await fetch(account.avatar);
const avatar = await avatarResponse.arrayBuffer();
await Promise.all(
TimelineReader.resizeAvatarPromiseMaker(
this.resizeAvatarPromiseMaker(
avatarFilenameBase,
50,
3,
@ -325,7 +321,7 @@ export class TimelineReader {
}
}
private static async saveSongThumbnails(songs: SongInfo[]) {
private async saveSongThumbnails(songs: SongInfo[]) {
for (const song of songs) {
if (!song.thumbnailUrl) {
continue;
@ -339,7 +335,7 @@ export class TimelineReader {
const imageResponse = await fetch(song.thumbnailUrl);
const avatar = await imageResponse.arrayBuffer();
await Promise.all(
TimelineReader.resizeThumbnailPromiseMaker(
this.resizeThumbnailPromiseMaker(
fileBaseName + '_large',
200,
3,
@ -350,7 +346,7 @@ export class TimelineReader {
)
);
await Promise.all(
TimelineReader.resizeThumbnailPromiseMaker(
this.resizeThumbnailPromiseMaker(
fileBaseName + '_small',
60,
3,
@ -375,27 +371,27 @@ export class TimelineReader {
const hashttags: string[] = HASHTAG_FILTER.split(',');
const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
const songs = await TimelineReader.getSongInfoInPost(post);
const songs = await this.getSongInfoInPost(post);
// If we don't have any tags or non-youtube urls, check youtube
// YT is handled separately, because it requires an API call and therefore is slower
if (songs.length === 0 && found_tags.length === 0) {
log.log('Ignoring post', post.url);
this.logger.log('Ignoring post', post.url);
return;
}
await savePost(post, songs);
await TimelineReader.saveAvatar(post.account);
await TimelineReader.saveSongThumbnails(songs);
await this.saveAvatar(post.account);
await this.saveSongThumbnails(songs);
log.debug('Saved post', post.url, 'songs', songs);
this.logger.debug('Saved post', post.url, 'songs', songs);
const posts = await getPosts(null, null, 100);
await saveAtomFeed(createFeed(posts));
for (let song of songs) {
log.debug('Adding to playlist', song);
this.logger.debug('Adding to playlist', song);
await this.addToPlaylist(song);
}
}
@ -455,9 +451,9 @@ export class TimelineReader {
const now = new Date().toISOString();
let latestPost = await getPosts(null, now, 1);
if (latestPost.length > 0) {
log.log('Last post in DB since', now, latestPost[0].created_at);
this.logger.log('Last post in DB since', now, latestPost[0].created_at);
} else {
log.log('No posts in DB since');
this.logger.log('No posts in DB since');
}
let u = new URL(`https://${MASTODON_INSTANCE}/api/v1/timelines/public?local=true&limit=40`);
if (latestPost.length > 0) {
@ -470,28 +466,28 @@ export class TimelineReader {
Authorization: `Bearer ${MASTODON_ACCESS_TOKEN}`
};
const latestPosts: Post[] = await fetch(u, { headers }).then((r) => r.json());
log.info('searched posts', latestPosts.length);
this.logger.info('searched posts', latestPosts.length);
for (const post of latestPosts) {
await this.checkAndSavePost(post);
}
}
private constructor() {
log.log('Constructing timeline object');
this.logger = new Logger('Timeline');
this.logger.log('Constructing timeline object');
this.playlistAdders = [new YoutubePlaylistAdder(), new SpotifyPlaylistAdder()];
this.startWebsocket();
this.loadPostsSinceLastRun()
.then((_) => {
log.info('loaded posts since last run');
this.logger.info('loaded posts since last run');
})
.catch((e) => {
log.error('cannot fetch latest posts', e);
this.logger.error('cannot fetch latest posts', e);
});
}
public static init() {
log.log('Timeline object init');
if (this._instance === undefined) {
this._instance = new TimelineReader();
}

10
src/routes/spotifyAuth/+page.server.ts
View File

@ -1,20 +1,22 @@
import { log } from '$lib/log';
import { Logger } from '$lib/log';
import { SpotifyPlaylistAdder } from '$lib/server/playlist/spotifyPlaylistAdder';
import { redirect } from '@sveltejs/kit';
import type { PageServerLoad } from './$types';
const logger = new Logger('SpotifyAuth');
export const load: PageServerLoad = async ({ url }) => {
const adder = new SpotifyPlaylistAdder();
let redirectUri = url;
if (url.hostname === 'localhost') {
redirectUri.hostname = '127.0.0.1';
}
log.debug(url.searchParams, url.hostname);
logger.debug(url.searchParams, url.hostname);
if (url.searchParams.has('code')) {
await adder.receivedAuthCode(url.searchParams.get('code') || '', url);
redirect(307, '/');
} else if (url.searchParams.has('error')) {
log.error('received error', url.searchParams.get('error'));
logger.error('received error', url.searchParams.get('error'));
return;
}
@ -23,6 +25,6 @@ export const load: PageServerLoad = async ({ url }) => {
}
const authUrl = adder.constructAuthUrl(url);
log.debug('+page.server.ts', authUrl.toString());
logger.debug('+page.server.ts', authUrl.toString());
redirect(307, authUrl);
};

10
src/routes/ytauth/+page.server.ts
View File

@ -1,16 +1,18 @@
import { log } from '$lib/log';
import { Logger } from '$lib/log';
import { YoutubePlaylistAdder } from '$lib/server/playlist/ytPlaylistAdder';
import { redirect } from '@sveltejs/kit';
import type { PageServerLoad } from './$types';
const logger = new Logger('YT Auth');
export const load: PageServerLoad = async ({ url }) => {
const adder = new YoutubePlaylistAdder();
if (url.searchParams.has('code')) {
log.debug(url.searchParams);
logger.debug(url.searchParams);
await adder.receivedAuthCode(url.searchParams.get('code') || '', url);
redirect(307, '/');
} else if (url.searchParams.has('error')) {
log.error('received error', url.searchParams.get('error'));
logger.error('received error', url.searchParams.get('error'));
return;
}
@ -19,6 +21,6 @@ export const load: PageServerLoad = async ({ url }) => {
}
const authUrl = adder.constructAuthUrl(url);
log.debug('+page.server.ts', authUrl.toString());
logger.debug('+page.server.ts', authUrl.toString());
redirect(307, authUrl);
};