update logging
This commit is contained in:
@ -5,7 +5,7 @@ import {
|
||||
ODESLI_API_KEY,
|
||||
YOUTUBE_API_KEY
|
||||
} from '$env/static/private';
|
||||
import { log, Logger } from '$lib/log';
|
||||
import { Logger } from '$lib/log';
|
||||
import type {
|
||||
Account,
|
||||
AccountAvatar,
|
||||
@ -47,11 +47,12 @@ export class TimelineReader {
|
||||
private static _instance: TimelineReader;
|
||||
private lastPosts: string[] = [];
|
||||
private playlistAdders: PlaylistAdder[];
|
||||
private logger: Logger;
|
||||
|
||||
private static async isMusicVideo(videoId: string) {
|
||||
private async isMusicVideo(videoId: string) {
|
||||
if (!YOUTUBE_API_KEY || YOUTUBE_API_KEY === 'CHANGE_ME') {
|
||||
// Assume that it *is* a music link when no YT API key is provided
|
||||
log.debug('YT API not configured');
|
||||
this.logger.debug('YT API not configured');
|
||||
return true;
|
||||
}
|
||||
const searchParams = new URLSearchParams([
|
||||
@ -63,13 +64,13 @@ export class TimelineReader {
|
||||
const resp = await fetch(youtubeVideoUrl);
|
||||
const respObj = await resp.json();
|
||||
if (!respObj.items.length) {
|
||||
log.warn('Could not find video with id', videoId);
|
||||
this.logger.warn('Could not find video with id', videoId);
|
||||
return false;
|
||||
}
|
||||
|
||||
const item = respObj.items[0];
|
||||
if (!item.snippet) {
|
||||
log.warn('Could not load snippet for video', videoId, item);
|
||||
this.logger.warn('Could not load snippet for video', videoId, item);
|
||||
return false;
|
||||
}
|
||||
if (item.snippet.tags?.includes('music')) {
|
||||
@ -87,16 +88,19 @@ export class TimelineReader {
|
||||
const categoryTitle: string = await fetch(youtubeCategoryUrl)
|
||||
.then((r) => r.json())
|
||||
.then((r) => r.items[0]?.snippet?.title);
|
||||
log.debug('YT category', categoryTitle);
|
||||
this.logger.debug('YT category', categoryTitle);
|
||||
return categoryTitle === 'Music';
|
||||
}
|
||||
|
||||
public static async getSongInfoInPost(post: Post): Promise<SongInfo[]> {
|
||||
public async getSongInfoInPost(post: Post): Promise<SongInfo[]> {
|
||||
const urlMatches = post.content.matchAll(URL_REGEX);
|
||||
const songs: SongInfo[] = [];
|
||||
for (const match of urlMatches) {
|
||||
if (match === undefined || match.groups === undefined) {
|
||||
log.warn('Match listed in allMatches, but either it or its groups are undefined', match);
|
||||
this.logger.warn(
|
||||
'Match listed in allMatches, but either it or its groups are undefined',
|
||||
match
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const urlMatch = match.groups.postUrl.toString();
|
||||
@ -104,14 +108,14 @@ export class TimelineReader {
|
||||
try {
|
||||
url = new URL(urlMatch);
|
||||
} catch (e) {
|
||||
log.error('URL found via Regex does not seem to be a valud url', urlMatch, e);
|
||||
this.logger.error('URL found via Regex does not seem to be a valud url', urlMatch, e);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check *all* found url and let odesli determine if it is music or not
|
||||
log.debug(`Checking ${url} if it contains song data`);
|
||||
const info = await TimelineReader.getSongInfo(url);
|
||||
//log.debug(`Found song info for ${url}?`, info);
|
||||
this.logger.debug(`Checking ${url} if it contains song data`);
|
||||
const info = await this.getSongInfo(url);
|
||||
//this.logger.debug(`Found song info for ${url}?`, info);
|
||||
if (info) {
|
||||
songs.push(info);
|
||||
}
|
||||
@ -119,9 +123,9 @@ export class TimelineReader {
|
||||
return songs;
|
||||
}
|
||||
|
||||
private static async getSongInfo(url: URL, remainingTries = 6): Promise<SongInfo | null> {
|
||||
private async getSongInfo(url: URL, remainingTries = 6): Promise<SongInfo | null> {
|
||||
if (remainingTries === 0) {
|
||||
log.error('No tries remaining. Lookup failed!');
|
||||
this.logger.error('No tries remaining. Lookup failed!');
|
||||
return null;
|
||||
}
|
||||
if (url.hostname === 'songwhip.com') {
|
||||
@ -153,7 +157,7 @@ export class TimelineReader {
|
||||
return null;
|
||||
}
|
||||
const info = odesliInfo.entitiesByUniqueId[odesliInfo.entityUniqueId];
|
||||
//log.debug('odesli response', info);
|
||||
//this.logger.debug('odesli response', info);
|
||||
const platform: Platform = 'youtube';
|
||||
if (info.platforms.includes(platform)) {
|
||||
const youtubeId =
|
||||
@ -161,12 +165,16 @@ export class TimelineReader {
|
||||
YOUTUBE_REGEX.exec(url.href)?.groups?.videoId ??
|
||||
new URL(odesliInfo.pageUrl).pathname.split('/y/').pop();
|
||||
if (youtubeId === undefined) {
|
||||
log.warn('Looks like a youtube video, but could not extract a video id', url, odesliInfo);
|
||||
this.logger.warn(
|
||||
'Looks like a youtube video, but could not extract a video id',
|
||||
url,
|
||||
odesliInfo
|
||||
);
|
||||
return null;
|
||||
}
|
||||
const isMusic = await TimelineReader.isMusicVideo(youtubeId);
|
||||
const isMusic = await this.isMusicVideo(youtubeId);
|
||||
if (!isMusic) {
|
||||
log.debug('Probably not a music video', youtubeId, url);
|
||||
this.logger.debug('Probably not a music video', youtubeId, url);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -181,11 +189,11 @@ export class TimelineReader {
|
||||
} as SongInfo;
|
||||
} catch (e) {
|
||||
if (e instanceof Error && e.cause === 429) {
|
||||
log.warn('song.link rate limit reached. Trying again in 10 seconds');
|
||||
this.logger.warn('song.link rate limit reached. Trying again in 10 seconds');
|
||||
await sleep(10_000);
|
||||
return await this.getSongInfo(url, remainingTries - 1);
|
||||
}
|
||||
log.error(`Failed to load ${url} info from song.link`, e);
|
||||
this.logger.error(`Failed to load ${url} info from song.link`, e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -196,7 +204,7 @@ export class TimelineReader {
|
||||
}
|
||||
}
|
||||
|
||||
private static async resizeAvatar(
|
||||
private async resizeAvatar(
|
||||
baseName: string,
|
||||
size: number,
|
||||
suffix: string,
|
||||
@ -209,15 +217,15 @@ export class TimelineReader {
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
if (exists) {
|
||||
log.debug('File already exists', fileName);
|
||||
this.logger.debug('File already exists', fileName);
|
||||
return null;
|
||||
}
|
||||
log.debug('Saving avatar', fileName);
|
||||
this.logger.debug('Saving avatar', fileName);
|
||||
await sharpAvatar.resize(size).toFile(fileName);
|
||||
return fileName;
|
||||
}
|
||||
|
||||
private static resizeAvatarPromiseMaker(
|
||||
private resizeAvatarPromiseMaker(
|
||||
avatarFilenameBase: string,
|
||||
baseSize: number,
|
||||
maxPixelDensity: number,
|
||||
@ -230,13 +238,7 @@ export class TimelineReader {
|
||||
for (let i = 1; i <= maxPixelDensity; i++) {
|
||||
promises.push(
|
||||
...formats.map((f) =>
|
||||
TimelineReader.resizeAvatar(
|
||||
avatarFilenameBase,
|
||||
baseSize * i,
|
||||
`${i}x.${f}`,
|
||||
'avatars',
|
||||
sharpAvatar
|
||||
)
|
||||
this.resizeAvatar(avatarFilenameBase, baseSize * i, `${i}x.${f}`, 'avatars', sharpAvatar)
|
||||
.then(
|
||||
(fn) =>
|
||||
({
|
||||
@ -252,7 +254,7 @@ export class TimelineReader {
|
||||
return promises;
|
||||
}
|
||||
|
||||
private static resizeThumbnailPromiseMaker(
|
||||
private resizeThumbnailPromiseMaker(
|
||||
filenameBase: string,
|
||||
baseSize: number,
|
||||
maxPixelDensity: number,
|
||||
@ -266,13 +268,7 @@ export class TimelineReader {
|
||||
for (let i = 1; i <= maxPixelDensity; i++) {
|
||||
promises.push(
|
||||
...formats.map((f) =>
|
||||
TimelineReader.resizeAvatar(
|
||||
filenameBase,
|
||||
baseSize * i,
|
||||
`${i}x.${f}`,
|
||||
'thumbnails',
|
||||
sharpAvatar
|
||||
)
|
||||
this.resizeAvatar(filenameBase, baseSize * i, `${i}x.${f}`, 'thumbnails', sharpAvatar)
|
||||
.then(
|
||||
(fn) =>
|
||||
({
|
||||
@ -289,7 +285,7 @@ export class TimelineReader {
|
||||
return promises;
|
||||
}
|
||||
|
||||
private static async saveAvatar(account: Account) {
|
||||
private async saveAvatar(account: Account) {
|
||||
try {
|
||||
const existingAvatars = await getAvatars(account.url, 1);
|
||||
const existingAvatarBase = existingAvatars.shift()?.file.split('/').pop()?.split('_').shift();
|
||||
@ -302,7 +298,7 @@ export class TimelineReader {
|
||||
const avatarsToDelete = (await fs.readdir('avatars'))
|
||||
.filter((x) => x.startsWith(existingAvatarBase + '_'))
|
||||
.map((x) => {
|
||||
log.debug('Removing existing avatar file', x);
|
||||
this.logger.debug('Removing existing avatar file', x);
|
||||
return x;
|
||||
})
|
||||
.map((x) => fs.unlink('avatars/' + x));
|
||||
@ -311,7 +307,7 @@ export class TimelineReader {
|
||||
const avatarResponse = await fetch(account.avatar);
|
||||
const avatar = await avatarResponse.arrayBuffer();
|
||||
await Promise.all(
|
||||
TimelineReader.resizeAvatarPromiseMaker(
|
||||
this.resizeAvatarPromiseMaker(
|
||||
avatarFilenameBase,
|
||||
50,
|
||||
3,
|
||||
@ -325,7 +321,7 @@ export class TimelineReader {
|
||||
}
|
||||
}
|
||||
|
||||
private static async saveSongThumbnails(songs: SongInfo[]) {
|
||||
private async saveSongThumbnails(songs: SongInfo[]) {
|
||||
for (const song of songs) {
|
||||
if (!song.thumbnailUrl) {
|
||||
continue;
|
||||
@ -339,7 +335,7 @@ export class TimelineReader {
|
||||
const imageResponse = await fetch(song.thumbnailUrl);
|
||||
const avatar = await imageResponse.arrayBuffer();
|
||||
await Promise.all(
|
||||
TimelineReader.resizeThumbnailPromiseMaker(
|
||||
this.resizeThumbnailPromiseMaker(
|
||||
fileBaseName + '_large',
|
||||
200,
|
||||
3,
|
||||
@ -350,7 +346,7 @@ export class TimelineReader {
|
||||
)
|
||||
);
|
||||
await Promise.all(
|
||||
TimelineReader.resizeThumbnailPromiseMaker(
|
||||
this.resizeThumbnailPromiseMaker(
|
||||
fileBaseName + '_small',
|
||||
60,
|
||||
3,
|
||||
@ -375,27 +371,27 @@ export class TimelineReader {
|
||||
const hashttags: string[] = HASHTAG_FILTER.split(',');
|
||||
const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
|
||||
|
||||
const songs = await TimelineReader.getSongInfoInPost(post);
|
||||
const songs = await this.getSongInfoInPost(post);
|
||||
|
||||
// If we don't have any tags or non-youtube urls, check youtube
|
||||
// YT is handled separately, because it requires an API call and therefore is slower
|
||||
if (songs.length === 0 && found_tags.length === 0) {
|
||||
log.log('Ignoring post', post.url);
|
||||
this.logger.log('Ignoring post', post.url);
|
||||
return;
|
||||
}
|
||||
|
||||
await savePost(post, songs);
|
||||
|
||||
await TimelineReader.saveAvatar(post.account);
|
||||
await TimelineReader.saveSongThumbnails(songs);
|
||||
await this.saveAvatar(post.account);
|
||||
await this.saveSongThumbnails(songs);
|
||||
|
||||
log.debug('Saved post', post.url, 'songs', songs);
|
||||
this.logger.debug('Saved post', post.url, 'songs', songs);
|
||||
|
||||
const posts = await getPosts(null, null, 100);
|
||||
await saveAtomFeed(createFeed(posts));
|
||||
|
||||
for (let song of songs) {
|
||||
log.debug('Adding to playlist', song);
|
||||
this.logger.debug('Adding to playlist', song);
|
||||
await this.addToPlaylist(song);
|
||||
}
|
||||
}
|
||||
@ -455,9 +451,9 @@ export class TimelineReader {
|
||||
const now = new Date().toISOString();
|
||||
let latestPost = await getPosts(null, now, 1);
|
||||
if (latestPost.length > 0) {
|
||||
log.log('Last post in DB since', now, latestPost[0].created_at);
|
||||
this.logger.log('Last post in DB since', now, latestPost[0].created_at);
|
||||
} else {
|
||||
log.log('No posts in DB since');
|
||||
this.logger.log('No posts in DB since');
|
||||
}
|
||||
let u = new URL(`https://${MASTODON_INSTANCE}/api/v1/timelines/public?local=true&limit=40`);
|
||||
if (latestPost.length > 0) {
|
||||
@ -470,28 +466,28 @@ export class TimelineReader {
|
||||
Authorization: `Bearer ${MASTODON_ACCESS_TOKEN}`
|
||||
};
|
||||
const latestPosts: Post[] = await fetch(u, { headers }).then((r) => r.json());
|
||||
log.info('searched posts', latestPosts.length);
|
||||
this.logger.info('searched posts', latestPosts.length);
|
||||
for (const post of latestPosts) {
|
||||
await this.checkAndSavePost(post);
|
||||
}
|
||||
}
|
||||
|
||||
private constructor() {
|
||||
log.log('Constructing timeline object');
|
||||
this.logger = new Logger('Timeline');
|
||||
this.logger.log('Constructing timeline object');
|
||||
this.playlistAdders = [new YoutubePlaylistAdder(), new SpotifyPlaylistAdder()];
|
||||
this.startWebsocket();
|
||||
|
||||
this.loadPostsSinceLastRun()
|
||||
.then((_) => {
|
||||
log.info('loaded posts since last run');
|
||||
this.logger.info('loaded posts since last run');
|
||||
})
|
||||
.catch((e) => {
|
||||
log.error('cannot fetch latest posts', e);
|
||||
this.logger.error('cannot fetch latest posts', e);
|
||||
});
|
||||
}
|
||||
|
||||
public static init() {
|
||||
log.log('Timeline object init');
|
||||
if (this._instance === undefined) {
|
||||
this._instance = new TimelineReader();
|
||||
}
|
||||
|
Reference in New Issue
Block a user