phlaym/moshing-mammut
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases 8 Activity

update dependencies, add songs to youtube playlist

Browse Source
This commit is contained in:
Max Nuding
2025-07-01 16:01:19 +02:00
parent 3c6e742e43
commit b7a930c69a
12 changed files with 652 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files

178
src/lib/server/timeline.ts
View File

@ -26,10 +26,13 @@ import {
saveSongThumbnail
} from '$lib/server/db';
import { createFeed, saveAtomFeed } from '$lib/server/rss';
import { YoutubePlaylistAdder } from '$lib/server/ytPlaylistAdder';
import { sleep } from '$lib/sleep';
import crypto from 'crypto';
import fs from 'fs/promises';
import { console } from 'inspector/promises';
import sharp from 'sharp';
import { URL, URLSearchParams } from 'url';
import { WebSocket } from 'ws';
const URL_REGEX = new RegExp(/href="(?<postUrl>[^>]+?)" target="_blank"/gm);
@ -40,10 +43,13 @@ const YOUTUBE_REGEX = new RegExp(
export class TimelineReader {
private static _instance: TimelineReader;
private lastPosts: string[] = [];
private youtubePlaylistAdder: YoutubePlaylistAdder;
private static async isMusicVideo(videoId: string) {
if (!YOUTUBE_API_KEY || YOUTUBE_API_KEY === 'CHANGE_ME') {
// Assume that it *is* a music link when no YT API key is provided
log.debug('YT API not configured');
return true;
}
const searchParams = new URLSearchParams([
@ -55,13 +61,13 @@ export class TimelineReader {
const resp = await fetch(youtubeVideoUrl);
const respObj = await resp.json();
if (!respObj.items.length) {
console.warn('Could not find video with id', videoId);
log.warn('Could not find video with id', videoId);
return false;
}
const item = respObj.items[0];
if (!item.snippet) {
console.warn('Could not load snippet for video', videoId, item);
log.warn('Could not load snippet for video', videoId, item);
return false;
}
if (item.snippet.tags?.includes('music')) {
@ -79,6 +85,7 @@ export class TimelineReader {
const categoryTitle: string = await fetch(youtubeCategoryUrl)
.then((r) => r.json())
.then((r) => r.items[0]?.snippet?.title);
log.debug('YT category', categoryTitle);
return categoryTitle === 'Music';
}
@ -102,7 +109,7 @@ export class TimelineReader {
// Check *all* found url and let odesli determine if it is music or not
log.debug(`Checking ${url} if it contains song data`);
const info = await TimelineReader.getSongInfo(url);
log.debug(`Found song info for ${url}?`, info);
//log.debug(`Found song info for ${url}?`, info);
if (info) {
songs.push(info);
}
@ -144,6 +151,7 @@ export class TimelineReader {
return null;
}
const info = odesliInfo.entitiesByUniqueId[odesliInfo.entityUniqueId];
//log.debug('odesli response', info);
const platform: Platform = 'youtube';
if (info.platforms.includes(platform)) {
const youtubeId =
@ -156,7 +164,7 @@ export class TimelineReader {
}
const isMusic = await TimelineReader.isMusicVideo(youtubeId);
if (!isMusic) {
log.debug('Probably not a music video', url);
log.debug('Probably not a music video', youtubeId, url);
return null;
}
}
@ -177,6 +185,88 @@ export class TimelineReader {
}
}
/*
private async addToYoutubePlaylist(song: SongInfo) {
log.debug('addToYoutubePlaylist');
let token: OauthResponse;
try {
const youtube_token_file = await fs.readFile('yt_auth_token', { encoding: 'utf8' });
token = JSON.parse(youtube_token_file);
log.debug('read youtube access token', token);
} catch (e) {
log.error('Could not read youtube access token', e);
return;
}
if (!YOUTUBE_PLAYLIST_ID || YOUTUBE_PLAYLIST_ID === 'CHANGE_ME') {
log.debug('no playlist ID configured');
return;
}
if (!song.youtubeUrl) {
log.debug('Skip adding song to YT playlist, no youtube Url', song);
return;
}
const songUrl = new URL(song.youtubeUrl);
const youtubeId = songUrl.searchParams.get('v');
if (!youtubeId) {
log.debug(
'Skip adding song to YT playlist, could not extract YT id from URL',
song.youtubeUrl
);
return;
}
log.debug('Found YT id from URL', song.youtubeUrl, youtubeId);
const playlistItemsUrl = new URL('https://www.googleapis.com/youtube/v3/playlistItems');
playlistItemsUrl.searchParams.append('videoId', youtubeId);
playlistItemsUrl.searchParams.append('playlistId', YOUTUBE_PLAYLIST_ID);
playlistItemsUrl.searchParams.append('part', 'id');
const existingPlaylistItem = await fetch(
'https://www.googleapis.com/youtube/v3/playlistItems',
{
headers: { Authorization: `${token.token_type} ${token.access_token}` }
}
).then((r) => r.json());
log.debug('existingPlaylistItem', existingPlaylistItem);
if (existingPlaylistItem.pageInfo && existingPlaylistItem.pageInfo.totalResults > 0) {
log.info('Item already in playlist');
return;
}
const searchParams = new URLSearchParams([
['part', 'snippet']
//['key', token.access_token]
]);
const options: RequestInit = {
method: 'POST',
headers: { Authorization: `${token.token_type} ${token.access_token}` },
body: JSON.stringify({
snippet: {
playlistId: YOUTUBE_PLAYLIST_ID,
resourceId: {
videoId: youtubeId,
kind: 'youtube#video'
}
}
})
};
const youtubeApiUrl = new URL(
`https://www.googleapis.com/youtube/v3/playlistItems?${searchParams}`
);
const resp = await fetch(youtubeApiUrl, options);
const respObj = await resp.json();
log.debug('Added to playlist', options, respObj);
if (respObj.error) {
log.debug('Add to playlist failed', respObj.error.errors);
}
}
*/
private async addToPlaylist(song: SongInfo) {
//await this.addToYoutubePlaylist(song);
await this.youtubePlaylistAdder.addToPlaylist(song);
}
private static async resizeAvatar(
baseName: string,
size: number,
@ -370,10 +460,15 @@ export class TimelineReader {
await TimelineReader.saveAvatar(post.account);
await TimelineReader.saveSongThumbnails(songs);
log.debug('Saved post', post.url);
log.debug('Saved post', post.url, 'songs', songs);
const posts = await getPosts(null, null, 100);
await saveAtomFeed(createFeed(posts));
for (let song of songs) {
log.debug('Adding to playlist', song);
await this.addToPlaylist(song);
}
}
private startWebsocket() {
@ -385,12 +480,76 @@ export class TimelineReader {
};
socket.onmessage = async (event) => {
try {
/*
let token: OauthResponse;
try {
const youtube_token_file = await fs.readFile('yt_auth_token', { encoding: 'utf8' });
token = JSON.parse(youtube_token_file);
if (token.expires) {
if (typeof token.expires === typeof '') {
token.expires = new Date(token.expires);
}
let now = new Date();
now.setTime(now.getTime() - 15 * 60 * 1000);
log.info('token expiry', token.expires, 'vs refresh @', now);
if (token.expires.getTime() <= now.getTime()) {
log.info(
'YT token expires',
token.expires,
token.expires.getTime(),
'which is less than 15 minutes from now',
now,
now.getTime()
);
const tokenUrl = new URL('https://oauth2.googleapis.com/token');
const params = new URLSearchParams();
params.append('client_id', YOUTUBE_CLIENT_ID);
params.append('client_secret', YOUTUBE_CLIENT_SECRET);
params.append('refresh_token', token.refresh_token || '');
params.append('grant_type', 'refresh_token');
params.append('redirect_uri', `${BASE_URL}/ytauth`);
if (token.refresh_token) {
log.debug('sending token req', params);
const resp = await fetch(tokenUrl, {
method: 'POST',
body: params
}).then((r) => r.json());
if (!resp.error) {
if (!resp.refresh_token) {
resp.refresh_token = token.refresh_token;
}
let expiration = new Date();
expiration.setSeconds(expiration.getSeconds() + resp.expires_in);
resp.expires = expiration;
await fs.writeFile('yt_auth_token', JSON.stringify(resp));
} else {
log.error('token resp error', resp);
}
} else {
log.error('no refresg token');
}
}
}
} catch (e) {
log.error('onmessage Could not read youtube access token', e);
}
*/
const data: TimelineEvent = JSON.parse(event.data.toString());
log.debug('ES event', data.event);
if (data.event !== 'update') {
log.log('Ignoring ES event', data.event);
return;
}
const post: Post = JSON.parse(data.payload);
if (this.lastPosts.includes(post.id)) {
log.log('Skipping post, already handled', post.id);
return;
}
this.lastPosts.push(post.id);
while (this.lastPosts.length > 10) {
this.lastPosts.shift();
}
await this.checkAndSavePost(post);
} catch (e) {
log.error('error message', event, event.data, e);
@ -416,7 +575,11 @@ export class TimelineReader {
private async loadPostsSinceLastRun() {
const now = new Date().toISOString();
let latestPost = await getPosts(null, now, 1);
log.log('Last post in DB since', now, latestPost);
if (latestPost.length > 0) {
log.log('Last post in DB since', now, latestPost[0].created_at);
} else {
log.log('No posts in DB since');
}
let u = new URL(`https://${MASTODON_INSTANCE}/api/v1/timelines/public?local=true&limit=40`);
if (latestPost.length > 0) {
u.searchParams.append('since_id', latestPost[0].id);
@ -428,7 +591,7 @@ export class TimelineReader {
Authorization: `Bearer ${MASTODON_ACCESS_TOKEN}`
};
const latestPosts: Post[] = await fetch(u, { headers }).then((r) => r.json());
log.info('searched posts', latestPosts);
log.info('searched posts', latestPosts.length);
for (const post of latestPosts) {
await this.checkAndSavePost(post);
}
@ -436,6 +599,7 @@ export class TimelineReader {
private constructor() {
log.log('Constructing timeline object');
this.youtubePlaylistAdder = new YoutubePlaylistAdder();
this.startWebsocket();
this.loadPostsSinceLastRun()