update dependencies, add songs to youtube playlist
This commit is contained in:
@ -26,10 +26,13 @@ import {
|
||||
saveSongThumbnail
|
||||
} from '$lib/server/db';
|
||||
import { createFeed, saveAtomFeed } from '$lib/server/rss';
|
||||
import { YoutubePlaylistAdder } from '$lib/server/ytPlaylistAdder';
|
||||
import { sleep } from '$lib/sleep';
|
||||
import crypto from 'crypto';
|
||||
import fs from 'fs/promises';
|
||||
import { console } from 'inspector/promises';
|
||||
import sharp from 'sharp';
|
||||
import { URL, URLSearchParams } from 'url';
|
||||
import { WebSocket } from 'ws';
|
||||
|
||||
const URL_REGEX = new RegExp(/href="(?<postUrl>[^>]+?)" target="_blank"/gm);
|
||||
@ -40,10 +43,13 @@ const YOUTUBE_REGEX = new RegExp(
|
||||
|
||||
export class TimelineReader {
|
||||
private static _instance: TimelineReader;
|
||||
private lastPosts: string[] = [];
|
||||
private youtubePlaylistAdder: YoutubePlaylistAdder;
|
||||
|
||||
private static async isMusicVideo(videoId: string) {
|
||||
if (!YOUTUBE_API_KEY || YOUTUBE_API_KEY === 'CHANGE_ME') {
|
||||
// Assume that it *is* a music link when no YT API key is provided
|
||||
log.debug('YT API not configured');
|
||||
return true;
|
||||
}
|
||||
const searchParams = new URLSearchParams([
|
||||
@ -55,13 +61,13 @@ export class TimelineReader {
|
||||
const resp = await fetch(youtubeVideoUrl);
|
||||
const respObj = await resp.json();
|
||||
if (!respObj.items.length) {
|
||||
console.warn('Could not find video with id', videoId);
|
||||
log.warn('Could not find video with id', videoId);
|
||||
return false;
|
||||
}
|
||||
|
||||
const item = respObj.items[0];
|
||||
if (!item.snippet) {
|
||||
console.warn('Could not load snippet for video', videoId, item);
|
||||
log.warn('Could not load snippet for video', videoId, item);
|
||||
return false;
|
||||
}
|
||||
if (item.snippet.tags?.includes('music')) {
|
||||
@ -79,6 +85,7 @@ export class TimelineReader {
|
||||
const categoryTitle: string = await fetch(youtubeCategoryUrl)
|
||||
.then((r) => r.json())
|
||||
.then((r) => r.items[0]?.snippet?.title);
|
||||
log.debug('YT category', categoryTitle);
|
||||
return categoryTitle === 'Music';
|
||||
}
|
||||
|
||||
@ -102,7 +109,7 @@ export class TimelineReader {
|
||||
// Check *all* found url and let odesli determine if it is music or not
|
||||
log.debug(`Checking ${url} if it contains song data`);
|
||||
const info = await TimelineReader.getSongInfo(url);
|
||||
log.debug(`Found song info for ${url}?`, info);
|
||||
//log.debug(`Found song info for ${url}?`, info);
|
||||
if (info) {
|
||||
songs.push(info);
|
||||
}
|
||||
@ -144,6 +151,7 @@ export class TimelineReader {
|
||||
return null;
|
||||
}
|
||||
const info = odesliInfo.entitiesByUniqueId[odesliInfo.entityUniqueId];
|
||||
//log.debug('odesli response', info);
|
||||
const platform: Platform = 'youtube';
|
||||
if (info.platforms.includes(platform)) {
|
||||
const youtubeId =
|
||||
@ -156,7 +164,7 @@ export class TimelineReader {
|
||||
}
|
||||
const isMusic = await TimelineReader.isMusicVideo(youtubeId);
|
||||
if (!isMusic) {
|
||||
log.debug('Probably not a music video', url);
|
||||
log.debug('Probably not a music video', youtubeId, url);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -177,6 +185,88 @@ export class TimelineReader {
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
private async addToYoutubePlaylist(song: SongInfo) {
|
||||
log.debug('addToYoutubePlaylist');
|
||||
let token: OauthResponse;
|
||||
try {
|
||||
const youtube_token_file = await fs.readFile('yt_auth_token', { encoding: 'utf8' });
|
||||
token = JSON.parse(youtube_token_file);
|
||||
log.debug('read youtube access token', token);
|
||||
} catch (e) {
|
||||
log.error('Could not read youtube access token', e);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!YOUTUBE_PLAYLIST_ID || YOUTUBE_PLAYLIST_ID === 'CHANGE_ME') {
|
||||
log.debug('no playlist ID configured');
|
||||
return;
|
||||
}
|
||||
if (!song.youtubeUrl) {
|
||||
log.debug('Skip adding song to YT playlist, no youtube Url', song);
|
||||
return;
|
||||
}
|
||||
|
||||
const songUrl = new URL(song.youtubeUrl);
|
||||
const youtubeId = songUrl.searchParams.get('v');
|
||||
if (!youtubeId) {
|
||||
log.debug(
|
||||
'Skip adding song to YT playlist, could not extract YT id from URL',
|
||||
song.youtubeUrl
|
||||
);
|
||||
return;
|
||||
}
|
||||
log.debug('Found YT id from URL', song.youtubeUrl, youtubeId);
|
||||
|
||||
const playlistItemsUrl = new URL('https://www.googleapis.com/youtube/v3/playlistItems');
|
||||
playlistItemsUrl.searchParams.append('videoId', youtubeId);
|
||||
playlistItemsUrl.searchParams.append('playlistId', YOUTUBE_PLAYLIST_ID);
|
||||
playlistItemsUrl.searchParams.append('part', 'id');
|
||||
const existingPlaylistItem = await fetch(
|
||||
'https://www.googleapis.com/youtube/v3/playlistItems',
|
||||
{
|
||||
headers: { Authorization: `${token.token_type} ${token.access_token}` }
|
||||
}
|
||||
).then((r) => r.json());
|
||||
log.debug('existingPlaylistItem', existingPlaylistItem);
|
||||
if (existingPlaylistItem.pageInfo && existingPlaylistItem.pageInfo.totalResults > 0) {
|
||||
log.info('Item already in playlist');
|
||||
return;
|
||||
}
|
||||
|
||||
const searchParams = new URLSearchParams([
|
||||
['part', 'snippet']
|
||||
//['key', token.access_token]
|
||||
]);
|
||||
const options: RequestInit = {
|
||||
method: 'POST',
|
||||
headers: { Authorization: `${token.token_type} ${token.access_token}` },
|
||||
body: JSON.stringify({
|
||||
snippet: {
|
||||
playlistId: YOUTUBE_PLAYLIST_ID,
|
||||
resourceId: {
|
||||
videoId: youtubeId,
|
||||
kind: 'youtube#video'
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
const youtubeApiUrl = new URL(
|
||||
`https://www.googleapis.com/youtube/v3/playlistItems?${searchParams}`
|
||||
);
|
||||
const resp = await fetch(youtubeApiUrl, options);
|
||||
const respObj = await resp.json();
|
||||
log.debug('Added to playlist', options, respObj);
|
||||
if (respObj.error) {
|
||||
log.debug('Add to playlist failed', respObj.error.errors);
|
||||
}
|
||||
}
|
||||
*/
|
||||
private async addToPlaylist(song: SongInfo) {
|
||||
//await this.addToYoutubePlaylist(song);
|
||||
await this.youtubePlaylistAdder.addToPlaylist(song);
|
||||
}
|
||||
|
||||
private static async resizeAvatar(
|
||||
baseName: string,
|
||||
size: number,
|
||||
@ -370,10 +460,15 @@ export class TimelineReader {
|
||||
await TimelineReader.saveAvatar(post.account);
|
||||
await TimelineReader.saveSongThumbnails(songs);
|
||||
|
||||
log.debug('Saved post', post.url);
|
||||
log.debug('Saved post', post.url, 'songs', songs);
|
||||
|
||||
const posts = await getPosts(null, null, 100);
|
||||
await saveAtomFeed(createFeed(posts));
|
||||
|
||||
for (let song of songs) {
|
||||
log.debug('Adding to playlist', song);
|
||||
await this.addToPlaylist(song);
|
||||
}
|
||||
}
|
||||
|
||||
private startWebsocket() {
|
||||
@ -385,12 +480,76 @@ export class TimelineReader {
|
||||
};
|
||||
socket.onmessage = async (event) => {
|
||||
try {
|
||||
/*
|
||||
let token: OauthResponse;
|
||||
try {
|
||||
const youtube_token_file = await fs.readFile('yt_auth_token', { encoding: 'utf8' });
|
||||
token = JSON.parse(youtube_token_file);
|
||||
if (token.expires) {
|
||||
if (typeof token.expires === typeof '') {
|
||||
token.expires = new Date(token.expires);
|
||||
}
|
||||
let now = new Date();
|
||||
now.setTime(now.getTime() - 15 * 60 * 1000);
|
||||
log.info('token expiry', token.expires, 'vs refresh @', now);
|
||||
if (token.expires.getTime() <= now.getTime()) {
|
||||
log.info(
|
||||
'YT token expires',
|
||||
token.expires,
|
||||
token.expires.getTime(),
|
||||
'which is less than 15 minutes from now',
|
||||
now,
|
||||
now.getTime()
|
||||
);
|
||||
const tokenUrl = new URL('https://oauth2.googleapis.com/token');
|
||||
const params = new URLSearchParams();
|
||||
params.append('client_id', YOUTUBE_CLIENT_ID);
|
||||
params.append('client_secret', YOUTUBE_CLIENT_SECRET);
|
||||
params.append('refresh_token', token.refresh_token || '');
|
||||
params.append('grant_type', 'refresh_token');
|
||||
params.append('redirect_uri', `${BASE_URL}/ytauth`);
|
||||
if (token.refresh_token) {
|
||||
log.debug('sending token req', params);
|
||||
const resp = await fetch(tokenUrl, {
|
||||
method: 'POST',
|
||||
body: params
|
||||
}).then((r) => r.json());
|
||||
if (!resp.error) {
|
||||
if (!resp.refresh_token) {
|
||||
resp.refresh_token = token.refresh_token;
|
||||
}
|
||||
let expiration = new Date();
|
||||
expiration.setSeconds(expiration.getSeconds() + resp.expires_in);
|
||||
resp.expires = expiration;
|
||||
await fs.writeFile('yt_auth_token', JSON.stringify(resp));
|
||||
} else {
|
||||
log.error('token resp error', resp);
|
||||
}
|
||||
} else {
|
||||
log.error('no refresg token');
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
log.error('onmessage Could not read youtube access token', e);
|
||||
}
|
||||
*/
|
||||
|
||||
const data: TimelineEvent = JSON.parse(event.data.toString());
|
||||
log.debug('ES event', data.event);
|
||||
if (data.event !== 'update') {
|
||||
log.log('Ignoring ES event', data.event);
|
||||
return;
|
||||
}
|
||||
const post: Post = JSON.parse(data.payload);
|
||||
if (this.lastPosts.includes(post.id)) {
|
||||
log.log('Skipping post, already handled', post.id);
|
||||
return;
|
||||
}
|
||||
this.lastPosts.push(post.id);
|
||||
while (this.lastPosts.length > 10) {
|
||||
this.lastPosts.shift();
|
||||
}
|
||||
await this.checkAndSavePost(post);
|
||||
} catch (e) {
|
||||
log.error('error message', event, event.data, e);
|
||||
@ -416,7 +575,11 @@ export class TimelineReader {
|
||||
private async loadPostsSinceLastRun() {
|
||||
const now = new Date().toISOString();
|
||||
let latestPost = await getPosts(null, now, 1);
|
||||
log.log('Last post in DB since', now, latestPost);
|
||||
if (latestPost.length > 0) {
|
||||
log.log('Last post in DB since', now, latestPost[0].created_at);
|
||||
} else {
|
||||
log.log('No posts in DB since');
|
||||
}
|
||||
let u = new URL(`https://${MASTODON_INSTANCE}/api/v1/timelines/public?local=true&limit=40`);
|
||||
if (latestPost.length > 0) {
|
||||
u.searchParams.append('since_id', latestPost[0].id);
|
||||
@ -428,7 +591,7 @@ export class TimelineReader {
|
||||
Authorization: `Bearer ${MASTODON_ACCESS_TOKEN}`
|
||||
};
|
||||
const latestPosts: Post[] = await fetch(u, { headers }).then((r) => r.json());
|
||||
log.info('searched posts', latestPosts);
|
||||
log.info('searched posts', latestPosts.length);
|
||||
for (const post of latestPosts) {
|
||||
await this.checkAndSavePost(post);
|
||||
}
|
||||
@ -436,6 +599,7 @@ export class TimelineReader {
|
||||
|
||||
private constructor() {
|
||||
log.log('Constructing timeline object');
|
||||
this.youtubePlaylistAdder = new YoutubePlaylistAdder();
|
||||
this.startWebsocket();
|
||||
|
||||
this.loadPostsSinceLastRun()
|
||||
|
Reference in New Issue
Block a user