import { HASHTAG_FILTER, MASTODON_INSTANCE, URL_FILTER, YOUTUBE_API_KEY } from '$env/static/private'; import type { Post, Tag, TimelineEvent } from '$lib/mastodon/response'; import { getPosts, savePost } from '$lib/server/db'; import { createFeed, saveAtomFeed } from '$lib/server/rss'; import { WebSocket } from "ws"; const YOUTUBE_REGEX = new RegExp(/https?:\/\/(www\.)?youtu((be.com\/.*?v=)|(\.be\/))(?[a-zA-Z_0-9-]+)/gm); export class TimelineReader { private static _instance: TimelineReader; private static async isMusicVideo(videoId: string) { const searchParams = new URLSearchParams([ ['part', 'snippet'], ['id', videoId], ['key', YOUTUBE_API_KEY]]); const youtubeVideoUrl = new URL(`https://www.googleapis.com/youtube/v3/videos?${searchParams}`); const resp = await fetch(youtubeVideoUrl); const respObj = await resp.json(); if (!respObj.items.length) { console.warn('Could not find video with id', videoId); return false; } const item = respObj.items[0]; if (item.tags?.includes('music')) { return true; } const categorySearchParams = new URLSearchParams([ ['part', 'snippet'], ['id', item.categoryId], ['key', YOUTUBE_API_KEY]]); const youtubeCategoryUrl = new URL(`https://www.googleapis.com/youtube/v3/videoCategories?${categorySearchParams}`); const categoryTitle: string = await fetch(youtubeCategoryUrl).then(r => r.json()).then(r => r.items[0]?.title); return categoryTitle === 'Music'; } private static async checkYoutubeMatches(postContent: string): Promise { const matches = postContent.matchAll(YOUTUBE_REGEX); for (let match of matches) { if (match === undefined || match.groups === undefined) { continue; } const videoId = match.groups.videoId.toString(); try { const isMusic = await TimelineReader.isMusicVideo(videoId); if (isMusic) { return true; } } catch (e) { console.error('Could not check if', videoId, 'is a music video', e); } } return false; } private constructor() { const socket = new WebSocket(`wss://${MASTODON_INSTANCE}/api/v1/streaming`); socket.onopen = (_event) => { socket.send('{ "type": "subscribe", "stream": "public:local"}'); }; socket.onmessage = (async (event) => { try { const data: TimelineEvent = JSON.parse(event.data.toString()); if (data.event !== 'update') { return; } const post: Post = JSON.parse(data.payload); const hashttags: string[] = HASHTAG_FILTER.split(','); const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name)); const urls: string[] = URL_FILTER.split(','); const found_urls = urls.filter(t => post.content.includes(t)); // If we don't have any tags or non-youtube urls, check youtube // YT is handled separately, because it requires an API call and therefore is slower if (found_urls.length === 0 && found_tags.length === 0 && !await TimelineReader.checkYoutubeMatches(post.content)) { return; } await savePost(post); const posts = await getPosts(null, null, 100); await saveAtomFeed(createFeed(posts)); } catch (e) { console.error("error message", event, event.data, e) } }); socket.onclose = (event) => { console.log("Closed", event, event.code, event.reason) }; socket.onerror = (event) => { console.log("error", event, event.message, event.error) }; } public static init() { if (this._instance === undefined) { this._instance = new TimelineReader(); } } public static get instance(): TimelineReader { TimelineReader.init(); return this._instance; } }