phlaym/moshing-mammut
1
0
Fork 0
Code Issues 4 Pull Requests Projects Releases 8 Activity

check for missed posts

Browse Source
This commit is contained in:
Max Nuding
2025-06-15 07:01:45 +02:00
parent 7296582b0d
commit 3c6e742e43
3 changed files with 723 additions and 565 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1204
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

8
package.json
View File

@ -14,8 +14,8 @@
"format": "prettier --plugin-search-dir . --write ." "format": "prettier --plugin-search-dir . --write ."
}, },
"devDependencies": { "devDependencies": {
"@sveltejs/adapter-node": "^5.0.0", "@sveltejs/adapter-node": "^5.2.12",
"@sveltejs/kit": "^2.7.0", "@sveltejs/kit": "^2.21.5",
"@sveltejs/vite-plugin-svelte": "^5.0.0", "@sveltejs/vite-plugin-svelte": "^5.0.0",
"@types/node": "^22.6.1", "@types/node": "^22.6.1",
"@types/sqlite3": "^3.0.0", "@types/sqlite3": "^3.0.0",
@ -37,8 +37,8 @@
"type": "module", "type": "module",
"dependencies": { "dependencies": {
"dotenv": "^16.0.3", "dotenv": "^16.0.3",
"feed": "^4.2.2", "feed": "^5.1.0",
"sharp": "^0.33.0", "sharp": "^0.34.2",
"sqlite3": "^5.0.0", "sqlite3": "^5.0.0",
"ws": "^8.18.0" "ws": "^8.18.0"
}, },

76
src/lib/server/timeline.ts
View File

@ -352,6 +352,30 @@ export class TimelineReader {
} }
} }
private async checkAndSavePost(post: Post) {
const hashttags: string[] = HASHTAG_FILTER.split(',');
const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
const songs = await TimelineReader.getSongInfoInPost(post);
// If we don't have any tags or non-youtube urls, check youtube
// YT is handled separately, because it requires an API call and therefore is slower
if (songs.length === 0 && found_tags.length === 0) {
log.log('Ignoring post', post.url);
return;
}
await savePost(post, songs);
await TimelineReader.saveAvatar(post.account);
await TimelineReader.saveSongThumbnails(songs);
log.debug('Saved post', post.url);
const posts = await getPosts(null, null, 100);
await saveAtomFeed(createFeed(posts));
}
private startWebsocket() { private startWebsocket() {
const socket = new WebSocket( const socket = new WebSocket(
`wss://${MASTODON_INSTANCE}/api/v1/streaming?type=subscribe&stream=public:local&access_token=${MASTODON_ACCESS_TOKEN}` `wss://${MASTODON_INSTANCE}/api/v1/streaming?type=subscribe&stream=public:local&access_token=${MASTODON_ACCESS_TOKEN}`
@ -367,28 +391,7 @@ export class TimelineReader {
return; return;
} }
const post: Post = JSON.parse(data.payload); const post: Post = JSON.parse(data.payload);
await this.checkAndSavePost(post);
const hashttags: string[] = HASHTAG_FILTER.split(',');
const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
const songs = await TimelineReader.getSongInfoInPost(post);
// If we don't have any tags or non-youtube urls, check youtube
// YT is handled separately, because it requires an API call and therefore is slower
if (songs.length === 0 && found_tags.length === 0) {
log.log('Ignoring post', post.url);
return;
}
await savePost(post, songs);
await TimelineReader.saveAvatar(post.account);
await TimelineReader.saveSongThumbnails(songs);
log.debug('Saved post', post.url);
const posts = await getPosts(null, null, 100);
await saveAtomFeed(createFeed(posts));
} catch (e) { } catch (e) {
log.error('error message', event, event.data, e); log.error('error message', event, event.data, e);
} }
@ -410,9 +413,38 @@ export class TimelineReader {
}; };
} }
private async loadPostsSinceLastRun() {
const now = new Date().toISOString();
let latestPost = await getPosts(null, now, 1);
log.log('Last post in DB since', now, latestPost);
let u = new URL(`https://${MASTODON_INSTANCE}/api/v1/timelines/public?local=true&limit=40`);
if (latestPost.length > 0) {
u.searchParams.append('since_id', latestPost[0].id);
}
for (let tag of HASHTAG_FILTER.split(',')) {
u.searchParams.append('q', '#' + tag);
}
const headers = {
Authorization: `Bearer ${MASTODON_ACCESS_TOKEN}`
};
const latestPosts: Post[] = await fetch(u, { headers }).then((r) => r.json());
log.info('searched posts', latestPosts);
for (const post of latestPosts) {
await this.checkAndSavePost(post);
}
}
private constructor() { private constructor() {
log.log('Constructing timeline object'); log.log('Constructing timeline object');
this.startWebsocket(); this.startWebsocket();
this.loadPostsSinceLastRun()
.then((_) => {
log.info('loaded posts since last run');
})
.catch((e) => {
log.error('cannot fetch latest posts', e);
});
} }
public static init() { public static init() {