phlaym/moshing-mammut
1
0
Fork 0
Code Issues 4 Pull Requests Projects Releases 8 Activity

check for missed posts

Browse Source
This commit is contained in:
Max Nuding 2025-06-15 07:01:45 +02:00
parent 7296582b0d
commit 3c6e742e43
Signed by: phlaym
SSH Key Fingerprint: SHA256:mionmF+5trOUI1AxqzAU1ZK3tv6IiDcdKGXcMWwa1nQ
3 changed files with 723 additions and 565 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1204
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

8
package.json
View File

@ -14,8 +14,8 @@
"format": "prettier --plugin-search-dir . --write ."
},
"devDependencies": {
"@sveltejs/adapter-node": "^5.0.0",
"@sveltejs/kit": "^2.7.0",
"@sveltejs/adapter-node": "^5.2.12",
"@sveltejs/kit": "^2.21.5",
"@sveltejs/vite-plugin-svelte": "^5.0.0",
"@types/node": "^22.6.1",
"@types/sqlite3": "^3.0.0",
@ -37,8 +37,8 @@
"type": "module",
"dependencies": {
"dotenv": "^16.0.3",
"feed": "^4.2.2",
"sharp": "^0.33.0",
"feed": "^5.1.0",
"sharp": "^0.34.2",
"sqlite3": "^5.0.0",
"ws": "^8.18.0"
},

76
src/lib/server/timeline.ts
View File

@ -352,6 +352,30 @@ export class TimelineReader {
}
}
private async checkAndSavePost(post: Post) {
const hashttags: string[] = HASHTAG_FILTER.split(',');
const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
const songs = await TimelineReader.getSongInfoInPost(post);
// If we don't have any tags or non-youtube urls, check youtube
// YT is handled separately, because it requires an API call and therefore is slower
if (songs.length === 0 && found_tags.length === 0) {
log.log('Ignoring post', post.url);
return;
}
await savePost(post, songs);
await TimelineReader.saveAvatar(post.account);
await TimelineReader.saveSongThumbnails(songs);
log.debug('Saved post', post.url);
const posts = await getPosts(null, null, 100);
await saveAtomFeed(createFeed(posts));
}
private startWebsocket() {
const socket = new WebSocket(
`wss://${MASTODON_INSTANCE}/api/v1/streaming?type=subscribe&stream=public:local&access_token=${MASTODON_ACCESS_TOKEN}`
@ -367,28 +391,7 @@ export class TimelineReader {
return;
}
const post: Post = JSON.parse(data.payload);
const hashttags: string[] = HASHTAG_FILTER.split(',');
const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
const songs = await TimelineReader.getSongInfoInPost(post);
// If we don't have any tags or non-youtube urls, check youtube
// YT is handled separately, because it requires an API call and therefore is slower
if (songs.length === 0 && found_tags.length === 0) {
log.log('Ignoring post', post.url);
return;
}
await savePost(post, songs);
await TimelineReader.saveAvatar(post.account);
await TimelineReader.saveSongThumbnails(songs);
log.debug('Saved post', post.url);
const posts = await getPosts(null, null, 100);
await saveAtomFeed(createFeed(posts));
await this.checkAndSavePost(post);
} catch (e) {
log.error('error message', event, event.data, e);
}
@ -410,9 +413,38 @@ export class TimelineReader {
};
}
private async loadPostsSinceLastRun() {
const now = new Date().toISOString();
let latestPost = await getPosts(null, now, 1);
log.log('Last post in DB since', now, latestPost);
let u = new URL(`https://${MASTODON_INSTANCE}/api/v1/timelines/public?local=true&limit=40`);
if (latestPost.length > 0) {
u.searchParams.append('since_id', latestPost[0].id);
}
for (let tag of HASHTAG_FILTER.split(',')) {
u.searchParams.append('q', '#' + tag);
}
const headers = {
Authorization: `Bearer ${MASTODON_ACCESS_TOKEN}`
};
const latestPosts: Post[] = await fetch(u, { headers }).then((r) => r.json());
log.info('searched posts', latestPosts);
for (const post of latestPosts) {
await this.checkAndSavePost(post);
}
}
private constructor() {
log.log('Constructing timeline object');
this.startWebsocket();
this.loadPostsSinceLastRun()
.then((_) => {
log.info('loaded posts since last run');
})
.catch((e) => {
log.error('cannot fetch latest posts', e);
});
}
public static init() {