phlaym/moshing-mammut
1
0
Fork 0
Code Issues 4 Pull Requests Projects Releases 8 Activity

Improve formatting

Browse Source
This commit is contained in:
Max Nuding
2023-04-11 16:02:54 +02:00
parent 268128c2f4
commit 6c9546b74a
22 changed files with 280 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
src/lib/server/db.ts
View File

@ -9,7 +9,7 @@ if (DEV && env.VERBOSE === 'true') {
sqlite3.verbose();
db.on('change', (t, d, table, rowid) => {
console.debug('DB change event', t, d, table, rowid);
})
});
db.on('trace', (sql) => {
console.debug('Running', sql);
@ -21,9 +21,9 @@ if (DEV && env.VERBOSE === 'true') {
}
interface Migration {
id: number,
name: string,
statement: string
id: number;
name: string;
statement: string;
}
db.on('open', () => {
@ -37,20 +37,24 @@ db.on('open', () => {
}
console.debug('Already applied migrations', rows);
const appliedMigrations: Set<number> = new Set(rows.map((row: any) => row['id']));
const toApply = getMigrations().filter(m => !appliedMigrations.has(m.id));
for (let migration of toApply) {
const toApply = getMigrations().filter((m) => !appliedMigrations.has(m.id));
for (const migration of toApply) {
db.exec(migration.statement, (err) => {
if (err !== null) {
console.error(`Failed to apply migration ${migration.name}`, err);
return;
}
db.run('INSERT INTO migrations (id, name) VALUES(?, ?)', [migration.id, migration.name], (e: Error) => {
if (e !== null) {
console.error(`Failed to mark migration ${migration.name} as applied`, e);
return;
db.run(
'INSERT INTO migrations (id, name) VALUES(?, ?)',
[migration.id, migration.name],
(e: Error) => {
if (e !== null) {
console.error(`Failed to mark migration ${migration.name} as applied`, e);
return;
}
console.info(`Applied migration ${migration.name}`);
}
console.info(`Applied migration ${migration.name}`);
});
);
});
}
});
@ -60,10 +64,11 @@ db.on('error', (err) => {
});
function getMigrations(): Migration[] {
return [{
id: 1,
name: 'initial',
statement: `
return [
{
id: 1,
name: 'initial',
statement: `
CREATE TABLE accounts (
id TEXT NOT NULL PRIMARY KEY,
acct TEXT,
@ -89,14 +94,16 @@ function getMigrations(): Migration[] {
FOREIGN KEY (post_id) REFERENCES posts(id),
FOREIGN KEY (tag_url) REFERENCES tags(url)
)`
}];
}
];
}
export async function savePost(post: Post): Promise<undefined> {
return new Promise((resolve, reject) => {
console.debug(`Saving post ${post.url}`);
const account = post.account;
db.run(`
db.run(
`
INSERT INTO accounts (id, acct, username, display_name, url, avatar, avatar_static)
VALUES(?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id)
@ -122,20 +129,15 @@ export async function savePost(post: Post): Promise<undefined> {
reject(err);
return;
}
db.run(`
db.run(
`
INSERT INTO posts (id, content, created_at, url, account_id)
VALUES (?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET
content=excluded.content,
created_at=excluded.created_at,
url=excluded.url,
account_id=excluded.account_id;`,
[
post.id,
post.content,
post.created_at,
post.url,
post.account.id
],
[post.id, post.content, post.created_at, post.url, post.account.id],
(postErr) => {
if (postErr !== null) {
console.error(`Could not insert post ${post.url}`, postErr);
@ -145,26 +147,28 @@ export async function savePost(post: Post): Promise<undefined> {
db.parallelize(() => {
let remaining = post.tags.length;
for (let tag of post.tags) {
db.run(`
for (const tag of post.tags) {
db.run(
`
INSERT INTO tags (url, tag) VALUES (?, ?)
ON CONFLICT(url) DO UPDATE SET
tag=excluded.tag;`,
[
tag.url,
tag.name
],
[tag.url, tag.name],
(tagErr) => {
if (tagErr !== null) {
console.error(`Could not insert/update tag ${tag.url}`, tagErr);
reject(tagErr);
return;
}
db.run('INSERT INTO poststags (post_id, tag_url) VALUES (?, ?)',
db.run(
'INSERT INTO poststags (post_id, tag_url) VALUES (?, ?)',
[post.id, tag.url],
(posttagserr) => {
if (posttagserr !== null) {
console.error(`Could not insert poststags ${tag.url}, ${post.url}`, posttagserr);
console.error(
`Could not insert poststags ${tag.url}, ${post.url}`,
posttagserr
);
reject(posttagserr);
return;
}
@ -180,15 +184,17 @@ export async function savePost(post: Post): Promise<undefined> {
);
}
});
});
});
}
);
}
);
});
}
export async function getPosts(since: string | null, before: string | null, limit: number) {
let promise = await new Promise<Post[]>((resolve, reject) => {
const promise = await new Promise<Post[]>((resolve, reject) => {
let filter_query;
let params: any = { $limit: limit };
const params: any = { $limit: limit };
if (since === null && before === null) {
filter_query = '';
} else if (since !== null) {
@ -207,66 +213,60 @@ export async function getPosts(since: string | null, before: string | null, limi
${filter_query}
ORDER BY created_at DESC
LIMIT $limit`;
db.all(
sql,
params,
(err, rows: any[]) => {
if (err != null) {
console.error('Error loading posts', err);
reject(err);
return;
}
if (rows.length === 0) {
// No need to check for tags
resolve([]);
return;
}
const postIdsParams = rows.map(() => '?').join(', ');
db.all(
`SELECT post_id, tags.url, tags.tag
db.all(sql, params, (err, rows: any[]) => {
if (err != null) {
console.error('Error loading posts', err);
reject(err);
return;
}
if (rows.length === 0) {
// No need to check for tags
resolve([]);
return;
}
const postIdsParams = rows.map(() => '?').join(', ');
db.all(
`SELECT post_id, tags.url, tags.tag
FROM poststags
JOIN tags ON poststags.tag_url = tags.url
WHERE post_id IN (${postIdsParams});`,
rows.map((r: any) => r.id),
(tagErr, tagRows: any[]) => {
if (tagErr != null) {
console.error('Error loading post tags', tagErr);
reject(tagErr);
return;
}
const tagMap: Map<string, Tag[]> = tagRows.reduce(
(result: Map<string, Tag[]>, item) => {
const tag: Tag = {
url: item.url,
name: item.tag
};
result.set(item.post_id, [...result.get(item.post_id) || [], tag]);
return result;
}, new Map());
const posts = rows.map(row => {
return {
id: row.id,
content: row.content,
created_at: row.created_at,
url: row.url,
tags: tagMap.get(row.id) || [],
account: {
id: row.account_id,
acct: row.acct,
username: row.username,
display_name: row.display_name,
url: row.account_url,
avatar: row.avatar,
avatar_static: ''
} as Account
} as Post
});
resolve(posts);
rows.map((r: any) => r.id),
(tagErr, tagRows: any[]) => {
if (tagErr != null) {
console.error('Error loading post tags', tagErr);
reject(tagErr);
return;
}
);
}
);
const tagMap: Map<string, Tag[]> = tagRows.reduce((result: Map<string, Tag[]>, item) => {
const tag: Tag = {
url: item.url,
name: item.tag
};
result.set(item.post_id, [...(result.get(item.post_id) || []), tag]);
return result;
}, new Map());
const posts = rows.map((row) => {
return {
id: row.id,
content: row.content,
created_at: row.created_at,
url: row.url,
tags: tagMap.get(row.id) || [],
account: {
id: row.account_id,
acct: row.acct,
username: row.username,
display_name: row.display_name,
url: row.account_url,
avatar: row.avatar,
avatar_static: ''
} as Account
} as Post;
});
resolve(posts);
}
);
});
});
return promise;
}

18
src/lib/server/rss.ts
View File

@ -22,24 +22,26 @@ export function createFeed(posts: Post[]): Feed {
author: {
name: '@aymm',
link: 'https://metalhead.club/@aymm'
},
}
});
posts.forEach(p => {
posts.forEach((p) => {
feed.addItem({
title: p.content,
id: p.url,
link: p.url,
content: p.content,
author: [{
name: p.account.acct,
link: p.account.url
}],
author: [
{
name: p.account.acct,
link: p.account.url
}
],
date: new Date(p.created_at)
})
});
});
feed.addCategory('Music');
return feed;
}
export async function saveAtomFeed(feed: Feed) {
await fs.writeFile('feed.xml', feed.atom1(), { encoding: 'utf8' });
}
}

54
src/lib/server/timeline.ts
View File

@ -1,10 +1,17 @@
import { HASHTAG_FILTER, MASTODON_INSTANCE, URL_FILTER, YOUTUBE_API_KEY } from '$env/static/private';
import {
HASHTAG_FILTER,
MASTODON_INSTANCE,
URL_FILTER,
YOUTUBE_API_KEY
} from '$env/static/private';
import type { Post, Tag, TimelineEvent } from '$lib/mastodon/response';
import { getPosts, savePost } from '$lib/server/db';
import { createFeed, saveAtomFeed } from '$lib/server/rss';
import { WebSocket } from "ws";
import { WebSocket } from 'ws';
const YOUTUBE_REGEX = new RegExp(/https?:\/\/(www\.)?youtu((be.com\/.*?v=)|(\.be\/))(?<videoId>[a-zA-Z_0-9-]+)/gm);
const YOUTUBE_REGEX = new RegExp(
/https?:\/\/(www\.)?youtu((be.com\/.*?v=)|(\.be\/))(?<videoId>[a-zA-Z_0-9-]+)/gm
);
export class TimelineReader {
private static _instance: TimelineReader;
@ -13,7 +20,8 @@ export class TimelineReader {
const searchParams = new URLSearchParams([
['part', 'snippet'],
['id', videoId],
['key', YOUTUBE_API_KEY]]);
['key', YOUTUBE_API_KEY]
]);
const youtubeVideoUrl = new URL(`https://www.googleapis.com/youtube/v3/videos?${searchParams}`);
const resp = await fetch(youtubeVideoUrl);
const respObj = await resp.json();
@ -30,15 +38,20 @@ export class TimelineReader {
const categorySearchParams = new URLSearchParams([
['part', 'snippet'],
['id', item.categoryId],
['key', YOUTUBE_API_KEY]]);
const youtubeCategoryUrl = new URL(`https://www.googleapis.com/youtube/v3/videoCategories?${categorySearchParams}`);
const categoryTitle: string = await fetch(youtubeCategoryUrl).then(r => r.json()).then(r => r.items[0]?.title);
['key', YOUTUBE_API_KEY]
]);
const youtubeCategoryUrl = new URL(
`https://www.googleapis.com/youtube/v3/videoCategories?${categorySearchParams}`
);
const categoryTitle: string = await fetch(youtubeCategoryUrl)
.then((r) => r.json())
.then((r) => r.items[0]?.title);
return categoryTitle === 'Music';
}
private static async checkYoutubeMatches(postContent: string): Promise<boolean> {
const matches = postContent.matchAll(YOUTUBE_REGEX);
for (let match of matches) {
for (const match of matches) {
if (match === undefined || match.groups === undefined) {
continue;
}
@ -57,10 +70,10 @@ export class TimelineReader {
private constructor() {
const socket = new WebSocket(`wss://${MASTODON_INSTANCE}/api/v1/streaming`);
socket.onopen = (_event) => {
socket.onopen = () => {
socket.send('{ "type": "subscribe", "stream": "public:local"}');
};
socket.onmessage = (async (event) => {
socket.onmessage = async (event) => {
try {
const data: TimelineEvent = JSON.parse(event.data.toString());
if (data.event !== 'update') {
@ -71,31 +84,30 @@ export class TimelineReader {
const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
const urls: string[] = URL_FILTER.split(',');
const found_urls = urls.filter(t => post.content.includes(t));
const found_urls = urls.filter((t) => post.content.includes(t));
// If we don't have any tags or non-youtube urls, check youtube
// YT is handled separately, because it requires an API call and therefore is slower
if (found_urls.length === 0 &&
if (
found_urls.length === 0 &&
found_tags.length === 0 &&
!await TimelineReader.checkYoutubeMatches(post.content)) {
!(await TimelineReader.checkYoutubeMatches(post.content))
) {
return;
}
await savePost(post);
const posts = await getPosts(null, null, 100);
await saveAtomFeed(createFeed(posts));
} catch (e) {
console.error("error message", event, event.data, e)
console.error('error message', event, event.data, e);
}
});
};
socket.onclose = (event) => {
console.log("Closed", event, event.code, event.reason)
console.log('Closed', event, event.code, event.reason);
};
socket.onerror = (event) => {
console.log("error", event, event.message, event.error)
console.log('error', event, event.message, event.error);
};
}
public static init() {
@ -108,4 +120,4 @@ export class TimelineReader {
TimelineReader.init();
return this._instance;
}
}
}