Basic post display

This commit is contained in:
Max Nuding 2023-04-01 14:31:29 +02:00
parent dccb94a792
commit d2f2214d65
Signed by: phlaym
GPG Key ID: A06651BAB6777237
13 changed files with 4325 additions and 2891 deletions

2
.gitignore vendored
View File

@ -1,3 +1,5 @@
*.db
node_modules
/build
/.svelte-kit

6719
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -27,5 +27,11 @@
"typescript": "^4.9.3",
"vite": "^4.0.0"
},
"type": "module"
"type": "module",
"dependencies": {
"@types/sqlite3": "^3.1.8",
"@types/ws": "^8.5.4",
"sqlite3": "^5.1.6",
"ws": "^8.13.0"
}
}

View File

@ -0,0 +1,7 @@
<script lang="ts">
import type { Account } from '$lib/mastodon/response';
export let account: Account;
</script>
<a href="{account.url}" target="_blank">{account.display_name} @{account.acct}</a>

View File

@ -0,0 +1,20 @@
<script lang="ts">
import type { Account } from '$lib/mastodon/response';
export let account: Account;
let avatarDescription: string;
$: avatarDescription = `Avatar for ${account.acct}`
</script>
<img src="{account.avatar}" alt={avatarDescription}/>
<style>
img {
max-width: 50px;
max-height: 50px;
width: auto;
height: auto;
object-fit: contain;
border-radius: 3px;;
}
</style>

View File

@ -0,0 +1,31 @@
<script lang="ts">
import type { Post } from '$lib/mastodon/response';
import AvatarComponent from '$lib/components/AvatarComponent.svelte';
import AccountComponent from '$lib/components/AccountComponent.svelte';
export let post: Post;
let dateCreated: string;
$: dateCreated = new Date(post.created_at).toLocaleString();
</script>
<div class="wrapper">
<div class="avatar"><AvatarComponent account={post.account} /></div>
<div class="post">
<AccountComponent account={post.account} />
<small><a href={post.url} target="_blank">{dateCreated}</a></small>
{@html post.content}
</div>
</div>
<style>
.wrapper {
display: flex;
}
.post {
display: flex;
flex-direction: column;
}
.avatar {
margin-right: 1em;
}
</style>

View File

@ -0,0 +1,29 @@
export interface TimelineEvent {
event: string,
payload: string
}
export interface Post {
id: string,
created_at: string,
tags: Tag[],
url: string,
content: string,
account: Account
}
export interface Tag {
name: string,
url: string
}
export interface Account {
id: string,
acct: string,
username: string,
display_name: string,
url: string,
avatar: string,
avatar_static: string
}

250
src/lib/server/db.ts Normal file
View File

@ -0,0 +1,250 @@
import type { Account, Post, Tag } from '$lib/mastodon/response';
import sqlite3 from 'sqlite3';
const { MODE } = import.meta.env;
const db: sqlite3.Database = new sqlite3.Database('moshingmammut.db');
if (MODE === 'development') {
sqlite3.verbose();
db.on('change', (t, d, table, rowid) => {
console.debug('DB change event', t, d, table, rowid);
})
db.on('trace', (sql) => {
console.debug('Running', sql);
});
db.on('profile', (sql) => {
console.debug('Finished', sql);
});
}
interface Migration {
id: number,
name: string,
statement: string
}
db.on('open', () => {
console.log('Opened database');
db.run('CREATE TABLE IF NOT EXISTS "migrations" ("id" integer,"name" TEXT, PRIMARY KEY (id))');
db.serialize();
db.all('SELECT id FROM migrations', (err, rows) => {
if (err !== null) {
console.error('Could not fetch existing migrations', err);
return;
}
console.debug('Already applied migrations', rows);
const appliedMigrations: Set<number> = new Set(rows.map((row: any) => row['id']));
const toApply = getMigrations().filter(m => !appliedMigrations.has(m.id));
for (let migration of toApply) {
db.exec(migration.statement, (err) => {
if (err !== null) {
console.error(`Failed to apply migration ${migration.name}`, err);
return;
}
db.run('INSERT INTO migrations (id, name) VALUES(?, ?)', [migration.id, migration.name], (e: Error) => {
if (e !== null) {
console.error(`Failed to mark migration ${migration.name} as applied`, e);
return;
}
console.info(`Applied migration ${migration.name}`);
});
});
}
});
});
db.on('error', (err) => {
console.error('Error opening database', err);
});
function getMigrations(): Migration[] {
return [{
id: 1,
name: 'initial',
statement: `
CREATE TABLE accounts (
id TEXT NOT NULL PRIMARY KEY,
acct TEXT,
username TEXT,
display_name TEXT,
url TEXT,
avatar TEXT,
avatar_static TEXT
);
CREATE TABLE tags (url TEXT NOT NULL PRIMARY KEY, tag TEXT NOT NULL);
CREATE TABLE posts (
id TEXT NOT NULL PRIMARY KEY,
content TEXT,
created_at TEXT,
url TEXT NOT NULL,
account_id TEXT NOT NULL,
FOREIGN KEY (account_id) REFERENCES accounts(id)
);
CREATE TABLE poststags (
id integer PRIMARY KEY,
post_id TEXT NOT NULL,
tag_url TEXT NOT NULL,
FOREIGN KEY (post_id) REFERENCES posts(id),
FOREIGN KEY (tag_url) REFERENCES tags(url)
)`
}];
}
export function savePost(post: Post): void {
console.debug(`Saving post ${post.url}`);
const account = post.account;
db.run(`
INSERT INTO accounts (id, acct, username, display_name, url, avatar, avatar_static)
VALUES(?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id)
DO UPDATE SET
acct=excluded.acct,
username=excluded.username,
display_name=excluded.display_name,
url=excluded.url,
avatar=excluded.avatar,
avatar_static=excluded.avatar_static;`,
[
account.id,
account.acct,
account.username,
account.display_name,
account.url,
account.avatar,
account.avatar_static
],
(err) => {
if (err !== null) {
console.error(`Could not insert/update account ${account.id}`, err);
return;
}
db.run(`
INSERT INTO posts (id, content, created_at, url, account_id)
VALUES (?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET
content=excluded.content,
created_at=excluded.created_at,
url=excluded.url,
account_id=excluded.account_id;`,
[
post.id,
post.content,
post.created_at,
post.url,
post.account.id
],
(postErr) => {
if (postErr !== null) {
console.error(`Could not insert post ${post.url}`, postErr);
return;
}
db.parallelize(() => {
for (let tag of post.tags) {
db.run(`
INSERT INTO tags (url, tag) VALUES (?, ?)
ON CONFLICT(url) DO UPDATE SET
tag=excluded.tag;`,
[
tag.url,
tag.name
],
(tagErr) => {
if (tagErr !== null) {
console.error(`Could not insert/update tag ${tag.url}`, tagErr);
return;
}
db.run('INSERT INTO poststags (post_id, tag_url) VALUES (?, ?)',
[post.id, tag.url],
(posttagserr) => {
if (posttagserr !== null) {
console.error(`Could not insert poststags ${tag.url}, ${post.url}`, posttagserr);
return;
}
}
);
}
);
}
});
});
});
}
export async function getPosts(since: string | null, limit: number) {
let promise = await new Promise<Post[]>((resolve, reject) => {
let filter_query;
let params: any = { $limit: limit };
if (since === null) {
filter_query = '';
} else {
filter_query = 'WHERE posts.created_at > $since';
params.$since = since;
console.debug('Filtering by created_at > ', since);
}
const sql = `SELECT posts.id, posts.content, posts.created_at, posts.url,
accounts.id AS account_id, accounts.acct, accounts.username, accounts.display_name,
accounts.url AS account_url, accounts.avatar
FROM posts
JOIN accounts ON posts.account_id = accounts.id
${filter_query}
ORDER BY created_at DESC
LIMIT $limit`;
db.all(
sql,
params,
(err, rows: any[]) => {
if (err != null) {
console.error('Error loading posts', err);
reject(err);
return;
}
const postIdsParams = rows.map(() => '?').join(', ');
db.all(
`SELECT post_id, tags.url, tags.tag
FROM poststags
JOIN tags ON poststags.tag_url = tags.url
WHERE post_id IN (${postIdsParams});`,
rows.map((r: any) => r.id),
(tagErr, tagRows: any[]) => {
if (tagErr != null) {
console.error('Error loading post tags', tagErr);
reject(tagErr);
return;
}
const tagMap: Map<string, Tag[]> = tagRows.reduce(
(result: Map<string, Tag[]>, item) => {
const tag: Tag = {
url: item.url,
name: item.tag
};
result.set(item.post_id, [...result.get(item.post_id) || [], tag]);
return result;
}, new Map());
const posts = rows.map(row => {
return {
id: row.id,
content: row.content,
created_at: row.created_at,
url: row.url,
tags: tagMap.get(row.id) || [],
account: {
id: row.account_id,
acct: row.acct,
username: row.username,
display_name: row.display_name,
url: row.account_url,
avatar: row.avatar,
avatar_static: ''
} as Account
} as Post
});
resolve(posts);
}
);
}
);
});
return promise;
}

View File

@ -0,0 +1,74 @@
import { HASHTAG_FILTER, URL_FILTER } from '$env/static/private';
import type { Post, Tag, TimelineEvent } from '$lib/mastodon/response';
import { savePost } from '$lib/server/db';
import { WebSocket } from "ws";
/*
Filter youtube: /v3/videos (part: snippet), /v3/videoCategories (part: snippet). Category 10 is Music
*/
const YOUTUBE_REGEX = new RegExp('https?:\/\/(www\.)?youtu((be.com\/.*v=)|(\.be\/))(?<videoId>[a-zA-Z_0-9-]+)');
export class TimelineReader {
private static _instance: TimelineReader;
private constructor() {
//const socket = new WebSocket("wss://metalhead.club/api/v1/streaming/public/local")
const socket = new WebSocket("wss://metalhead.club/api/v1/streaming")
socket.onopen = (_event) => {
socket.send('{ "type": "subscribe", "stream": "public:local"}');
};
socket.onmessage = (event) => {
try {
const data: TimelineEvent = JSON.parse(event.data.toString());
if (data.event !== 'update') {
return;
}
const post: Post = JSON.parse(data.payload);
const hashttags: string[] = HASHTAG_FILTER.split(',');
const found_tags: Tag[] = post.tags.filter((t: Tag) => hashttags.includes(t.name));
const urls: string[] = URL_FILTER.split(',');
const found_urls = urls.filter(t => post.content.includes(t));
if (found_urls.length === 0 && found_tags.length === 0) {
//return;
}
const youtubeMatches = found_urls.map(u => u.match(YOUTUBE_REGEX)).filter(i => i !== null);
if (found_urls.length > 0 && youtubeMatches.length === found_urls.length) {
// TODO: Check with YouTube API if it is in YT music or labeled as music
console.info('Found youtube urls', youtubeMatches);
}
console.log(
"message",
`Update by @${post.account?.username}: ${post.content}`,
'tags',
post.tags,
found_tags,
found_urls);
savePost(post);
} catch (e) {
console.error("error message", event, event.data, e)
}
};
socket.onclose = (event) => {
console.log("Closed", event, event.code, event.reason)
};
socket.onerror = (event) => {
console.log("error", event, event.message, event.error)
};
}
public static init() {
if (this._instance === undefined) {
this._instance = new TimelineReader();
}
}
public static get instance(): TimelineReader {
TimelineReader.init();
return this._instance;
}
}

View File

@ -1,2 +1,41 @@
<h1>Welcome to SvelteKit</h1>
<p>Visit <a href="https://kit.svelte.dev">kit.svelte.dev</a> to read the documentation</p>
<script lang="ts">
import { onMount } from "svelte";
import type { PageData } from './$types';
import type { Post } from '$lib/mastodon/response';
import { PUBLIC_REFRESH_INTERVAL } from '$env/static/public';
import PostComponent from '$lib/components/PostComponent.svelte';
export let data: PageData;
let interval: NodeJS.Timer | null = null;
onMount(async () => {
interval = setInterval(async () => {
const params = new URLSearchParams();
if (data.posts.length > 0) {
params.set('since', data.posts[0].created_at);
}
await fetch(`/api/posts?${params}`)
.then(r => r.json())
.then((resp: Post[]) => {
if (resp.length > 0) {
data.posts = resp.concat(data.posts);
console.log('updated data', resp, data.posts);
}
})
.catch(e => {
// TODO: Show error in UI
console.error('Error loading newest posts', e);
});
}, parseInt(PUBLIC_REFRESH_INTERVAL));
return () => {
if (interval !== null) {
clearInterval(interval)
}
}
})
</script>
{#each data.posts as post (post.id)}
<PostComponent {post}></PostComponent>
{/each}

10
src/routes/+page.ts Normal file
View File

@ -0,0 +1,10 @@
import type { Post } from '$lib/mastodon/response';
import type { PageLoad } from './$types';
export const load = (async ({ fetch }) => {
const p = await fetch('/');
//const posts: Post[] = await p.json();
return {
posts: await p.json() as Post[]
};
}) satisfies PageLoad;

8
src/routes/+server.ts Normal file
View File

@ -0,0 +1,8 @@
import { TimelineReader } from '$lib/server/timeline';
import type { RequestHandler } from './$types';
TimelineReader.init();
export const GET = (async ({ fetch }) => {
return await fetch('api/posts');
}) satisfies RequestHandler;

View File

@ -0,0 +1,15 @@
import { getPosts } from '$lib/server/db';
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
export const GET = (async ({ url }) => {
const since = url.searchParams.get('since');
let count = Number.parseInt(url.searchParams.get('count') || '');
if (isNaN(count)) {
count = 20;
}
count = Math.min(count, 100);
const posts = await getPosts(since, count);
return json(posts);
}) satisfies RequestHandler;