8 Commits

9 changed files with 706 additions and 454 deletions

2
.gitignore vendored
View File

@ -3,6 +3,8 @@ yt_auth_token
spotify_auth_token
tidal_auth_token
*.db
*.db-shm
*.db-wal
feed.xml
playbook.yml
inventory.yml

View File

@ -32,6 +32,10 @@
Header set Cache-Control "public,max-age=31536000,immutable"
</Directory>
Header set X-Content-Type-Options "nosniff"
Header set Strict-Transport-Security "max-age=15768000; includeSubDomains"
Header set X-Frame-Options "DENY"
ProxyPass / http://localhost:3000/
ProxyPassReverse / http://localhost:3000/

937
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,6 +1,6 @@
{
"name": "moshing-mammut",
"version": "2.0.1",
"version": "2.1.1",
"private": true,
"license": "LGPL-3.0-or-later",
"scripts": {
@ -18,7 +18,7 @@
"@eslint/js": "^9.30.1",
"@sveltejs/adapter-node": "^5.2.12",
"@sveltejs/kit": "^2.22.2",
"@sveltejs/vite-plugin-svelte": "^5.1.0",
"@sveltejs/vite-plugin-svelte": "^6.1.0",
"@types/node": "^22.9.0",
"@types/sqlite3": "^3.0.0",
"@types/ws": "^8.5.0",
@ -32,6 +32,7 @@
"prettier": "^3.1.0",
"prettier-plugin-svelte": "^3.2.6",
"svelte": "^5",
"svelte-adapter-bun": "^0.5.2",
"svelte-check": "^4.0.0",
"tslib": "^2.0.0",
"typescript": "^5.0.0"
@ -44,6 +45,11 @@
"sqlite3": "^5.0.0",
"ws": "^8.18.0"
},
"overrides": {
"@sveltejs/kit": {
"cookie": ">=0.7.0"
}
},
"engines": {
"node": ">=20.0.0"
}

View File

@ -67,8 +67,12 @@ type Migration = {
name: string;
statement: string;
};
const db: sqlite3.Database = new sqlite3.Database('moshingmammut.db');
const db: sqlite3.Database = new sqlite3.cached.Database('moshingmammut.db');
const maxThumbnailCacheSize = 100;
const maxThumbnailCacheAge = 1 * 60 * 60 * 1000; //1h in ms
const thumbnailCache = new Map<string, { data: SongThumbnailImage[]; ts: number }>();
let thumbnailCacheHits = 0;
let thumbnailCacheMisses = 0;
export function close() {
try {
@ -93,6 +97,7 @@ const ignoredUsers: string[] =
let databaseReady = false;
if (enableVerboseLog) {
logger.verbose('Enabling verbose DB log');
sqlite3.verbose();
db.on('change', (t, d, table, rowid) => {
logger.verbose('DB change event', t, d, table, rowid);
@ -102,8 +107,16 @@ if (enableVerboseLog) {
logger.verbose('Running', sql);
});
db.on('profile', (sql) => {
logger.verbose('Finished', sql);
db.on('profile', (sql, time) => {
if (sql.startsWith('EXPLAIN')) {
return;
}
logger.verbose('Finished', sql, 'time', time);
if (sql.startsWith('SELECT')) {
db.get('EXPLAIN QUERY PLAN ' + sql, (a: sqlite3.RunResult, b: Error) => {
logger.verbose(sql, a, b);
});
}
});
}
@ -149,6 +162,9 @@ async function applyMigration(migration: Migration) {
db.on('open', () => {
logger.info('Opened database');
db.serialize();
db.run('pragma journal_mode = WAL;');
db.run('pragma synchronous = normal;');
db.run('pragma journal_size_limit = 6144000;');
db.run('CREATE TABLE IF NOT EXISTS "migrations" ("id" integer,"name" TEXT, PRIMARY KEY (id))');
db.all('SELECT id FROM migrations', (err, rows: Migration[]) => {
if (err !== null) {
@ -340,6 +356,26 @@ function getMigrations(): Migration[] {
name: 'song tidal id',
statement: `
ALTER TABLE songs ADD COLUMN tidalId TEXT NULL;`
},
{
id: 10,
name: 'add indexes',
statement: `
CREATE INDEX posts_created_at ON posts(created_at);
CREATE INDEX accounts_acct ON accounts(acct);
CREATE INDEX accounts_username ON accounts(username);`
},
{
id: 11,
name: 'add FK indexes',
statement: `
CREATE INDEX migrations_id ON migrations(id);
CREATE INDEX accountsavatars_account_url ON accountsavatars(account_url);
CREATE INDEX posts_account_id ON posts(account_id);
CREATE INDEX poststags_post_id ON poststags(post_id);
CREATE INDEX poststags_tag_url ON poststags(tag_url);
CREATE INDEX songs_post_url ON songs(post_url);
CREATE INDEX songsthumbnails_song_thumbnailUrl ON songsthumbnails(song_thumbnailUrl);`
}
];
}
@ -614,7 +650,6 @@ function getSongData(postIdsParams: string, postIds: string[]): Promise<Map<stri
},
new Map()
);
logger.verbose('songMap', songMap);
resolve(songMap);
}
);
@ -660,12 +695,15 @@ function getSongThumbnailData(
thumbUrls: string[]
): Promise<Map<string, SongThumbnailImage[]>> {
return new Promise((resolve, reject) => {
const start = performance.now();
db.all(
`SELECT song_thumbnailUrl, file, sizeDescriptor, kind
FROM songsthumbnails
WHERE song_thumbnailUrl IN (${thumbUrlsParams});`,
thumbUrls,
(err, rows: SongThumbnailAvatarRow[]) => {
const afterQuery = performance.now();
logger.verbose('thumbnail query took', afterQuery - start, 'ms');
if (err != null) {
logger.error('Error loading avatars', err);
reject(err);
@ -687,6 +725,8 @@ function getSongThumbnailData(
},
new Map()
);
const afterReduce = performance.now();
logger.verbose('thumbnail reduce took', afterReduce - afterQuery, 'ms');
resolve(thumbnailMap);
}
);
@ -730,10 +770,16 @@ export async function getPosts(
before: string | null,
limit: number
): Promise<Post[]> {
const start = performance.now();
if (!databaseReady) {
await waitReady();
}
return await getPostsInternal(since, before, limit);
const ready = performance.now();
logger.debug('DB ready took', ready - start, 'ms');
const posts = await getPostsInternal(since, before, limit);
const afterPosts = performance.now();
logger.debug('DB posts', afterPosts - ready, 'ms');
return posts;
}
async function getPostsInternal(
@ -741,6 +787,7 @@ async function getPostsInternal(
before: string | null,
limit: number
): Promise<Post[]> {
const start = performance.now();
let filterQuery = '';
const params: FilterParameter = { $limit: limit };
if (since === null && before === null) {
@ -763,8 +810,12 @@ async function getPostsInternal(
params[acctParam] = ignoredUser;
params[usernameParam] = ignoredUser;
});
const afterFilter = performance.now();
logger.debug('filterQuery took', afterFilter - start, 'ms');
const rows = await getPostData(filterQuery, params);
const afterRows = performance.now();
logger.debug('rows took', afterRows - afterFilter, 'ms');
if (rows.length === 0) {
// No need to check for tags and songs
return [];
@ -772,19 +823,50 @@ async function getPostsInternal(
const postIdsParams = rows.map(() => '?').join(', ');
const postIds = rows.map((r: PostRow) => r.url);
const afterParams = performance.now();
logger.debug('params took', afterParams - afterRows, 'ms');
const tagMap = await getTagData(postIdsParams, postIds);
const afterTag = performance.now();
logger.debug('rows took', afterTag - afterRows, 'ms');
const songMap = await getSongData(postIdsParams, postIds);
for (const entry of songMap) {
for (const songInfo of entry[1]) {
const thumbs = await getSongThumbnails(songInfo);
songInfo.resizedThumbnails = thumbs;
const afterSong = performance.now();
logger.debug('rows took', afterSong - afterTag, 'ms');
const turls = songMap
.values()
.flatMap((x) => x)
.map((x) => x.thumbnailUrl)
.filter((x) => x !== undefined)
.filter((x) => getCachedThumbnail(x) === null)
.toArray();
if (turls) {
const tMap = await getSongThumbnailData('?, '.repeat(turls.length).slice(0, -2), turls);
for (const entry of songMap) {
for (const songInfo of entry[1]) {
if (songInfo.thumbnailUrl === undefined) {
continue;
}
let thumbs = getCachedThumbnail(songInfo.thumbnailUrl);
if (thumbs === null) {
thumbs = tMap.get(songInfo.thumbnailUrl) ?? [];
cacheThumbnail(songInfo.thumbnailUrl, thumbs);
}
songInfo.resizedThumbnails = thumbs;
}
}
}
const afterThumbs2 = performance.now();
logger.debug('thumbs took', afterThumbs2 - afterSong, 'ms');
const accountUrls = [...new Set(rows.map((r: PostRow) => r.account_url))];
const accountUrlsParams = accountUrls.map(() => '?').join(', ');
const avatars = await getAvatarData(accountUrlsParams, accountUrls);
const afterAvatar = performance.now();
logger.debug('avatar took', afterAvatar - afterThumbs2, 'ms');
const posts = rows.map((row) => {
return {
id: row.id,
@ -804,6 +886,8 @@ async function getPostsInternal(
songs: songMap.get(row.url) || []
} as Post;
});
const afterMap = performance.now();
logger.debug('map took', afterMap - afterAvatar, 'ms');
return posts;
}
@ -926,6 +1010,73 @@ export async function getSongThumbnails(song: SongInfo): Promise<SongThumbnailIm
if (!song.thumbnailUrl) {
return [];
}
const cachedThumbnail = getCachedThumbnail(song.thumbnailUrl);
if (cachedThumbnail !== null) {
return cachedThumbnail;
}
const rows = await getSongThumbnailData('?', [song.thumbnailUrl]);
return rows.get(song.thumbnailUrl) ?? [];
const data = rows.get(song.thumbnailUrl) ?? [];
cacheThumbnail(song.thumbnailUrl, data);
return data;
}
function getCachedThumbnail(thumbnailUrl: string): SongThumbnailImage[] | null {
const cachedThumbnail = thumbnailCache.get(thumbnailUrl);
if (cachedThumbnail == undefined) {
thumbnailCacheMisses++;
return null;
}
const age = new Date().getTime() - cachedThumbnail.ts;
if (age <= maxThumbnailCacheAge) {
thumbnailCacheHits++;
return cachedThumbnail.data;
}
thumbnailCache.delete(thumbnailUrl);
thumbnailCacheMisses++;
return null;
}
function cacheThumbnail(thumbnailUrl: string, thumbnails: SongThumbnailImage[]) {
const now = new Date().getTime();
const initialSize = thumbnailCache.size;
if (initialSize >= maxThumbnailCacheSize) {
logger.debug('Sweeping cache. Current size', initialSize);
const deleteTimestampsOlderThan =
thumbnailCache
.values()
.map((x) => x.ts)
.toArray()
.sort((a, b) => a - b)
.slice(0, 20)
.pop() ?? 0;
const timestampExpired = now - maxThumbnailCacheAge;
const threshold = Math.max(timestampExpired, deleteTimestampsOlderThan);
thumbnailCache.forEach((v, k) => {
if (v.ts <= threshold) {
thumbnailCache.delete(k);
}
});
if (Logger.isDebugEnabled()) {
logger.debug(
'Swept cache. New size',
thumbnailCache.size,
'deleted',
initialSize - thumbnailCache.size,
'entries'
);
logger.debug(
'Current hitrate',
thumbnailCacheHits,
'/',
thumbnailCacheHits + thumbnailCacheMisses,
'=',
((100.0 * thumbnailCacheHits) / (thumbnailCacheHits + thumbnailCacheMisses)).toFixed(2),
'%'
);
}
}
thumbnailCache.set(thumbnailUrl, {
data: thumbnails,
ts: now
});
}

View File

@ -6,7 +6,8 @@ export const load = (async ({ fetch, setHeaders }) => {
setHeaders({
'cache-control': 'public,max-age=60'
});
const j: Post[] = await p.json();
return {
posts: (await p.json()) as Post[]
posts: j
};
}) satisfies PageLoad;

View File

@ -1,8 +1,17 @@
import { Logger } from '$lib/log';
import type { RequestHandler } from './$types';
const logger = new Logger('+server.ts /');
export const GET = (async ({ fetch, setHeaders }) => {
const start = performance.now();
setHeaders({
'cache-control': 'max-age=10'
});
return await fetch('api/posts');
const afterHeaders = performance.now();
logger.debug('Headers took', afterHeaders - start, 'ms');
const f = await fetch('api/posts?count=5');
const afterFetch = performance.now();
logger.debug('Fetch took', afterFetch - afterHeaders, 'ms');
return f;
}) satisfies RequestHandler;

View File

@ -1,16 +1,28 @@
import { getPosts } from '$lib/server/db';
import { json } from '@sveltejs/kit';
import { Logger } from '$lib/log';
import type { RequestHandler } from './$types';
import { performance } from 'perf_hooks';
const logger = new Logger('+server.ts API');
export const GET = (async ({ url }) => {
const start = performance.now();
const since = url.searchParams.get('since');
const before = url.searchParams.get('before');
let count = Number.parseInt(url.searchParams.get('count') || '');
if (isNaN(count)) {
count = 20;
count = 10;
}
count = Math.min(count, 100);
const afterCount = performance.now();
logger.debug('Count took', afterCount - start, 'ms');
const posts = await getPosts(since, before, count);
return json(posts);
const afterFetch = performance.now();
logger.debug('DB took', afterFetch - afterCount, 'ms');
const resp = json(posts);
const afterResponse = performance.now();
logger.debug('Response took', afterResponse - afterFetch, 'ms');
return resp;
}) satisfies RequestHandler;

View File

@ -1,3 +1,5 @@
// Does not seem to work
//import adapter from 'svelte-adapter-bun';
import adapter from '@sveltejs/adapter-node';
import { vitePreprocess } from '@sveltejs/vite-plugin-svelte';
@ -17,7 +19,7 @@ const config = {
},
csp: {
directives: {
'script-src': ['self', 'unsafe-inline'],
'script-src': ['self'],
'base-uri': ['self'],
'object-src': ['none']
}