Compare commits
8 Commits
Author | SHA1 | Date | |
---|---|---|---|
74bc0a18da
|
|||
03605149d5
|
|||
509f76e064
|
|||
e1644b636b
|
|||
5cff2cdc86
|
|||
1077491423
|
|||
348fc929d9
|
|||
258a813c94
|
2
.gitignore
vendored
2
.gitignore
vendored
@ -3,6 +3,8 @@ yt_auth_token
|
||||
spotify_auth_token
|
||||
tidal_auth_token
|
||||
*.db
|
||||
*.db-shm
|
||||
*.db-wal
|
||||
feed.xml
|
||||
playbook.yml
|
||||
inventory.yml
|
||||
|
@ -32,6 +32,10 @@
|
||||
Header set Cache-Control "public,max-age=31536000,immutable"
|
||||
</Directory>
|
||||
|
||||
Header set X-Content-Type-Options "nosniff"
|
||||
Header set Strict-Transport-Security "max-age=15768000; includeSubDomains"
|
||||
Header set X-Frame-Options "DENY"
|
||||
|
||||
ProxyPass / http://localhost:3000/
|
||||
ProxyPassReverse / http://localhost:3000/
|
||||
|
||||
|
937
package-lock.json
generated
937
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
10
package.json
10
package.json
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "moshing-mammut",
|
||||
"version": "2.0.1",
|
||||
"version": "2.1.1",
|
||||
"private": true,
|
||||
"license": "LGPL-3.0-or-later",
|
||||
"scripts": {
|
||||
@ -18,7 +18,7 @@
|
||||
"@eslint/js": "^9.30.1",
|
||||
"@sveltejs/adapter-node": "^5.2.12",
|
||||
"@sveltejs/kit": "^2.22.2",
|
||||
"@sveltejs/vite-plugin-svelte": "^5.1.0",
|
||||
"@sveltejs/vite-plugin-svelte": "^6.1.0",
|
||||
"@types/node": "^22.9.0",
|
||||
"@types/sqlite3": "^3.0.0",
|
||||
"@types/ws": "^8.5.0",
|
||||
@ -32,6 +32,7 @@
|
||||
"prettier": "^3.1.0",
|
||||
"prettier-plugin-svelte": "^3.2.6",
|
||||
"svelte": "^5",
|
||||
"svelte-adapter-bun": "^0.5.2",
|
||||
"svelte-check": "^4.0.0",
|
||||
"tslib": "^2.0.0",
|
||||
"typescript": "^5.0.0"
|
||||
@ -44,6 +45,11 @@
|
||||
"sqlite3": "^5.0.0",
|
||||
"ws": "^8.18.0"
|
||||
},
|
||||
"overrides": {
|
||||
"@sveltejs/kit": {
|
||||
"cookie": ">=0.7.0"
|
||||
}
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
|
@ -67,8 +67,12 @@ type Migration = {
|
||||
name: string;
|
||||
statement: string;
|
||||
};
|
||||
|
||||
const db: sqlite3.Database = new sqlite3.Database('moshingmammut.db');
|
||||
const db: sqlite3.Database = new sqlite3.cached.Database('moshingmammut.db');
|
||||
const maxThumbnailCacheSize = 100;
|
||||
const maxThumbnailCacheAge = 1 * 60 * 60 * 1000; //1h in ms
|
||||
const thumbnailCache = new Map<string, { data: SongThumbnailImage[]; ts: number }>();
|
||||
let thumbnailCacheHits = 0;
|
||||
let thumbnailCacheMisses = 0;
|
||||
|
||||
export function close() {
|
||||
try {
|
||||
@ -93,6 +97,7 @@ const ignoredUsers: string[] =
|
||||
let databaseReady = false;
|
||||
|
||||
if (enableVerboseLog) {
|
||||
logger.verbose('Enabling verbose DB log');
|
||||
sqlite3.verbose();
|
||||
db.on('change', (t, d, table, rowid) => {
|
||||
logger.verbose('DB change event', t, d, table, rowid);
|
||||
@ -102,8 +107,16 @@ if (enableVerboseLog) {
|
||||
logger.verbose('Running', sql);
|
||||
});
|
||||
|
||||
db.on('profile', (sql) => {
|
||||
logger.verbose('Finished', sql);
|
||||
db.on('profile', (sql, time) => {
|
||||
if (sql.startsWith('EXPLAIN')) {
|
||||
return;
|
||||
}
|
||||
logger.verbose('Finished', sql, 'time', time);
|
||||
if (sql.startsWith('SELECT')) {
|
||||
db.get('EXPLAIN QUERY PLAN ' + sql, (a: sqlite3.RunResult, b: Error) => {
|
||||
logger.verbose(sql, a, b);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@ -149,6 +162,9 @@ async function applyMigration(migration: Migration) {
|
||||
db.on('open', () => {
|
||||
logger.info('Opened database');
|
||||
db.serialize();
|
||||
db.run('pragma journal_mode = WAL;');
|
||||
db.run('pragma synchronous = normal;');
|
||||
db.run('pragma journal_size_limit = 6144000;');
|
||||
db.run('CREATE TABLE IF NOT EXISTS "migrations" ("id" integer,"name" TEXT, PRIMARY KEY (id))');
|
||||
db.all('SELECT id FROM migrations', (err, rows: Migration[]) => {
|
||||
if (err !== null) {
|
||||
@ -340,6 +356,26 @@ function getMigrations(): Migration[] {
|
||||
name: 'song tidal id',
|
||||
statement: `
|
||||
ALTER TABLE songs ADD COLUMN tidalId TEXT NULL;`
|
||||
},
|
||||
{
|
||||
id: 10,
|
||||
name: 'add indexes',
|
||||
statement: `
|
||||
CREATE INDEX posts_created_at ON posts(created_at);
|
||||
CREATE INDEX accounts_acct ON accounts(acct);
|
||||
CREATE INDEX accounts_username ON accounts(username);`
|
||||
},
|
||||
{
|
||||
id: 11,
|
||||
name: 'add FK indexes',
|
||||
statement: `
|
||||
CREATE INDEX migrations_id ON migrations(id);
|
||||
CREATE INDEX accountsavatars_account_url ON accountsavatars(account_url);
|
||||
CREATE INDEX posts_account_id ON posts(account_id);
|
||||
CREATE INDEX poststags_post_id ON poststags(post_id);
|
||||
CREATE INDEX poststags_tag_url ON poststags(tag_url);
|
||||
CREATE INDEX songs_post_url ON songs(post_url);
|
||||
CREATE INDEX songsthumbnails_song_thumbnailUrl ON songsthumbnails(song_thumbnailUrl);`
|
||||
}
|
||||
];
|
||||
}
|
||||
@ -614,7 +650,6 @@ function getSongData(postIdsParams: string, postIds: string[]): Promise<Map<stri
|
||||
},
|
||||
new Map()
|
||||
);
|
||||
logger.verbose('songMap', songMap);
|
||||
resolve(songMap);
|
||||
}
|
||||
);
|
||||
@ -660,12 +695,15 @@ function getSongThumbnailData(
|
||||
thumbUrls: string[]
|
||||
): Promise<Map<string, SongThumbnailImage[]>> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const start = performance.now();
|
||||
db.all(
|
||||
`SELECT song_thumbnailUrl, file, sizeDescriptor, kind
|
||||
FROM songsthumbnails
|
||||
WHERE song_thumbnailUrl IN (${thumbUrlsParams});`,
|
||||
thumbUrls,
|
||||
(err, rows: SongThumbnailAvatarRow[]) => {
|
||||
const afterQuery = performance.now();
|
||||
logger.verbose('thumbnail query took', afterQuery - start, 'ms');
|
||||
if (err != null) {
|
||||
logger.error('Error loading avatars', err);
|
||||
reject(err);
|
||||
@ -687,6 +725,8 @@ function getSongThumbnailData(
|
||||
},
|
||||
new Map()
|
||||
);
|
||||
const afterReduce = performance.now();
|
||||
logger.verbose('thumbnail reduce took', afterReduce - afterQuery, 'ms');
|
||||
resolve(thumbnailMap);
|
||||
}
|
||||
);
|
||||
@ -730,10 +770,16 @@ export async function getPosts(
|
||||
before: string | null,
|
||||
limit: number
|
||||
): Promise<Post[]> {
|
||||
const start = performance.now();
|
||||
if (!databaseReady) {
|
||||
await waitReady();
|
||||
}
|
||||
return await getPostsInternal(since, before, limit);
|
||||
const ready = performance.now();
|
||||
logger.debug('DB ready took', ready - start, 'ms');
|
||||
const posts = await getPostsInternal(since, before, limit);
|
||||
const afterPosts = performance.now();
|
||||
logger.debug('DB posts', afterPosts - ready, 'ms');
|
||||
return posts;
|
||||
}
|
||||
|
||||
async function getPostsInternal(
|
||||
@ -741,6 +787,7 @@ async function getPostsInternal(
|
||||
before: string | null,
|
||||
limit: number
|
||||
): Promise<Post[]> {
|
||||
const start = performance.now();
|
||||
let filterQuery = '';
|
||||
const params: FilterParameter = { $limit: limit };
|
||||
if (since === null && before === null) {
|
||||
@ -763,8 +810,12 @@ async function getPostsInternal(
|
||||
params[acctParam] = ignoredUser;
|
||||
params[usernameParam] = ignoredUser;
|
||||
});
|
||||
const afterFilter = performance.now();
|
||||
logger.debug('filterQuery took', afterFilter - start, 'ms');
|
||||
|
||||
const rows = await getPostData(filterQuery, params);
|
||||
const afterRows = performance.now();
|
||||
logger.debug('rows took', afterRows - afterFilter, 'ms');
|
||||
if (rows.length === 0) {
|
||||
// No need to check for tags and songs
|
||||
return [];
|
||||
@ -772,19 +823,50 @@ async function getPostsInternal(
|
||||
|
||||
const postIdsParams = rows.map(() => '?').join(', ');
|
||||
const postIds = rows.map((r: PostRow) => r.url);
|
||||
const afterParams = performance.now();
|
||||
logger.debug('params took', afterParams - afterRows, 'ms');
|
||||
|
||||
const tagMap = await getTagData(postIdsParams, postIds);
|
||||
const afterTag = performance.now();
|
||||
logger.debug('rows took', afterTag - afterRows, 'ms');
|
||||
|
||||
const songMap = await getSongData(postIdsParams, postIds);
|
||||
for (const entry of songMap) {
|
||||
for (const songInfo of entry[1]) {
|
||||
const thumbs = await getSongThumbnails(songInfo);
|
||||
songInfo.resizedThumbnails = thumbs;
|
||||
const afterSong = performance.now();
|
||||
logger.debug('rows took', afterSong - afterTag, 'ms');
|
||||
|
||||
const turls = songMap
|
||||
.values()
|
||||
.flatMap((x) => x)
|
||||
.map((x) => x.thumbnailUrl)
|
||||
.filter((x) => x !== undefined)
|
||||
.filter((x) => getCachedThumbnail(x) === null)
|
||||
.toArray();
|
||||
if (turls) {
|
||||
const tMap = await getSongThumbnailData('?, '.repeat(turls.length).slice(0, -2), turls);
|
||||
for (const entry of songMap) {
|
||||
for (const songInfo of entry[1]) {
|
||||
if (songInfo.thumbnailUrl === undefined) {
|
||||
continue;
|
||||
}
|
||||
let thumbs = getCachedThumbnail(songInfo.thumbnailUrl);
|
||||
if (thumbs === null) {
|
||||
thumbs = tMap.get(songInfo.thumbnailUrl) ?? [];
|
||||
cacheThumbnail(songInfo.thumbnailUrl, thumbs);
|
||||
}
|
||||
songInfo.resizedThumbnails = thumbs;
|
||||
}
|
||||
}
|
||||
}
|
||||
const afterThumbs2 = performance.now();
|
||||
logger.debug('thumbs took', afterThumbs2 - afterSong, 'ms');
|
||||
|
||||
const accountUrls = [...new Set(rows.map((r: PostRow) => r.account_url))];
|
||||
const accountUrlsParams = accountUrls.map(() => '?').join(', ');
|
||||
|
||||
const avatars = await getAvatarData(accountUrlsParams, accountUrls);
|
||||
const afterAvatar = performance.now();
|
||||
logger.debug('avatar took', afterAvatar - afterThumbs2, 'ms');
|
||||
|
||||
const posts = rows.map((row) => {
|
||||
return {
|
||||
id: row.id,
|
||||
@ -804,6 +886,8 @@ async function getPostsInternal(
|
||||
songs: songMap.get(row.url) || []
|
||||
} as Post;
|
||||
});
|
||||
const afterMap = performance.now();
|
||||
logger.debug('map took', afterMap - afterAvatar, 'ms');
|
||||
return posts;
|
||||
}
|
||||
|
||||
@ -926,6 +1010,73 @@ export async function getSongThumbnails(song: SongInfo): Promise<SongThumbnailIm
|
||||
if (!song.thumbnailUrl) {
|
||||
return [];
|
||||
}
|
||||
const cachedThumbnail = getCachedThumbnail(song.thumbnailUrl);
|
||||
if (cachedThumbnail !== null) {
|
||||
return cachedThumbnail;
|
||||
}
|
||||
const rows = await getSongThumbnailData('?', [song.thumbnailUrl]);
|
||||
return rows.get(song.thumbnailUrl) ?? [];
|
||||
const data = rows.get(song.thumbnailUrl) ?? [];
|
||||
cacheThumbnail(song.thumbnailUrl, data);
|
||||
return data;
|
||||
}
|
||||
|
||||
function getCachedThumbnail(thumbnailUrl: string): SongThumbnailImage[] | null {
|
||||
const cachedThumbnail = thumbnailCache.get(thumbnailUrl);
|
||||
if (cachedThumbnail == undefined) {
|
||||
thumbnailCacheMisses++;
|
||||
return null;
|
||||
}
|
||||
const age = new Date().getTime() - cachedThumbnail.ts;
|
||||
if (age <= maxThumbnailCacheAge) {
|
||||
thumbnailCacheHits++;
|
||||
return cachedThumbnail.data;
|
||||
}
|
||||
thumbnailCache.delete(thumbnailUrl);
|
||||
thumbnailCacheMisses++;
|
||||
return null;
|
||||
}
|
||||
|
||||
function cacheThumbnail(thumbnailUrl: string, thumbnails: SongThumbnailImage[]) {
|
||||
const now = new Date().getTime();
|
||||
const initialSize = thumbnailCache.size;
|
||||
if (initialSize >= maxThumbnailCacheSize) {
|
||||
logger.debug('Sweeping cache. Current size', initialSize);
|
||||
const deleteTimestampsOlderThan =
|
||||
thumbnailCache
|
||||
.values()
|
||||
.map((x) => x.ts)
|
||||
.toArray()
|
||||
.sort((a, b) => a - b)
|
||||
.slice(0, 20)
|
||||
.pop() ?? 0;
|
||||
const timestampExpired = now - maxThumbnailCacheAge;
|
||||
const threshold = Math.max(timestampExpired, deleteTimestampsOlderThan);
|
||||
thumbnailCache.forEach((v, k) => {
|
||||
if (v.ts <= threshold) {
|
||||
thumbnailCache.delete(k);
|
||||
}
|
||||
});
|
||||
if (Logger.isDebugEnabled()) {
|
||||
logger.debug(
|
||||
'Swept cache. New size',
|
||||
thumbnailCache.size,
|
||||
'deleted',
|
||||
initialSize - thumbnailCache.size,
|
||||
'entries'
|
||||
);
|
||||
logger.debug(
|
||||
'Current hitrate',
|
||||
thumbnailCacheHits,
|
||||
'/',
|
||||
thumbnailCacheHits + thumbnailCacheMisses,
|
||||
'=',
|
||||
((100.0 * thumbnailCacheHits) / (thumbnailCacheHits + thumbnailCacheMisses)).toFixed(2),
|
||||
'%'
|
||||
);
|
||||
}
|
||||
}
|
||||
thumbnailCache.set(thumbnailUrl, {
|
||||
data: thumbnails,
|
||||
ts: now
|
||||
});
|
||||
}
|
||||
|
@ -6,7 +6,8 @@ export const load = (async ({ fetch, setHeaders }) => {
|
||||
setHeaders({
|
||||
'cache-control': 'public,max-age=60'
|
||||
});
|
||||
const j: Post[] = await p.json();
|
||||
return {
|
||||
posts: (await p.json()) as Post[]
|
||||
posts: j
|
||||
};
|
||||
}) satisfies PageLoad;
|
||||
|
@ -1,8 +1,17 @@
|
||||
import { Logger } from '$lib/log';
|
||||
import type { RequestHandler } from './$types';
|
||||
|
||||
const logger = new Logger('+server.ts /');
|
||||
|
||||
export const GET = (async ({ fetch, setHeaders }) => {
|
||||
const start = performance.now();
|
||||
setHeaders({
|
||||
'cache-control': 'max-age=10'
|
||||
});
|
||||
return await fetch('api/posts');
|
||||
const afterHeaders = performance.now();
|
||||
logger.debug('Headers took', afterHeaders - start, 'ms');
|
||||
const f = await fetch('api/posts?count=5');
|
||||
const afterFetch = performance.now();
|
||||
logger.debug('Fetch took', afterFetch - afterHeaders, 'ms');
|
||||
return f;
|
||||
}) satisfies RequestHandler;
|
||||
|
@ -1,16 +1,28 @@
|
||||
import { getPosts } from '$lib/server/db';
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { Logger } from '$lib/log';
|
||||
|
||||
import type { RequestHandler } from './$types';
|
||||
import { performance } from 'perf_hooks';
|
||||
|
||||
const logger = new Logger('+server.ts API');
|
||||
|
||||
export const GET = (async ({ url }) => {
|
||||
const start = performance.now();
|
||||
const since = url.searchParams.get('since');
|
||||
const before = url.searchParams.get('before');
|
||||
let count = Number.parseInt(url.searchParams.get('count') || '');
|
||||
if (isNaN(count)) {
|
||||
count = 20;
|
||||
count = 10;
|
||||
}
|
||||
count = Math.min(count, 100);
|
||||
const afterCount = performance.now();
|
||||
logger.debug('Count took', afterCount - start, 'ms');
|
||||
const posts = await getPosts(since, before, count);
|
||||
return json(posts);
|
||||
const afterFetch = performance.now();
|
||||
logger.debug('DB took', afterFetch - afterCount, 'ms');
|
||||
const resp = json(posts);
|
||||
const afterResponse = performance.now();
|
||||
logger.debug('Response took', afterResponse - afterFetch, 'ms');
|
||||
return resp;
|
||||
}) satisfies RequestHandler;
|
||||
|
@ -1,3 +1,5 @@
|
||||
// Does not seem to work
|
||||
//import adapter from 'svelte-adapter-bun';
|
||||
import adapter from '@sveltejs/adapter-node';
|
||||
import { vitePreprocess } from '@sveltejs/vite-plugin-svelte';
|
||||
|
||||
@ -17,7 +19,7 @@ const config = {
|
||||
},
|
||||
csp: {
|
||||
directives: {
|
||||
'script-src': ['self', 'unsafe-inline'],
|
||||
'script-src': ['self'],
|
||||
'base-uri': ['self'],
|
||||
'object-src': ['none']
|
||||
}
|
||||
|
Reference in New Issue
Block a user