Compare commits
No commits in common. "42d91a097f5e1e781e626f8599fc3f74547db11e" and "1cd9d8391079e81f724b3db2d2803dd303ffbcb7" have entirely different histories.
42d91a097f
...
1cd9d83910
@ -1,11 +1,8 @@
|
||||
{
|
||||
"apexskier.eslint.config.eslintConfigPath" : ".eslintrc.cjs",
|
||||
"apexskier.eslint.config.eslintPath" : "node_modules\/@eslint\/eslintrc\/dist\/eslintrc.cjs",
|
||||
"apexskier.eslint.config.fixOnSave" : "Enable",
|
||||
"apexskier.typescript.config.formatDocumentOnSave" : "false",
|
||||
"apexskier.typescript.config.formatDocumentOnSave" : "true",
|
||||
"apexskier.typescript.config.isEnabledForJavascript" : "Enable",
|
||||
"apexskier.typescript.config.organizeImportsOnSave" : "true",
|
||||
"apexskier.typescript.config.userPreferences.quotePreference" : "single",
|
||||
"apexskier.typescript.config.userPreferences.useLabelDetailsInCompletionEntries" : true,
|
||||
"prettier.format-on-save" : "Global Default"
|
||||
"apexskier.typescript.config.userPreferences.useLabelDetailsInCompletionEntries" : true
|
||||
}
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
import { log } from '$lib/log';
|
||||
import { TimelineReader } from '$lib/server/timeline';
|
||||
import type { HandleServerError } from '@sveltejs/kit';
|
||||
import fs from 'fs/promises';
|
||||
@ -7,7 +6,7 @@ TimelineReader.init();
|
||||
|
||||
export const handleError = (({ error }) => {
|
||||
if (error instanceof Error) {
|
||||
log.error('Something went wrong: ', error.name, error.message);
|
||||
console.error('Something went wrong: ', error.name, error.message);
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@ -1,32 +0,0 @@
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { isTruthy } from '$lib/truthyString';
|
||||
const { DEV } = import.meta.env;
|
||||
|
||||
export const enableVerboseLog = isTruthy(env.VERBOSE);
|
||||
|
||||
export const log = {
|
||||
verbose: (...params: any[]) => {
|
||||
if (!enableVerboseLog) {
|
||||
return;
|
||||
}
|
||||
console.debug(new Date().toISOString(), ...params);
|
||||
},
|
||||
debug: (...params: any[]) => {
|
||||
if (!DEV) {
|
||||
return;
|
||||
}
|
||||
console.debug(new Date().toISOString(), ...params);
|
||||
},
|
||||
log: (...params: any[]) => {
|
||||
console.log(new Date().toISOString(), ...params);
|
||||
},
|
||||
info: (...params: any[]) => {
|
||||
console.info(new Date().toISOString(), ...params);
|
||||
},
|
||||
warn: (...params: any[]) => {
|
||||
console.warn(new Date().toISOString(), ...params);
|
||||
},
|
||||
error: (...params: any[]) => {
|
||||
console.error(new Date().toISOString(), ...params);
|
||||
}
|
||||
};
|
||||
@ -1,5 +1,3 @@
|
||||
import type { SongInfo } from '$lib/odesliResponse';
|
||||
|
||||
export interface TimelineEvent {
|
||||
event: string;
|
||||
payload: string;
|
||||
@ -13,7 +11,6 @@ export interface Post {
|
||||
content: string;
|
||||
account: Account;
|
||||
card?: PreviewCard;
|
||||
songs?: SongInfo[];
|
||||
}
|
||||
|
||||
export interface PreviewCard {
|
||||
|
||||
@ -5,7 +5,6 @@ export type SongInfo = {
|
||||
title?: string;
|
||||
artistName?: string;
|
||||
thumbnailUrl?: string;
|
||||
postedUrl: string;
|
||||
};
|
||||
|
||||
export type SongwhipReponse = {
|
||||
|
||||
@ -1,47 +1,10 @@
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { IGNORE_USERS, MASTODON_INSTANCE } from '$env/static/private';
|
||||
import { enableVerboseLog, log } from '$lib/log';
|
||||
import type { Account, Post, Tag } from '$lib/mastodon/response';
|
||||
import type { SongInfo } from '$lib/odesliResponse';
|
||||
import { isTruthy } from '$lib/truthyString';
|
||||
import sqlite3 from 'sqlite3';
|
||||
|
||||
const { DEV } = import.meta.env;
|
||||
|
||||
type FilterParameter = {
|
||||
$limit: number | undefined | null;
|
||||
$since?: string | undefined | null;
|
||||
$before?: string | undefined | null;
|
||||
[x: string]: string | number | undefined | null;
|
||||
};
|
||||
|
||||
type PostRow = {
|
||||
id: string;
|
||||
content: string;
|
||||
created_at: string;
|
||||
url: string;
|
||||
account_id: string;
|
||||
acct: string;
|
||||
username: string;
|
||||
display_name: string;
|
||||
account_url: string;
|
||||
avatar: string;
|
||||
};
|
||||
|
||||
type PostTagRow = {
|
||||
post_id: string;
|
||||
tag: string;
|
||||
url: string;
|
||||
};
|
||||
|
||||
type SongRow = SongInfo & {
|
||||
post_url: string;
|
||||
};
|
||||
|
||||
type Migration = {
|
||||
id: number;
|
||||
name: string;
|
||||
statement: string;
|
||||
};
|
||||
|
||||
const db: sqlite3.Database = new sqlite3.Database('moshingmammut.db');
|
||||
// for the local masto instance, the instance name is *not* saved
|
||||
// as part of the username or acct, so it needs to be stripped
|
||||
@ -57,32 +20,38 @@ const ignoredUsers: string[] =
|
||||
);
|
||||
let databaseReady = false;
|
||||
|
||||
if (enableVerboseLog) {
|
||||
if (DEV && isTruthy(env.VERBOSE)) {
|
||||
sqlite3.verbose();
|
||||
db.on('change', (t, d, table, rowid) => {
|
||||
log.verbose('DB change event', t, d, table, rowid);
|
||||
console.debug('DB change event', t, d, table, rowid);
|
||||
});
|
||||
|
||||
db.on('trace', (sql) => {
|
||||
log.verbose('Running', sql);
|
||||
console.debug('Running', sql);
|
||||
});
|
||||
|
||||
db.on('profile', (sql) => {
|
||||
log.verbose('Finished', sql);
|
||||
console.debug('Finished', sql);
|
||||
});
|
||||
}
|
||||
|
||||
interface Migration {
|
||||
id: number;
|
||||
name: string;
|
||||
statement: string;
|
||||
}
|
||||
|
||||
db.on('open', () => {
|
||||
log.info('Opened database');
|
||||
console.log('Opened database');
|
||||
db.serialize();
|
||||
db.run('CREATE TABLE IF NOT EXISTS "migrations" ("id" integer,"name" TEXT, PRIMARY KEY (id))');
|
||||
db.all('SELECT id FROM migrations', (err, rows: Migration[]) => {
|
||||
if (err !== null) {
|
||||
log.error('Could not fetch existing migrations', err);
|
||||
console.error('Could not fetch existing migrations', err);
|
||||
databaseReady = true;
|
||||
return;
|
||||
}
|
||||
log.debug('Already applied migrations', rows);
|
||||
console.debug('Already applied migrations', rows);
|
||||
const appliedMigrations: Set<number> = new Set(rows.map((row) => row['id']));
|
||||
const toApply = getMigrations().filter((m) => !appliedMigrations.has(m.id));
|
||||
let remaining = toApply.length;
|
||||
@ -99,7 +68,7 @@ db.on('open', () => {
|
||||
databaseReady = true;
|
||||
}
|
||||
if (err !== null) {
|
||||
log.error(`Failed to apply migration ${migration.name}`, err);
|
||||
console.error(`Failed to apply migration ${migration.name}`, err);
|
||||
return;
|
||||
}
|
||||
db.run(
|
||||
@ -107,10 +76,10 @@ db.on('open', () => {
|
||||
[migration.id, migration.name],
|
||||
(e: Error) => {
|
||||
if (e !== null) {
|
||||
log.error(`Failed to mark migration ${migration.name} as applied`, e);
|
||||
console.error(`Failed to mark migration ${migration.name} as applied`, e);
|
||||
return;
|
||||
}
|
||||
log.info(`Applied migration ${migration.name}`);
|
||||
console.info(`Applied migration ${migration.name}`);
|
||||
}
|
||||
);
|
||||
});
|
||||
@ -118,7 +87,7 @@ db.on('open', () => {
|
||||
});
|
||||
});
|
||||
db.on('error', (err) => {
|
||||
log.error('Error opening database', err);
|
||||
console.error('Error opening database', err);
|
||||
});
|
||||
|
||||
function getMigrations(): Migration[] {
|
||||
@ -201,23 +170,6 @@ function getMigrations(): Migration[] {
|
||||
DROP TABLE poststags;
|
||||
ALTER TABLE poststags_new RENAME TO poststags;
|
||||
`
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
name: 'song info for posts',
|
||||
statement: `
|
||||
CREATE TABLE songs (
|
||||
id integer PRIMARY KEY,
|
||||
postedUrl TEXT NOT NULL,
|
||||
overviewUrl TEXT,
|
||||
type TEXT CHECK ( type in ('album', 'song') ),
|
||||
youtubeUrl TEXT,
|
||||
title TEXT,
|
||||
artistName TEXT,
|
||||
thumbnailUrl TEXT,
|
||||
post_url TEXT,
|
||||
FOREIGN KEY (post_url) REFERENCES posts(url)
|
||||
);`
|
||||
}
|
||||
];
|
||||
}
|
||||
@ -227,11 +179,11 @@ async function waitReady(): Promise<undefined> {
|
||||
return new Promise((resolve) => {
|
||||
const interval = setInterval(() => {
|
||||
if (DEV) {
|
||||
log.debug('Waiting for database to be ready');
|
||||
console.debug('Waiting for database to be ready');
|
||||
}
|
||||
if (databaseReady) {
|
||||
if (DEV) {
|
||||
log.debug('DB is ready');
|
||||
console.debug('DB is ready');
|
||||
}
|
||||
clearInterval(interval);
|
||||
resolve(undefined);
|
||||
@ -240,8 +192,13 @@ async function waitReady(): Promise<undefined> {
|
||||
});
|
||||
}
|
||||
|
||||
function saveAccountData(account: Account): Promise<undefined> {
|
||||
return new Promise<undefined>((resolve, reject) => {
|
||||
export async function savePost(post: Post): Promise<undefined> {
|
||||
if (!databaseReady) {
|
||||
await waitReady();
|
||||
}
|
||||
return await new Promise<undefined>((resolve, reject) => {
|
||||
console.debug(`Saving post ${post.url}`);
|
||||
const account = post.account;
|
||||
db.run(
|
||||
`
|
||||
INSERT INTO accounts (id, acct, username, display_name, url, avatar)
|
||||
@ -263,281 +220,192 @@ function saveAccountData(account: Account): Promise<undefined> {
|
||||
],
|
||||
(err) => {
|
||||
if (err !== null) {
|
||||
log.error(`Could not insert/update account ${account.id}`, err);
|
||||
console.error(`Could not insert/update account ${account.id}`, err);
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
resolve(undefined);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function savePostData(post: Post): Promise<undefined> {
|
||||
return new Promise<undefined>((resolve, reject) => {
|
||||
db.run(
|
||||
`
|
||||
INSERT INTO posts (id, content, created_at, url, account_id)
|
||||
VALUES (?, ?, ?, ?, ?) ON CONFLICT(url) DO UPDATE SET
|
||||
content=excluded.content,
|
||||
created_at=excluded.created_at,
|
||||
id=excluded.id,
|
||||
account_id=excluded.account_id;`,
|
||||
[post.id, post.content, post.created_at, post.url, post.account.url],
|
||||
(postErr) => {
|
||||
if (postErr !== null) {
|
||||
log.error(`Could not insert post ${post.url}`, postErr);
|
||||
reject(postErr);
|
||||
return;
|
||||
}
|
||||
resolve(undefined);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function savePostTagData(post: Post): Promise<undefined> {
|
||||
return new Promise<undefined>((resolve, reject) => {
|
||||
if (!post.tags.length) {
|
||||
resolve(undefined);
|
||||
return;
|
||||
}
|
||||
|
||||
db.parallelize(() => {
|
||||
let remaining = post.tags.length;
|
||||
for (const tag of post.tags) {
|
||||
db.run(
|
||||
`
|
||||
INSERT INTO tags (url, tag) VALUES (?, ?)
|
||||
ON CONFLICT(url) DO UPDATE SET
|
||||
tag=excluded.tag;`,
|
||||
[tag.url, tag.name],
|
||||
(tagErr) => {
|
||||
if (tagErr !== null) {
|
||||
log.error(`Could not insert/update tag ${tag.url}`, tagErr);
|
||||
reject(tagErr);
|
||||
INSERT INTO posts (id, content, created_at, url, account_id)
|
||||
VALUES (?, ?, ?, ?, ?) ON CONFLICT(url) DO UPDATE SET
|
||||
content=excluded.content,
|
||||
created_at=excluded.created_at,
|
||||
id=excluded.id,
|
||||
account_id=excluded.account_id;`,
|
||||
[post.id, post.content, post.created_at, post.url, post.account.url],
|
||||
(postErr) => {
|
||||
if (postErr !== null) {
|
||||
console.error(`Could not insert post ${post.url}`, postErr);
|
||||
reject(postErr);
|
||||
return;
|
||||
}
|
||||
db.run(
|
||||
'INSERT INTO poststags (post_id, tag_url) VALUES (?, ?)',
|
||||
[post.url, tag.url],
|
||||
(posttagserr) => {
|
||||
if (posttagserr !== null) {
|
||||
log.error(`Could not insert poststags ${tag.url}, ${post.url}`, posttagserr);
|
||||
reject(posttagserr);
|
||||
return;
|
||||
}
|
||||
// Don't decrease on fail
|
||||
remaining--;
|
||||
// Only resolve after all have been inserted
|
||||
if (remaining === 0) {
|
||||
resolve(undefined);
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function saveSongInfoData(postUrl: string, songs: SongInfo[]): Promise<undefined> {
|
||||
return new Promise<undefined>((resolve, reject) => {
|
||||
if (songs.length === 0) {
|
||||
resolve(undefined);
|
||||
return;
|
||||
}
|
||||
db.parallelize(() => {
|
||||
let remaining = songs.length;
|
||||
for (const song of songs) {
|
||||
db.run(
|
||||
`
|
||||
INSERT INTO songs (postedUrl, overviewUrl, type, youtubeUrl, title, artistName, thumbnailUrl, post_url)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`,
|
||||
[
|
||||
song.postedUrl,
|
||||
song.pageUrl,
|
||||
song.type,
|
||||
song.youtubeUrl,
|
||||
song.title,
|
||||
song.artistName,
|
||||
song.thumbnailUrl,
|
||||
postUrl
|
||||
],
|
||||
(songErr) => {
|
||||
if (songErr !== null) {
|
||||
log.error(`Could not insert song ${song.postedUrl}`, songErr);
|
||||
reject(songErr);
|
||||
return;
|
||||
}
|
||||
// Don't decrease on fail
|
||||
remaining--;
|
||||
// Only resolve after all have been inserted
|
||||
if (remaining === 0) {
|
||||
if (!post.tags.length) {
|
||||
resolve(undefined);
|
||||
return;
|
||||
}
|
||||
|
||||
db.parallelize(() => {
|
||||
let remaining = post.tags.length;
|
||||
for (const tag of post.tags) {
|
||||
db.run(
|
||||
`
|
||||
INSERT INTO tags (url, tag) VALUES (?, ?)
|
||||
ON CONFLICT(url) DO UPDATE SET
|
||||
tag=excluded.tag;`,
|
||||
[tag.url, tag.name],
|
||||
(tagErr) => {
|
||||
if (tagErr !== null) {
|
||||
console.error(`Could not insert/update tag ${tag.url}`, tagErr);
|
||||
reject(tagErr);
|
||||
return;
|
||||
}
|
||||
db.run(
|
||||
'INSERT INTO poststags (post_id, tag_url) VALUES (?, ?)',
|
||||
[post.url, tag.url],
|
||||
(posttagserr) => {
|
||||
if (posttagserr !== null) {
|
||||
console.error(
|
||||
`Could not insert poststags ${tag.url}, ${post.url}`,
|
||||
posttagserr
|
||||
);
|
||||
reject(posttagserr);
|
||||
return;
|
||||
}
|
||||
// Don't decrease on fail
|
||||
remaining--;
|
||||
// Only resolve after all have been inserted
|
||||
if (remaining === 0) {
|
||||
resolve(undefined);
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export async function savePost(post: Post, songs: SongInfo[]) {
|
||||
type FilterParameter = {
|
||||
$limit: number | undefined | null;
|
||||
$since?: string | undefined | null;
|
||||
$before?: string | undefined | null;
|
||||
[x: string]: string | number | undefined | null;
|
||||
};
|
||||
|
||||
export async function getPosts(since: string | null, before: string | null, limit: number) {
|
||||
if (!databaseReady) {
|
||||
await waitReady();
|
||||
}
|
||||
const promise = await new Promise<Post[]>((resolve, reject) => {
|
||||
let filter_query = '';
|
||||
const params: FilterParameter = { $limit: limit };
|
||||
if (since === null && before === null) {
|
||||
filter_query = '';
|
||||
} else if (since !== null) {
|
||||
filter_query = 'WHERE posts.created_at > $since';
|
||||
params.$since = since;
|
||||
} else if (before !== null) {
|
||||
// Setting both, before and since doesn't make sense, so this case is not explicitly handled
|
||||
filter_query = 'WHERE posts.created_at < $before';
|
||||
params.$before = before;
|
||||
}
|
||||
|
||||
log.debug(`Saving post ${post.url}`);
|
||||
const account = post.account;
|
||||
await saveAccountData(account);
|
||||
log.debug(`Saved account data ${post.url}`);
|
||||
await savePostData(post);
|
||||
log.debug(`Saved post data ${post.url}`);
|
||||
await savePostTagData(post);
|
||||
log.debug(`Saved ${post.tags.length} tag data ${post.url}`);
|
||||
await saveSongInfoData(post.url, songs);
|
||||
log.debug(`Saved ${songs.length} song info data ${post.url}`);
|
||||
}
|
||||
ignoredUsers.forEach((ignoredUser, index) => {
|
||||
const userParam = `$user_${index}`;
|
||||
const acctParam = userParam + 'a';
|
||||
const usernameParam = userParam + 'u';
|
||||
const prefix = filter_query === '' ? ' WHERE' : ' AND';
|
||||
filter_query += `${prefix} acct != ${acctParam} AND username != ${usernameParam} `;
|
||||
params[acctParam] = ignoredUser;
|
||||
params[usernameParam] = ignoredUser;
|
||||
});
|
||||
|
||||
function getPostData(filterQuery: string, params: FilterParameter): Promise<PostRow[]> {
|
||||
const sql = `SELECT posts.id, posts.content, posts.created_at, posts.url,
|
||||
accounts.id AS account_id, accounts.acct, accounts.username, accounts.display_name,
|
||||
accounts.url AS account_url, accounts.avatar
|
||||
FROM posts
|
||||
JOIN accounts ON posts.account_id = accounts.url
|
||||
${filterQuery}
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $limit`;
|
||||
type PostResult = {
|
||||
id: string;
|
||||
content: string;
|
||||
created_at: string;
|
||||
url: string;
|
||||
account_id: string;
|
||||
acct: string;
|
||||
username: string;
|
||||
display_name: string;
|
||||
account_url: string;
|
||||
avatar: string;
|
||||
};
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
db.all(sql, params, (err, rows: PostRow[]) => {
|
||||
type PostTagResult = {
|
||||
post_id: string;
|
||||
tag: string;
|
||||
url: string;
|
||||
};
|
||||
|
||||
const sql = `SELECT posts.id, posts.content, posts.created_at, posts.url,
|
||||
accounts.id AS account_id, accounts.acct, accounts.username, accounts.display_name,
|
||||
accounts.url AS account_url, accounts.avatar
|
||||
FROM posts
|
||||
JOIN accounts ON posts.account_id = accounts.url
|
||||
${filter_query}
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $limit`;
|
||||
db.all(sql, params, (err, rows: PostResult[]) => {
|
||||
if (err != null) {
|
||||
log.error('Error loading posts', err);
|
||||
console.error('Error loading posts', err);
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
resolve(rows);
|
||||
if (rows.length === 0) {
|
||||
// No need to check for tags
|
||||
resolve([]);
|
||||
return;
|
||||
}
|
||||
const postIdsParams = rows.map(() => '?').join(', ');
|
||||
db.all(
|
||||
`SELECT post_id, tags.url, tags.tag
|
||||
FROM poststags
|
||||
JOIN tags ON poststags.tag_url = tags.url
|
||||
WHERE post_id IN (${postIdsParams});`,
|
||||
rows.map((r: PostResult) => r.url),
|
||||
(tagErr, tagRows: PostTagResult[]) => {
|
||||
if (tagErr != null) {
|
||||
console.error('Error loading post tags', tagErr);
|
||||
reject(tagErr);
|
||||
return;
|
||||
}
|
||||
const tagMap: Map<string, Tag[]> = tagRows.reduce((result: Map<string, Tag[]>, item) => {
|
||||
const tag: Tag = {
|
||||
url: item.url,
|
||||
name: item.tag
|
||||
};
|
||||
result.set(item.post_id, [...(result.get(item.post_id) || []), tag]);
|
||||
return result;
|
||||
}, new Map());
|
||||
const posts = rows.map((row) => {
|
||||
return {
|
||||
id: row.id,
|
||||
content: row.content,
|
||||
created_at: row.created_at,
|
||||
url: row.url,
|
||||
tags: tagMap.get(row.id) || [],
|
||||
account: {
|
||||
id: row.account_id,
|
||||
acct: row.acct,
|
||||
username: row.username,
|
||||
display_name: row.display_name,
|
||||
url: row.account_url,
|
||||
avatar: row.avatar
|
||||
} as Account
|
||||
} as Post;
|
||||
});
|
||||
resolve(posts);
|
||||
}
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function getTagData(postIdsParams: String, postIds: string[]): Promise<Map<string, Tag[]>> {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.all(
|
||||
`SELECT post_id, tags.url, tags.tag
|
||||
FROM poststags
|
||||
JOIN tags ON poststags.tag_url = tags.url
|
||||
WHERE post_id IN (${postIdsParams});`,
|
||||
postIds,
|
||||
(tagErr, tagRows: PostTagRow[]) => {
|
||||
if (tagErr != null) {
|
||||
log.error('Error loading post tags', tagErr);
|
||||
reject(tagErr);
|
||||
return;
|
||||
}
|
||||
const tagMap: Map<string, Tag[]> = tagRows.reduce((result: Map<string, Tag[]>, item) => {
|
||||
const tag: Tag = {
|
||||
url: item.url,
|
||||
name: item.tag
|
||||
};
|
||||
result.set(item.post_id, [...(result.get(item.post_id) || []), tag]);
|
||||
return result;
|
||||
}, new Map());
|
||||
resolve(tagMap);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function getSongData(postIdsParams: String, postIds: string[]): Promise<Map<string, SongInfo[]>> {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.all(
|
||||
`SELECT post_url, songs.postedUrl, songs.overviewUrl, songs.type, songs.youtubeUrl,
|
||||
songs.title, songs.artistName, songs.thumbnailUrl, songs.post_url
|
||||
FROM songs
|
||||
WHERE post_url IN (${postIdsParams});`,
|
||||
postIds,
|
||||
(tagErr, tagRows: SongRow[]) => {
|
||||
if (tagErr != null) {
|
||||
log.error('Error loading post tags', tagErr);
|
||||
reject(tagErr);
|
||||
return;
|
||||
}
|
||||
const songMap: Map<string, SongInfo[]> = tagRows.reduce(
|
||||
(result: Map<string, SongInfo[]>, item) => {
|
||||
result.set(item.post_url, [...(result.get(item.post_url) || []), item]);
|
||||
return result;
|
||||
},
|
||||
new Map()
|
||||
);
|
||||
resolve(songMap);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export async function getPosts(
|
||||
since: string | null,
|
||||
before: string | null,
|
||||
limit: number
|
||||
): Promise<Post[]> {
|
||||
if (!databaseReady) {
|
||||
await waitReady();
|
||||
}
|
||||
|
||||
let filterQuery = '';
|
||||
const params: FilterParameter = { $limit: limit };
|
||||
if (since === null && before === null) {
|
||||
filterQuery = '';
|
||||
} else if (since !== null) {
|
||||
filterQuery = 'WHERE posts.created_at > $since';
|
||||
params.$since = since;
|
||||
} else if (before !== null) {
|
||||
// Setting both, before and since doesn't make sense, so this case is not explicitly handled
|
||||
filterQuery = 'WHERE posts.created_at < $before';
|
||||
params.$before = before;
|
||||
}
|
||||
|
||||
ignoredUsers.forEach((ignoredUser, index) => {
|
||||
const userParam = `$user_${index}`;
|
||||
const acctParam = userParam + 'a';
|
||||
const usernameParam = userParam + 'u';
|
||||
const prefix = filterQuery === '' ? ' WHERE' : ' AND';
|
||||
filterQuery += `${prefix} acct != ${acctParam} AND username != ${usernameParam} `;
|
||||
params[acctParam] = ignoredUser;
|
||||
params[usernameParam] = ignoredUser;
|
||||
});
|
||||
|
||||
const rows = await getPostData(filterQuery, params);
|
||||
if (rows.length === 0) {
|
||||
// No need to check for tags and songs
|
||||
return [];
|
||||
}
|
||||
|
||||
const postIdsParams = rows.map(() => '?').join(', ');
|
||||
const postIds = rows.map((r: PostRow) => r.url);
|
||||
const tagMap = await getTagData(postIdsParams, postIds);
|
||||
const songMap = await getSongData(postIdsParams, postIds);
|
||||
const posts = rows.map((row) => {
|
||||
return {
|
||||
id: row.id,
|
||||
content: row.content,
|
||||
created_at: row.created_at,
|
||||
url: row.url,
|
||||
tags: tagMap.get(row.url) || [],
|
||||
account: {
|
||||
id: row.account_id,
|
||||
acct: row.acct,
|
||||
username: row.username,
|
||||
display_name: row.display_name,
|
||||
url: row.account_url,
|
||||
avatar: row.avatar
|
||||
} as Account,
|
||||
songs: songMap.get(row.url) || []
|
||||
} as Post;
|
||||
});
|
||||
return posts;
|
||||
return promise;
|
||||
}
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
import { BASE_URL, WEBSUB_HUB } from '$env/static/private';
|
||||
import { PUBLIC_MASTODON_INSTANCE_DISPLAY_NAME } from '$env/static/public';
|
||||
import type { Post } from '$lib//mastodon/response';
|
||||
import { log } from '$lib/log';
|
||||
import { Feed } from 'feed';
|
||||
import fs from 'fs/promises';
|
||||
|
||||
@ -60,6 +59,6 @@ export async function saveAtomFeed(feed: Feed) {
|
||||
body: params
|
||||
});
|
||||
} catch (e) {
|
||||
log.error('Failed to update WebSub hub', e);
|
||||
console.error('Failed to update WebSub hub', e);
|
||||
}
|
||||
}
|
||||
|
||||
@ -6,7 +6,6 @@ import {
|
||||
YOUTUBE_API_KEY,
|
||||
YOUTUBE_DISABLE
|
||||
} from '$env/static/private';
|
||||
import { log } from '$lib/log';
|
||||
import type { Post, Tag, TimelineEvent } from '$lib/mastodon/response';
|
||||
import type { OdesliResponse, Platform, SongInfo } from '$lib/odesliResponse';
|
||||
import { getPosts, savePost } from '$lib/server/db';
|
||||
@ -39,7 +38,7 @@ export class TimelineReader {
|
||||
const resp = await fetch(youtubeVideoUrl);
|
||||
const respObj = await resp.json();
|
||||
if (!respObj.items.length) {
|
||||
log.warn('Could not find video with id', videoId);
|
||||
console.warn('Could not find video with id', videoId);
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -78,18 +77,25 @@ export class TimelineReader {
|
||||
return match[0];
|
||||
}
|
||||
} catch (e) {
|
||||
log.error('Could not check if', videoId, 'is a music video', e);
|
||||
console.error('Could not check if', videoId, 'is a music video', e);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static async getSongInfo(url: URL, remainingTries = 6): Promise<SongInfo | null> {
|
||||
private static async getSongInfo(url: string, remainingTries = 6): Promise<SongInfo | null> {
|
||||
if (remainingTries === 0) {
|
||||
log.error('No tries remaining. Lookup failed!');
|
||||
console.error('No tries remaining. Lookup failed!');
|
||||
return null;
|
||||
}
|
||||
if (url.hostname === 'songwhip.com') {
|
||||
let hostname: string;
|
||||
try {
|
||||
hostname = new URL(url).hostname;
|
||||
} catch (e) {
|
||||
console.error(`Could not construct URL ${url}`, e);
|
||||
return null;
|
||||
}
|
||||
if (hostname === 'songwhip.com') {
|
||||
// song.link doesn't support songwhip links and songwhip themselves will provide metadata if you pass in a
|
||||
// Apple Music/Spotify/etc link, but won't when provided with their own link, so no way to extract song info
|
||||
// except maybe scraping their HTML
|
||||
@ -97,7 +103,7 @@ export class TimelineReader {
|
||||
}
|
||||
|
||||
const odesliParams = new URLSearchParams();
|
||||
odesliParams.append('url', url.toString());
|
||||
odesliParams.append('url', url);
|
||||
odesliParams.append('userCountry', 'DE');
|
||||
odesliParams.append('songIfSingle', 'true');
|
||||
if (ODESLI_API_KEY && ODESLI_API_KEY !== 'CHANGE_ME') {
|
||||
@ -115,18 +121,17 @@ export class TimelineReader {
|
||||
return {
|
||||
...info,
|
||||
pageUrl: odesliInfo.pageUrl,
|
||||
youtubeUrl: odesliInfo.linksByPlatform[platform]?.url,
|
||||
postedUrl: url.toString()
|
||||
youtubeUrl: odesliInfo.linksByPlatform[platform]?.url
|
||||
} as SongInfo;
|
||||
});
|
||||
});
|
||||
} catch (e) {
|
||||
if (e instanceof Error && e.cause === 429) {
|
||||
log.warn('song.link rate limit reached. Trying again in 10 seconds');
|
||||
console.warn('song.link rate limit reached. Trying again in 10 seconds');
|
||||
await sleep(10_000);
|
||||
return await this.getSongInfo(url, remainingTries - 1);
|
||||
}
|
||||
log.error(`Failed to load ${url} info from song.link`, e);
|
||||
console.error(`Failed to load ${url} info from song.link`, e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -144,7 +149,7 @@ export class TimelineReader {
|
||||
).json();
|
||||
return status.card?.url;
|
||||
} catch (e) {
|
||||
log.error(`Could not fetch status ${post.url}`, e);
|
||||
console.error(`Could not fetch status ${post.url}`, e);
|
||||
}
|
||||
*/
|
||||
}
|
||||
@ -152,7 +157,7 @@ export class TimelineReader {
|
||||
private startWebsocket() {
|
||||
const socket = new WebSocket(`wss://${MASTODON_INSTANCE}/api/v1/streaming`);
|
||||
socket.onopen = () => {
|
||||
log.log('Connected to WS');
|
||||
console.log('Connected to WS');
|
||||
socket.send('{ "type": "subscribe", "stream": "public:local"}');
|
||||
};
|
||||
socket.onmessage = async (event) => {
|
||||
@ -167,97 +172,86 @@ export class TimelineReader {
|
||||
|
||||
const urls: string[] = URL_FILTER.split(',');
|
||||
const found_urls = urls.filter((t) => post.content.includes(t));
|
||||
const urlsToCheck: string[] = [];
|
||||
// If we don't have any tags or non-youtube urls, check youtube
|
||||
// YT is handled separately, because it requires an API call and therefore is slower
|
||||
if (found_urls.length === 0 && found_tags.length === 0) {
|
||||
const youtubeUrl = await TimelineReader.checkYoutubeMatches(post.content);
|
||||
if (youtubeUrl === null) {
|
||||
log.log('Ignoring post', post.url);
|
||||
console.log('Ignoring post', post.url);
|
||||
return;
|
||||
}
|
||||
log.debug('Found YT URL', youtubeUrl, found_urls, found_urls.length);
|
||||
urlsToCheck.push(youtubeUrl);
|
||||
console.log('Found YT URL', youtubeUrl, found_urls, found_urls.length);
|
||||
}
|
||||
|
||||
// TODO: Change URL detection above to use this regex.
|
||||
// Looks like we're stuck with regex for now instead of using preview cards.
|
||||
// Might as well use it to find URLs. Could also use this for YouTube: If Odesli finds something, it's a song,
|
||||
// if not, ignore it. No need to consult the YT API and give those links a special handling
|
||||
const musicUrls: URL[] = [];
|
||||
const musicUrls: string[] = [];
|
||||
const musicUrl = await TimelineReader.getUrlFromPreviewCard(post);
|
||||
if (musicUrl) {
|
||||
try {
|
||||
musicUrls.push(new URL(musicUrl));
|
||||
} catch (e) {
|
||||
log.error(
|
||||
'URL received from preview card does not seem to be a valid URL',
|
||||
musicUrl,
|
||||
e
|
||||
);
|
||||
}
|
||||
musicUrls.push(musicUrl);
|
||||
} else {
|
||||
const urlMatches = post.content.matchAll(URL_REGEX);
|
||||
for (const match of urlMatches) {
|
||||
if (match === undefined || match.groups === undefined) {
|
||||
console.warn(
|
||||
'Match listed in allMatches, but either it or its groups are undefined',
|
||||
match
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const urlMatch = match.groups.postUrl.toString();
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(urlMatch);
|
||||
} catch (e) {
|
||||
log.error('URL found via Regex does not seem to be a valud url', urlMatch, e);
|
||||
continue;
|
||||
const musicUrl = urls.find((u) => urlMatch.includes(u));
|
||||
if (musicUrl) {
|
||||
musicUrls.push(urlMatch);
|
||||
}
|
||||
|
||||
// Check *all* found url and let odesli determine if it is music or not
|
||||
musicUrls.push(url);
|
||||
}
|
||||
}
|
||||
|
||||
const songs: SongInfo[] = [];
|
||||
log.debug(`Checking ${musicUrls.length} URLs if they contain song data`);
|
||||
for (const url of musicUrls) {
|
||||
let hostname: string | null = null;
|
||||
try {
|
||||
hostname = new URL(url).hostname;
|
||||
} catch (e) {
|
||||
log.error(`Could not check hostname for URL ${url}`, e);
|
||||
console.error(`Could not check hostname for URL ${url}`, e);
|
||||
}
|
||||
if (hostname === 'songwhip.com') {
|
||||
// TODO: Implement checking the songwhip API
|
||||
continue;
|
||||
}
|
||||
const info = await TimelineReader.getSongInfo(url);
|
||||
log.debug(`Found song info for ${url}?`, info);
|
||||
if (info) {
|
||||
songs.push(info);
|
||||
console.info(
|
||||
'Got song info for',
|
||||
post.url,
|
||||
url,
|
||||
info.artistName,
|
||||
info.title,
|
||||
info.thumbnailUrl,
|
||||
info.pageUrl,
|
||||
info.youtubeUrl
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await savePost(post, songs);
|
||||
log.debug('Saved post', post.url);
|
||||
await savePost(post);
|
||||
|
||||
const posts = await getPosts(null, null, 100);
|
||||
await saveAtomFeed(createFeed(posts));
|
||||
} catch (e) {
|
||||
log.error('error message', event, event.data, e);
|
||||
console.error('error message', event, event.data, e);
|
||||
}
|
||||
};
|
||||
socket.onclose = (event) => {
|
||||
log.warn(
|
||||
console.warn(
|
||||
`Websocket connection to ${MASTODON_INSTANCE} closed. Code: ${event.code}, reason: '${event.reason}'`
|
||||
);
|
||||
setTimeout(() => {
|
||||
log.info(`Attempting to reconenct to WS`);
|
||||
console.info(`Attempting to reconenct to WS`);
|
||||
this.startWebsocket();
|
||||
}, 10000);
|
||||
};
|
||||
socket.onerror = (event) => {
|
||||
log.error(
|
||||
console.error(
|
||||
`Websocket connection to ${MASTODON_INSTANCE} failed. ${event.type}: ${event.error}, message: '${event.message}'`
|
||||
);
|
||||
};
|
||||
|
||||
Loading…
Reference in New Issue
Block a user