Fix #44, additional minor enhncements

.gitignore

@@ -1,6 +1,7 @@
 moshing-mammut.pid
 yt_auth_token
 spotify_auth_token
+tidal_auth_token
 *.db
 feed.xml
 playbook.yml

src/hooks.server.ts

@@ -37,12 +37,6 @@ export const handleError = (({ error, status }) => {
 }) satisfies HandleServerError;
 
 export const handle = (async ({ event, resolve }) => {
-  const searchParams = event.url.searchParams;
-  const authCode = searchParams.get('code');
-  if (authCode) {
-    logger.debug('received GET hook', event.url.searchParams);
-  }
-
   // Reeder *insists* on checking /feed instead of /feed.xml
   if (event.url.pathname === '/feed') {
     return new Response('', { status: 301, headers: { Location: '/feed.xml' } });

src/lib/log.ts

@@ -48,25 +48,25 @@ export class Logger {
     if (!enableVerboseLog) {
       return;
     }
-    console.debug(new Date().toISOString(), `- ${this.name} -`, '- [VRBSE] -', ...params);
+    console.debug(new Date().toISOString(), '- [VRBSE]', `- ${this.name} -`, ...params);
   }
   public debug(...params: any[]) {
     if (!Logger.isDebugEnabled()) {
       return;
     }
-    console.debug(new Date().toISOString(), `- ${this.name} -`, '- [DEBUG] -', ...params);
+    console.debug(new Date().toISOString(), '- [DEBUG]', `- ${this.name} -`, ...params);
   }
   public log(...params: any[]) {
-    console.log(new Date().toISOString(), `- ${this.name} -`, '- [ LOG ] -', ...params);
+    console.log(new Date().toISOString(), '- [ LOG ]', `- ${this.name} -`, ...params);
   }
   public info(...params: any[]) {
-    console.info(new Date().toISOString(), `- ${this.name} -`, '- [INFO ] -', ...params);
+    console.info(new Date().toISOString(), '- [INFO ]', `- ${this.name} -`, ...params);
   }
   public warn(...params: any[]) {
-    console.warn(new Date().toISOString(), `- ${this.name} -`, '- [WARN ] -', ...params);
+    console.warn(new Date().toISOString(), '- [WARN ]', `- ${this.name} -`, ...params);
   }
   public error(...params: any[]) {
-    console.error(new Date().toISOString(), `- ${this.name} -`, '- [ERROR] -', ...params);
+    console.error(new Date().toISOString(), '- [ERROR]', `- ${this.name} -`, ...params);
   }
 
   public static error(...params: any[]) {

src/lib/odesliResponse.ts

@@ -5,6 +5,7 @@ export type SongInfo = {
   youtubeUrl?: string;
   spotifyUrl?: string;
   spotifyUri?: string;
+  tidalUri?: string;
   type: 'song' | 'album';
   title?: string;
   artistName?: string;

src/lib/server/playlist/oauthPlaylistAdder.ts

@@ -54,7 +54,8 @@ export abstract class OauthPlaylistAdder {
     code: string,
     redirectUri: URL,
     client_secret?: string,
-    customHeader?: HeadersInit
+    customHeader?: HeadersInit,
+    code_verifier?: string
   ) {
     this.logger.debug('received code');
     const params = new URLSearchParams();
@@ -65,6 +66,9 @@ export abstract class OauthPlaylistAdder {
     if (client_secret) {
       params.append('client_secret', client_secret);
     }
+    if (code_verifier) {
+      params.append('code_verifier', code_verifier);
+    }
     this.logger.debug('sending token req', params);
     const resp: OauthResponse = await fetch(tokenUrl, {
       method: 'POST',
@@ -98,12 +102,22 @@ export abstract class OauthPlaylistAdder {
   protected async shouldRefreshToken(): Promise<{ token: OauthResponse; refresh: boolean } | null> {
     const token = await this.auth();
     if (token == null || !token?.expires) {
+      this.logger.warn('Cannot check if token should be refreshed. Token expiry is unreadablle');
       return null;
     }
-    let refreshAt = new Date();
+    if (token.error) {
+      this.logger.error('Access token is invalid, should refresh');
+      return {
+        token: token,
+        refresh: true
+      };
+    }
+    let refreshAt = new Date(token.expires);
+    // Refresh token this.refresh_time minutes before it expires
     refreshAt.setTime(refreshAt.getTime() - this.refresh_time * 60 * 1000);
-    this.logger.info('token expiry', token.expires, 'vs refresh @', refreshAt);
-    if (token.expires.getTime() > refreshAt.getTime()) {
+
+    this.logger.info('refresh @', refreshAt, 'token expires', token.expires);
+    if (refreshAt.getTime() > new Date().getTime()) {
       return {
         token: token,
         refresh: false
@@ -114,7 +128,7 @@ export abstract class OauthPlaylistAdder {
       'Token expires',
       token.expires,
       token.expires.getTime(),
-      `which is after the refresh time`,
+      `so it should be refreshed before or at`,
       refreshAt,
       refreshAt.getTime()
     );

src/lib/server/playlist/tidalPlaylistAdder.ts

@@ -0,0 +1,187 @@
+import { TIDAL_PLAYLIST_ID, TIDAL_CLIENT_ID, TIDAL_CLIENT_SECRET } from '$env/static/private';
+import { Logger } from '$lib/log';
+import type { OauthResponse } from '$lib/mastodon/response';
+import type { SongInfo } from '$lib/odesliResponse';
+import { createHash } from 'crypto';
+import { OauthPlaylistAdder } from './oauthPlaylistAdder';
+import type { PlaylistAdder } from './playlistAdder';
+import type { TidalAddToPlaylistResponse } from './tidalResponse';
+
+export class TidalPlaylistAdder extends OauthPlaylistAdder implements PlaylistAdder {
+  private static code_verifier?: string;
+
+  public constructor() {
+    super('https://openapi.tidal.com/v2', 'tidal_auth_token');
+    //super('https://api.tidal.com/v2', 'tidal_auth_token');
+    this.logger = new Logger('TidalPlaylistAdder');
+  }
+
+  public constructAuthUrl(redirectUri: URL): URL {
+    const endpoint = 'https://login.tidal.com/authorize';
+    const verifier = Buffer.from(crypto.getRandomValues(new Uint8Array(100)).toString(), 'ascii')
+      .toString('base64url')
+      .slice(0, 128);
+    const code_challenge = createHash('sha256').update(verifier).digest('base64url');
+    TidalPlaylistAdder.code_verifier = verifier;
+
+    let additionalParameters = new Map([
+      ['code_challenge_method', 'S256'],
+      ['code_challenge', code_challenge]
+    ]);
+    return this.constructAuthUrlInternal(
+      endpoint,
+      TIDAL_CLIENT_ID,
+      'playlists.write playlists.read user.read', //r_usr w_usr
+      redirectUri,
+      additionalParameters
+    );
+  }
+
+  public async receivedAuthCode(code: string, url: URL) {
+    this.logger.debug('received code');
+    const tokenUrl = new URL('https://auth.tidal.com/v1/oauth2/token');
+    await this.receivedAuthCodeInternal(
+      tokenUrl,
+      TIDAL_CLIENT_ID,
+      code,
+      url,
+      TIDAL_CLIENT_SECRET,
+      undefined,
+      TidalPlaylistAdder.code_verifier
+    );
+  }
+
+  private async refreshToken(force: boolean = false): Promise<OauthResponse | null> {
+    const tokenInfo = await this.shouldRefreshToken();
+    if (tokenInfo == null) {
+      return null;
+    }
+    let token = tokenInfo.token;
+    if (!tokenInfo.refresh && !force) {
+      return token;
+    }
+
+    if (!token.refresh_token) {
+      this.logger.error('Need to refresh access token, but no refresh token provided');
+      return null;
+    }
+
+    const tokenUrl = new URL('https://auth.tidal.com/v1/oauth2/token');
+    return await this.requestRefreshToken(tokenUrl, TIDAL_CLIENT_ID, token.refresh_token);
+  }
+
+  private async addToPlaylistRetry(song: SongInfo, remaning: number = 3) {
+    if (remaning < 0) {
+      this.logger.error('max retries reached, song will not be added to playlist');
+    }
+    this.logger.debug('addToTidalPlaylist', remaning);
+    const token = await this.refreshToken();
+    if (token == null) {
+      return;
+    }
+    this.logger.debug('token check successful');
+
+    if (!TIDAL_PLAYLIST_ID || TIDAL_PLAYLIST_ID === 'CHANGE_ME') {
+      this.logger.debug('no playlist ID configured');
+      return;
+    }
+    if (!song.tidalUri) {
+      this.logger.info('Skip adding song to playlist, no Uri', song);
+      return;
+    }
+
+    // This would be API v2, but that's still in beta and only allows adding an item *before* another one
+    const options: RequestInit = {
+      method: 'POST',
+      headers: {
+        Authorization: `${token.token_type} ${token.access_token}`,
+        'Content-Type': 'application/vnd.api+json',
+        Accept: 'application/vnd.api+json'
+      },
+      body: JSON.stringify({
+        data: [
+          {
+            id: song.tidalUri,
+            type: 'tracks'
+          }
+        ],
+        meta: {
+          positionBefore: 'ffb6286e-237a-4dfc-bbf1-2fb0eb004ed5' // Hardcoded last element of list
+        }
+      })
+    };
+    const apiUrl = new URL(`${this.apiBase}/playlists/${TIDAL_PLAYLIST_ID}/relationships/items`);
+    const request = new Request(apiUrl, options);
+    this.logger.debug('Adding to playlist request', request);
+
+    // This would be API v1 (or api v2, but *not* the OpenAPI v2),
+    // but that requires r_usr and w_usr permission scopes which are impossible to request
+    /*
+    const options: RequestInit = {
+      method: 'POST',
+      headers: {
+        Authorization: `${token.token_type} ${token.access_token}`,
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({
+        onArtifactNotFound: 'SKIP',
+        trackIds: song.tidalUri,
+        //toIndex: -1
+        onDupes: 'SKIP'
+      })
+    };
+    const apiUrl = new URL(`${this.apiBase}/playlists/${TIDAL_PLAYLIST_ID}/items`);
+    try {
+      const r = await fetch(new URL(`${this.apiBase}/playlists/${TIDAL_PLAYLIST_ID}`), {
+        headers: {
+          Authorization: `${token.token_type} ${token.access_token}`
+        }
+      });
+      const txt = await r.text();
+      this.logger.debug('playlist', r.status, txt);
+      const rj = JSON.parse(txt);
+      this.logger.debug('playlist', rj);
+    } catch (e) {
+      this.logger.error('playlist fetch failed', e);
+    }
+
+    const request = new Request(apiUrl, options);
+    this.logger.debug('Adding to playlist request', request);
+    */
+
+    let resp: Response | null = null;
+    try {
+      resp = await fetch(request);
+      let respObj: TidalAddToPlaylistResponse | null = null;
+      // If the request was successful, a 201 with no content is received
+      // Errors will have content and a different status code
+      if (resp.status !== 201) {
+        respObj = await resp.json();
+      }
+      if (respObj !== null && respObj.errors) {
+        this.logger.error('Add to playlist failed', song.tidalUri, resp.status, respObj.errors);
+        if (respObj.errors.some((x) => x.status === 401)) {
+          const token = await this.refreshToken(true);
+          if (token == null) {
+            return;
+          }
+          this.addToPlaylistRetry(song, remaning--);
+        }
+      } else if (respObj === null && resp.status === 201) {
+        this.logger.info('Added to playlist', song.tidalUri, song.title);
+      } else {
+        this.logger.info(
+          'Add to playlist result is neither 201 nor error',
+          song.tidalUri,
+          song.title,
+          respObj
+        );
+      }
+    } catch (e) {
+      this.logger.error('Add to playlist request failed', resp?.status, e);
+    }
+  }
+  public async addToPlaylist(song: SongInfo) {
+    await this.addToPlaylistRetry(song);
+  }
+}

src/lib/server/playlist/tidalResponse.ts

@@ -0,0 +1,28 @@
+export type TidalAddToPlaylistResponse = {
+  errors: TidalAddToPlaylistError[];
+};
+
+export type TidalAddToPlaylistError = {
+  id: string;
+  status: number;
+  code: TidalErrorCode;
+  detail: string;
+  source: TidalAddToPlaylistErrorSource;
+  meta: TidalAddToPlaylistErrorMeta;
+};
+
+export type TidalAddToPlaylistErrorSource = {
+  parameter: string;
+};
+export type TidalAddToPlaylistErrorMeta = {
+  category: string;
+};
+export type TidalErrorCode =
+  | 'INVALID_ENUM_VALUE'
+  | 'VALUE_REGEX_MISMATCH'
+  | 'NOT_FOUND'
+  | 'METHOD_NOT_SUPPORTED'
+  | 'NOT_ACCEPTABLE'
+  | 'UNSUPPORTED_MEDIA_TYPE'
+  | 'UNAVAILABLE_FOR_LEGAL_REASONS_RESPONSE'
+  | 'INTERNAL_SERVER_ERROR';

src/lib/server/playlist/ytPlaylistAdder.ts

@@ -78,7 +78,9 @@ export class YoutubePlaylistAdder extends OauthPlaylistAdder implements Playlist
     );
   }
 
-  public async addToPlaylist(song: SongInfo) {}
+  public async addToPlaylist(song: SongInfo) {
+    await this.addToPlaylistRetry(song);
+  }
 
   private async addToPlaylistRetry(song: SongInfo, remaning: number = 3) {
     if (remaning < 0) {

src/lib/server/rss.ts

@@ -54,12 +54,16 @@ export async function saveAtomFeed(feed: Feed) {
     return;
   }
   try {
-    const params = new URLSearchParams();
-    params.append('hub.mode', 'publish');
-    params.append('hub.url', `${BASE_URL}/feed.xml`);
+    const param = new FormData();
+    param.append('hub.mode', 'publish');
+    param.append('hub.url', `${BASE_URL}/feed.xml`);
+    //const params = new URLSearchParams();
+    //params.append('hub.mode', 'publish');
+    //params.append('hub.url', `${BASE_URL}/feed.xml`);
     await fetch(WEBSUB_HUB, {
       method: 'POST',
-      body: params
+      headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+      body: param
     });
   } catch (e) {
     logger.error('Failed to update WebSub hub', e);

src/lib/server/timeline.ts

@@ -37,6 +37,7 @@ import sharp from 'sharp';
 import { URL, URLSearchParams } from 'url';
 import { WebSocket } from 'ws';
 import type { PlaylistAdder } from './playlist/playlistAdder';
+import { TidalPlaylistAdder } from './playlist/tidalPlaylistAdder';
 
 const URL_REGEX = new RegExp(/href="(?<postUrl>[^>]+?)" target="_blank"/gm);
 const INVIDIOUS_REGEX = new RegExp(/invidious.*?watch.*?v=(?<videoId>[a-zA-Z_0-9-]+)/gm);
@@ -176,22 +177,24 @@ export class TimelineReader {
         }
         const isMusic = await this.isMusicVideo(youtubeId);
         if (!isMusic) {
-          this.logger.debug('Probably not a music video', youtubeId, url);
+          this.logger.debug('Probably not a music video', youtubeId);
           return null;
         }
       }
       const spotify: Platform = 'spotify';
+      const tidal: Platform = 'tidal';
+      const tidalId = odesliInfo.linksByPlatform[tidal]?.entityUniqueId;
+      const tidalUri = tidalId ? odesliInfo.entitiesByUniqueId[tidalId].id : undefined;
+
       const songInfo = {
         ...info,
         pageUrl: odesliInfo.pageUrl,
         youtubeUrl: odesliInfo.linksByPlatform[platform]?.url,
         spotifyUrl: odesliInfo.linksByPlatform[spotify]?.url,
         spotifyUri: odesliInfo.linksByPlatform[spotify]?.nativeAppUriDesktop,
+        tidalUri: tidalUri,
         postedUrl: url.toString()
       } as SongInfo;
-      if (songInfo.youtubeUrl && !songInfo.spotifyUrl) {
-        this.logger.warn('SongInfo with YT, but no spotify URL', odesliInfo);
-      }
       return songInfo;
     } catch (e) {
       if (e instanceof Error && e.cause === 429) {
@@ -374,18 +377,16 @@ export class TimelineReader {
   }
 
   private async checkAndSavePost(post: Post) {
-    if (IGNORE_USERS !== undefined && IGNORE_USERS !== '' && IGNORE_USERS !== 'CHANGE_ME') {
-      const ignorelist = IGNORE_USERS.split(',');
-      const isIgnored = ignorelist.includes(post.account.username);
-      const isIgnoredDb = this.ignoredUsers.includes(post.account.username);
-      this.logger.debug(
-        'Check if user',
-        post.account.username,
+    const isIgnored = this.ignoredUsers.includes(post.account.acct);
+    if (isIgnored) {
+      this.logger.info(
+        'Ignoring post by ignored user',
+        post.account.acct,
         'is ignored',
         this.ignoredUsers,
-        isIgnored,
-        isIgnoredDb
+        isIgnored
       );
+      return;
     }
 
     const hashttags: string[] = HASHTAG_FILTER.split(',');
@@ -421,11 +422,42 @@ export class TimelineReader {
     const socket = new WebSocket(
       `wss://${MASTODON_INSTANCE}/api/v1/streaming?type=subscribe&stream=public:local&access_token=${MASTODON_ACCESS_TOKEN}`
     );
+
+    // Sometimes, the app just stops receiving WS updates.
+    // Regularly check if it is necessary to reset it
+    const wsTimeout = 5;
+    let timeoutId = setTimeout(
+      () => {
+        socketLogger.warn(
+          'Websocket has not received a new post in',
+          wsTimeout,
+          'hours. Resetting, it might be stuck'
+        );
+        socket.close();
+        this.startWebsocket();
+      },
+      1000 * 60 * 60 * wsTimeout
+    ); // 5 hours
     socket.onopen = () => {
       socketLogger.log('Connected to WS');
     };
     socket.onmessage = async (event) => {
       try {
+        // Reset timer
+        clearTimeout(timeoutId);
+        timeoutId = setTimeout(
+          () => {
+            socketLogger.warn(
+              'Websocket has not received a new post in',
+              wsTimeout,
+              'hours. Resetting, it might be stuck'
+            );
+            socket.close();
+            this.startWebsocket();
+          },
+          1000 * 60 * 60 * wsTimeout
+        );
+
         const data: TimelineEvent = JSON.parse(event.data.toString());
         socketLogger.debug('ES event', data.event);
         if (data.event !== 'update') {
@@ -495,9 +527,13 @@ export class TimelineReader {
   private constructor() {
     this.logger = new Logger('Timeline');
     this.logger.log('Constructing timeline object');
-    this.playlistAdders = [new YoutubePlaylistAdder(), new SpotifyPlaylistAdder()];
+    this.playlistAdders = [
+      new YoutubePlaylistAdder(),
+      new SpotifyPlaylistAdder(),
+      new TidalPlaylistAdder()
+    ];
     this.ignoredUsers =
-      IGNORE_USERS === undefined
+      IGNORE_USERS === undefined || IGNORE_USERS === 'CHANGE_ME' || !!IGNORE_USERS
         ? []
         : IGNORE_USERS.split(',')
             .map((u) => (u.startsWith('@') ? u.substring(1) : u))

src/routes/+page.ts

@@ -1,8 +1,11 @@
 import type { Post } from '$lib/mastodon/response';
 import type { PageLoad } from './$types';
 
-export const load = (async ({ fetch }) => {
+export const load = (async ({ fetch, setHeaders }) => {
   const p = await fetch('/');
+  setHeaders({
+    'cache-control': 'public,max-age=60'
+  });
   return {
     posts: (await p.json()) as Post[]
   };

src/routes/+server.ts

@@ -1,5 +1,8 @@
 import type { RequestHandler } from './$types';
 
-export const GET = (async ({ fetch }) => {
+export const GET = (async ({ fetch, setHeaders }) => {
+  setHeaders({
+    'cache-control': 'max-age=10'
+  });
   return await fetch('api/posts');
 }) satisfies RequestHandler;

src/routes/spotifyAuth/+page.server.ts

@@ -8,9 +8,18 @@ const { DEV } = import.meta.env;
 const logger = new Logger('SpotifyAuth');
 
 export const load: PageServerLoad = async ({ url, request }) => {
-  const baseUrl = request.headers.get('X-Forwarded-Host') ?? BASE_URL;
+  const forwardedHost = request.headers.get('X-Forwarded-Host');
+  let redirect_base;
+  if (DEV) {
+    redirect_base = url.origin;
+  } else if (forwardedHost) {
+    redirect_base = `${url.protocol}//${forwardedHost}`;
+  } else {
+    redirect_base = BASE_URL;
+  }
+  const redirect_uri = new URL(`${redirect_base}${url.pathname}`);
+
   const adder = new SpotifyPlaylistAdder();
-  let redirect_uri = new URL(`${new URL(BASE_URL).protocol}//${baseUrl}/spotifyAuth`);
   if (url.hostname === 'localhost' && DEV) {
     redirect_uri.hostname = '127.0.0.1';
   }

src/routes/tidalAuth/+page.server.ts

@@ -0,0 +1,39 @@
+import { BASE_URL } from '$env/static/private';
+import { Logger } from '$lib/log';
+import { redirect } from '@sveltejs/kit';
+import type { PageServerLoad } from './$types';
+import { TidalPlaylistAdder } from '$lib/server/playlist/tidalPlaylistAdder';
+import { URL } from 'node:url';
+const { DEV } = import.meta.env;
+
+const logger = new Logger('TidalAuth');
+
+export const load: PageServerLoad = async ({ url, request }) => {
+  const forwardedHost = request.headers.get('X-Forwarded-Host');
+  let redirect_base;
+  if (DEV) {
+    redirect_base = url.origin;
+  } else if (forwardedHost) {
+    redirect_base = `${url.protocol}//${forwardedHost}`;
+  } else {
+    redirect_base = BASE_URL;
+  }
+  const redirect_uri = new URL(`${redirect_base}${url.pathname}`);
+  const adder = new TidalPlaylistAdder();
+  logger.debug(url.searchParams, url.hostname, redirect_uri);
+  if (url.searchParams.has('code')) {
+    await adder.receivedAuthCode(url.searchParams.get('code') || '', redirect_uri);
+    redirect(307, '/');
+  } else if (url.searchParams.has('error')) {
+    logger.error('received error', url.searchParams.get('error'));
+    return;
+  }
+
+  if (await adder.authCodeExists()) {
+    redirect(307, '/');
+  }
+
+  const authUrl = adder.constructAuthUrl(redirect_uri);
+  logger.debug('+page.server.ts', authUrl.toString());
+  redirect(307, authUrl);
+};

src/routes/tidalAuth/+page.svelte

@@ -0,0 +1 @@
+<h1>Something went wrong</h1>

src/routes/ytauth/+page.server.ts

@@ -3,14 +3,24 @@ import { Logger } from '$lib/log';
 import { YoutubePlaylistAdder } from '$lib/server/playlist/ytPlaylistAdder';
 import { redirect } from '@sveltejs/kit';
 import type { PageServerLoad } from './$types';
+const { DEV } = import.meta.env;
 
 const logger = new Logger('YT Auth');
 
 export const load: PageServerLoad = async ({ url, request }) => {
-  const baseUrl = request.headers.get('X-Forwarded-Host') ?? BASE_URL;
+  const forwardedHost = request.headers.get('X-Forwarded-Host');
+  let redirect_base;
+  if (DEV) {
+    redirect_base = url.origin;
+  } else if (forwardedHost) {
+    redirect_base = `${url.protocol}//${forwardedHost}`;
+  } else {
+    redirect_base = BASE_URL;
+  }
+  const redirect_uri = new URL(`${redirect_base}${url.pathname}`);
+
   const adder = new YoutubePlaylistAdder();
-  logger.debug('redirect URL', `${new URL(BASE_URL).protocol}//${baseUrl}/ytauth`);
-  const redirect_uri = new URL(`${new URL(BASE_URL).protocol}//${baseUrl}/ytauth`);
+  logger.debug('redirect URL', redirect_uri);
   if (url.searchParams.has('code')) {
     logger.debug(url.searchParams);
     await adder.receivedAuthCode(url.searchParams.get('code') || '', redirect_uri);