fix: Fix corrupt database, Add soundcloud and spotify embeds

This commit is contained in:
2026-04-17 19:41:16 +02:00
parent 28797a0141
commit 3ba8a2c9eb
17 changed files with 463 additions and 39 deletions

View File

@@ -1,3 +1,4 @@
import { randomBytes } from 'crypto';
import { app } from 'electron';
import * as fs from 'fs';
import * as fsp from 'fs/promises';
@@ -20,23 +21,93 @@ import {
import { settings } from '../settings';
let applicationDataSource: DataSource | undefined;
let dbFilePath = '';
let dbBackupPath = '';
// SQLite files start with this 16-byte header string.
const SQLITE_MAGIC = 'SQLite format 3\0';
export function getDataSource(): DataSource | undefined {
return applicationDataSource;
}
/**
* Returns true when `data` looks like a valid SQLite file
* (correct header magic and at least one complete page).
*/
function isValidSqlite(data: Uint8Array): boolean {
if (data.length < 100)
return false;
const header = Buffer.from(data.buffer, data.byteOffset, 16).toString('ascii');
return header === SQLITE_MAGIC;
}
/**
* Back up the current DB file so there is always a recovery point.
* If the main file is corrupted/empty but a valid backup exists,
* restore the backup before the app loads the database.
*/
function safeguardDbFile(): Uint8Array | undefined {
if (!fs.existsSync(dbFilePath))
return undefined;
const data = new Uint8Array(fs.readFileSync(dbFilePath));
if (isValidSqlite(data)) {
fs.copyFileSync(dbFilePath, dbBackupPath);
console.log('[DB] Backed up database to', dbBackupPath);
return data;
}
console.warn(`[DB] ${dbFilePath} appears corrupt (${data.length} bytes) - checking backup`);
if (fs.existsSync(dbBackupPath)) {
const backup = new Uint8Array(fs.readFileSync(dbBackupPath));
if (isValidSqlite(backup)) {
fs.copyFileSync(dbBackupPath, dbFilePath);
console.warn('[DB] Restored database from backup', dbBackupPath);
return backup;
}
console.error('[DB] Backup is also invalid - starting with a fresh database');
} else {
console.error('[DB] No backup available - starting with a fresh database');
}
return undefined;
}
/**
* Write the database to disk atomically: write a temp file first,
* then rename it over the real file. rename() is atomic on the same
* filesystem, so a crash mid-write can never leave a half-written DB.
*/
async function atomicSave(data: Uint8Array): Promise<void> {
const tmpPath = dbFilePath + '.tmp-' + randomBytes(6).toString('hex');
try {
await fsp.writeFile(tmpPath, Buffer.from(data));
await fsp.rename(tmpPath, dbFilePath);
} catch (err) {
await fsp.unlink(tmpPath).catch(() => {});
throw err;
}
}
export async function initializeDatabase(): Promise<void> {
const userDataPath = app.getPath('userData');
const dbDir = path.join(userDataPath, 'metoyou');
await fsp.mkdir(dbDir, { recursive: true });
const databaseFilePath = path.join(dbDir, settings.databaseName);
dbFilePath = path.join(dbDir, settings.databaseName);
dbBackupPath = dbFilePath + '.bak';
let database: Uint8Array | undefined;
if (fs.existsSync(databaseFilePath)) {
database = fs.readFileSync(databaseFilePath);
}
const database = safeguardDbFile();
applicationDataSource = new DataSource({
type: 'sqljs',
@@ -59,12 +130,12 @@ export async function initializeDatabase(): Promise<void> {
synchronize: false,
logging: false,
autoSave: true,
location: databaseFilePath
autoSaveCallback: atomicSave
});
try {
await applicationDataSource.initialize();
console.log('[DB] Connection initialised at:', databaseFilePath);
console.log('[DB] Connection initialised at:', dbFilePath);
try {
await applicationDataSource.runMigrations();

View File

@@ -15,8 +15,37 @@ let mainWindow: BrowserWindow | null = null;
let tray: Tray | null = null;
let closeToTrayEnabled = true;
let appQuitting = false;
let youtubeRequestHeadersConfigured = false;
const WINDOW_STATE_CHANGED_CHANNEL = 'window-state-changed';
const YOUTUBE_EMBED_REFERRER = 'https://toju.app/';
function ensureYoutubeEmbedRequestHeaders(): void {
if (youtubeRequestHeadersConfigured || !app.isPackaged) {
return;
}
youtubeRequestHeadersConfigured = true;
session.defaultSession.webRequest.onBeforeSendHeaders(
{
urls: [
'https://www.youtube-nocookie.com/*',
'https://www.youtube.com/*',
'https://*.youtube.com/*',
'https://*.googlevideo.com/*',
'https://*.ytimg.com/*'
]
},
(details, callback) => {
const requestHeaders = { ...details.requestHeaders };
requestHeaders['Referer'] ??= YOUTUBE_EMBED_REFERRER;
callback({ requestHeaders });
}
);
}
function getAssetPath(...segments: string[]): string {
const basePath = app.isPackaged
@@ -163,6 +192,7 @@ export async function createWindow(): Promise<void> {
closeToTrayEnabled = readDesktopSettings().closeToTray;
ensureTray();
ensureYoutubeEmbedRequestHeaders();
mainWindow = new BrowserWindow({
width: 1400,

View File

@@ -1,4 +1,6 @@
import { randomBytes } from 'crypto';
import fs from 'fs';
import fsp from 'fs/promises';
import path from 'path';
import { DataSource } from 'typeorm';
import {
@@ -101,6 +103,23 @@ function resolveSqlJsConfig(): { locateFile: (file: string) => string } {
};
}
/**
* Write the database to disk atomically: write a temp file first,
* then rename it over the real file. rename() is atomic on the same
* filesystem, so a crash mid-write can never leave a half-written DB.
*/
async function atomicSave(data: Uint8Array): Promise<void> {
const tmpPath = DB_FILE + '.tmp-' + randomBytes(6).toString('hex');
try {
await fsp.writeFile(tmpPath, Buffer.from(data));
await fsp.rename(tmpPath, DB_FILE);
} catch (err) {
await fsp.unlink(tmpPath).catch(() => {});
throw err;
}
}
export function getDataSource(): DataSource {
if (!applicationDataSource?.isInitialized) {
throw new Error('DataSource not initialised');
@@ -136,7 +155,7 @@ export async function initDatabase(): Promise<void> {
synchronize: process.env.DB_SYNCHRONIZE === 'true',
logging: false,
autoSave: true,
location: DB_FILE,
autoSaveCallback: atomicSave,
sqlJsConfig: resolveSqlJsConfig()
});
} catch (error) {

View File

@@ -0,0 +1,149 @@
export type SpotifyResourceType = 'album' | 'artist' | 'episode' | 'playlist' | 'show' | 'track';
export type SoundcloudResourceType = 'playlist' | 'track';
export interface SpotifyResource {
type: SpotifyResourceType;
id: string;
}
export interface SoundcloudResource {
canonicalUrl: string;
type: SoundcloudResourceType;
}
const SPOTIFY_RESOURCE_TYPES = new Set<SpotifyResourceType>([
'album',
'artist',
'episode',
'playlist',
'show',
'track'
]);
const SPOTIFY_URI_PATTERN = /^spotify:(album|artist|episode|playlist|show|track):([a-zA-Z0-9]+)$/i;
const SOUNDCLOUD_HOST_PATTERN = /^(?:www\.|m\.)?soundcloud\.com$/i;
const YOUTUBE_HOST_PATTERN = /^(?:www\.|m\.|music\.)?youtube\.com$/i;
const YOUTU_BE_HOST_PATTERN = /^(?:www\.)?youtu\.be$/i;
const YOUTUBE_VIDEO_ID_PATTERN = /^[\w-]{11}$/;
function parseUrl(url: string): URL | null {
try {
return new URL(url);
} catch {
return null;
}
}
export function extractYoutubeVideoId(url: string): string | null {
const parsedUrl = parseUrl(url);
if (!parsedUrl) {
return null;
}
if (YOUTU_BE_HOST_PATTERN.test(parsedUrl.hostname)) {
const shortId = parsedUrl.pathname.split('/').filter(Boolean)[0] ?? '';
return YOUTUBE_VIDEO_ID_PATTERN.test(shortId) ? shortId : null;
}
if (!YOUTUBE_HOST_PATTERN.test(parsedUrl.hostname)) {
return null;
}
const pathSegments = parsedUrl.pathname.split('/').filter(Boolean);
if (parsedUrl.pathname === '/watch') {
const queryId = parsedUrl.searchParams.get('v') ?? '';
return YOUTUBE_VIDEO_ID_PATTERN.test(queryId) ? queryId : null;
}
if (pathSegments.length >= 2 && (pathSegments[0] === 'embed' || pathSegments[0] === 'shorts')) {
const pathId = pathSegments[1];
return YOUTUBE_VIDEO_ID_PATTERN.test(pathId) ? pathId : null;
}
return null;
}
export function isYoutubeUrl(url?: string): boolean {
return !!url && extractYoutubeVideoId(url) !== null;
}
export function extractSpotifyResource(url: string): SpotifyResource | null {
const spotifyUriMatch = url.match(SPOTIFY_URI_PATTERN);
if (spotifyUriMatch?.[1] && spotifyUriMatch[2]) {
return {
type: spotifyUriMatch[1].toLowerCase() as SpotifyResourceType,
id: spotifyUriMatch[2]
};
}
const parsedUrl = parseUrl(url);
if (!parsedUrl || !/^(?:open|play)\.spotify\.com$/i.test(parsedUrl.hostname)) {
return null;
}
const segments = parsedUrl.pathname.split('/').filter(Boolean);
if (segments.length >= 2 && SPOTIFY_RESOURCE_TYPES.has(segments[0] as SpotifyResourceType)) {
return {
type: segments[0] as SpotifyResourceType,
id: segments[1]
};
}
if (segments.length >= 4 && segments[0] === 'user' && segments[2] === 'playlist') {
return {
type: 'playlist',
id: segments[3]
};
}
return null;
}
export function isSpotifyUrl(url?: string): boolean {
return !!url && extractSpotifyResource(url) !== null;
}
export function extractSoundcloudResource(url: string): SoundcloudResource | null {
const parsedUrl = parseUrl(url);
if (!parsedUrl || !SOUNDCLOUD_HOST_PATTERN.test(parsedUrl.hostname)) {
return null;
}
const segments = parsedUrl.pathname.split('/').filter(Boolean);
if (segments.length === 2) {
const canonicalUrl = new URL(`https://soundcloud.com/${segments[0]}/${segments[1]}`);
return {
canonicalUrl: canonicalUrl.toString(),
type: 'track'
};
}
if (segments.length === 3 && segments[1] === 'sets') {
const canonicalUrl = new URL(`https://soundcloud.com/${segments[0]}/sets/${segments[2]}`);
return {
canonicalUrl: canonicalUrl.toString(),
type: 'playlist'
};
}
return null;
}
export function isSoundcloudUrl(url?: string): boolean {
return !!url && extractSoundcloudResource(url) !== null;
}
export function hasDedicatedChatEmbed(url?: string): boolean {
return isYoutubeUrl(url) || isSpotifyUrl(url) || isSoundcloudUrl(url);
}

View File

@@ -100,11 +100,13 @@
@if (msg.linkMetadata?.length) {
@for (meta of msg.linkMetadata; track meta.url) {
<app-chat-link-embed
[metadata]="meta"
[canRemove]="isOwnMessage() || isAdmin()"
(removed)="removeEmbed(meta.url)"
/>
@if (shouldShowLinkEmbed(meta.url)) {
<app-chat-link-embed
[metadata]="meta"
[canRemove]="isOwnMessage() || isAdmin()"
(removed)="removeEmbed(meta.url)"
/>
}
}
}

View File

@@ -31,6 +31,7 @@ import {
MAX_AUTO_SAVE_SIZE_BYTES
} from '../../../../../attachment';
import { KlipyService } from '../../../../application/services/klipy.service';
import { hasDedicatedChatEmbed } from '../../../../domain/rules/link-embed.rules';
import {
DELETED_MESSAGE_CONTENT,
Message,
@@ -278,6 +279,10 @@ export class ChatMessageItemComponent {
});
}
shouldShowLinkEmbed(url?: string): boolean {
return !hasDedicatedChatEmbed(url);
}
requestReferenceScroll(messageId: string): void {
this.referenceRequested.emit(messageId);
}

View File

@@ -45,6 +45,14 @@
<div class="block">
<app-chat-youtube-embed [url]="node.url" />
</div>
} @else if (isSpotifyUrl(node.url)) {
<div class="block">
<app-chat-spotify-embed [url]="node.url" />
</div>
} @else if (isSoundcloudUrl(node.url)) {
<div class="block">
<app-chat-soundcloud-embed [url]="node.url" />
</div>
}
</ng-template>
</remark>

View File

@@ -5,8 +5,15 @@ import remarkBreaks from 'remark-breaks';
import remarkGfm from 'remark-gfm';
import remarkParse from 'remark-parse';
import { unified } from 'unified';
import {
isSoundcloudUrl,
isSpotifyUrl,
isYoutubeUrl
} from '../../../../../domain/rules/link-embed.rules';
import { ChatImageProxyFallbackDirective } from '../../../../chat-image-proxy-fallback.directive';
import { ChatYoutubeEmbedComponent, isYoutubeUrl } from '../chat-youtube-embed/chat-youtube-embed.component';
import { ChatSoundcloudEmbedComponent } from '../chat-soundcloud-embed/chat-soundcloud-embed.component';
import { ChatSpotifyEmbedComponent } from '../chat-spotify-embed/chat-spotify-embed.component';
import { ChatYoutubeEmbedComponent } from '../chat-youtube-embed/chat-youtube-embed.component';
const PRISM_LANGUAGE_ALIASES: Record<string, string> = {
cs: 'csharp',
@@ -40,6 +47,8 @@ const REMARK_PROCESSOR = unified()
RemarkModule,
MermaidComponent,
ChatImageProxyFallbackDirective,
ChatSpotifyEmbedComponent,
ChatSoundcloudEmbedComponent,
ChatYoutubeEmbedComponent
],
templateUrl: './chat-message-markdown.component.html'
@@ -63,6 +72,14 @@ export class ChatMessageMarkdownComponent {
return isYoutubeUrl(url);
}
isSpotifyUrl(url?: string): boolean {
return isSpotifyUrl(url);
}
isSoundcloudUrl(url?: string): boolean {
return isSoundcloudUrl(url);
}
isMermaidCodeBlock(lang?: string): boolean {
return this.normalizeCodeLanguage(lang) === 'mermaid';
}

View File

@@ -0,0 +1,11 @@
@if (embedUrl(); as soundcloudEmbedUrl) {
<div class="mt-2 w-[480px] max-w-full overflow-hidden rounded-md border border-border/60 bg-secondary/20">
<iframe
[src]="soundcloudEmbedUrl"
[style.height.px]="embedHeight()"
class="w-full border-0"
loading="lazy"
title="SoundCloud player"
></iframe>
</div>
}

View File

@@ -0,0 +1,44 @@
import {
Component,
computed,
inject,
input
} from '@angular/core';
import { DomSanitizer } from '@angular/platform-browser';
import { extractSoundcloudResource } from '../../../../../domain/rules/link-embed.rules';
@Component({
selector: 'app-chat-soundcloud-embed',
standalone: true,
templateUrl: './chat-soundcloud-embed.component.html'
})
export class ChatSoundcloudEmbedComponent {
readonly url = input.required<string>();
readonly resource = computed(() => extractSoundcloudResource(this.url()));
readonly embedHeight = computed(() => this.resource()?.type === 'playlist' ? 352 : 166);
readonly embedUrl = computed(() => {
const resource = this.resource();
if (!resource) {
return '';
}
const embedUrl = new URL('https://w.soundcloud.com/player/');
embedUrl.searchParams.set('url', resource.canonicalUrl);
embedUrl.searchParams.set('auto_play', 'false');
embedUrl.searchParams.set('hide_related', 'false');
embedUrl.searchParams.set('show_comments', 'false');
embedUrl.searchParams.set('show_user', 'true');
embedUrl.searchParams.set('show_reposts', 'false');
embedUrl.searchParams.set('show_teaser', 'true');
embedUrl.searchParams.set('visual', resource.type === 'playlist' ? 'true' : 'false');
return this.sanitizer.bypassSecurityTrustResourceUrl(embedUrl.toString());
});
private readonly sanitizer = inject(DomSanitizer);
}

View File

@@ -0,0 +1,12 @@
@if (embedUrl(); as spotifyEmbedUrl) {
<div class="mt-2 w-[480px] max-w-full overflow-hidden rounded-md border border-border/60 bg-secondary/20">
<iframe
[src]="spotifyEmbedUrl"
[style.height.px]="embedHeight()"
class="w-full border-0"
loading="lazy"
title="Spotify player"
allowfullscreen
></iframe>
</div>
}

View File

@@ -0,0 +1,51 @@
import {
Component,
computed,
inject,
input
} from '@angular/core';
import { DomSanitizer } from '@angular/platform-browser';
import { extractSpotifyResource } from '../../../../../domain/rules/link-embed.rules';
@Component({
selector: 'app-chat-spotify-embed',
standalone: true,
templateUrl: './chat-spotify-embed.component.html'
})
export class ChatSpotifyEmbedComponent {
readonly url = input.required<string>();
readonly resource = computed(() => extractSpotifyResource(this.url()));
readonly embedHeight = computed(() => {
const resource = this.resource();
if (!resource) {
return 0;
}
switch (resource.type) {
case 'track':
case 'episode':
return 152;
default:
return 352;
}
});
readonly embedUrl = computed(() => {
const resource = this.resource();
if (!resource) {
return '';
}
const embedUrl = new URL(`https://open.spotify.com/embed/${resource.type}/${encodeURIComponent(resource.id)}`);
embedUrl.searchParams.set('utm_source', 'generator');
return this.sanitizer.bypassSecurityTrustResourceUrl(embedUrl.toString());
});
private readonly sanitizer = inject(DomSanitizer);
}

View File

@@ -3,7 +3,6 @@
<iframe
[src]="embedUrl()"
class="aspect-video w-full"
allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
allowfullscreen
loading="lazy"
></iframe>

View File

@@ -5,8 +5,21 @@ import {
input
} from '@angular/core';
import { DomSanitizer } from '@angular/platform-browser';
import { extractYoutubeVideoId } from '../../../../../domain/rules/link-embed.rules';
const YOUTUBE_URL_PATTERN = /(?:youtube\.com\/(?:watch\?.*v=|embed\/|shorts\/)|youtu\.be\/)([\w-]{11})/;
const YOUTUBE_EMBED_FALLBACK_ORIGIN = 'https://toju.app';
function resolveYoutubeClientOrigin(): string {
if (typeof window === 'undefined') {
return YOUTUBE_EMBED_FALLBACK_ORIGIN;
}
const origin = window.location.origin;
return /^https?:\/\//.test(origin)
? origin
: YOUTUBE_EMBED_FALLBACK_ORIGIN;
}
@Component({
selector: 'app-chat-youtube-embed',
@@ -16,11 +29,7 @@ const YOUTUBE_URL_PATTERN = /(?:youtube\.com\/(?:watch\?.*v=|embed\/|shorts\/)|y
export class ChatYoutubeEmbedComponent {
readonly url = input.required<string>();
readonly videoId = computed(() => {
const match = this.url().match(YOUTUBE_URL_PATTERN);
return match?.[1] ?? null;
});
readonly videoId = computed(() => extractYoutubeVideoId(this.url()));
readonly embedUrl = computed(() => {
const id = this.videoId();
@@ -28,14 +37,16 @@ export class ChatYoutubeEmbedComponent {
if (!id)
return '';
const clientOrigin = resolveYoutubeClientOrigin();
const embedUrl = new URL(`https://www.youtube-nocookie.com/embed/${encodeURIComponent(id)}`);
embedUrl.searchParams.set('origin', clientOrigin);
embedUrl.searchParams.set('widget_referrer', clientOrigin);
return this.sanitizer.bypassSecurityTrustResourceUrl(
`https://www.youtube-nocookie.com/embed/${encodeURIComponent(id)}`
embedUrl.toString()
);
});
private readonly sanitizer = inject(DomSanitizer);
}
export function isYoutubeUrl(url?: string): boolean {
return !!url && YOUTUBE_URL_PATTERN.test(url);
}

View File

@@ -489,16 +489,9 @@ export class ServersRailComponent {
ensureEndpoint: !!resolvedRoom.sourceUrl
});
const authoritativeServer = (
selector
? await firstValueFrom(this.serverDirectory.getServer(room.id, selector))
: null
) ?? await firstValueFrom(this.serverDirectory.findServerAcrossActiveEndpoints(room.id, {
sourceId: resolvedRoom.sourceId,
sourceName: resolvedRoom.sourceName,
sourceUrl: resolvedRoom.sourceUrl,
fallbackName: resolvedRoom.sourceName ?? resolvedRoom.name
}));
const authoritativeServer = selector
? await firstValueFrom(this.serverDirectory.getServer(room.id, selector))
: null;
if (!authoritativeServer) {
return {

View File

@@ -37,6 +37,7 @@ import { DatabaseService } from '../../infrastructure/persistence';
import { reportDebuggingError, trackDebuggingTaskFailure } from '../../core/helpers/debugging-helpers';
import { DebuggingService } from '../../core/services';
import { AttachmentFacade } from '../../domains/attachment';
import { hasDedicatedChatEmbed } from '../../domains/chat/domain/rules/link-embed.rules';
import { LinkMetadataService } from '../../domains/chat/application/services/link-metadata.service';
import { TimeSyncService } from '../../core/services/time-sync.service';
import {
@@ -388,7 +389,8 @@ export class MessagesEffects {
if (message.isDeleted || message.linkMetadata?.length)
return EMPTY;
const urls = this.linkMetadata.extractUrls(message.content);
const urls = this.linkMetadata.extractUrls(message.content)
.filter((url) => !hasDedicatedChatEmbed(url));
if (urls.length === 0)
return EMPTY;

View File

@@ -10,7 +10,7 @@
/>
<meta
http-equiv="Content-Security-Policy"
content="default-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; connect-src 'self' blob: ws: wss: http: https:; media-src 'self' blob:; img-src 'self' data: blob: http: https:; frame-src https://www.youtube-nocookie.com;"
content="default-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; connect-src 'self' blob: ws: wss: http: https:; media-src 'self' blob:; img-src 'self' data: blob: http: https:; frame-src https://www.youtube-nocookie.com https://open.spotify.com https://w.soundcloud.com;"
/>
<link
rel="icon"