myxelium/Toju
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects 1 Releases 14 Wiki Activity

feat: Add chat embeds v1

Browse Source 
Youtube and Website metadata embeds
This commit is contained in:
SocksOnHead
2026-04-04 04:47:04 +02:00
parent 35352923a5
commit 84fa45985a
25 changed files with 759 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
server/src/config/variables.ts
View File

@@ -4,18 +4,28 @@ import { resolveRuntimePath } from '../runtime-paths';
export type ServerHttpProtocol = 'http' | 'https';
export interface LinkPreviewConfig {
enabled: boolean;
cacheTtlMinutes: number;
maxCacheSizeMb: number;
}
export interface ServerVariablesConfig {
klipyApiKey: string;
releaseManifestUrl: string;
serverPort: number;
serverProtocol: ServerHttpProtocol;
serverHost: string;
linkPreview: LinkPreviewConfig;
}
const DATA_DIR = resolveRuntimePath('data');
const VARIABLES_FILE = path.join(DATA_DIR, 'variables.json');
const DEFAULT_SERVER_PORT = 3001;
const DEFAULT_SERVER_PROTOCOL: ServerHttpProtocol = 'http';
const DEFAULT_LINK_PREVIEW_CACHE_TTL_MINUTES = 7200;
const DEFAULT_LINK_PREVIEW_MAX_CACHE_SIZE_MB = 50;
const HARD_MAX_CACHE_SIZE_MB = 50;
function normalizeKlipyApiKey(value: unknown): string {
return typeof value === 'string' ? value.trim() : '';
@@ -66,6 +76,27 @@ function normalizeServerPort(value: unknown, fallback = DEFAULT_SERVER_PORT): nu
: fallback;
}
function normalizeLinkPreviewConfig(value: unknown): LinkPreviewConfig {
const raw = (value && typeof value === 'object' && !Array.isArray(value))
? value as Record<string, unknown>
: {};
const enabled = typeof raw.enabled === 'boolean'
? raw.enabled
: true;
const cacheTtl = typeof raw.cacheTtlMinutes === 'number'
&& Number.isFinite(raw.cacheTtlMinutes)
&& raw.cacheTtlMinutes >= 0
? raw.cacheTtlMinutes
: DEFAULT_LINK_PREVIEW_CACHE_TTL_MINUTES;
const maxSize = typeof raw.maxCacheSizeMb === 'number'
&& Number.isFinite(raw.maxCacheSizeMb)
&& raw.maxCacheSizeMb >= 0
? Math.min(raw.maxCacheSizeMb, HARD_MAX_CACHE_SIZE_MB)
: DEFAULT_LINK_PREVIEW_MAX_CACHE_SIZE_MB;
return { enabled, cacheTtlMinutes: cacheTtl, maxCacheSizeMb: maxSize };
}
function hasEnvironmentOverride(value: string | undefined): value is string {
return typeof value === 'string' && value.trim().length > 0;
}
@@ -111,7 +142,8 @@ export function ensureVariablesConfig(): ServerVariablesConfig {
releaseManifestUrl: normalizeReleaseManifestUrl(remainingParsed.releaseManifestUrl),
serverPort: normalizeServerPort(remainingParsed.serverPort),
serverProtocol: normalizeServerProtocol(remainingParsed.serverProtocol),
serverHost: normalizeServerHost(remainingParsed.serverHost ?? legacyServerIpAddress)
serverHost: normalizeServerHost(remainingParsed.serverHost ?? legacyServerIpAddress),
linkPreview: normalizeLinkPreviewConfig(remainingParsed.linkPreview)
};
const nextContents = JSON.stringify(normalized, null, 2) + '\n';
@@ -124,7 +156,8 @@ export function ensureVariablesConfig(): ServerVariablesConfig {
releaseManifestUrl: normalized.releaseManifestUrl,
serverPort: normalized.serverPort,
serverProtocol: normalized.serverProtocol,
serverHost: normalized.serverHost
serverHost: normalized.serverHost,
linkPreview: normalized.linkPreview
};
}
@@ -169,3 +202,7 @@ export function getServerHost(): string | undefined {
export function isHttpsServerEnabled(): boolean {
return getServerProtocol() === 'https';
}
export function getLinkPreviewConfig(): LinkPreviewConfig {
return getVariablesConfig().linkPreview;
}

2
server/src/routes/index.ts
View File

@@ -1,6 +1,7 @@
import { Express } from 'express';
import healthRouter from './health';
import klipyRouter from './klipy';
import linkMetadataRouter from './link-metadata';
import proxyRouter from './proxy';
import usersRouter from './users';
import serversRouter from './servers';
@@ -10,6 +11,7 @@ import { invitesApiRouter, invitePageRouter } from './invites';
export function registerRoutes(app: Express): void {
app.use('/api', healthRouter);
app.use('/api', klipyRouter);
app.use('/api', linkMetadataRouter);
app.use('/api', proxyRouter);
app.use('/api/users', usersRouter);
app.use('/api/servers', serversRouter);

292
server/src/routes/link-metadata.ts Normal file
View File

@@ -0,0 +1,292 @@
import { Router } from 'express';
import { getLinkPreviewConfig } from '../config/variables';
import { resolveAndValidateHost, safeFetch } from './ssrf-guard';
const router = Router();
const REQUEST_TIMEOUT_MS = 8000;
const MAX_HTML_BYTES = 512 * 1024;
const BYTES_PER_MB = 1024 * 1024;
const MAX_FIELD_LENGTH = 512;
interface CachedMetadata {
title?: string;
description?: string;
imageUrl?: string;
siteName?: string;
failed?: boolean;
cachedAt: number;
}
const metadataCache = new Map<string, CachedMetadata>();
let cacheByteEstimate = 0;
function estimateEntryBytes(key: string, entry: CachedMetadata): number {
let bytes = key.length * 2;
if (entry.title)
bytes += entry.title.length * 2;
if (entry.description)
bytes += entry.description.length * 2;
if (entry.imageUrl)
bytes += entry.imageUrl.length * 2;
if (entry.siteName)
bytes += entry.siteName.length * 2;
return bytes + 64;
}
function cacheSet(key: string, entry: CachedMetadata): void {
const config = getLinkPreviewConfig();
const maxBytes = config.maxCacheSizeMb * BYTES_PER_MB;
if (metadataCache.has(key)) {
const existing = metadataCache.get(key) as CachedMetadata;
cacheByteEstimate -= estimateEntryBytes(key, existing);
}
const entryBytes = estimateEntryBytes(key, entry);
while (cacheByteEstimate + entryBytes > maxBytes && metadataCache.size > 0) {
const oldest = metadataCache.keys().next().value as string;
const oldestEntry = metadataCache.get(oldest) as CachedMetadata;
cacheByteEstimate -= estimateEntryBytes(oldest, oldestEntry);
metadataCache.delete(oldest);
}
metadataCache.set(key, entry);
cacheByteEstimate += entryBytes;
}
function truncateField(value: string | undefined): string | undefined {
if (!value)
return value;
if (value.length <= MAX_FIELD_LENGTH)
return value;
return value.slice(0, MAX_FIELD_LENGTH);
}
function sanitizeImageUrl(rawUrl: string | undefined, baseUrl: string): string | undefined {
if (!rawUrl)
return undefined;
try {
const resolved = new URL(rawUrl, baseUrl);
if (resolved.protocol !== 'http:' && resolved.protocol !== 'https:')
return undefined;
return resolved.href;
} catch {
return undefined;
}
}
function getMetaContent(html: string, patterns: RegExp[]): string | undefined {
for (const pattern of patterns) {
const match = pattern.exec(html);
if (match?.[1])
return decodeHtmlEntities(match[1].trim());
}
return undefined;
}
function decodeHtmlEntities(text: string): string {
return text
.replace(/&amp;/g, '&')
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&#x27;/g, "'")
.replace(/&#x2F;/g, '/');
}
function parseMetadata(html: string, url: string): CachedMetadata {
const title = getMetaContent(html, [
/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i,
/<meta[^>]+content=["']([^"']+)["'][^>]+property=["']og:title["']/i,
/<meta[^>]+name=["']twitter:title["'][^>]+content=["']([^"']+)["']/i,
/<meta[^>]+content=["']([^"']+)["'][^>]+name=["']twitter:title["']/i,
/<title[^>]*>([^<]+)<\/title>/i
]);
const description = getMetaContent(html, [
/<meta[^>]+property=["']og:description["'][^>]+content=["']([^"']+)["']/i,
/<meta[^>]+content=["']([^"']+)["'][^>]+property=["']og:description["']/i,
/<meta[^>]+name=["']twitter:description["'][^>]+content=["']([^"']+)["']/i,
/<meta[^>]+content=["']([^"']+)["'][^>]+name=["']twitter:description["']/i,
/<meta[^>]+name=["']description["'][^>]+content=["']([^"']+)["']/i,
/<meta[^>]+content=["']([^"']+)["'][^>]+name=["']description["']/i
]);
const rawImageUrl = getMetaContent(html, [
/<meta[^>]+property=["']og:image["'][^>]+content=["']([^"']+)["']/i,
/<meta[^>]+content=["']([^"']+)["'][^>]+property=["']og:image["']/i,
/<meta[^>]+name=["']twitter:image["'][^>]+content=["']([^"']+)["']/i,
/<meta[^>]+content=["']([^"']+)["'][^>]+name=["']twitter:image["']/i
]);
const siteNamePatterns = [
// eslint-disable-next-line @stylistic/js/array-element-newline
/<meta[^>]+property=["']og:site_name["'][^>]+content=["']([^"']+)["']/i,
/<meta[^>]+content=["']([^"']+)["'][^>]+property=["']og:site_name["']/i
];
const siteName = getMetaContent(html, siteNamePatterns);
const imageUrl = sanitizeImageUrl(rawImageUrl, url);
return {
title: truncateField(title),
description: truncateField(description),
imageUrl,
siteName: truncateField(siteName),
cachedAt: Date.now()
};
}
function evictExpired(): void {
const config = getLinkPreviewConfig();
if (config.cacheTtlMinutes === 0) {
cacheByteEstimate = 0;
metadataCache.clear();
return;
}
const ttlMs = config.cacheTtlMinutes * 60 * 1000;
const now = Date.now();
for (const [key, entry] of metadataCache) {
if (now - entry.cachedAt > ttlMs) {
cacheByteEstimate -= estimateEntryBytes(key, entry);
metadataCache.delete(key);
}
}
}
router.get('/link-metadata', async (req, res) => {
try {
const config = getLinkPreviewConfig();
if (!config.enabled) {
return res.status(403).json({ error: 'Link previews are disabled' });
}
const url = String(req.query.url || '');
if (!/^https?:\/\//i.test(url)) {
return res.status(400).json({ error: 'Invalid URL' });
}
const hostAllowed = await resolveAndValidateHost(url);
if (!hostAllowed) {
return res.status(400).json({ error: 'URL resolves to a blocked address' });
}
evictExpired();
const cached = metadataCache.get(url);
if (cached) {
const { cachedAt, ...metadata } = cached;
console.log(`[Link Metadata] Cache hit for ${url} (cached at ${new Date(cachedAt).toISOString()})`);
return res.json(metadata);
}
console.log(`[Link Metadata] Cache miss for ${url}. Fetching...`);
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
const response = await safeFetch(url, {
signal: controller.signal,
headers: {
'Accept': 'text/html',
'User-Agent': 'MetoYou-LinkPreview/1.0'
}
});
clearTimeout(timeout);
if (!response || !response.ok) {
const failed: CachedMetadata = { failed: true, cachedAt: Date.now() };
cacheSet(url, failed);
return res.json({ failed: true });
}
const contentType = response.headers.get('content-type') || '';
if (!contentType.includes('text/html')) {
const failed: CachedMetadata = { failed: true, cachedAt: Date.now() };
cacheSet(url, failed);
return res.json({ failed: true });
}
const reader = response.body?.getReader();
if (!reader) {
const failed: CachedMetadata = { failed: true, cachedAt: Date.now() };
cacheSet(url, failed);
return res.json({ failed: true });
}
const chunks: Uint8Array[] = [];
let totalBytes = 0;
let done = false;
while (!done) {
const result = await reader.read();
done = result.done;
if (result.value) {
chunks.push(result.value);
totalBytes += result.value.length;
if (totalBytes > MAX_HTML_BYTES) {
reader.cancel();
break;
}
}
}
const html = Buffer.concat(chunks).toString('utf-8');
const metadata = parseMetadata(html, url);
cacheSet(url, metadata);
const { cachedAt, ...result } = metadata;
res.json(result);
} catch (err) {
const url = String(req.query.url || '');
if (url) {
cacheSet(url, { failed: true, cachedAt: Date.now() });
}
if ((err as { name?: string })?.name === 'AbortError') {
return res.json({ failed: true });
}
console.error('Link metadata error:', err);
res.json({ failed: true });
}
});
export default router;

13
server/src/routes/proxy.ts
View File

@@ -1,4 +1,5 @@
import { Router } from 'express';
import { resolveAndValidateHost, safeFetch } from './ssrf-guard';
const router = Router();
@@ -10,14 +11,20 @@ router.get('/image-proxy', async (req, res) => {
return res.status(400).json({ error: 'Invalid URL' });
}
const hostAllowed = await resolveAndValidateHost(url);
if (!hostAllowed) {
return res.status(400).json({ error: 'URL resolves to a blocked address' });
}
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 8000);
const response = await fetch(url, { redirect: 'follow', signal: controller.signal });
const response = await safeFetch(url, { signal: controller.signal });
clearTimeout(timeout);
if (!response.ok) {
return res.status(response.status).end();
if (!response || !response.ok) {
return res.status(response?.status ?? 502).end();
}
const contentType = response.headers.get('content-type') || '';

119
server/src/routes/ssrf-guard.ts Normal file
View File

@@ -0,0 +1,119 @@
import { lookup } from 'dns/promises';
const MAX_REDIRECTS = 5;
function isPrivateIp(ip: string): boolean {
if (
ip === '127.0.0.1' ||
ip === '::1' ||
ip === '0.0.0.0' ||
ip === '::'
)
return true;
// 10.x.x.x
if (ip.startsWith('10.'))
return true;
// 172.16.0.0 - 172.31.255.255
if (ip.startsWith('172.')) {
const second = parseInt(ip.split('.')[1], 10);
if (second >= 16 && second <= 31)
return true;
}
// 192.168.x.x
if (ip.startsWith('192.168.'))
return true;
// 169.254.x.x (link-local, AWS metadata)
if (ip.startsWith('169.254.'))
return true;
// IPv6 private ranges (fc00::/7, fe80::/10)
const lower = ip.toLowerCase();
if (lower.startsWith('fc') || lower.startsWith('fd') || lower.startsWith('fe80'))
return true;
return false;
}
export async function resolveAndValidateHost(url: string): Promise<boolean> {
let hostname: string;
try {
hostname = new URL(url).hostname;
} catch {
return false;
}
// Block obvious private hostnames
if (hostname === 'localhost' || hostname === 'metadata.google.internal')
return false;
// If hostname is already an IP literal, check it directly
if (/^[\d.]+$/.test(hostname) || hostname.startsWith('['))
return !isPrivateIp(hostname.replace(/[[\]]/g, ''));
try {
const { address } = await lookup(hostname);
return !isPrivateIp(address);
} catch {
return false;
}
}
export interface SafeFetchOptions {
signal?: AbortSignal;
headers?: Record<string, string>;
}
/**
* Fetches a URL while following redirects safely, validating each
* hop against SSRF (private/reserved IPs, blocked hostnames).
*
* The caller must validate the initial URL with `resolveAndValidateHost`
* before calling this function.
*/
export async function safeFetch(url: string, options: SafeFetchOptions = {}): Promise<Response | undefined> {
let currentUrl = url;
let response: Response | undefined;
for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
response = await fetch(currentUrl, {
redirect: 'manual',
signal: options.signal,
headers: options.headers
});
const location = response.headers.get('location');
if (response.status >= 300 && response.status < 400 && location) {
let nextUrl: string;
try {
nextUrl = new URL(location, currentUrl).href;
} catch {
break;
}
if (!/^https?:\/\//i.test(nextUrl))
break;
const redirectAllowed = await resolveAndValidateHost(nextUrl);
if (!redirectAllowed)
break;
currentUrl = nextUrl;
continue;
}
break;
}
return response;
}