fix: security, performance and correctness pass; add sidecar archive support
Security - Fix path traversal in GET /api/archives/:name/files. Express decodes route params after segment matching, so `..%2f..%2fetc` escaped ARCHIVES_DIR and returned a recursive listing of arbitrary directories. - Add CSP and baseline security headers; disable x-powered-by. - Stop baking GEMINI_API_KEY into the client bundle (the SDK was unused). - Run the container as `node` instead of root. Performance - Add a directory-mtime-keyed archive index, warmed in the background and persisted. Listing 110k files went from ~52s to ~0.1s; the largest archive (24k files) serves in ~0.3s. Per-file stat over CIFS costs ~1.4ms and does not parallelise, so it is now done once rather than per request. - Build media URLs from the File directly instead of `new Blob([await file.arrayBuffer()])`, which read every media file fully into memory (a 20GB archive tried to become 20GB of resident blobs). - Track and revoke object URLs; previously none were ever revoked. - Give `requestThumbnail` a stable identity so a completed thumbnail stops re-running the effect in every mounted thumbnail. - Namespace IndexedDB keys so listing archives no longer deserializes every cached thumbnail blob, and thumbnails no longer collide across archives. - Serve real file sizes: RemoteArchiveFile was constructed with size 0, which silently disabled high-res thumbnailing for every server archive. Correctness - Local archives cached media as blob: URLs, which die with the document, so a cached local archive restored as an archive of broken images. Media now carries a stable path and is rehydrated from a persisted directory handle (File System Access API), falling back to re-prompting for the folder. - Fix permalinks: the URL-writing effect erased ?a= on mount before the archive list arrived to consume it, so deep links never resolved. - Make cache invalidation detect nested changes via a directory signature. - Add an error boundary and tolerate unparseable dates, which previously threw a RangeError and blanked the app. - Default video to muted so autoplay is not blocked by Safari/Firefox. Features - Fold sidecar directories into their base profile: `<user> - reels`, `story - <user>` and `story highlights - <user> - <title>` now appear as reels, the story ring and Instagram-style highlight circles rather than as separate archives. Housekeeping - Add @types/react; React was previously type-checked against its JavaScript source, so `npm run lint` gave almost no type safety on components. - Vendor fonts and PWA icons locally; the app made third-party CDN requests despite advertising offline support and local-only processing. - Drop unused better-sqlite3 (a native module that broke `npm install`). - Add vitest with 36 tests over the filename and directory-naming rules. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011uBWhwV3wFQ5MBCcMHHem7
This commit is contained in:
ergosteur
co-authored by
Claude Opus 5
Claude Opus 5
parent
0ba7a0d9ad
commit
0db4274f46
@@ -0,0 +1,232 @@
|
||||
import fs from 'fs';
|
||||
import fsp from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { ArchiveSource, SourceKind, groupArchiveDirectories } from './archive-grouping.js';
|
||||
|
||||
/**
|
||||
* On-disk archive index.
|
||||
*
|
||||
* Archives live on network storage where per-file `stat` costs ~1.4ms and does
|
||||
* not parallelise well, so walking every file on each request is unaffordable:
|
||||
* measured against a real 110k-file archive root, listing took ~52s.
|
||||
*
|
||||
* Directory `stat` is effectively free, so each source directory is indexed
|
||||
* once and re-used until its mtime changes. The index is warmed in the
|
||||
* background at startup and persisted, making steady-state requests instant.
|
||||
*/
|
||||
|
||||
export interface IndexedFile {
|
||||
path: string;
|
||||
size: number;
|
||||
mtime: number;
|
||||
kind: SourceKind;
|
||||
title?: string;
|
||||
}
|
||||
|
||||
interface DirIndex {
|
||||
dir: string;
|
||||
/** Directory mtime the index was built from; the cache key. */
|
||||
mtimeMs: number;
|
||||
files: IndexedFile[];
|
||||
}
|
||||
|
||||
const MEDIA_RE = /\.(jpg|jpeg|png|webp|gif|bmp|tiff|mp4|webm|ogv|mov)$/i;
|
||||
const STAT_CONCURRENCY = 16;
|
||||
|
||||
export class ArchiveIndex {
|
||||
private dirs = new Map<string, DirIndex>();
|
||||
private inFlight = new Map<string, Promise<DirIndex>>();
|
||||
private dirty = false;
|
||||
|
||||
constructor(private archivesDir: string, private cachePath: string) {}
|
||||
|
||||
/** Visible (non-system) directories at the archive root. */
|
||||
private listRootDirs(): string[] {
|
||||
return fs.readdirSync(this.archivesDir, { withFileTypes: true })
|
||||
.filter(e => e.isDirectory() && !/^[.@_]/.test(e.name))
|
||||
.map(e => e.name);
|
||||
}
|
||||
|
||||
groups(): Map<string, ArchiveSource[]> {
|
||||
return groupArchiveDirectories(this.listRootDirs());
|
||||
}
|
||||
|
||||
private dirMtime(dir: string): number {
|
||||
try {
|
||||
return fs.statSync(path.join(this.archivesDir, dir)).mtimeMs;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** Recursively list relative file paths without stat()ing them. */
|
||||
private walk(absDir: string, base = ''): string[] {
|
||||
let out: string[] = [];
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = fs.readdirSync(absDir, { withFileTypes: true });
|
||||
} catch {
|
||||
return out;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
const rel = base ? `${base}/${entry.name}` : entry.name;
|
||||
if (entry.isDirectory()) out = out.concat(this.walk(path.join(absDir, entry.name), rel));
|
||||
else if (entry.isFile()) out.push(rel);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private async buildDir(source: ArchiveSource): Promise<DirIndex> {
|
||||
const started = Date.now();
|
||||
const mtimeMs = this.dirMtime(source.dir);
|
||||
const absDir = path.join(this.archivesDir, source.dir);
|
||||
const relPaths = this.walk(absDir);
|
||||
|
||||
// Only media needs a size (the client thumbnails anything over 1MiB), and
|
||||
// only highlights need an mtime (their filenames carry no date). Skipping
|
||||
// the rest avoids thousands of pointless round trips.
|
||||
const needsStat = (rel: string) => MEDIA_RE.test(rel) || source.kind === 'highlight';
|
||||
|
||||
const files: IndexedFile[] = relPaths.map(rel => ({
|
||||
path: `${source.dir}/${rel}`,
|
||||
size: 0,
|
||||
mtime: 0,
|
||||
kind: source.kind,
|
||||
...(source.title ? { title: source.title } : {}),
|
||||
}));
|
||||
|
||||
const targets = files.filter((_, i) => needsStat(relPaths[i]));
|
||||
let cursor = 0;
|
||||
const worker = async () => {
|
||||
while (cursor < targets.length) {
|
||||
const file = targets[cursor++];
|
||||
try {
|
||||
const stat = await fsp.stat(path.join(this.archivesDir, file.path));
|
||||
file.size = stat.size;
|
||||
file.mtime = stat.mtimeMs;
|
||||
} catch { /* raced with a delete */ }
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: STAT_CONCURRENCY }, worker));
|
||||
|
||||
console.log(
|
||||
`[Index] ${source.dir}: ${files.length} files (${targets.length} statted) in ${((Date.now() - started) / 1000).toFixed(1)}s`
|
||||
);
|
||||
this.dirty = true;
|
||||
return { dir: source.dir, mtimeMs, files };
|
||||
}
|
||||
|
||||
/** Index for one source directory, rebuilding only if its mtime moved. */
|
||||
private async ensureDir(source: ArchiveSource): Promise<DirIndex> {
|
||||
const cached = this.dirs.get(source.dir);
|
||||
const mtimeMs = this.dirMtime(source.dir);
|
||||
if (cached && cached.mtimeMs === mtimeMs) return cached;
|
||||
|
||||
// Collapse concurrent requests for the same directory into one walk.
|
||||
const existing = this.inFlight.get(source.dir);
|
||||
if (existing) return existing;
|
||||
|
||||
const build = this.buildDir(source).then(index => {
|
||||
this.dirs.set(source.dir, index);
|
||||
this.inFlight.delete(source.dir);
|
||||
return index;
|
||||
}).catch(err => {
|
||||
this.inFlight.delete(source.dir);
|
||||
throw err;
|
||||
});
|
||||
this.inFlight.set(source.dir, build);
|
||||
return build;
|
||||
}
|
||||
|
||||
/** All files for one profile, across its base and sidecar directories. */
|
||||
async filesFor(owner: string): Promise<IndexedFile[] | null> {
|
||||
const sources = this.groups().get(owner);
|
||||
if (!sources?.length) return null;
|
||||
const indexes = await Promise.all(sources.map(s => this.ensureDir(s)));
|
||||
return indexes.flatMap(i => i.files);
|
||||
}
|
||||
|
||||
/**
|
||||
* A cheap change signature for a profile, used by the client to decide
|
||||
* whether its cached copy is stale. Built from directory mtimes only, so it
|
||||
* costs one stat per source directory rather than a full walk.
|
||||
*/
|
||||
signatureFor(sources: ArchiveSource[]): string {
|
||||
return sources.map(s => `${s.dir}:${this.dirMtime(s.dir)}`).join('|');
|
||||
}
|
||||
|
||||
/** File count for a profile, if its directories are already indexed. */
|
||||
countFor(sources: ArchiveSource[]): number | null {
|
||||
let total = 0;
|
||||
for (const source of sources) {
|
||||
const cached = this.dirs.get(source.dir);
|
||||
if (!cached) return null;
|
||||
total += cached.files.length;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort profile picture.
|
||||
*
|
||||
* Probes the conventional filenames first (one stat each) and only falls back
|
||||
* to the indexed listing, so an unindexed archive still gets a thumbnail
|
||||
* without triggering a walk.
|
||||
*/
|
||||
thumbnailFor(owner: string, sources: ArchiveSource[]): string {
|
||||
const base = sources.find(s => s.kind === 'posts') ?? sources[0];
|
||||
if (!base) return '';
|
||||
|
||||
for (const candidate of [`${owner}.jpg`, `${owner}_profile_pic.jpg`, `${owner}.jpeg`, `${owner}.png`]) {
|
||||
if (fs.existsSync(path.join(this.archivesDir, base.dir, candidate))) {
|
||||
return `/archives/${encodeURI(`${base.dir}/${candidate}`)}`;
|
||||
}
|
||||
}
|
||||
|
||||
const cached = this.dirs.get(base.dir);
|
||||
if (cached) {
|
||||
const pick = cached.files.find(f => /_profile_pic\.jpg$/i.test(f.path))
|
||||
?? cached.files.find(f => /\.(jpg|jpeg|png|webp)$/i.test(f.path));
|
||||
if (pick) return `/archives/${encodeURI(pick.path)}`;
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
/** Walk every directory once, in the background, so first opens are fast. */
|
||||
async warm(): Promise<void> {
|
||||
const started = Date.now();
|
||||
const sources = [...this.groups().values()].flat();
|
||||
console.log(`[Index] Warming ${sources.length} source directories...`);
|
||||
for (const source of sources) {
|
||||
try {
|
||||
await this.ensureDir(source);
|
||||
} catch (err) {
|
||||
console.error(`[Index] Failed to index ${source.dir}:`, err);
|
||||
}
|
||||
}
|
||||
await this.save();
|
||||
console.log(`[Index] Warm complete in ${((Date.now() - started) / 1000).toFixed(1)}s`);
|
||||
}
|
||||
|
||||
async load(): Promise<void> {
|
||||
try {
|
||||
const raw = await fsp.readFile(this.cachePath, 'utf8');
|
||||
const parsed: DirIndex[] = JSON.parse(raw);
|
||||
for (const entry of parsed) this.dirs.set(entry.dir, entry);
|
||||
console.log(`[Index] Loaded ${this.dirs.size} directories from ${this.cachePath}`);
|
||||
} catch {
|
||||
console.log('[Index] No usable index cache; will build from scratch.');
|
||||
}
|
||||
}
|
||||
|
||||
async save(): Promise<void> {
|
||||
if (!this.dirty) return;
|
||||
try {
|
||||
await fsp.writeFile(this.cachePath, JSON.stringify([...this.dirs.values()]), 'utf8');
|
||||
this.dirty = false;
|
||||
console.log(`[Index] Persisted ${this.dirs.size} directories to ${this.cachePath}`);
|
||||
} catch (err) {
|
||||
console.warn('[Index] Could not persist index (continuing in memory):', err);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user