fix: never descend into NAS metadata directories when indexing
Docker Build and Publish / build-and-push (push) Failing after 10s
Docker Build and Publish / build-and-push (push) Failing after 10s
The archive root was filtered by prefix, but the recursive walk below it was not, so anything inside a profile directory got indexed. NAS filesystems put sidecar metadata *inside* every folder rather than only at the share root: Synology writes @eaDir (thumbnails and indexing data), #recycle holds deletions, .sync is Resilio state. On the live share those account for 12,516 of 123,023 files. None currently sit inside a profile directory, so nothing was miscounted yet — but the moment that share gets indexed for Photos, every generated thumbnail would be counted as archive media and stat'd one by one over the network, which is the cost the index exists to avoid. One isSystemDirectory rule now applies at every level, and the root listing uses it too instead of keeping a second copy of the pattern. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011uBWhwV3wFQ5MBCcMHHem7
This commit is contained in:
co-authored by
Claude Opus 5
parent
843607f47e
commit
faadc3e586
Generated
+2
-2
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "instaarchive-viewer",
|
"name": "instaarchive-viewer",
|
||||||
"version": "1.5.0",
|
"version": "1.5.1",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "instaarchive-viewer",
|
"name": "instaarchive-viewer",
|
||||||
"version": "1.5.0",
|
"version": "1.5.1",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@tailwindcss/vite": "^4.1.14",
|
"@tailwindcss/vite": "^4.1.14",
|
||||||
"@vitejs/plugin-react": "^5.0.4",
|
"@vitejs/plugin-react": "^5.0.4",
|
||||||
|
|||||||
+1
-1
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "instaarchive-viewer",
|
"name": "instaarchive-viewer",
|
||||||
"private": true,
|
"private": true,
|
||||||
"version": "1.5.0",
|
"version": "1.5.1",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "vite --port=3000 --host=0.0.0.0",
|
"dev": "vite --port=3000 --host=0.0.0.0",
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { isSystemDirectory } from './archive-index';
|
||||||
|
|
||||||
|
describe('isSystemDirectory', () => {
|
||||||
|
it.each([
|
||||||
|
['@eaDir', 'Synology thumbnail/index metadata, written inside every folder'],
|
||||||
|
['@tmp', 'Synology scratch'],
|
||||||
|
['.sync', 'Resilio state'],
|
||||||
|
['.DS_Store', 'macOS'],
|
||||||
|
['#recycle', 'Synology deletions'],
|
||||||
|
['#snapshot', 'Synology snapshots'],
|
||||||
|
])('skips %s (%s)', name => {
|
||||||
|
expect(isSystemDirectory(name)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
'0ct0ber19',
|
||||||
|
'0ct0ber19 - reels',
|
||||||
|
'story - cher_ryppo',
|
||||||
|
'story highlights - official_artms - G.C.I',
|
||||||
|
'story highlights - theoldtaylorswiftinsta - 💙2014-1989 era',
|
||||||
|
'Heejin_Bubble heejinmedia',
|
||||||
|
'gallery-dl',
|
||||||
|
'posts',
|
||||||
|
])('keeps %s', name => {
|
||||||
|
expect(isSystemDirectory(name)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not treat a leading underscore as a system directory', () => {
|
||||||
|
// `_gemini-plans` is filtered separately at the archive root only; nothing
|
||||||
|
// below the root should be excluded just for starting with an underscore.
|
||||||
|
expect(isSystemDirectory('_gemini-plans')).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -33,6 +33,21 @@ interface DirIndex {
|
|||||||
const MEDIA_RE = /\.(jpg|jpeg|png|webp|gif|bmp|tiff|mp4|webm|ogv|mov)$/i;
|
const MEDIA_RE = /\.(jpg|jpeg|png|webp|gif|bmp|tiff|mp4|webm|ogv|mov)$/i;
|
||||||
const STAT_CONCURRENCY = 16;
|
const STAT_CONCURRENCY = 16;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Directories the walk must never descend into.
|
||||||
|
*
|
||||||
|
* NAS filesystems scatter sidecar metadata *inside* every folder, not just at
|
||||||
|
* the share root: Synology writes `@eaDir` (thumbnails and indexing data),
|
||||||
|
* `#recycle` holds deletions, and `.sync` is Resilio's state. Indexing those
|
||||||
|
* would count NAS thumbnails as archive media and spend a stat on each one —
|
||||||
|
* measured on a real share, `@eaDir` accounted for 12,516 of 123,023 files.
|
||||||
|
*
|
||||||
|
* The archive root is already filtered by prefix; this is the same rule applied
|
||||||
|
* at every level below it.
|
||||||
|
*/
|
||||||
|
export const isSystemDirectory = (name: string): boolean =>
|
||||||
|
name.startsWith('@') || name.startsWith('.') || name === '#recycle' || name === '#snapshot';
|
||||||
|
|
||||||
export class ArchiveIndex {
|
export class ArchiveIndex {
|
||||||
private dirs = new Map<string, DirIndex>();
|
private dirs = new Map<string, DirIndex>();
|
||||||
private inFlight = new Map<string, Promise<DirIndex>>();
|
private inFlight = new Map<string, Promise<DirIndex>>();
|
||||||
@@ -43,7 +58,7 @@ export class ArchiveIndex {
|
|||||||
/** Visible (non-system) directories at the archive root. */
|
/** Visible (non-system) directories at the archive root. */
|
||||||
private listRootDirs(): string[] {
|
private listRootDirs(): string[] {
|
||||||
return fs.readdirSync(this.archivesDir, { withFileTypes: true })
|
return fs.readdirSync(this.archivesDir, { withFileTypes: true })
|
||||||
.filter(e => e.isDirectory() && !/^[.@_]/.test(e.name))
|
.filter(e => e.isDirectory() && !isSystemDirectory(e.name) && !e.name.startsWith('_'))
|
||||||
.map(e => e.name);
|
.map(e => e.name);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -69,6 +84,7 @@ export class ArchiveIndex {
|
|||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
for (const entry of entries) {
|
for (const entry of entries) {
|
||||||
|
if (isSystemDirectory(entry.name)) continue;
|
||||||
const rel = base ? `${base}/${entry.name}` : entry.name;
|
const rel = base ? `${base}/${entry.name}` : entry.name;
|
||||||
if (entry.isDirectory()) out = out.concat(this.walk(path.join(absDir, entry.name), rel));
|
if (entry.isDirectory()) out = out.concat(this.walk(path.join(absDir, entry.name), rel));
|
||||||
else if (entry.isFile()) out.push(rel);
|
else if (entry.isFile()) out.push(rel);
|
||||||
|
|||||||
Reference in New Issue
Block a user