feat: generate JDownloader crawljobs from the archives on disk

The manual flow is: paste a profile URL into JDownloader, paste the /reels URL
separately (the profile page misses some reels), set the output folder by hand,
repeat per profile. scripts/jd2-sync.ts emits one crawljob per source with the
folder already pointed at the right directory, so folder-watch picks up the
whole batch at once.

Profiles and sidecars are derived with the same grouping logic the server uses,
so output folders always match what the viewer expects to find. --download-base
maps the path for a JDownloader running on another machine (Windows paths
included), since it typically runs on a desktop against the share.

Directories that aren't Instagram profiles are skipped: an archive root also
collects tool output and exports from other services, and pointing a crawl at
those spends requests on instagram.com to be told the profile doesn't exist —
exactly the traffic worth not spending. Filtering is by username shape, plus
--skip and a .jd2ignore file for names that look like usernames but aren't.

Defaults are conservative: chunks=1, because multi-chunk ranged requests are the
one CDN-side pattern that doesn't resemble a browser, and links park in the
LinkGrabber for review rather than auto-starting.

Only posts and reels are emitted; highlight URLs need a numeric id and story
URLs expire, so those stay manual.

Format verified against JDownloader's own explain.txt for the folderwatch
extension, read from the daily SVN mirror rather than one of the decade-stale
GitHub copies.

Also refreshes CLAUDE.md, whose URL-state section still described the query
parameters replaced in 1.4.0, and documents the mobile feed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011uBWhwV3wFQ5MBCcMHHem7

scripts/jd2-sync.ts

@@ -0,0 +1,277 @@
+/**
+ * Generate JDownloader2 .crawljob files for the archives on disk.
+ *
+ * The manual flow is: paste a profile URL into JDownloader, paste the /reels
+ * URL separately (the profile page misses some reels), and set the output
+ * folder by hand — times however many profiles you keep. This emits one
+ * crawljob per source with the folder already pointed at the right directory,
+ * so JDownloader's folder-watch picks the whole batch up at once.
+ *
+ * Profiles and their sidecar directories are derived with the same grouping
+ * logic the server uses, so the output folders always match what the viewer
+ * expects to find.
+ *
+ * Only posts and reels are emitted. Story and highlight URLs can't be rebuilt
+ * from a directory name — highlights need their numeric id and stories expire —
+ * so those stay manual.
+ *
+ * Crawljob format verified against JDownloader's own docs for the extension:
+ * src/org/jdownloader/extensions/folderwatchV2/explain.txt. JDownloader
+ * develops on SVN; read it via the daily mirror at
+ * https://github.com/mycodedoesnotcompile2/jdownloader_mirror (svn_trunk/),
+ * not one of the abandoned GitHub copies — several are a decade stale.
+ *
+ * Entries are separated by `->NEW ENTRY<-` and any property may be omitted.
+ * There is also a `setBeforePackagizerEnabled` companion to
+ * `overwritePackagizerEnabled`, if the Packagizer ever needs to see these
+ * values before they're applied.
+ *
+ * Usage:
+ *   npx tsx scripts/jd2-sync.ts --archives <dir> [options]
+ *
+ *   --archives <dir>       Archive root to scan (default: $ARCHIVES_DIR)
+ *   --out <dir>            JDownloader folder-watch directory to write into
+ *   --download-base <dir>  Root path as *JDownloader* sees it, when it runs on
+ *                          a different machine than this script (e.g. a mapped
+ *                          drive). Defaults to --archives.
+ *   --user <name>          Only this profile (repeatable)
+ *   --skip <name>          Never emit jobs for this directory (repeatable).
+ *                          Also read from a `.jd2ignore` file in the archive
+ *                          root, one name per line.
+ *   --chunks <n>           Connections per file (default 1: multi-chunk ranged
+ *                          requests are the one CDN pattern that doesn't look
+ *                          like a browser)
+ *   --auto-start           Start downloads immediately instead of parking them
+ *                          in the LinkGrabber for review
+ *   --all-reels            Emit a reels job even where no reels directory
+ *                          exists yet
+ *   --dry-run              Print the crawljob instead of writing it
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { groupArchiveDirectories, ArchiveSource } from '../src/lib/archive-grouping.js';
+
+interface Options {
+  archives: string;
+  out: string | null;
+  downloadBase: string;
+  users: string[];
+  skip: Set<string>;
+  chunks: number;
+  autoStart: boolean;
+  allReels: boolean;
+  dryRun: boolean;
+}
+
+const parseArgs = (argv: string[]): Options => {
+  const opts: Options = {
+    archives: process.env.ARCHIVES_DIR ?? '',
+    out: null,
+    downloadBase: '',
+    users: [],
+    skip: new Set(),
+    chunks: 1,
+    autoStart: false,
+    allReels: false,
+    dryRun: false,
+  };
+
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    const next = () => argv[++i];
+    switch (arg) {
+      case '--archives': opts.archives = path.resolve(next()); break;
+      case '--out': opts.out = path.resolve(next()); break;
+      case '--download-base': opts.downloadBase = next(); break;
+      case '--user': opts.users.push(next()); break;
+      case '--skip': opts.skip.add(next()); break;
+      case '--chunks': opts.chunks = parseInt(next(), 10); break;
+      case '--auto-start': opts.autoStart = true; break;
+      case '--all-reels': opts.allReels = true; break;
+      case '--dry-run': opts.dryRun = true; break;
+      case '--help': case '-h': printUsage(); process.exit(0);
+      default:
+        console.error(`Unknown argument: ${arg}`);
+        process.exit(1);
+    }
+  }
+
+  if (!opts.archives) {
+    console.error('No archive root. Pass --archives <dir> or set ARCHIVES_DIR.');
+    process.exit(1);
+  }
+  if (!opts.downloadBase) opts.downloadBase = opts.archives;
+  if (!opts.out && !opts.dryRun) {
+    console.error('No destination. Pass --out <folder-watch dir>, or --dry-run to preview.');
+    process.exit(1);
+  }
+  return opts;
+};
+
+const printUsage = () => {
+  const header = readHeaderComment();
+  console.log(header);
+};
+
+/** Print the usage block from this file's own header comment. */
+const readHeaderComment = () => {
+  try {
+    const self = fs.readFileSync(new URL(import.meta.url), 'utf8');
+    const usage = self.slice(self.indexOf(' * Usage:'), self.indexOf(' */'));
+    return usage.split('\n').map(l => l.replace(/^ \* ?/, '')).join('\n');
+  } catch {
+    return 'See the comment at the top of scripts/jd2-sync.ts';
+  }
+};
+
+/**
+ * JDownloader escapes nothing in crawljob values, so a stray newline would
+ * silently split a property. Paths with spaces are fine as-is.
+ */
+const sanitise = (value: string) => value.replace(/[\r\n]+/g, ' ').trim();
+
+/**
+ * Instagram usernames are 1–30 characters of letters, digits, dots and
+ * underscores. Archive roots also collect directories that aren't profiles at
+ * all — tool output, exports from other services — and pointing a crawl at
+ * those spends requests on instagram.com to be told the profile doesn't exist.
+ * That's the exact traffic worth not spending.
+ */
+const USERNAME_RE = /^[A-Za-z0-9._]{1,30}$/;
+
+/** Directory names to skip, from `.jd2ignore` in the archive root. */
+const readIgnoreFile = (archives: string): string[] => {
+  try {
+    return fs.readFileSync(path.join(archives, '.jd2ignore'), 'utf8')
+      .split('\n').map(l => l.trim()).filter(l => l && !l.startsWith('#'));
+  } catch {
+    return [];
+  }
+};
+
+interface Job {
+  user: string;
+  kind: 'posts' | 'reels';
+  url: string;
+  packageName: string;
+  downloadFolder: string;
+  fileCount: number | null;
+}
+
+const buildJobs = (opts: Options): Job[] => {
+  const dirNames = fs.readdirSync(opts.archives, { withFileTypes: true })
+    .filter(e => e.isDirectory() && !/^[.@_]/.test(e.name))
+    .map(e => e.name);
+
+  const groups = groupArchiveDirectories(dirNames);
+  const jobs: Job[] = [];
+  const skipped: string[] = [];
+
+  for (const name of readIgnoreFile(opts.archives)) opts.skip.add(name);
+
+  const countFiles = (dir: string): number | null => {
+    try {
+      return fs.readdirSync(path.join(opts.archives, dir)).length;
+    } catch {
+      return null;
+    }
+  };
+
+  // JDownloader must be given the path *it* can see, which differs from the
+  // scan path whenever the archive lives on a share.
+  const downloadFolderFor = (dir: string) =>
+    opts.downloadBase.includes('\\')
+      ? `${opts.downloadBase.replace(/\\$/, '')}\\${dir}`
+      : path.posix.join(opts.downloadBase, dir);
+
+  for (const [user, sources] of [...groups].sort(([a], [b]) => a.localeCompare(b))) {
+    if (opts.users.length && !opts.users.includes(user)) continue;
+
+    if (opts.skip.has(user)) { skipped.push(`${user} (ignored)`); continue; }
+    if (!USERNAME_RE.test(user)) { skipped.push(`${user} (not a username)`); continue; }
+
+    const has = (kind: ArchiveSource['kind']) => sources.find(s => s.kind === kind);
+    const base = has('posts');
+    if (!base) continue; // sidecar-only group: nothing sensible to point a URL at
+
+    jobs.push({
+      user, kind: 'posts',
+      url: `https://www.instagram.com/${encodeURIComponent(user)}/`,
+      packageName: base.dir,
+      downloadFolder: downloadFolderFor(base.dir),
+      fileCount: countFiles(base.dir),
+    });
+
+    const reels = has('reels');
+    if (reels || opts.allReels) {
+      const dir = reels?.dir ?? `${user} - reels`;
+      jobs.push({
+        user, kind: 'reels',
+        url: `https://www.instagram.com/${encodeURIComponent(user)}/reels/`,
+        packageName: dir,
+        downloadFolder: downloadFolderFor(dir),
+        fileCount: reels ? countFiles(dir) : null,
+      });
+    }
+  }
+
+  if (skipped.length) {
+    console.error(`Skipped ${skipped.length} director${skipped.length === 1 ? 'y' : 'ies'}:`);
+    for (const s of skipped) console.error(`  - ${s}`);
+    console.error('');
+  }
+
+  return jobs;
+};
+
+const renderCrawljob = (jobs: Job[], opts: Options): string =>
+  jobs.map(job => [
+    `text=${sanitise(job.url)}`,
+    `packageName=${sanitise(job.packageName)}`,
+    `downloadFolder=${sanitise(job.downloadFolder)}`,
+    `chunks=${opts.chunks}`,
+    // Without this a Packagizer rule can override downloadFolder and scatter
+    // files away from the directory the viewer reads.
+    'overwritePackagizerEnabled=TRUE',
+    `autoStart=${opts.autoStart ? 'TRUE' : 'FALSE'}`,
+    `autoConfirm=${opts.autoStart ? 'TRUE' : 'FALSE'}`,
+    'enabled=TRUE',
+    `comment=instaarchive jd2-sync (${job.kind})`,
+  ].join('\n')).join('\n->NEW ENTRY<-\n');
+
+const main = () => {
+  const opts = parseArgs(process.argv.slice(2));
+  const jobs = buildJobs(opts);
+
+  if (!jobs.length) {
+    console.error('No profiles matched.');
+    process.exit(1);
+  }
+
+  console.error(`Archive root : ${opts.archives}`);
+  console.error(`JD sees root : ${opts.downloadBase}`);
+  console.error(`Jobs         : ${jobs.length} (${new Set(jobs.map(j => j.user)).size} profiles)\n`);
+  for (const job of jobs) {
+    const count = job.fileCount === null ? 'new' : `${job.fileCount} files`;
+    console.error(`  ${job.kind.padEnd(5)} ${job.user.padEnd(24)} -> ${job.packageName}  (${count})`);
+  }
+  console.error('');
+
+  const body = renderCrawljob(jobs, opts);
+
+  if (opts.dryRun || !opts.out) {
+    console.log(body);
+    return;
+  }
+
+  fs.mkdirSync(opts.out, { recursive: true });
+  const file = path.join(opts.out, `instaarchive-${new Date().toISOString().replace(/[:.]/g, '-')}.crawljob`);
+  fs.writeFileSync(file, body, 'utf8');
+  console.error(`Wrote ${file}`);
+  console.error(opts.autoStart
+    ? 'Downloads will start automatically.'
+    : 'Links land in the LinkGrabber for review; start them when ready.');
+};
+
+main();