diff --git a/docs/gallery-dl.md b/docs/gallery-dl.md index 57908d7..cab85ca 100644 --- a/docs/gallery-dl.md +++ b/docs/gallery-dl.md @@ -251,6 +251,60 @@ The one trap, which silently seeds almost nothing if you get it backwards: `live_key()` encodes this. Matching on the wrong field seeded 5 of 2275. +## Publishing + +The fetch host stages to local disk and rsyncs afterwards. `rsync +--ignore-existing` is not an optimisation but the safety property: the archive +deliberately outlives Instagram, so publishing must only ever **add**. No +`--delete`, and nothing already present is overwritten — including sidecars, +which are rewritten every run and would otherwise churn the synced share. + +Publishing happens once at the end of a run, so a profile that fails midway +never reaches the archive half-written. + +## Verified run + +`withaseul`, all four surfaces, staged locally and published to a scratch +directory (never the live archive): + +``` +==> withaseul / posts seeded 915 of 984 live items +==> withaseul / reels seeded 28 of 34 live items +==> withaseul / stories no results (none active) +==> withaseul / highlights no results +``` + +Output landed correctly, including the collab-reel case — `withaseul - reels` +contains 53 files owned by `withaseul`, 10 by `cher_ryppo`, 3 by `0ct0ber19` +and 2 by `official_artms`, all with the owner in the filename and the crawl +scope as the directory. + +### The CDN rate-limits, and the first run tripped it + +At `rate: 3M` with `sleep: [1.0, 3.0]`, `scontent-*.cdninstagram.com` returned +**`429 Too Many Requests`** and two videos were lost (gallery-dl retried, then +gave up with exit 4). This is the *tolerant* surface complaining, which is a +clear signal the pacing was too aggressive. + +Defaults are now: + +| option | value | +|---|---| +| `--rate` | `1M` | +| `--sleep-request` | 6–10 s | +| `--sleep` | 3–6 s | +| `sleep-429` | 120 s | +| `retries` (extractor and downloader) | 8 | + +Re-running with those recovered both videos and produced **0 failures and 0 +429s**. Do not raise them for speed; an archive sync has no deadline. + +### yt-dlp is worth installing + +Without it, gallery-dl logs `Cannot import yt-dlp or youtube-dl` and falls back +to a progressive URL for DASH videos. The fallback mostly works but is what the +429s hit hardest. `pipx install yt-dlp` on the fetch host. + ## Known quirks - **`count` is not the emitted file count.** For 135 of 214 posts it was exactly diff --git a/scripts/gdl-sync.py b/scripts/gdl-sync.py index 468c5e5..0049c80 100755 --- a/scripts/gdl-sync.py +++ b/scripts/gdl-sync.py @@ -6,16 +6,23 @@ The CLI replacement for the JDownloader2 workflow. See docs/gallery-dl.md for the measurements behind every choice here — especially the safety model, which is the reason this script exists in this shape rather than a simpler one. -STATUS: skeleton. The config generation and planning are complete and tested; -`--execute` is deliberately gated behind an explicit flag and has not been run -against the live archive. +STATUS: exercised end-to-end against `withaseul` (posts, reels, stories, +highlights) publishing to a scratch directory. It has never written to the live +archive. + +The fetch host needs no copy of the archive. It stages locally and rsyncs +afterwards; what it already holds is learned from a *file listing* alone +(`--index`), which the viewer's own API serves. Usage: - ./scripts/gdl-sync.py --archives --profile 0ct0ber19 --dry-run - ./scripts/gdl-sync.py --archives --all --dry-run - ./scripts/gdl-sync.py --archives --profile 0ct0ber19 --execute + ./scripts/gdl-sync.py --index https://instaarchive.ergosteur.com \\ + --staging /var/tmp/gdl --publish user@host:/path/to/archives \\ + --profile 0ct0ber19 --dry-run -Run it from the host whose public IP matches the browser the cookie came from. + # ...then swap --dry-run for --execute. --index also accepts a local path. + +Run it from the host whose public IP matches the browser the cookie came from; +using the cookie from elsewhere is what session-hijack detection looks for. """ from __future__ import annotations @@ -107,6 +114,59 @@ def scan_archives(root: Path) -> dict[str, Profile]: return profiles +class ArchiveIndex: + """ + What the archive already holds, as filenames only. + + Deliberately never reads file *contents*, so the fetch host does not need a + copy of the archive — it can stage locally and rsync afterwards. Backed + either by a local directory or by the viewer's own API, which already + serves exactly this listing and is the cheaper option when the archive + lives on network storage (a full walk there took ~52s). + """ + + def __init__(self, source: str): + self.remote = source.startswith(("http://", "https://")) + self.source = source.rstrip("/") if self.remote else None + self.root = None if self.remote else Path(source) + if self.root and not self.root.is_dir(): + raise SystemExit(f"archive index not found: {source}") + self._cache: dict[str, list[str]] = {} + + def _get(self, path: str): + from urllib.request import urlopen + with urlopen(f"{self.source}{path}", timeout=60) as resp: + return json.load(resp) + + def profiles(self) -> set[str]: + if self.remote: + return {a["name"] for a in self._get("/api/archives")} + return set(scan_archives(self.root)) + + def listing(self, user: str) -> list[str]: + """Every filename belonging to a profile, across all its sidecars.""" + if user in self._cache: + return self._cache[user] + + names: list[str] = [] + if self.remote: + try: + data = self._get(f"/api/archives/{user}/files") + except Exception: + data = [] + files = data if isinstance(data, list) else data.get("files", []) + names = [f["path"] for f in files] + else: + prof = scan_archives(self.root).get(user) + for dirname in (prof.existing.values() if prof else ()): + d = self.root / dirname + if d.is_dir(): + names += [f"{dirname}/{n}" for n in os.listdir(d)] + + self._cache[user] = names + return names + + # -------------------------------------------------------------------------- # gallery-dl configuration # -------------------------------------------------------------------------- @@ -181,6 +241,11 @@ def build_config(rate: str, sleep_request: list[float], "api": "rest", # never "graphql" -- see docstring "sleep-request": sleep_request, "sleep": sleep, + # The CDN does rate-limit: a first run at 3M/1-3s drew + # '429 Too Many Requests' from scontent-*.cdninstagram.com and + # lost two videos. Back off hard rather than retry fast. + "sleep-429": 120.0, + "retries": 8, "videos": True, "include": "", # never "all"; sources are explicit # Directory is forced per-invocation with -D, because a reels @@ -198,7 +263,8 @@ def build_config(rate: str, sleep_request: list[float], }, }, }, - "downloader": {"http": {"rate": rate}}, + # `retries` here is the CDN-side counterpart to sleep-429 above. + "downloader": {"http": {"rate": rate, "retries": 8}}, "output": {"mode": "null"}, } @@ -207,7 +273,7 @@ def build_config(rate: str, sleep_request: list[float], # Planning and execution # -------------------------------------------------------------------------- -def gdl_command(src: Source, root: Path, config: Path, cookies: str, +def gdl_command(src: Source, staging: Path, config: Path, cookies: str, archive_db: Path | None) -> list[str]: cmd = [ "gallery-dl", @@ -215,13 +281,15 @@ def gdl_command(src: Source, root: Path, config: Path, cookies: str, "--cookies-from-browser", cookies, ] if archive_db: - # gallery-dl's skip-archive. Seed it before the first real run or the - # whole 110k-file tree is re-downloaded; see --seed-archive. + # Without a seeded skip-archive, staging is empty and every file is + # re-downloaded; see seed_archive_db. cmd += ["--download-archive", str(archive_db)] - if src.subcategory != "highlights": - cmd += ["--destination", str(root / src.directory)] - else: - cmd += ["--destination", str(root)] + # Forced destination -- never `{username}` -- because a reels tab returns + # collab reels owned by other accounts, which would otherwise be filed + # under the wrong profile. Highlights are the exception: their directory + # embeds a title only known mid-extraction, so the config formats it. + dest = staging if src.subcategory == "highlights" else staging / src.directory + cmd += ["--destination", str(dest)] cmd.append(src.url) return cmd @@ -329,11 +397,48 @@ def probe_live(src: Source, config: Path, cookies: str) -> list[dict]: return items +def rsync_command(staging: Path, dest: str, dry_run: bool) -> list[str]: + """ + Publish a staging tree into the archive. + + `--ignore-existing` is not an optimisation, it is the safety property: the + archive deliberately outlives Instagram (posts exist here that Instagram no + longer serves), so publishing must only ever *add*. No `--delete`, and + nothing already present is overwritten — including sidecars, which get + rewritten on every run and would otherwise churn the synced share. + + `dest` may be a local path or any rsync destination (`user@host:/path`), + because the archive usually is not writable from the fetch host. + """ + cmd = ["rsync", "-a", "--ignore-existing", "--partial", "--info=stats2"] + if dry_run: + cmd.append("--dry-run") + # Trailing slash: copy the *contents* of staging into dest. + cmd += [f"{staging}/", dest if dest.endswith("/") else dest + "/"] + return cmd + + +def publish(staging: Path, dest: str, dry_run: bool) -> int: + if not any(staging.iterdir()): + print(" nothing staged; skipping publish") + return 0 + cmd = rsync_command(staging, dest, dry_run) + print(" " + " ".join(cmd)) + return subprocess.run(cmd).returncode + + def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--archives", required=True, type=Path, - help="archive root (one directory per profile plus sidecars)") + ap.add_argument("--index", required=True, + help="existing archive listing: a local root, or the " + "viewer's base URL (only a FILE LISTING is needed, " + "never the contents)") + ap.add_argument("--publish", required=True, + help="rsync destination for fetched files; a local path or " + "user@host:/path") + ap.add_argument("--staging", type=Path, required=True, + help="local scratch directory gallery-dl writes into") g = ap.add_mutually_exclusive_group(required=True) g.add_argument("--profile", action="append", default=[], help="profile to sync; repeatable") @@ -342,10 +447,10 @@ def main() -> int: help="gallery-dl --cookies-from-browser value") ap.add_argument("--archive-db", type=Path, default=None, help="gallery-dl skip-archive sqlite path") - ap.add_argument("--rate", default="3M", help="per-download rate cap") - ap.add_argument("--sleep-request", nargs=2, type=float, default=[4.0, 7.0], + ap.add_argument("--rate", default="1M", help="per-download rate cap") + ap.add_argument("--sleep-request", nargs=2, type=float, default=[6.0, 10.0], metavar=("MIN", "MAX")) - ap.add_argument("--sleep", nargs=2, type=float, default=[1.0, 3.0], + ap.add_argument("--sleep", nargs=2, type=float, default=[3.0, 6.0], metavar=("MIN", "MAX")) ap.add_argument("--no-stories", action="store_true", help="skip stories and highlights (posts and reels only)") @@ -356,25 +461,26 @@ def main() -> int: help="actually run gallery-dl") args = ap.parse_args() - if not args.archives.is_dir(): - print(f"archives root not found: {args.archives}", file=sys.stderr) - return 2 if not shutil.which("gallery-dl"): print("gallery-dl not on PATH", file=sys.stderr) return 2 + if not shutil.which("rsync"): + print("rsync not on PATH", file=sys.stderr) + return 2 - profiles = scan_archives(args.archives) + index = ArchiveIndex(args.index) + names = index.profiles() if args.profile: - missing = [p for p in args.profile if p not in profiles] - for p in missing: - print(f"note: {p} has no directory yet; it will be created") - profiles.setdefault(p, Profile(p)) - selected = [profiles[p] for p in args.profile] + for p in args.profile: + if p not in names: + print(f"note: {p} is not in the index yet; it will be created") + selected = [Profile(p) for p in args.profile] else: - selected = list(profiles.values()) + selected = [Profile(p) for p in sorted(names)] config = build_config(args.rate, list(args.sleep_request), list(args.sleep)) - config_path = args.archives / ".gdl-sync.config.json" + args.staging.mkdir(parents=True, exist_ok=True) + config_path = args.staging / "gdl-sync.config.json" plan: list[tuple[Profile, Source]] = [ (prof, src) @@ -386,34 +492,57 @@ def main() -> int: print(f"sources : {len(plan)}") print(f"pacing : {args.sleep_request[0]}-{args.sleep_request[1]}s between " f"requests, rate cap {args.rate}") + print(f"staging : {args.staging}") + print(f"publish : {args.publish}") print() if not args.execute: - print(json.dumps(config, indent=2)) - print() for prof, src in plan: dest = src.directory or "(per-highlight)" print(f" {prof.user:<20} {src.kind:<11} -> {dest}") + print() + print(" " + " ".join(rsync_command(args.staging, args.publish, True))) print("\ndry run; nothing fetched. pass --execute to run.") return 0 config_path.write_text(json.dumps(config, indent=2)) failures = 0 + for prof, src in plan: - if src.subcategory != "highlights": - (args.archives / src.directory).mkdir(parents=True, exist_ok=True) - cmd = gdl_command(src, args.archives, config_path, args.cookies, - args.archive_db) print(f"==> {prof.user} / {src.kind}") + stage_dir = args.staging / (src.directory or ".") + stage_dir.mkdir(parents=True, exist_ok=True) + + # Seed the skip-archive from what the archive already holds, so + # fetching into an empty staging directory pulls only what is missing. + # The listing pass this needs is one we have to make anyway. + if args.archive_db: + try: + live = probe_live(src, config_path, args.cookies) + held = index_existing(index.listing(prof.user)) + seeded = seed_archive_db(args.archive_db, held, live, + src.subcategory) + print(f" seeded {seeded} of {len(live)} live items") + except subprocess.CalledProcessError as exc: + failures += 1 + print(f" probe FAILED: {exc}", file=sys.stderr) + continue + + cmd = gdl_command(src, args.staging, config_path, args.cookies, + args.archive_db) result = subprocess.run(cmd) if result.returncode != 0: failures += 1 - # Keep going: one private/renamed profile must not abort the run. + # Keep going: one private or renamed profile must not abort the run. print(f" FAILED (exit {result.returncode})", file=sys.stderr) - # A sync NEVER deletes. The archive deliberately outlives Instagram -- - # 2 posts in 0ct0ber19 exist only here now. - print(f"\ndone; {failures} source(s) failed") + # Publish once, at the end, so a partially-fetched profile never reaches + # the archive mid-run. Only ever adds -- see rsync_command. + print("\n==> publish") + if publish(args.staging, args.publish, dry_run=False) != 0: + failures += 1 + + print(f"\ndone; {failures} step(s) failed") return 1 if failures else 0