Files
instaarchive-viewer/scripts/gdl-sync.py
T
ergosteurandClaude Opus 5 3a34e4359e feat: publish fetched media by rsync, and back off from the CDN's 429s
Wires up the staging -> rsync step and replaces --archives with --index
(a listing source: local path or the viewer's API), --staging and
--publish, so the fetch host needs no copy of the archive.

rsync runs --ignore-existing with no --delete. That is a safety property
rather than an optimisation: the archive deliberately outlives Instagram,
so publishing must only ever add. It runs once at the end so a profile
that fails midway never reaches the archive half-written.

Exercised end-to-end against withaseul across all four surfaces,
publishing to a scratch directory. Seeding worked as designed (915 of 984
post items and 28 of 34 reel items already held), stories and highlights
returned no results cleanly, and the collab-reel case landed correctly:
"withaseul - reels" holds files owned by cher_ryppo, 0ct0ber19 and
official_artms, each with the owner in the filename and the crawl scope
as the directory.

The first run drew '429 Too Many Requests' from the CDN at 3M with 1-3s
sleeps and lost two videos. That is the tolerant surface complaining, so
the defaults are now 1M, 6-10s between requests, 3-6s between downloads,
sleep-429 of 120s and 8 retries. Re-running recovered both videos with
zero failures and zero 429s. Installing yt-dlp on the fetch host also
matters: without it DASH videos fall back to a progressive URL, which is
what the rate limiting hit hardest.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-16 21:37:38 -04:00

551 lines
22 KiB
Python
Executable File

#!/usr/bin/env python3
"""
Fetch Instagram profiles into the archive layout using gallery-dl.
The CLI replacement for the JDownloader2 workflow. See docs/gallery-dl.md for
the measurements behind every choice here — especially the safety model, which
is the reason this script exists in this shape rather than a simpler one.
STATUS: exercised end-to-end against `withaseul` (posts, reels, stories,
highlights) publishing to a scratch directory. It has never written to the live
archive.
The fetch host needs no copy of the archive. It stages locally and rsyncs
afterwards; what it already holds is learned from a *file listing* alone
(`--index`), which the viewer's own API serves.
Usage:
./scripts/gdl-sync.py --index https://instaarchive.ergosteur.com \\
--staging /var/tmp/gdl --publish user@host:/path/to/archives \\
--profile 0ct0ber19 --dry-run
# ...then swap --dry-run for --execute. --index also accepts a local path.
Run it from the host whose public IP matches the browser the cookie came from;
using the cookie from elsewhere is what session-hijack detection looks for.
"""
from __future__ import annotations
import argparse
import json
import os
import re
import shutil
import subprocess
import sys
from dataclasses import dataclass, field
from pathlib import Path
# --------------------------------------------------------------------------
# Archive layout
# --------------------------------------------------------------------------
# Mirrors src/lib/archive-grouping.ts. Instagram usernames cannot contain
# spaces, which is what makes the username separable from a highlight title.
RE_HIGHLIGHT = re.compile(r"^story highlights - ([^ ]+) - (.+)$")
RE_STORIES = re.compile(r"^story - ([^ ]+)$")
RE_REELS = re.compile(r"^([^ ]+) - reels$")
DATE_FMT = "{date:Olocal/%Y-%m-%d}"
"""Local-time date. JD2 stamped US Eastern, NOT UTC (0/212 mismatches vs 19 for
UTC). `Olocal` is DST-aware per timestamp. The trailing separator must be
omitted or it lands in the strftime format and sanitises to an underscore."""
POST_STEM = DATE_FMT + "_{username} - {post_shortcode}"
ITEM_STEM = DATE_FMT + "_{username} - {shortcode}"
@dataclass
class Source:
"""One gallery-dl invocation: a URL fetched into a specific directory."""
kind: str # posts | reels | stories | highlights
url: str
directory: str # relative to the archives root
subcategory: str # gallery-dl config key
title: str | None = None # highlight title, when known
@dataclass
class Profile:
user: str
existing: dict[str, str] = field(default_factory=dict) # kind -> dirname
def sources(self, include_stories: bool) -> list[Source]:
u = self.user
base = f"https://www.instagram.com/{u}"
out = [
Source("posts", f"{base}/posts/", u, "posts"),
Source("reels", f"{base}/reels/", f"{u} - reels", "reels"),
]
if include_stories:
# Stories expire after 24h, so these can only ever be captured
# live. There is no backfill and no re-fetch.
out.append(Source(
"stories", f"https://www.instagram.com/stories/{u}/",
f"story - {u}", "stories"))
# Highlight directories embed the title, which gallery-dl only
# learns mid-extraction -- so this one source fans out into many
# directories and is handled with a directory format string.
out.append(Source(
"highlights", f"{base}/highlights", "", "highlights"))
return out
def scan_archives(root: Path) -> dict[str, Profile]:
"""Group existing directories into profiles, as the server does."""
profiles: dict[str, Profile] = {}
def get(user: str) -> Profile:
return profiles.setdefault(user, Profile(user))
for entry in sorted(os.listdir(root)):
if not (root / entry).is_dir() or entry.startswith("."):
continue
if m := RE_HIGHLIGHT.match(entry):
get(m.group(1)).existing.setdefault("highlights", entry)
elif m := RE_STORIES.match(entry):
get(m.group(1)).existing["stories"] = entry
elif m := RE_REELS.match(entry):
get(m.group(1)).existing["reels"] = entry
else:
get(entry).existing["posts"] = entry
return profiles
class ArchiveIndex:
"""
What the archive already holds, as filenames only.
Deliberately never reads file *contents*, so the fetch host does not need a
copy of the archive — it can stage locally and rsync afterwards. Backed
either by a local directory or by the viewer's own API, which already
serves exactly this listing and is the cheaper option when the archive
lives on network storage (a full walk there took ~52s).
"""
def __init__(self, source: str):
self.remote = source.startswith(("http://", "https://"))
self.source = source.rstrip("/") if self.remote else None
self.root = None if self.remote else Path(source)
if self.root and not self.root.is_dir():
raise SystemExit(f"archive index not found: {source}")
self._cache: dict[str, list[str]] = {}
def _get(self, path: str):
from urllib.request import urlopen
with urlopen(f"{self.source}{path}", timeout=60) as resp:
return json.load(resp)
def profiles(self) -> set[str]:
if self.remote:
return {a["name"] for a in self._get("/api/archives")}
return set(scan_archives(self.root))
def listing(self, user: str) -> list[str]:
"""Every filename belonging to a profile, across all its sidecars."""
if user in self._cache:
return self._cache[user]
names: list[str] = []
if self.remote:
try:
data = self._get(f"/api/archives/{user}/files")
except Exception:
data = []
files = data if isinstance(data, list) else data.get("files", [])
names = [f["path"] for f in files]
else:
prof = scan_archives(self.root).get(user)
for dirname in (prof.existing.values() if prof else ()):
d = self.root / dirname
if d.is_dir():
names += [f"{dirname}/{n}" for n in os.listdir(d)]
self._cache[user] = names
return names
# --------------------------------------------------------------------------
# gallery-dl configuration
# --------------------------------------------------------------------------
def build_config(rate: str, sleep_request: list[float],
sleep: list[float]) -> dict:
"""
The config is generated rather than checked in so the safety-critical
options cannot drift out of sync with the docs.
`api: rest` is the single most important line in this file. The graphql
backend issues one request PER POST for every video and carousel, which is
the pattern that got this account banned once already.
"""
caption_pp = {
"name": "metadata",
"event": "post",
"mode": "custom",
"content-format": "{description}",
"extension": "txt",
# JD2 wrote no .txt when the caption was empty; "empty": false (the
# default) reproduces that.
}
meta_pp = {
"name": "metadata",
"event": "post",
"mode": "json",
# `include`, NOT `fields` -- `fields` applies to mode:custom and
# silently does nothing here, dumping audio_user blobs that contain
# unrelated users' profile picture URLs.
"include": [
"post_shortcode", "post_id", "type", "date", "post_date",
"username", "fullname", "owner_id", "description", "count",
"likes", "post_url", "sidecar_shortcode",
],
}
def post_like(stem: str) -> dict:
"""Naming for surfaces whose unit is a post (posts, reels)."""
return {
# `sidecar_shortcode` is set only for carousels, so it is the
# carousel discriminator. First matching condition wins.
"filename": {
"sidecar_shortcode and count >= 10":
stem + " - {num:02}.{extension}",
"sidecar_shortcode":
stem + " - {num}.{extension}",
"":
stem + ".{extension}",
},
"postprocessors": [
{**caption_pp, "filename": stem + ".txt"},
{**meta_pp, "filename": stem + ".json"},
],
}
def item_like(stem: str) -> dict:
"""
Naming for surfaces whose unit is an item inside a reel (stories,
highlights). `{shortcode}` is per item; `{post_shortcode}` is the
reel's id and is shared by every item in it.
"""
return {
"filename": stem + ".{extension}",
"postprocessors": [{**meta_pp, "filename": stem + ".json"}],
}
return {
"extractor": {
"base-directory": ".",
"instagram": {
"api": "rest", # never "graphql" -- see docstring
"sleep-request": sleep_request,
"sleep": sleep,
# The CDN does rate-limit: a first run at 3M/1-3s drew
# '429 Too Many Requests' from scontent-*.cdninstagram.com and
# lost two videos. Back off hard rather than retry fast.
"sleep-429": 120.0,
"retries": 8,
"videos": True,
"include": "", # never "all"; sources are explicit
# Directory is forced per-invocation with -D, because a reels
# tab returns collab reels owned by OTHER accounts and
# {username} would scatter them into the wrong profile.
"directory": [],
"posts": post_like(POST_STEM),
"reels": post_like(POST_STEM),
"stories": item_like(ITEM_STEM),
"highlights": {
**item_like(ITEM_STEM),
# The only surface that must derive its own directory,
# since the title is not known until extraction.
"directory": ["story highlights - {username} - {highlight_title}"],
},
},
},
# `retries` here is the CDN-side counterpart to sleep-429 above.
"downloader": {"http": {"rate": rate, "retries": 8}},
"output": {"mode": "null"},
}
# --------------------------------------------------------------------------
# Planning and execution
# --------------------------------------------------------------------------
def gdl_command(src: Source, staging: Path, config: Path, cookies: str,
archive_db: Path | None) -> list[str]:
cmd = [
"gallery-dl",
"--config", str(config),
"--cookies-from-browser", cookies,
]
if archive_db:
# Without a seeded skip-archive, staging is empty and every file is
# re-downloaded; see seed_archive_db.
cmd += ["--download-archive", str(archive_db)]
# Forced destination -- never `{username}` -- because a reels tab returns
# collab reels owned by other accounts, which would otherwise be filed
# under the wrong profile. Highlights are the exception: their directory
# embeds a title only known mid-extraction, so the config formats it.
dest = staging if src.subcategory == "highlights" else staging / src.directory
cmd += ["--destination", str(dest)]
cmd.append(src.url)
return cmd
# gallery-dl keys its skip-archive on `archive_prefix + archive_fmt`, which for
# this extractor is the literal "instagram" followed by the per-media numeric
# pk (`instagram.py:25`, `job.py:713-719`). Verified against a real run: a
# 3-image carousel produced 3 rows, one per item.
ARCHIVE_KEY = "instagram{}".format
ARCHIVE_SCHEMA = "CREATE TABLE IF NOT EXISTS archive (entry TEXT PRIMARY KEY)"
RE_ARCHIVED = re.compile(
r"^(\d{4}-\d{2}-\d{2})_(.+?) - ([A-Za-z0-9_-]+?)(?: - (\d+))?\.(\w+)$")
NON_MEDIA = {"txt", "json"}
def index_existing(listing: list[str]) -> set[tuple[str, int]]:
"""
Reduce a flat list of filenames to the (shortcode, index) pairs already
held. Only names matter — never the bytes — which is what lets the sync run
on a host that has no copy of the archive.
"""
have: set[tuple[str, int]] = set()
for name in listing:
m = RE_ARCHIVED.match(name.rsplit("/", 1)[-1])
if not m or m.group(5).lower() in NON_MEDIA:
continue
# An absent index means a single-media post, which is index 1 — the
# same normalisation the viewer's EXPORT_RE applies.
have.add((m.group(3), int(m.group(4) or 1)))
return have
def live_key(item: dict, kind: str) -> tuple[str, int]:
"""
The (shortcode, index) a live item *would* be filed under, mirroring the
filename template exactly.
The two surfaces disagree about which shortcode identifies a file, and
getting this wrong silently seeds almost nothing:
posts/reels filed under {post_shortcode} — for a carousel, each
child item ALSO has its own `shortcode`, which is not
what appears in the filename.
stories/highlights filed under the per-item {shortcode}, because
`post_shortcode` there is the containing reel's id and
is shared by every item in it.
"""
if kind in ("stories", "highlights"):
return (item.get("shortcode"), 1)
return (item.get("post_shortcode"), item.get("num"))
def seed_archive_db(db: Path, existing: set[tuple[str, int]],
live: list[dict], kind: str) -> int:
"""
Mark everything already held as downloaded, so a fetch into an empty
directory pulls only what is missing.
`live` is the metadata of one listing pass — the pass we have to make
anyway — each entry carrying at least `media_id` plus the shortcode fields
`live_key` needs. Seeding costs no additional Instagram requests, and needs
only a *listing* of the archive, never its contents.
"""
import sqlite3
db.parent.mkdir(parents=True, exist_ok=True)
con = sqlite3.connect(db)
con.execute(ARCHIVE_SCHEMA)
rows = [
(ARCHIVE_KEY(item["media_id"]),)
for item in live
if live_key(item, kind) in existing
]
con.executemany("INSERT OR IGNORE INTO archive (entry) VALUES (?)", rows)
con.commit()
con.close()
return len(rows)
def probe_live(src: Source, config: Path, cookies: str) -> list[dict]:
"""
One metadata-only listing pass. `sleep` is forced to 0 because it otherwise
applies per *file* even with no download — 2275 files at 1-3s each is over
an hour for a single profile.
"""
out = subprocess.run(
["gallery-dl", "-j", "--config", str(config),
"--cookies-from-browser", cookies, "-o", "sleep=0", src.url],
capture_output=True, text=True, check=True,
)
items: list[dict] = []
def walk(node):
if isinstance(node, dict):
if "media_id" in node and "shortcode" in node:
items.append(node)
for value in node.values():
walk(value)
elif isinstance(node, list):
for value in node:
walk(value)
walk(json.loads(out.stdout))
return items
def rsync_command(staging: Path, dest: str, dry_run: bool) -> list[str]:
"""
Publish a staging tree into the archive.
`--ignore-existing` is not an optimisation, it is the safety property: the
archive deliberately outlives Instagram (posts exist here that Instagram no
longer serves), so publishing must only ever *add*. No `--delete`, and
nothing already present is overwritten — including sidecars, which get
rewritten on every run and would otherwise churn the synced share.
`dest` may be a local path or any rsync destination (`user@host:/path`),
because the archive usually is not writable from the fetch host.
"""
cmd = ["rsync", "-a", "--ignore-existing", "--partial", "--info=stats2"]
if dry_run:
cmd.append("--dry-run")
# Trailing slash: copy the *contents* of staging into dest.
cmd += [f"{staging}/", dest if dest.endswith("/") else dest + "/"]
return cmd
def publish(staging: Path, dest: str, dry_run: bool) -> int:
if not any(staging.iterdir()):
print(" nothing staged; skipping publish")
return 0
cmd = rsync_command(staging, dest, dry_run)
print(" " + " ".join(cmd))
return subprocess.run(cmd).returncode
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--index", required=True,
help="existing archive listing: a local root, or the "
"viewer's base URL (only a FILE LISTING is needed, "
"never the contents)")
ap.add_argument("--publish", required=True,
help="rsync destination for fetched files; a local path or "
"user@host:/path")
ap.add_argument("--staging", type=Path, required=True,
help="local scratch directory gallery-dl writes into")
g = ap.add_mutually_exclusive_group(required=True)
g.add_argument("--profile", action="append", default=[],
help="profile to sync; repeatable")
g.add_argument("--all", action="store_true", help="every profile on disk")
ap.add_argument("--cookies", default="chrome:/home/matt/.config/google-chrome-devtools",
help="gallery-dl --cookies-from-browser value")
ap.add_argument("--archive-db", type=Path, default=None,
help="gallery-dl skip-archive sqlite path")
ap.add_argument("--rate", default="1M", help="per-download rate cap")
ap.add_argument("--sleep-request", nargs=2, type=float, default=[6.0, 10.0],
metavar=("MIN", "MAX"))
ap.add_argument("--sleep", nargs=2, type=float, default=[3.0, 6.0],
metavar=("MIN", "MAX"))
ap.add_argument("--no-stories", action="store_true",
help="skip stories and highlights (posts and reels only)")
mode = ap.add_mutually_exclusive_group()
mode.add_argument("--dry-run", action="store_true", default=True,
help="print the plan and the config; default")
mode.add_argument("--execute", action="store_true",
help="actually run gallery-dl")
args = ap.parse_args()
if not shutil.which("gallery-dl"):
print("gallery-dl not on PATH", file=sys.stderr)
return 2
if not shutil.which("rsync"):
print("rsync not on PATH", file=sys.stderr)
return 2
index = ArchiveIndex(args.index)
names = index.profiles()
if args.profile:
for p in args.profile:
if p not in names:
print(f"note: {p} is not in the index yet; it will be created")
selected = [Profile(p) for p in args.profile]
else:
selected = [Profile(p) for p in sorted(names)]
config = build_config(args.rate, list(args.sleep_request), list(args.sleep))
args.staging.mkdir(parents=True, exist_ok=True)
config_path = args.staging / "gdl-sync.config.json"
plan: list[tuple[Profile, Source]] = [
(prof, src)
for prof in selected
for src in prof.sources(include_stories=not args.no_stories)
]
print(f"profiles : {len(selected)}")
print(f"sources : {len(plan)}")
print(f"pacing : {args.sleep_request[0]}-{args.sleep_request[1]}s between "
f"requests, rate cap {args.rate}")
print(f"staging : {args.staging}")
print(f"publish : {args.publish}")
print()
if not args.execute:
for prof, src in plan:
dest = src.directory or "(per-highlight)"
print(f" {prof.user:<20} {src.kind:<11} -> {dest}")
print()
print(" " + " ".join(rsync_command(args.staging, args.publish, True)))
print("\ndry run; nothing fetched. pass --execute to run.")
return 0
config_path.write_text(json.dumps(config, indent=2))
failures = 0
for prof, src in plan:
print(f"==> {prof.user} / {src.kind}")
stage_dir = args.staging / (src.directory or ".")
stage_dir.mkdir(parents=True, exist_ok=True)
# Seed the skip-archive from what the archive already holds, so
# fetching into an empty staging directory pulls only what is missing.
# The listing pass this needs is one we have to make anyway.
if args.archive_db:
try:
live = probe_live(src, config_path, args.cookies)
held = index_existing(index.listing(prof.user))
seeded = seed_archive_db(args.archive_db, held, live,
src.subcategory)
print(f" seeded {seeded} of {len(live)} live items")
except subprocess.CalledProcessError as exc:
failures += 1
print(f" probe FAILED: {exc}", file=sys.stderr)
continue
cmd = gdl_command(src, args.staging, config_path, args.cookies,
args.archive_db)
result = subprocess.run(cmd)
if result.returncode != 0:
failures += 1
# Keep going: one private or renamed profile must not abort the run.
print(f" FAILED (exit {result.returncode})", file=sys.stderr)
# Publish once, at the end, so a partially-fetched profile never reaches
# the archive mid-run. Only ever adds -- see rsync_command.
print("\n==> publish")
if publish(args.staging, args.publish, dry_run=False) != 0:
failures += 1
print(f"\ndone; {failures} step(s) failed")
return 1 if failures else 0
if __name__ == "__main__":
sys.exit(main())