diff --git a/docs/gallery-dl.md b/docs/gallery-dl.md index 915b157..57908d7 100644 --- a/docs/gallery-dl.md +++ b/docs/gallery-dl.md @@ -204,6 +204,53 @@ viewer retire the lone-video heuristic in `src/lib/post-tabs.ts` — see `/p//` URL leaves it `null`. Sync always uses listing URLs, so this only matters when testing by hand. +## Incremental sync — why the fetch host needs no copy of the archive + +gallery-dl can skip already-held media two ways, and the difference decides +whether the fetcher needs the archive mounted: + +- **By file existence** (default). Needs the destination to already contain the + files, so it only works if the archive is mounted where gallery-dl writes. +- **By skip-archive** (`--download-archive`). A sqlite DB of ids. Needs nothing + on disk. + +We use the second, so the fetch host can write to **local disk and rsync +afterwards**. That avoids writing tens of thousands of small files over CIFS, +and keeps a mid-sync failure from leaving partial files on the live Resilio +share. + +The key is `archive_prefix + archive_fmt`, which for this extractor is the +literal `instagram` plus the per-media numeric pk (`instagram.py:25`, +`job.py:713-719`). Verified: a 3-image carousel produced + +``` +instagram3079387627521318672 +instagram3079387627521429433 +instagram3079387627529716672 +``` + +and a second run skipped every media file, rewriting only the idempotent +`.txt`/`.json` sidecars. + +**Seeding.** `media_id` is not in our filenames, so the DB cannot be built from +names alone — but one listing pass (the pass we make anyway) maps every live +item to its `media_id`, and the archive's *file listing* says which we already +hold. No extra Instagram requests, and no archive content — a listing is +enough, which `GET /api/archives/:name/files` already serves. + +Measured on `0ct0ber19`: 2275 live media items, 2248 seeded from the existing +listing, **27 left to download** — precisely the media of the two posts added +since the last crawl. + +The one trap, which silently seeds almost nothing if you get it backwards: + +| surface | filed under | why | +|---|---|---| +| posts, reels | `post_shortcode` | carousel children each have their own `shortcode`, which never appears in a filename | +| stories, highlights | `shortcode` (per item) | `post_shortcode` is the containing reel's id, shared by every item | + +`live_key()` encodes this. Matching on the wrong field seeded 5 of 2275. + ## Known quirks - **`count` is not the emitted file count.** For 135 of 214 posts it was exactly diff --git a/scripts/__pycache__/gdl-sync.cpython-314.pyc b/scripts/__pycache__/gdl-sync.cpython-314.pyc new file mode 100644 index 0000000..e90de1c Binary files /dev/null and b/scripts/__pycache__/gdl-sync.cpython-314.pyc differ diff --git a/scripts/gdl-sync.py b/scripts/gdl-sync.py index b624624..468c5e5 100755 --- a/scripts/gdl-sync.py +++ b/scripts/gdl-sync.py @@ -226,19 +226,107 @@ def gdl_command(src: Source, root: Path, config: Path, cookies: str, return cmd -def seed_archive_db(root: Path, profiles: dict[str, Profile], - db: Path) -> int: - """ - TODO: populate the skip-archive from filenames already on disk. +# gallery-dl keys its skip-archive on `archive_prefix + archive_fmt`, which for +# this extractor is the literal "instagram" followed by the per-media numeric +# pk (`instagram.py:25`, `job.py:713-719`). Verified against a real run: a +# 3-image carousel produced 3 rows, one per item. +ARCHIVE_KEY = "instagram{}".format +ARCHIVE_SCHEMA = "CREATE TABLE IF NOT EXISTS archive (entry TEXT PRIMARY KEY)" - gallery-dl keys its archive on an extractor-specific id string, so this - needs the exact format `InstagramExtractor.archive_fmt` produces. Until - it is implemented, the first run of any profile re-downloads everything — - bandwidth on the CDN (the tolerant surface), but hours of it. +RE_ARCHIVED = re.compile( + r"^(\d{4}-\d{2}-\d{2})_(.+?) - ([A-Za-z0-9_-]+?)(?: - (\d+))?\.(\w+)$") +NON_MEDIA = {"txt", "json"} + + +def index_existing(listing: list[str]) -> set[tuple[str, int]]: """ - raise NotImplementedError( - "seed_archive_db is unimplemented; run without --archive-db and accept " - "a full re-download, or implement this first") + Reduce a flat list of filenames to the (shortcode, index) pairs already + held. Only names matter — never the bytes — which is what lets the sync run + on a host that has no copy of the archive. + """ + have: set[tuple[str, int]] = set() + for name in listing: + m = RE_ARCHIVED.match(name.rsplit("/", 1)[-1]) + if not m or m.group(5).lower() in NON_MEDIA: + continue + # An absent index means a single-media post, which is index 1 — the + # same normalisation the viewer's EXPORT_RE applies. + have.add((m.group(3), int(m.group(4) or 1))) + return have + + +def live_key(item: dict, kind: str) -> tuple[str, int]: + """ + The (shortcode, index) a live item *would* be filed under, mirroring the + filename template exactly. + + The two surfaces disagree about which shortcode identifies a file, and + getting this wrong silently seeds almost nothing: + + posts/reels filed under {post_shortcode} — for a carousel, each + child item ALSO has its own `shortcode`, which is not + what appears in the filename. + stories/highlights filed under the per-item {shortcode}, because + `post_shortcode` there is the containing reel's id and + is shared by every item in it. + """ + if kind in ("stories", "highlights"): + return (item.get("shortcode"), 1) + return (item.get("post_shortcode"), item.get("num")) + + +def seed_archive_db(db: Path, existing: set[tuple[str, int]], + live: list[dict], kind: str) -> int: + """ + Mark everything already held as downloaded, so a fetch into an empty + directory pulls only what is missing. + + `live` is the metadata of one listing pass — the pass we have to make + anyway — each entry carrying at least `media_id` plus the shortcode fields + `live_key` needs. Seeding costs no additional Instagram requests, and needs + only a *listing* of the archive, never its contents. + """ + import sqlite3 + + db.parent.mkdir(parents=True, exist_ok=True) + con = sqlite3.connect(db) + con.execute(ARCHIVE_SCHEMA) + rows = [ + (ARCHIVE_KEY(item["media_id"]),) + for item in live + if live_key(item, kind) in existing + ] + con.executemany("INSERT OR IGNORE INTO archive (entry) VALUES (?)", rows) + con.commit() + con.close() + return len(rows) + + +def probe_live(src: Source, config: Path, cookies: str) -> list[dict]: + """ + One metadata-only listing pass. `sleep` is forced to 0 because it otherwise + applies per *file* even with no download — 2275 files at 1-3s each is over + an hour for a single profile. + """ + out = subprocess.run( + ["gallery-dl", "-j", "--config", str(config), + "--cookies-from-browser", cookies, "-o", "sleep=0", src.url], + capture_output=True, text=True, check=True, + ) + items: list[dict] = [] + + def walk(node): + if isinstance(node, dict): + if "media_id" in node and "shortcode" in node: + items.append(node) + for value in node.values(): + walk(value) + elif isinstance(node, list): + for value in node: + walk(value) + + walk(json.loads(out.stdout)) + return items def main() -> int: