diff --git a/docs/gallery-dl.md b/docs/gallery-dl.md index 7507dc1..011c3d7 100644 --- a/docs/gallery-dl.md +++ b/docs/gallery-dl.md @@ -359,6 +359,71 @@ The one trap, which silently seeds almost nothing if you get it backwards: `live_key()` encodes this. Matching on the wrong field seeded 5 of 2275. +### The skip-archive saves the CDN, not `instagram.com` + +Worth being exact about, because the two costs land on different surfaces and +only one of them bans accounts: + +| what | which surface | scales with | +|---|---|---| +| downloading media | `scontent-*.cdninstagram.com` | how much is **new** | +| enumerating the profile to find it | `instagram.com` | how **big** the profile is | + +The skip-archive suppresses the first. It does nothing about the second, so a +2275-post profile costs ~76 pages of pagination every run, forever, whether it +has three new posts or none. Seeding (above) saved a *second* full pass, not +the first. + +Measured on the 2026-08-20 run, from sidecar write times in staging — free, +since the run was paying for the listing anyway: + +``` +1787248852 2026-08-19 … DcOeoVxkthi new, +0s +1787248944 2026-08-18 … DcLpfoJCZtp new, +92s +1787249058 2026-08-17 … DcIlGbxCUk0 new, +114s +1787249162 2026-07-24 … DbKr1TxlPSX ┐ all one second: nothing +1787249162 2026-08-15 … DcD-FdBCYGm ┘ downloaded, sidecars only +``` + +Three posts took ~100s each; the remaining 2272 were enumeration with nothing +to show for it. + +**Pinned posts do not break early abort.** Test case 16 previously claimed +`0ct0ber19` returns its 3 pinned posts out of date order — that is true of the +*web grid*, but the REST `/posts/` listing came back strictly +reverse-chronological, newest first, no hoisting. That matters because +front-loaded old posts are the one thing that would make `skip: abort:N` +dangerous: it would trip on them and abort before reaching anything new. + +So `skip: abort:N` is viable, and cuts ~420 requests per run to ~40-60: + +| surface | live items | pages | with `abort:50` | +|---|---:|---:|---:| +| posts, 6 profiles | 11,248 | ~377 | ~12 | +| reels, 6 profiles | 1,080 | ~24 | ~8 | +| stories + highlights | — | ~20 | ~20 | + +N counts consecutive skipped **files**, not posts, so it must clear the largest +already-held carousel — `DcD-FdBCYGm` alone is 22 media. 50 is comfortable; 5 +would not be. + +**The tradeoff is edited carousels.** Test case 15 is a post that gained items +after we archived it, and only a full enumeration finds those. Suggested +policy: `abort:50` for routine runs, a full sweep occasionally. + +Measured the same day, resuming a stopped run with `--abort 50`: + +| source | live items | enumerated | +|---|---:|---:| +| `cher_ryppo` posts | 2,151 | **7** | +| `cher_ryppo` reels | 92 | 53 | + +One page instead of 72, and every new post was still caught. The 7 is roughly +3 new posts plus 4 already-held carousels making up the 50 skipped files. +Reels need 53 because they are single-media, so 50 consecutive skips really is +50 reels — another reminder that N counts files, and that the same N behaves +very differently on a carousel-heavy surface than on a reels tab. + ## Publishing The fetch host stages to local disk and rsyncs afterwards. `rsync @@ -493,7 +558,7 @@ Real subjects, all present in the archive today. See | 13 | empty caption | `CrdsY5CrSsO` | media written, `.txt` absent | | 14 | deleted post | `C0TgI7sphfZ` | on disk, absent live — must not be removed | | 15 | edited carousel | `C7zG7-jJMlq` | 18 on disk, 8 live — must not be removed | -| 16 | pinned posts | `0ct0ber19` | 3 pinned, returned out of date order | +| 16 | pinned posts | `0ct0ber19` | REST listing is strictly reverse-chronological; see below | | 17 | profile avatar | `0ct0ber19.jpg` | base dir, undated | Cases 14–16 are reconciliation, not naming: **a sync must never delete**, since diff --git a/scripts/gdl-sync.py b/scripts/gdl-sync.py index 36e385a..0ae6dec 100755 --- a/scripts/gdl-sync.py +++ b/scripts/gdl-sync.py @@ -219,7 +219,7 @@ class ArchiveIndex: # -------------------------------------------------------------------------- def build_config(rate: str, sleep_request: list[float], - sleep: list[float]) -> dict: + sleep: list[float], abort: int = 0) -> dict: """ The config is generated rather than checked in so the safety-critical options cannot drift out of sync with the docs. @@ -253,7 +253,26 @@ def build_config(rate: str, sleep_request: list[float], def post_like(stem: str) -> dict: """Naming for surfaces whose unit is a post (posts, reels).""" + skip: dict = {} + if abort: + # Stop enumerating once `abort` consecutive files are already in + # the skip-archive. The listing pass -- not the downloading -- is + # what costs `instagram.com` requests, and it otherwise walks the + # whole profile every run to find three new posts. + # + # Safe here only because the REST listing is strictly + # reverse-chronological: the web grid hoists pinned posts to the + # front, but this endpoint does not (measured 2026-08-20), so old + # posts never appear before new ones. + # + # Counted in FILES, not posts, so it must clear the largest + # already-held carousel -- 22 media for one real post in this + # archive. It also means edited carousels (test case 15) stop + # being noticed, so a full sweep is still worth running + # occasionally. + skip["skip"] = f"abort:{abort}" return { + **skip, # `sidecar_shortcode` is set only for carousels, so it is the # carousel discriminator. First matching condition wins. "filename": { @@ -666,6 +685,12 @@ def main() -> int: ap.add_argument("--max-sources", type=int, default=0, metavar="N", help="hard ceiling on sources touched in one run " "(0 = no limit)") + ap.add_argument("--abort", type=int, default=0, metavar="N", + help="stop enumerating posts/reels after N consecutive " + "already-archived FILES (0 = walk everything, the " + "default). 50 is a safe routine value; it cuts the " + "per-run listing cost by roughly 85%%, at the price " + "of no longer noticing edited carousels") ap.add_argument("--probe-ttl", type=float, default=24.0, metavar="HOURS", help="reuse cached listing results younger than this") ap.add_argument("--force", action="store_true", @@ -704,7 +729,8 @@ def main() -> int: else: selected = [Profile(p) for p in sorted(names)] - config = build_config(args.rate, list(args.sleep_request), list(args.sleep)) + config = build_config(args.rate, list(args.sleep_request), + list(args.sleep), args.abort) args.staging.mkdir(parents=True, exist_ok=True) # Deliberately a SIBLING of the staging directory, not inside it: staging is # rsynced wholesale into the archive, and a dry run caught this file being