ergosteur/instaarchive-viewer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: give the sync a memory, so it stops paying for the same listing twice

Browse Source 
Nothing in this tool had any memory: every invocation started from zero
and would happily re-enumerate a profile it had listed minutes earlier.
That is what suspended the account -- the listing passes, not the
downloads -- and an aborted run re-enumerating five profiles on restart
was a large part of the bill.

Three changes, in order of how much they save:

- Seeding is now a one-time bootstrap per source. After the first
  successful sync the archive DB records everything gallery-dl has seen,
  so the source is never probed again. A second full sync costs roughly
  half what the first did.
- Stories never seed at all. A story cannot be in the archive before it
  is fetched, so there is nothing to seed from, and probing would double
  the cost of the cheapest surface we have.
- A source fetched within --min-interval (20h) is refused, and listing
  results are cached for --probe-ttl (24h), so a restart mid-run is free
  rather than a repeat. --force overrides both.

--only replaces --no-stories and takes any subset of the surfaces, which
is what makes a daily stories-only run possible: one source per profile,
no seeding, a handful of requests. Everything else stays monthly.

Tested with stdlib unittest -- no new dependencies, and it runs anywhere
the sync does. The cases include the aborted-restart scenario, which now
plans zero work.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
ergosteur
2026-08-17 13:53:18 -04:00
co-authored by Claude Opus 5
parent 7a085d2272
commit 4ce5cf048e
5 changed files with 476 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/gdl-sync.py
+203 -26
View File
@@ -28,6 +28,7 @@ using the cookie from elsewhere is what session-hijack detection looks for.
from __future__ import annotations
import argparse
import datetime as dt
import json
import os
import re
@@ -72,25 +73,23 @@ class Profile:
user: str
existing: dict[str, str] = field(default_factory=dict) # kind -> dirname
def sources(self, include_stories: bool) -> list[Source]:
def sources(self, kinds: set[str]) -> list[Source]:
u = self.user
base = f"https://www.instagram.com/{u}"
out = [
all_sources = [
Source("posts", f"{base}/posts/", u, "posts"),
Source("reels", f"{base}/reels/", f"{u} - reels", "reels"),
]
if include_stories:
# Stories expire after 24h, so these can only ever be captured
# live. There is no backfill and no re-fetch.
out.append(Source(
"stories", f"https://www.instagram.com/stories/{u}/",
f"story - {u}", "stories"))
# live. There is no backfill and no re-fetch -- which is why they
# are the one surface worth visiting daily.
Source("stories", f"https://www.instagram.com/stories/{u}/",
f"story - {u}", "stories"),
# Highlight directories embed the title, which gallery-dl only
# learns mid-extraction -- so this one source fans out into many
# directories and is handled with a directory format string.
out.append(Source(
"highlights", f"{base}/highlights", "", "highlights"))
return out
Source("highlights", f"{base}/highlights", "", "highlights"),
]
return [s for s in all_sources if s.kind in kinds]
def scan_archives(root: Path) -> dict[str, Profile]:
@@ -456,6 +455,133 @@ def probe_live(src: Source, config: Path, cookies: str) -> list[dict]:
return items
ALL_KINDS = ("posts", "reels", "stories", "highlights")
# Stories cannot be backfilled and expire in 24h, so a run that only wants
# stories is both cheap and the one worth scheduling daily.
STORIES_ONLY = {"stories"}
class SyncState:
"""
What has already been spent against `instagram.com`.
Exists because nothing else in this tool has any memory: every invocation
used to start from zero and happily re-enumerate profiles it had listed
minutes earlier. That is what suspended the account — the listing passes,
not the downloads.
Two facts are tracked per source:
seeded the skip-archive has been primed from the archive listing.
This is a ONE-TIME bootstrap: afterwards the archive DB records
every item gallery-dl has seen, so the source never needs
probing again. This is the single biggest request saving here.
fetched when it was last downloaded, so a re-run soon after is refused
rather than silently repeating the whole pass.
"""
VERSION = 1
def __init__(self, path: Path):
self.path = path
self.data = {"version": self.VERSION, "sources": {}}
if path.is_file():
try:
loaded = json.loads(path.read_text())
if loaded.get("version") == self.VERSION:
self.data = loaded
except Exception:
pass # a corrupt state file must never block a sync
def _entry(self, url: str) -> dict:
return self.data.setdefault("sources", {}).setdefault(url, {})
def needs_seed(self, url: str) -> bool:
return not self._entry(url).get("seeded")
def mark_seeded(self, url: str, stamp: str) -> None:
self._entry(url)["seeded"] = stamp
def last_fetch(self, url: str) -> str | None:
return self._entry(url).get("fetched")
def mark_fetched(self, url: str, stamp: str) -> None:
self._entry(url)["fetched"] = stamp
def save(self) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
self.path.write_text(json.dumps(self.data, indent=1, sort_keys=True))
def hours_since(stamp: str | None, now: float) -> float:
"""Hours between an ISO stamp and `now`; infinite when never."""
if not stamp:
return float("inf")
try:
then = dt.datetime.fromisoformat(stamp)
except ValueError:
return float("inf")
if then.tzinfo is None:
then = then.replace(tzinfo=dt.timezone.utc)
return (now - then.timestamp()) / 3600.0
def plan_source(src: Source, state: SyncState, now: float,
min_interval: float) -> tuple[bool, bool, str]:
"""
Decide what a source needs: (fetch, seed, reason).
Seeding is skipped once done, and skipped entirely for stories — a story
cannot exist in the archive before it is fetched, so there is nothing to
seed from, and probing would double the request cost of the cheapest
surface we have.
"""
since = hours_since(state.last_fetch(src.url), now)
if since < min_interval:
return (False, False, f"fetched {since:.1f}h ago, under the "
f"{min_interval:g}h floor")
if src.kind == "stories":
return (True, False, "stories: no seed needed")
if state.needs_seed(src.url):
return (True, True, "first run: seeding from the archive listing")
return (True, False, "already seeded; the skip-archive knows what we hold")
class ProbeCache:
"""
Listing-pass results, kept so an interrupted run does not pay for them
twice. Yesterday an aborted sync re-enumerated five profiles on restart.
"""
def __init__(self, path: Path, ttl_hours: float):
self.path = path
self.ttl = ttl_hours
self.data: dict = {}
if path.is_file():
try:
self.data = json.loads(path.read_text())
except Exception:
self.data = {}
def get(self, url: str, now: float) -> list[dict] | None:
entry = self.data.get(url)
if not entry or hours_since(entry.get("at"), now) > self.ttl:
return None
return entry.get("items")
def put(self, url: str, items: list[dict], stamp: str) -> None:
# Only the fields seeding needs, so the cache stays small.
self.data[url] = {"at": stamp, "items": [
{k: i.get(k) for k in ("shortcode", "post_shortcode", "num", "media_id")}
for i in items
]}
def save(self) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
self.path.write_text(json.dumps(self.data))
def rsync_command(staging: Path, dest: str, dry_run: bool) -> list[str]:
"""
Publish a staging tree into the archive.
@@ -529,8 +655,21 @@ def main() -> int:
metavar=("MIN", "MAX"))
ap.add_argument("--sleep", nargs=2, type=float, default=[3.0, 6.0],
metavar=("MIN", "MAX"))
ap.add_argument("--no-stories", action="store_true",
help="skip stories and highlights (posts and reels only)")
ap.add_argument("--only", default=",".join(ALL_KINDS),
help="comma-separated surfaces to sync: "
"posts,reels,stories,highlights. Use --only stories "
"for the cheap daily run.")
ap.add_argument("--min-interval", type=float, default=20.0, metavar="HOURS",
help="refuse to re-fetch a source touched more recently "
"than this (default 20h); the guard that makes a "
"restart cheap instead of a repeat")
ap.add_argument("--max-sources", type=int, default=0, metavar="N",
help="hard ceiling on sources touched in one run "
"(0 = no limit)")
ap.add_argument("--probe-ttl", type=float, default=24.0, metavar="HOURS",
help="reuse cached listing results younger than this")
ap.add_argument("--force", action="store_true",
help="ignore --min-interval and the probe cache")
mode = ap.add_mutually_exclusive_group()
mode.add_argument("--dry-run", action="store_true", default=True,
help="print the plan and the config; default")
@@ -572,14 +711,36 @@ def main() -> int:
# published to the archive root.
config_path = args.staging.parent / f"{args.staging.name}.gdl-config.json"
plan: list[tuple[Profile, Source]] = [
(prof, src)
for prof in selected
for src in prof.sources(include_stories=not args.no_stories)
]
kinds = {k.strip() for k in args.only.split(",") if k.strip()}
unknown = kinds - set(ALL_KINDS)
if unknown:
print(f"unknown surface(s): {', '.join(sorted(unknown))}", file=sys.stderr)
return 2
state_path = (args.archive_db.with_suffix(".state.json") if args.archive_db
else args.staging.parent / f"{args.staging.name}.state.json")
state = SyncState(state_path)
now = dt.datetime.now(dt.timezone.utc)
now_ts, stamp = now.timestamp(), now.isoformat()
min_interval = 0.0 if args.force else args.min_interval
plan: list[tuple[Profile, Source, bool]] = []
skipped = 0
for prof in selected:
for src in prof.sources(kinds):
fetch, seed, reason = plan_source(src, state, now_ts, min_interval)
if not fetch:
skipped += 1
print(f" skip {prof.user}/{src.kind}: {reason}")
continue
if args.max_sources and len(plan) >= args.max_sources:
skipped += 1
continue
plan.append((prof, src, seed))
print(f"profiles : {len(selected)}")
print(f"sources : {len(plan)}")
print(f"surfaces : {','.join(k for k in ALL_KINDS if k in kinds)}")
print(f"sources : {len(plan)} to sync, {skipped} skipped")
print(f"pacing : {args.sleep_request[0]}-{args.sleep_request[1]}s between "
f"requests, rate cap {args.rate}")
print(f"staging : {args.staging}")
@@ -587,32 +748,44 @@ def main() -> int:
print()
if not args.execute:
for prof, src in plan:
for prof, src, seed in plan:
dest = src.directory or "(per-highlight)"
print(f" {prof.user:<20} {src.kind:<11} -> {dest}")
note = " [will seed]" if seed else ""
print(f" {prof.user:<20} {src.kind:<11} -> {dest}{note}")
print()
print(" " + " ".join(rsync_command(args.staging, args.publish, True)))
print("\ndry run; nothing fetched. pass --execute to run.")
return 0
config_path.write_text(json.dumps(config, indent=2))
probes = ProbeCache(state_path.with_suffix(".probes.json"),
0.0 if args.force else args.probe_ttl)
failures = 0
for prof, src in plan:
for prof, src, seed in plan:
print(f"==> {prof.user} / {src.kind}")
stage_dir = args.staging / (src.directory or ".")
stage_dir.mkdir(parents=True, exist_ok=True)
# Seed the skip-archive from what the archive already holds, so
# Prime the skip-archive from what the archive already holds, so
# fetching into an empty staging directory pulls only what is missing.
# The listing pass this needs is one we have to make anyway.
if args.archive_db:
# Done once per source, ever: afterwards the archive DB records
# everything gallery-dl has seen and no listing pass is needed.
if seed and args.archive_db:
try:
live = probe_live(src, config_path, args.cookies)
live = probes.get(src.url, now_ts)
if live is None:
live = probe_live(src, config_path, args.cookies)
probes.put(src.url, live, stamp)
probes.save()
else:
print(f" reusing {len(live)} cached listing items")
held = index_existing(index.listing(prof.user))
seeded = seed_archive_db(args.archive_db, held, live,
src.subcategory)
print(f" seeded {seeded} of {len(live)} live items")
state.mark_seeded(src.url, stamp)
state.save()
except subprocess.CalledProcessError as exc:
failures += 1
print(f" probe FAILED: {exc}", file=sys.stderr)
@@ -625,6 +798,10 @@ def main() -> int:
failures += 1
# Keep going: one private or renamed profile must not abort the run.
print(f" FAILED (exit {result.returncode})", file=sys.stderr)
else:
# Recorded even for an empty fetch: the request was still spent.
state.mark_fetched(src.url, stamp)
state.save()
# Publish once, at the end, so a partially-fetched profile never reaches
# the archive mid-run. Only ever adds -- see rsync_command.