ergosteurandClaude Opus 5
7b63ba6a76
Every claim in docs/gallery-dl.md was measured against the live site and
the archive rather than taken from documentation, because two of the
assumptions turned out to be wrong.
The safety model is the reason the config looks the way it does.
gallery-dl has two API backends: the graphql one issues a request PER
POST for every video and carousel -- the pattern that got this account
banned via Instaloader -- while the default rest one paginates listings
at 30-50 items and carries carousel_media, video_versions and
product_type inline. A 300-post profile costs ~10 requests.
Findings worth recording:
- JD2 stamped filenames in desktop LOCAL time (US Eastern), not UTC.
Across 212 comparable posts: UTC 19 mismatches, UTC-5 10, UTC-4 zero.
{date:Olocal/%Y-%m-%d} reproduces it; the trailing separator must be
omitted or it lands in the strftime format.
- A profile's reels tab returns collab reels owned by OTHER accounts, so
the directory must be forced with -D. JD2 did the same: chuuo3o and
official_artms filenames sit inside "0ct0ber19 - reels".
- Stories and highlights need per-item {shortcode}; {post_shortcode} is
the reel's id and is shared by every item. {date} is per-item, verified
on a 154-item highlight with distinct times.
- gallery-dl reproduces JD2's caption .txt exactly, including writing
nothing for an empty caption and omitting the trailing newline.
- The json sidecar needs `include`, not `fields`; `fields` silently does
nothing in mode:json and leaks audio_user blobs. It yields `type`
(post/reel) -- Instagram's own flag, which can retire the lone-video
heuristic once the scanner reads it.
Naming differences between the two tools are cosmetic: EXPORT_RE already
makes the index optional and parseInt normalises zero-padding, so a mixed
archive parses identically. Tests pin that down.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
334 lines
13 KiB
Python
Executable File
334 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Fetch Instagram profiles into the archive layout using gallery-dl.
|
|
|
|
The CLI replacement for the JDownloader2 workflow. See docs/gallery-dl.md for
|
|
the measurements behind every choice here — especially the safety model, which
|
|
is the reason this script exists in this shape rather than a simpler one.
|
|
|
|
STATUS: skeleton. The config generation and planning are complete and tested;
|
|
`--execute` is deliberately gated behind an explicit flag and has not been run
|
|
against the live archive.
|
|
|
|
Usage:
|
|
./scripts/gdl-sync.py --archives <dir> --profile 0ct0ber19 --dry-run
|
|
./scripts/gdl-sync.py --archives <dir> --all --dry-run
|
|
./scripts/gdl-sync.py --archives <dir> --profile 0ct0ber19 --execute
|
|
|
|
Run it from the host whose public IP matches the browser the cookie came from.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Archive layout
|
|
# --------------------------------------------------------------------------
|
|
|
|
# Mirrors src/lib/archive-grouping.ts. Instagram usernames cannot contain
|
|
# spaces, which is what makes the username separable from a highlight title.
|
|
RE_HIGHLIGHT = re.compile(r"^story highlights - ([^ ]+) - (.+)$")
|
|
RE_STORIES = re.compile(r"^story - ([^ ]+)$")
|
|
RE_REELS = re.compile(r"^([^ ]+) - reels$")
|
|
|
|
DATE_FMT = "{date:Olocal/%Y-%m-%d}"
|
|
"""Local-time date. JD2 stamped US Eastern, NOT UTC (0/212 mismatches vs 19 for
|
|
UTC). `Olocal` is DST-aware per timestamp. The trailing separator must be
|
|
omitted or it lands in the strftime format and sanitises to an underscore."""
|
|
|
|
POST_STEM = DATE_FMT + "_{username} - {post_shortcode}"
|
|
ITEM_STEM = DATE_FMT + "_{username} - {shortcode}"
|
|
|
|
|
|
@dataclass
|
|
class Source:
|
|
"""One gallery-dl invocation: a URL fetched into a specific directory."""
|
|
|
|
kind: str # posts | reels | stories | highlights
|
|
url: str
|
|
directory: str # relative to the archives root
|
|
subcategory: str # gallery-dl config key
|
|
title: str | None = None # highlight title, when known
|
|
|
|
|
|
@dataclass
|
|
class Profile:
|
|
user: str
|
|
existing: dict[str, str] = field(default_factory=dict) # kind -> dirname
|
|
|
|
def sources(self, include_stories: bool) -> list[Source]:
|
|
u = self.user
|
|
base = f"https://www.instagram.com/{u}"
|
|
out = [
|
|
Source("posts", f"{base}/posts/", u, "posts"),
|
|
Source("reels", f"{base}/reels/", f"{u} - reels", "reels"),
|
|
]
|
|
if include_stories:
|
|
# Stories expire after 24h, so these can only ever be captured
|
|
# live. There is no backfill and no re-fetch.
|
|
out.append(Source(
|
|
"stories", f"https://www.instagram.com/stories/{u}/",
|
|
f"story - {u}", "stories"))
|
|
# Highlight directories embed the title, which gallery-dl only
|
|
# learns mid-extraction -- so this one source fans out into many
|
|
# directories and is handled with a directory format string.
|
|
out.append(Source(
|
|
"highlights", f"{base}/highlights", "", "highlights"))
|
|
return out
|
|
|
|
|
|
def scan_archives(root: Path) -> dict[str, Profile]:
|
|
"""Group existing directories into profiles, as the server does."""
|
|
profiles: dict[str, Profile] = {}
|
|
|
|
def get(user: str) -> Profile:
|
|
return profiles.setdefault(user, Profile(user))
|
|
|
|
for entry in sorted(os.listdir(root)):
|
|
if not (root / entry).is_dir() or entry.startswith("."):
|
|
continue
|
|
if m := RE_HIGHLIGHT.match(entry):
|
|
get(m.group(1)).existing.setdefault("highlights", entry)
|
|
elif m := RE_STORIES.match(entry):
|
|
get(m.group(1)).existing["stories"] = entry
|
|
elif m := RE_REELS.match(entry):
|
|
get(m.group(1)).existing["reels"] = entry
|
|
else:
|
|
get(entry).existing["posts"] = entry
|
|
return profiles
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# gallery-dl configuration
|
|
# --------------------------------------------------------------------------
|
|
|
|
def build_config(rate: str, sleep_request: list[float],
|
|
sleep: list[float]) -> dict:
|
|
"""
|
|
The config is generated rather than checked in so the safety-critical
|
|
options cannot drift out of sync with the docs.
|
|
|
|
`api: rest` is the single most important line in this file. The graphql
|
|
backend issues one request PER POST for every video and carousel, which is
|
|
the pattern that got this account banned once already.
|
|
"""
|
|
caption_pp = {
|
|
"name": "metadata",
|
|
"event": "post",
|
|
"mode": "custom",
|
|
"content-format": "{description}",
|
|
"extension": "txt",
|
|
# JD2 wrote no .txt when the caption was empty; "empty": false (the
|
|
# default) reproduces that.
|
|
}
|
|
meta_pp = {
|
|
"name": "metadata",
|
|
"event": "post",
|
|
"mode": "json",
|
|
# `include`, NOT `fields` -- `fields` applies to mode:custom and
|
|
# silently does nothing here, dumping audio_user blobs that contain
|
|
# unrelated users' profile picture URLs.
|
|
"include": [
|
|
"post_shortcode", "post_id", "type", "date", "post_date",
|
|
"username", "fullname", "owner_id", "description", "count",
|
|
"likes", "post_url", "sidecar_shortcode",
|
|
],
|
|
}
|
|
|
|
def post_like(stem: str) -> dict:
|
|
"""Naming for surfaces whose unit is a post (posts, reels)."""
|
|
return {
|
|
# `sidecar_shortcode` is set only for carousels, so it is the
|
|
# carousel discriminator. First matching condition wins.
|
|
"filename": {
|
|
"sidecar_shortcode and count >= 10":
|
|
stem + " - {num:02}.{extension}",
|
|
"sidecar_shortcode":
|
|
stem + " - {num}.{extension}",
|
|
"":
|
|
stem + ".{extension}",
|
|
},
|
|
"postprocessors": [
|
|
{**caption_pp, "filename": stem + ".txt"},
|
|
{**meta_pp, "filename": stem + ".json"},
|
|
],
|
|
}
|
|
|
|
def item_like(stem: str) -> dict:
|
|
"""
|
|
Naming for surfaces whose unit is an item inside a reel (stories,
|
|
highlights). `{shortcode}` is per item; `{post_shortcode}` is the
|
|
reel's id and is shared by every item in it.
|
|
"""
|
|
return {
|
|
"filename": stem + ".{extension}",
|
|
"postprocessors": [{**meta_pp, "filename": stem + ".json"}],
|
|
}
|
|
|
|
return {
|
|
"extractor": {
|
|
"base-directory": ".",
|
|
"instagram": {
|
|
"api": "rest", # never "graphql" -- see docstring
|
|
"sleep-request": sleep_request,
|
|
"sleep": sleep,
|
|
"videos": True,
|
|
"include": "", # never "all"; sources are explicit
|
|
# Directory is forced per-invocation with -D, because a reels
|
|
# tab returns collab reels owned by OTHER accounts and
|
|
# {username} would scatter them into the wrong profile.
|
|
"directory": [],
|
|
"posts": post_like(POST_STEM),
|
|
"reels": post_like(POST_STEM),
|
|
"stories": item_like(ITEM_STEM),
|
|
"highlights": {
|
|
**item_like(ITEM_STEM),
|
|
# The only surface that must derive its own directory,
|
|
# since the title is not known until extraction.
|
|
"directory": ["story highlights - {username} - {highlight_title}"],
|
|
},
|
|
},
|
|
},
|
|
"downloader": {"http": {"rate": rate}},
|
|
"output": {"mode": "null"},
|
|
}
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Planning and execution
|
|
# --------------------------------------------------------------------------
|
|
|
|
def gdl_command(src: Source, root: Path, config: Path, cookies: str,
|
|
archive_db: Path | None) -> list[str]:
|
|
cmd = [
|
|
"gallery-dl",
|
|
"--config", str(config),
|
|
"--cookies-from-browser", cookies,
|
|
]
|
|
if archive_db:
|
|
# gallery-dl's skip-archive. Seed it before the first real run or the
|
|
# whole 110k-file tree is re-downloaded; see --seed-archive.
|
|
cmd += ["--download-archive", str(archive_db)]
|
|
if src.subcategory != "highlights":
|
|
cmd += ["--destination", str(root / src.directory)]
|
|
else:
|
|
cmd += ["--destination", str(root)]
|
|
cmd.append(src.url)
|
|
return cmd
|
|
|
|
|
|
def seed_archive_db(root: Path, profiles: dict[str, Profile],
|
|
db: Path) -> int:
|
|
"""
|
|
TODO: populate the skip-archive from filenames already on disk.
|
|
|
|
gallery-dl keys its archive on an extractor-specific id string, so this
|
|
needs the exact format `InstagramExtractor.archive_fmt` produces. Until
|
|
it is implemented, the first run of any profile re-downloads everything —
|
|
bandwidth on the CDN (the tolerant surface), but hours of it.
|
|
"""
|
|
raise NotImplementedError(
|
|
"seed_archive_db is unimplemented; run without --archive-db and accept "
|
|
"a full re-download, or implement this first")
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("--archives", required=True, type=Path,
|
|
help="archive root (one directory per profile plus sidecars)")
|
|
g = ap.add_mutually_exclusive_group(required=True)
|
|
g.add_argument("--profile", action="append", default=[],
|
|
help="profile to sync; repeatable")
|
|
g.add_argument("--all", action="store_true", help="every profile on disk")
|
|
ap.add_argument("--cookies", default="chrome:/home/matt/.config/google-chrome-devtools",
|
|
help="gallery-dl --cookies-from-browser value")
|
|
ap.add_argument("--archive-db", type=Path, default=None,
|
|
help="gallery-dl skip-archive sqlite path")
|
|
ap.add_argument("--rate", default="3M", help="per-download rate cap")
|
|
ap.add_argument("--sleep-request", nargs=2, type=float, default=[4.0, 7.0],
|
|
metavar=("MIN", "MAX"))
|
|
ap.add_argument("--sleep", nargs=2, type=float, default=[1.0, 3.0],
|
|
metavar=("MIN", "MAX"))
|
|
ap.add_argument("--no-stories", action="store_true",
|
|
help="skip stories and highlights (posts and reels only)")
|
|
mode = ap.add_mutually_exclusive_group()
|
|
mode.add_argument("--dry-run", action="store_true", default=True,
|
|
help="print the plan and the config; default")
|
|
mode.add_argument("--execute", action="store_true",
|
|
help="actually run gallery-dl")
|
|
args = ap.parse_args()
|
|
|
|
if not args.archives.is_dir():
|
|
print(f"archives root not found: {args.archives}", file=sys.stderr)
|
|
return 2
|
|
if not shutil.which("gallery-dl"):
|
|
print("gallery-dl not on PATH", file=sys.stderr)
|
|
return 2
|
|
|
|
profiles = scan_archives(args.archives)
|
|
if args.profile:
|
|
missing = [p for p in args.profile if p not in profiles]
|
|
for p in missing:
|
|
print(f"note: {p} has no directory yet; it will be created")
|
|
profiles.setdefault(p, Profile(p))
|
|
selected = [profiles[p] for p in args.profile]
|
|
else:
|
|
selected = list(profiles.values())
|
|
|
|
config = build_config(args.rate, list(args.sleep_request), list(args.sleep))
|
|
config_path = args.archives / ".gdl-sync.config.json"
|
|
|
|
plan: list[tuple[Profile, Source]] = [
|
|
(prof, src)
|
|
for prof in selected
|
|
for src in prof.sources(include_stories=not args.no_stories)
|
|
]
|
|
|
|
print(f"profiles : {len(selected)}")
|
|
print(f"sources : {len(plan)}")
|
|
print(f"pacing : {args.sleep_request[0]}-{args.sleep_request[1]}s between "
|
|
f"requests, rate cap {args.rate}")
|
|
print()
|
|
|
|
if not args.execute:
|
|
print(json.dumps(config, indent=2))
|
|
print()
|
|
for prof, src in plan:
|
|
dest = src.directory or "(per-highlight)"
|
|
print(f" {prof.user:<20} {src.kind:<11} -> {dest}")
|
|
print("\ndry run; nothing fetched. pass --execute to run.")
|
|
return 0
|
|
|
|
config_path.write_text(json.dumps(config, indent=2))
|
|
failures = 0
|
|
for prof, src in plan:
|
|
if src.subcategory != "highlights":
|
|
(args.archives / src.directory).mkdir(parents=True, exist_ok=True)
|
|
cmd = gdl_command(src, args.archives, config_path, args.cookies,
|
|
args.archive_db)
|
|
print(f"==> {prof.user} / {src.kind}")
|
|
result = subprocess.run(cmd)
|
|
if result.returncode != 0:
|
|
failures += 1
|
|
# Keep going: one private/renamed profile must not abort the run.
|
|
print(f" FAILED (exit {result.returncode})", file=sys.stderr)
|
|
|
|
# A sync NEVER deletes. The archive deliberately outlives Instagram --
|
|
# 2 posts in 0ct0ber19 exist only here now.
|
|
print(f"\ndone; {failures} source(s) failed")
|
|
return 1 if failures else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|