feat: keep the archive-fetching tooling on a branch of its own
The scripts and docs for fetching from Instagram now live here rather than on main, which is the branch published to GitHub. They carry things that do not belong in a public repo: the fetch host's public IP, the browser profile path the cookie is read from, the NAS archive path, and the list of accounts being archived. This branch is a superset of main — the viewer plus the tooling — so it can take main's changes by merging, and the npm script and CLAUDE.md entries that reference the tooling live here where the files actually exist. Restored with the sync work from the 2026-08-20 run already in place: the --abort flag, the corrected yt-dlp install advice, and the measurements behind both. Note that main's history was rewritten to strip these paths, so the tooling's own per-file history does not exist on this branch. It is preserved on gitea as pre-rewrite-20260820 and pre-rewrite-tooling-20260820. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UXfdJu7QhSJLr47K7koTDF
This commit is contained in:
Claude Opus 5
Executable
+843
@@ -0,0 +1,843 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fetch Instagram profiles into the archive layout using gallery-dl.
|
||||
|
||||
The CLI replacement for the JDownloader2 workflow. See docs/gallery-dl.md for
|
||||
the measurements behind every choice here — especially the safety model, which
|
||||
is the reason this script exists in this shape rather than a simpler one.
|
||||
|
||||
The fetch host needs no copy of the archive. It stages locally and rsyncs
|
||||
afterwards; what it already holds is learned from a *file listing* alone
|
||||
(`--index`), which the viewer's own API serves.
|
||||
|
||||
Usage:
|
||||
./scripts/gdl-sync.py --index https://instaarchive.ergosteur.com \\
|
||||
--staging /var/tmp/gdl --publish user@host:/path/to/archives \\
|
||||
--urls-file artms_account_links.txt --dry-run
|
||||
|
||||
# ...then swap --dry-run for --execute. --index also accepts a local path,
|
||||
# and --profile / --all work instead of --urls-file.
|
||||
|
||||
Always --dry-run first: it prints the plan, and the publish step it reports is
|
||||
the one that would touch the archive.
|
||||
|
||||
Run it from the host whose public IP matches the browser the cookie came from;
|
||||
using the cookie from elsewhere is what session-hijack detection looks for.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Archive layout
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
# Mirrors src/lib/archive-grouping.ts. Instagram usernames cannot contain
|
||||
# spaces, which is what makes the username separable from a highlight title.
|
||||
RE_HIGHLIGHT = re.compile(r"^story highlights - ([^ ]+) - (.+)$")
|
||||
RE_STORIES = re.compile(r"^story - ([^ ]+)$")
|
||||
RE_REELS = re.compile(r"^([^ ]+) - reels$")
|
||||
|
||||
DATE_FMT = "{date:Olocal/%Y-%m-%d}"
|
||||
"""Local-time date. JD2 stamped US Eastern, NOT UTC (0/212 mismatches vs 19 for
|
||||
UTC). `Olocal` is DST-aware per timestamp. The trailing separator must be
|
||||
omitted or it lands in the strftime format and sanitises to an underscore."""
|
||||
|
||||
POST_STEM = DATE_FMT + "_{username} - {post_shortcode}"
|
||||
ITEM_STEM = DATE_FMT + "_{username} - {shortcode}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Source:
|
||||
"""One gallery-dl invocation: a URL fetched into a specific directory."""
|
||||
|
||||
kind: str # posts | reels | stories | highlights
|
||||
url: str
|
||||
directory: str # relative to the archives root
|
||||
subcategory: str # gallery-dl config key
|
||||
title: str | None = None # highlight title, when known
|
||||
|
||||
|
||||
@dataclass
|
||||
class Profile:
|
||||
user: str
|
||||
existing: dict[str, str] = field(default_factory=dict) # kind -> dirname
|
||||
|
||||
def sources(self, kinds: set[str]) -> list[Source]:
|
||||
u = self.user
|
||||
base = f"https://www.instagram.com/{u}"
|
||||
all_sources = [
|
||||
Source("posts", f"{base}/posts/", u, "posts"),
|
||||
Source("reels", f"{base}/reels/", f"{u} - reels", "reels"),
|
||||
# Stories expire after 24h, so these can only ever be captured
|
||||
# live. There is no backfill and no re-fetch -- which is why they
|
||||
# are the one surface worth visiting daily.
|
||||
Source("stories", f"https://www.instagram.com/stories/{u}/",
|
||||
f"story - {u}", "stories"),
|
||||
# Highlight directories embed the title, which gallery-dl only
|
||||
# learns mid-extraction -- so this one source fans out into many
|
||||
# directories and is handled with a directory format string.
|
||||
Source("highlights", f"{base}/highlights", "", "highlights"),
|
||||
]
|
||||
return [s for s in all_sources if s.kind in kinds]
|
||||
|
||||
|
||||
def scan_archives(root: Path) -> dict[str, Profile]:
|
||||
"""Group existing directories into profiles, as the server does."""
|
||||
profiles: dict[str, Profile] = {}
|
||||
|
||||
def get(user: str) -> Profile:
|
||||
return profiles.setdefault(user, Profile(user))
|
||||
|
||||
for entry in sorted(os.listdir(root)):
|
||||
if not (root / entry).is_dir() or entry.startswith("."):
|
||||
continue
|
||||
if m := RE_HIGHLIGHT.match(entry):
|
||||
get(m.group(1)).existing.setdefault("highlights", entry)
|
||||
elif m := RE_STORIES.match(entry):
|
||||
get(m.group(1)).existing["stories"] = entry
|
||||
elif m := RE_REELS.match(entry):
|
||||
get(m.group(1)).existing["reels"] = entry
|
||||
else:
|
||||
get(entry).existing["posts"] = entry
|
||||
return profiles
|
||||
|
||||
|
||||
RE_PROFILE_URL = re.compile(
|
||||
r"^(?:https?://)?(?:www\.)?instagram\.com/(?P<user>[^/?#\s]+)/?", re.I)
|
||||
|
||||
# Path segments that are Instagram features, not profiles. A line like
|
||||
# ".../p/ABC123/" names a post, and treating "p" as a username would silently
|
||||
# sync nothing under a nonsense directory.
|
||||
RESERVED_SEGMENTS = {
|
||||
"p", "reel", "reels", "stories", "explore", "accounts", "direct",
|
||||
"tv", "s", "invites", "challenge", "about", "developer",
|
||||
}
|
||||
|
||||
|
||||
def read_urls_file(path: Path) -> list[str]:
|
||||
"""
|
||||
Read profile URLs (or bare usernames) from a file, one per line.
|
||||
|
||||
Written for hand-maintained lists: blank lines are skipped, `#` starts a
|
||||
comment, and either a full URL or a bare username works. Order is kept and
|
||||
duplicates dropped, so a list can be appended to without care.
|
||||
"""
|
||||
users: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for lineno, raw in enumerate(path.read_text().splitlines(), 1):
|
||||
line = raw.split("#", 1)[0].strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
m = RE_PROFILE_URL.match(line)
|
||||
user = m.group("user") if m else line.strip("/")
|
||||
|
||||
if not user or "/" in user or " " in user:
|
||||
print(f"{path}:{lineno}: cannot read a username from {raw.strip()!r}",
|
||||
file=sys.stderr)
|
||||
continue
|
||||
if user.lower() in RESERVED_SEGMENTS:
|
||||
print(f"{path}:{lineno}: {user!r} is an Instagram path, not a "
|
||||
f"profile — skipping", file=sys.stderr)
|
||||
continue
|
||||
if user in seen:
|
||||
continue
|
||||
|
||||
seen.add(user)
|
||||
users.append(user)
|
||||
|
||||
return users
|
||||
|
||||
|
||||
class ArchiveIndex:
|
||||
"""
|
||||
What the archive already holds, as filenames only.
|
||||
|
||||
Deliberately never reads file *contents*, so the fetch host does not need a
|
||||
copy of the archive — it can stage locally and rsync afterwards. Backed
|
||||
either by a local directory or by the viewer's own API, which already
|
||||
serves exactly this listing and is the cheaper option when the archive
|
||||
lives on network storage (a full walk there took ~52s).
|
||||
"""
|
||||
|
||||
def __init__(self, source: str):
|
||||
self.remote = source.startswith(("http://", "https://"))
|
||||
self.source = source.rstrip("/") if self.remote else None
|
||||
self.root = None if self.remote else Path(source)
|
||||
if self.root and not self.root.is_dir():
|
||||
raise SystemExit(f"archive index not found: {source}")
|
||||
self._cache: dict[str, list[str]] = {}
|
||||
|
||||
def _get(self, path: str):
|
||||
from urllib.request import urlopen
|
||||
with urlopen(f"{self.source}{path}", timeout=60) as resp:
|
||||
return json.load(resp)
|
||||
|
||||
def profiles(self) -> set[str]:
|
||||
if self.remote:
|
||||
return {a["name"] for a in self._get("/api/archives")}
|
||||
return set(scan_archives(self.root))
|
||||
|
||||
def listing(self, user: str) -> list[str]:
|
||||
"""Every filename belonging to a profile, across all its sidecars."""
|
||||
if user in self._cache:
|
||||
return self._cache[user]
|
||||
|
||||
names: list[str] = []
|
||||
if self.remote:
|
||||
try:
|
||||
data = self._get(f"/api/archives/{user}/files")
|
||||
except Exception:
|
||||
data = []
|
||||
files = data if isinstance(data, list) else data.get("files", [])
|
||||
names = [f["path"] for f in files]
|
||||
else:
|
||||
prof = scan_archives(self.root).get(user)
|
||||
for dirname in (prof.existing.values() if prof else ()):
|
||||
d = self.root / dirname
|
||||
if d.is_dir():
|
||||
names += [f"{dirname}/{n}" for n in os.listdir(d)]
|
||||
|
||||
self._cache[user] = names
|
||||
return names
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# gallery-dl configuration
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def build_config(rate: str, sleep_request: list[float],
|
||||
sleep: list[float], abort: int = 0) -> dict:
|
||||
"""
|
||||
The config is generated rather than checked in so the safety-critical
|
||||
options cannot drift out of sync with the docs.
|
||||
|
||||
`api: rest` is the single most important line in this file. The graphql
|
||||
backend issues one request PER POST for every video and carousel, which is
|
||||
the pattern that got this account banned once already.
|
||||
"""
|
||||
caption_pp = {
|
||||
"name": "metadata",
|
||||
"event": "post",
|
||||
"mode": "custom",
|
||||
"content-format": "{description}",
|
||||
"extension": "txt",
|
||||
# JD2 wrote no .txt when the caption was empty; "empty": false (the
|
||||
# default) reproduces that.
|
||||
}
|
||||
meta_pp = {
|
||||
"name": "metadata",
|
||||
"event": "post",
|
||||
"mode": "json",
|
||||
# `include`, NOT `fields` -- `fields` applies to mode:custom and
|
||||
# silently does nothing here, dumping audio_user blobs that contain
|
||||
# unrelated users' profile picture URLs.
|
||||
"include": [
|
||||
"post_shortcode", "post_id", "type", "date", "post_date",
|
||||
"username", "fullname", "owner_id", "description", "count",
|
||||
"likes", "post_url", "sidecar_shortcode",
|
||||
],
|
||||
}
|
||||
|
||||
def post_like(stem: str) -> dict:
|
||||
"""Naming for surfaces whose unit is a post (posts, reels)."""
|
||||
skip: dict = {}
|
||||
if abort:
|
||||
# Stop enumerating once `abort` consecutive files are already in
|
||||
# the skip-archive. The listing pass -- not the downloading -- is
|
||||
# what costs `instagram.com` requests, and it otherwise walks the
|
||||
# whole profile every run to find three new posts.
|
||||
#
|
||||
# Safe here only because the REST listing is strictly
|
||||
# reverse-chronological: the web grid hoists pinned posts to the
|
||||
# front, but this endpoint does not (measured 2026-08-20), so old
|
||||
# posts never appear before new ones.
|
||||
#
|
||||
# Counted in FILES, not posts, so it must clear the largest
|
||||
# already-held carousel -- 22 media for one real post in this
|
||||
# archive. It also means edited carousels (test case 15) stop
|
||||
# being noticed, so a full sweep is still worth running
|
||||
# occasionally.
|
||||
skip["skip"] = f"abort:{abort}"
|
||||
return {
|
||||
**skip,
|
||||
# `sidecar_shortcode` is set only for carousels, so it is the
|
||||
# carousel discriminator. First matching condition wins.
|
||||
"filename": {
|
||||
"sidecar_shortcode and count >= 10":
|
||||
stem + " - {num:02}.{extension}",
|
||||
"sidecar_shortcode":
|
||||
stem + " - {num}.{extension}",
|
||||
"":
|
||||
stem + ".{extension}",
|
||||
},
|
||||
"postprocessors": [
|
||||
{**caption_pp, "filename": stem + ".txt"},
|
||||
{**meta_pp, "filename": stem + ".json"},
|
||||
],
|
||||
}
|
||||
|
||||
def item_like(stem: str) -> dict:
|
||||
"""
|
||||
Naming for surfaces whose unit is an item inside a reel (stories,
|
||||
highlights). `{shortcode}` is per item; `{post_shortcode}` is the
|
||||
reel's id and is shared by every item in it.
|
||||
|
||||
The media filename uses the per-item shortcode, but the sidecar cannot:
|
||||
it runs at `event: post`, where the kwdict describes the *reel* and has
|
||||
no `shortcode` at all -- which silently formatted as the literal
|
||||
"None", producing one "<date>_<user> - None.json" per reel. It is keyed
|
||||
by `post_shortcode` instead, and is genuinely reel-level data (the
|
||||
reel's own date and item count); per-item dates live in the media
|
||||
filenames, which is the more precise source anyway.
|
||||
"""
|
||||
return {
|
||||
"filename": stem + ".{extension}",
|
||||
"postprocessors": [
|
||||
{**meta_pp,
|
||||
"filename": DATE_FMT + "_{username} - {post_shortcode}.json"},
|
||||
],
|
||||
}
|
||||
|
||||
return {
|
||||
"extractor": {
|
||||
"base-directory": ".",
|
||||
"instagram": {
|
||||
"api": "rest", # never "graphql" -- see docstring
|
||||
"sleep-request": sleep_request,
|
||||
"sleep": sleep,
|
||||
# The CDN does rate-limit: a first run at 3M/1-3s drew
|
||||
# '429 Too Many Requests' from scontent-*.cdninstagram.com and
|
||||
# lost two videos. Back off hard rather than retry fast.
|
||||
"sleep-429": 120.0,
|
||||
"retries": 8,
|
||||
"videos": True,
|
||||
"include": "", # never "all"; sources are explicit
|
||||
# Directory is forced per-invocation with -D, because a reels
|
||||
# tab returns collab reels owned by OTHER accounts and
|
||||
# {username} would scatter them into the wrong profile.
|
||||
"directory": [],
|
||||
"posts": post_like(POST_STEM),
|
||||
"reels": post_like(POST_STEM),
|
||||
"stories": item_like(ITEM_STEM),
|
||||
"highlights": {
|
||||
**item_like(ITEM_STEM),
|
||||
# The only surface that must derive its own directory,
|
||||
# since the title is not known until extraction.
|
||||
"directory": ["story highlights - {username} - {highlight_title}"],
|
||||
},
|
||||
},
|
||||
},
|
||||
# `retries` here is the CDN-side counterpart to sleep-429 above.
|
||||
"downloader": {"http": {"rate": rate, "retries": 8}},
|
||||
"output": {"mode": "null"},
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Planning and execution
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def gdl_command(src: Source, staging: Path, config: Path, cookies: str,
|
||||
archive_db: Path | None) -> list[str]:
|
||||
cmd = [
|
||||
"gallery-dl",
|
||||
"--config", str(config),
|
||||
"--cookies-from-browser", cookies,
|
||||
]
|
||||
if archive_db:
|
||||
# Without a seeded skip-archive, staging is empty and every file is
|
||||
# re-downloaded; see seed_archive_db.
|
||||
cmd += ["--download-archive", str(archive_db)]
|
||||
# Forced destination -- never `{username}` -- because a reels tab returns
|
||||
# collab reels owned by other accounts, which would otherwise be filed
|
||||
# under the wrong profile. Highlights are the exception: their directory
|
||||
# embeds a title only known mid-extraction, so the config formats it.
|
||||
dest = staging if src.subcategory == "highlights" else staging / src.directory
|
||||
cmd += ["--destination", str(dest)]
|
||||
cmd.append(src.url)
|
||||
return cmd
|
||||
|
||||
|
||||
# gallery-dl keys its skip-archive on `archive_prefix + archive_fmt`, which for
|
||||
# this extractor is the literal "instagram" followed by the per-media numeric
|
||||
# pk (`instagram.py:25`, `job.py:713-719`). Verified against a real run: a
|
||||
# 3-image carousel produced 3 rows, one per item.
|
||||
ARCHIVE_KEY = "instagram{}".format
|
||||
ARCHIVE_SCHEMA = "CREATE TABLE IF NOT EXISTS archive (entry TEXT PRIMARY KEY)"
|
||||
|
||||
RE_ARCHIVED = re.compile(
|
||||
r"^(\d{4}-\d{2}-\d{2})_(.+?) - ([A-Za-z0-9_-]+?)(?: - (\d+))?\.(\w+)$")
|
||||
NON_MEDIA = {"txt", "json"}
|
||||
|
||||
|
||||
def index_existing(listing: list[str]) -> set[tuple[str, int]]:
|
||||
"""
|
||||
Reduce a flat list of filenames to the (shortcode, index) pairs already
|
||||
held. Only names matter — never the bytes — which is what lets the sync run
|
||||
on a host that has no copy of the archive.
|
||||
"""
|
||||
have: set[tuple[str, int]] = set()
|
||||
for name in listing:
|
||||
m = RE_ARCHIVED.match(name.rsplit("/", 1)[-1])
|
||||
if not m or m.group(5).lower() in NON_MEDIA:
|
||||
continue
|
||||
# An absent index means a single-media post, which is index 1 — the
|
||||
# same normalisation the viewer's EXPORT_RE applies.
|
||||
have.add((m.group(3), int(m.group(4) or 1)))
|
||||
return have
|
||||
|
||||
|
||||
def live_key(item: dict, kind: str) -> tuple[str, int]:
|
||||
"""
|
||||
The (shortcode, index) a live item *would* be filed under, mirroring the
|
||||
filename template exactly.
|
||||
|
||||
The two surfaces disagree about which shortcode identifies a file, and
|
||||
getting this wrong silently seeds almost nothing:
|
||||
|
||||
posts/reels filed under {post_shortcode} — for a carousel, each
|
||||
child item ALSO has its own `shortcode`, which is not
|
||||
what appears in the filename.
|
||||
stories/highlights filed under the per-item {shortcode}, because
|
||||
`post_shortcode` there is the containing reel's id and
|
||||
is shared by every item in it.
|
||||
"""
|
||||
if kind in ("stories", "highlights"):
|
||||
return (item.get("shortcode"), 1)
|
||||
return (item.get("post_shortcode"), item.get("num"))
|
||||
|
||||
|
||||
def seed_archive_db(db: Path, existing: set[tuple[str, int]],
|
||||
live: list[dict], kind: str) -> int:
|
||||
"""
|
||||
Mark everything already held as downloaded, so a fetch into an empty
|
||||
directory pulls only what is missing.
|
||||
|
||||
`live` is the metadata of one listing pass — the pass we have to make
|
||||
anyway — each entry carrying at least `media_id` plus the shortcode fields
|
||||
`live_key` needs. Seeding costs no additional Instagram requests, and needs
|
||||
only a *listing* of the archive, never its contents.
|
||||
"""
|
||||
import sqlite3
|
||||
|
||||
db.parent.mkdir(parents=True, exist_ok=True)
|
||||
con = sqlite3.connect(db)
|
||||
con.execute(ARCHIVE_SCHEMA)
|
||||
rows = [
|
||||
(ARCHIVE_KEY(item["media_id"]),)
|
||||
for item in live
|
||||
if live_key(item, kind) in existing
|
||||
]
|
||||
con.executemany("INSERT OR IGNORE INTO archive (entry) VALUES (?)", rows)
|
||||
con.commit()
|
||||
con.close()
|
||||
return len(rows)
|
||||
|
||||
|
||||
def probe_live(src: Source, config: Path, cookies: str) -> list[dict]:
|
||||
"""
|
||||
One metadata-only listing pass. `sleep` is forced to 0 because it otherwise
|
||||
applies per *file* even with no download — 2275 files at 1-3s each is over
|
||||
an hour for a single profile.
|
||||
"""
|
||||
out = subprocess.run(
|
||||
["gallery-dl", "-j", "--config", str(config),
|
||||
"--cookies-from-browser", cookies, "-o", "sleep=0", src.url],
|
||||
capture_output=True, text=True, check=True,
|
||||
)
|
||||
items: list[dict] = []
|
||||
|
||||
def walk(node):
|
||||
if isinstance(node, dict):
|
||||
if "media_id" in node and "shortcode" in node:
|
||||
items.append(node)
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for value in node:
|
||||
walk(value)
|
||||
|
||||
walk(json.loads(out.stdout))
|
||||
return items
|
||||
|
||||
|
||||
ALL_KINDS = ("posts", "reels", "stories", "highlights")
|
||||
|
||||
# Stories cannot be backfilled and expire in 24h, so a run that only wants
|
||||
# stories is both cheap and the one worth scheduling daily.
|
||||
STORIES_ONLY = {"stories"}
|
||||
|
||||
|
||||
class SyncState:
|
||||
"""
|
||||
What has already been spent against `instagram.com`.
|
||||
|
||||
Exists because nothing else in this tool has any memory: every invocation
|
||||
used to start from zero and happily re-enumerate profiles it had listed
|
||||
minutes earlier. That is what suspended the account — the listing passes,
|
||||
not the downloads.
|
||||
|
||||
Two facts are tracked per source:
|
||||
|
||||
seeded the skip-archive has been primed from the archive listing.
|
||||
This is a ONE-TIME bootstrap: afterwards the archive DB records
|
||||
every item gallery-dl has seen, so the source never needs
|
||||
probing again. This is the single biggest request saving here.
|
||||
fetched when it was last downloaded, so a re-run soon after is refused
|
||||
rather than silently repeating the whole pass.
|
||||
"""
|
||||
|
||||
VERSION = 1
|
||||
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
self.data = {"version": self.VERSION, "sources": {}}
|
||||
if path.is_file():
|
||||
try:
|
||||
loaded = json.loads(path.read_text())
|
||||
if loaded.get("version") == self.VERSION:
|
||||
self.data = loaded
|
||||
except Exception:
|
||||
pass # a corrupt state file must never block a sync
|
||||
|
||||
def _entry(self, url: str) -> dict:
|
||||
return self.data.setdefault("sources", {}).setdefault(url, {})
|
||||
|
||||
def needs_seed(self, url: str) -> bool:
|
||||
return not self._entry(url).get("seeded")
|
||||
|
||||
def mark_seeded(self, url: str, stamp: str) -> None:
|
||||
self._entry(url)["seeded"] = stamp
|
||||
|
||||
def last_fetch(self, url: str) -> str | None:
|
||||
return self._entry(url).get("fetched")
|
||||
|
||||
def mark_fetched(self, url: str, stamp: str) -> None:
|
||||
self._entry(url)["fetched"] = stamp
|
||||
|
||||
def save(self) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.path.write_text(json.dumps(self.data, indent=1, sort_keys=True))
|
||||
|
||||
|
||||
def hours_since(stamp: str | None, now: float) -> float:
|
||||
"""Hours between an ISO stamp and `now`; infinite when never."""
|
||||
if not stamp:
|
||||
return float("inf")
|
||||
try:
|
||||
then = dt.datetime.fromisoformat(stamp)
|
||||
except ValueError:
|
||||
return float("inf")
|
||||
if then.tzinfo is None:
|
||||
then = then.replace(tzinfo=dt.timezone.utc)
|
||||
return (now - then.timestamp()) / 3600.0
|
||||
|
||||
|
||||
def plan_source(src: Source, state: SyncState, now: float,
|
||||
min_interval: float) -> tuple[bool, bool, str]:
|
||||
"""
|
||||
Decide what a source needs: (fetch, seed, reason).
|
||||
|
||||
Seeding is skipped once done, and skipped entirely for stories — a story
|
||||
cannot exist in the archive before it is fetched, so there is nothing to
|
||||
seed from, and probing would double the request cost of the cheapest
|
||||
surface we have.
|
||||
"""
|
||||
since = hours_since(state.last_fetch(src.url), now)
|
||||
if since < min_interval:
|
||||
return (False, False, f"fetched {since:.1f}h ago, under the "
|
||||
f"{min_interval:g}h floor")
|
||||
if src.kind == "stories":
|
||||
return (True, False, "stories: no seed needed")
|
||||
if state.needs_seed(src.url):
|
||||
return (True, True, "first run: seeding from the archive listing")
|
||||
return (True, False, "already seeded; the skip-archive knows what we hold")
|
||||
|
||||
|
||||
class ProbeCache:
|
||||
"""
|
||||
Listing-pass results, kept so an interrupted run does not pay for them
|
||||
twice. Yesterday an aborted sync re-enumerated five profiles on restart.
|
||||
"""
|
||||
|
||||
def __init__(self, path: Path, ttl_hours: float):
|
||||
self.path = path
|
||||
self.ttl = ttl_hours
|
||||
self.data: dict = {}
|
||||
if path.is_file():
|
||||
try:
|
||||
self.data = json.loads(path.read_text())
|
||||
except Exception:
|
||||
self.data = {}
|
||||
|
||||
def get(self, url: str, now: float) -> list[dict] | None:
|
||||
entry = self.data.get(url)
|
||||
if not entry or hours_since(entry.get("at"), now) > self.ttl:
|
||||
return None
|
||||
return entry.get("items")
|
||||
|
||||
def put(self, url: str, items: list[dict], stamp: str) -> None:
|
||||
# Only the fields seeding needs, so the cache stays small.
|
||||
self.data[url] = {"at": stamp, "items": [
|
||||
{k: i.get(k) for k in ("shortcode", "post_shortcode", "num", "media_id")}
|
||||
for i in items
|
||||
]}
|
||||
|
||||
def save(self) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.path.write_text(json.dumps(self.data))
|
||||
|
||||
|
||||
def rsync_command(staging: Path, dest: str, dry_run: bool) -> list[str]:
|
||||
"""
|
||||
Publish a staging tree into the archive.
|
||||
|
||||
`--ignore-existing` is not an optimisation, it is the safety property: the
|
||||
archive deliberately outlives Instagram (posts exist here that Instagram no
|
||||
longer serves), so publishing must only ever *add*. No `--delete`, and
|
||||
nothing already present is overwritten — including sidecars, which get
|
||||
rewritten on every run and would otherwise churn the synced share.
|
||||
|
||||
`dest` may be a local path or any rsync destination (`user@host:/path`),
|
||||
because the archive usually is not writable from the fetch host.
|
||||
"""
|
||||
cmd = ["rsync", "-a", "--ignore-existing", "--partial", "--info=stats2",
|
||||
# Belt and braces: the config lives outside staging, but nothing
|
||||
# resembling tooling output should ever reach the archive. Archive
|
||||
# sidecars are always "<date>_<user> - <code>.json", so none of
|
||||
# these can match real content.
|
||||
"--exclude", "gdl-sync*.json",
|
||||
"--exclude", "*.gdl-config.json",
|
||||
"--exclude", ".gdl-*",
|
||||
"--exclude", "*.sqlite", "--exclude", "*.db"]
|
||||
if dry_run:
|
||||
cmd.append("--dry-run")
|
||||
# Trailing slash: copy the *contents* of staging into dest.
|
||||
cmd += [f"{staging}/", dest if dest.endswith("/") else dest + "/"]
|
||||
return cmd
|
||||
|
||||
|
||||
def publish(staging: Path, dest: str, dry_run: bool) -> int:
|
||||
if not any(staging.iterdir()):
|
||||
print(" nothing staged; skipping publish")
|
||||
return 0
|
||||
cmd = rsync_command(staging, dest, dry_run)
|
||||
print(" " + " ".join(cmd))
|
||||
return subprocess.run(cmd).returncode
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# A sync runs for hours and is normally watched through a redirected log,
|
||||
# where Python's block buffering would withhold progress until it happened
|
||||
# to flush -- and the gallery-dl subprocesses write to the same descriptor
|
||||
# unbuffered, so the log would also interleave out of order.
|
||||
sys.stdout.reconfigure(line_buffering=True)
|
||||
sys.stderr.reconfigure(line_buffering=True)
|
||||
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--index", required=True,
|
||||
help="existing archive listing: a local root, or the "
|
||||
"viewer's base URL (only a FILE LISTING is needed, "
|
||||
"never the contents)")
|
||||
ap.add_argument("--publish", required=True,
|
||||
help="rsync destination for fetched files; a local path or "
|
||||
"user@host:/path")
|
||||
ap.add_argument("--staging", type=Path, required=True,
|
||||
help="local scratch directory gallery-dl writes into")
|
||||
g = ap.add_mutually_exclusive_group(required=True)
|
||||
g.add_argument("--profile", action="append", default=[],
|
||||
help="profile to sync; repeatable")
|
||||
g.add_argument("--all", action="store_true", help="every profile on disk")
|
||||
g.add_argument("--urls-file", type=Path,
|
||||
help="file of Instagram profile URLs or usernames, one per "
|
||||
"line; # comments and blank lines allowed")
|
||||
ap.add_argument("--cookies", default="chrome:/home/matt/.config/google-chrome-devtools",
|
||||
help="gallery-dl --cookies-from-browser value")
|
||||
ap.add_argument("--archive-db", type=Path, default=None,
|
||||
help="gallery-dl skip-archive sqlite path")
|
||||
ap.add_argument("--rate", default="1M", help="per-download rate cap")
|
||||
ap.add_argument("--sleep-request", nargs=2, type=float, default=[6.0, 10.0],
|
||||
metavar=("MIN", "MAX"))
|
||||
ap.add_argument("--sleep", nargs=2, type=float, default=[3.0, 6.0],
|
||||
metavar=("MIN", "MAX"))
|
||||
ap.add_argument("--only", default=",".join(ALL_KINDS),
|
||||
help="comma-separated surfaces to sync: "
|
||||
"posts,reels,stories,highlights. Use --only stories "
|
||||
"for the cheap daily run.")
|
||||
ap.add_argument("--min-interval", type=float, default=20.0, metavar="HOURS",
|
||||
help="refuse to re-fetch a source touched more recently "
|
||||
"than this (default 20h); the guard that makes a "
|
||||
"restart cheap instead of a repeat")
|
||||
ap.add_argument("--max-sources", type=int, default=0, metavar="N",
|
||||
help="hard ceiling on sources touched in one run "
|
||||
"(0 = no limit)")
|
||||
ap.add_argument("--abort", type=int, default=0, metavar="N",
|
||||
help="stop enumerating posts/reels after N consecutive "
|
||||
"already-archived FILES (0 = walk everything, the "
|
||||
"default). 50 is a safe routine value; it cuts the "
|
||||
"per-run listing cost by roughly 85%%, at the price "
|
||||
"of no longer noticing edited carousels")
|
||||
ap.add_argument("--probe-ttl", type=float, default=24.0, metavar="HOURS",
|
||||
help="reuse cached listing results younger than this")
|
||||
ap.add_argument("--force", action="store_true",
|
||||
help="ignore --min-interval and the probe cache")
|
||||
mode = ap.add_mutually_exclusive_group()
|
||||
mode.add_argument("--dry-run", action="store_true", default=True,
|
||||
help="print the plan and the config; default")
|
||||
mode.add_argument("--execute", action="store_true",
|
||||
help="actually run gallery-dl")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not shutil.which("gallery-dl"):
|
||||
print("gallery-dl not on PATH", file=sys.stderr)
|
||||
return 2
|
||||
if not shutil.which("rsync"):
|
||||
print("rsync not on PATH", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
index = ArchiveIndex(args.index)
|
||||
names = index.profiles()
|
||||
if args.urls_file:
|
||||
if not args.urls_file.is_file():
|
||||
print(f"urls file not found: {args.urls_file}", file=sys.stderr)
|
||||
return 2
|
||||
wanted = read_urls_file(args.urls_file)
|
||||
if not wanted:
|
||||
print(f"no usable profiles in {args.urls_file}", file=sys.stderr)
|
||||
return 2
|
||||
print(f"read {len(wanted)} profile(s) from {args.urls_file}")
|
||||
selected = [Profile(p) for p in wanted]
|
||||
elif args.profile:
|
||||
for p in args.profile:
|
||||
if p not in names:
|
||||
print(f"note: {p} is not in the index yet; it will be created")
|
||||
selected = [Profile(p) for p in args.profile]
|
||||
else:
|
||||
selected = [Profile(p) for p in sorted(names)]
|
||||
|
||||
config = build_config(args.rate, list(args.sleep_request),
|
||||
list(args.sleep), args.abort)
|
||||
args.staging.mkdir(parents=True, exist_ok=True)
|
||||
# Deliberately a SIBLING of the staging directory, not inside it: staging is
|
||||
# rsynced wholesale into the archive, and a dry run caught this file being
|
||||
# published to the archive root.
|
||||
config_path = args.staging.parent / f"{args.staging.name}.gdl-config.json"
|
||||
|
||||
kinds = {k.strip() for k in args.only.split(",") if k.strip()}
|
||||
unknown = kinds - set(ALL_KINDS)
|
||||
if unknown:
|
||||
print(f"unknown surface(s): {', '.join(sorted(unknown))}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
state_path = (args.archive_db.with_suffix(".state.json") if args.archive_db
|
||||
else args.staging.parent / f"{args.staging.name}.state.json")
|
||||
state = SyncState(state_path)
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
now_ts, stamp = now.timestamp(), now.isoformat()
|
||||
min_interval = 0.0 if args.force else args.min_interval
|
||||
|
||||
plan: list[tuple[Profile, Source, bool]] = []
|
||||
skipped = 0
|
||||
for prof in selected:
|
||||
for src in prof.sources(kinds):
|
||||
fetch, seed, reason = plan_source(src, state, now_ts, min_interval)
|
||||
if not fetch:
|
||||
skipped += 1
|
||||
print(f" skip {prof.user}/{src.kind}: {reason}")
|
||||
continue
|
||||
if args.max_sources and len(plan) >= args.max_sources:
|
||||
skipped += 1
|
||||
continue
|
||||
plan.append((prof, src, seed))
|
||||
|
||||
print(f"profiles : {len(selected)}")
|
||||
print(f"surfaces : {','.join(k for k in ALL_KINDS if k in kinds)}")
|
||||
print(f"sources : {len(plan)} to sync, {skipped} skipped")
|
||||
print(f"pacing : {args.sleep_request[0]}-{args.sleep_request[1]}s between "
|
||||
f"requests, rate cap {args.rate}")
|
||||
print(f"staging : {args.staging}")
|
||||
print(f"publish : {args.publish}")
|
||||
print()
|
||||
|
||||
if not args.execute:
|
||||
for prof, src, seed in plan:
|
||||
dest = src.directory or "(per-highlight)"
|
||||
note = " [will seed]" if seed else ""
|
||||
print(f" {prof.user:<20} {src.kind:<11} -> {dest}{note}")
|
||||
print()
|
||||
print(" " + " ".join(rsync_command(args.staging, args.publish, True)))
|
||||
print("\ndry run; nothing fetched. pass --execute to run.")
|
||||
return 0
|
||||
|
||||
config_path.write_text(json.dumps(config, indent=2))
|
||||
probes = ProbeCache(state_path.with_suffix(".probes.json"),
|
||||
0.0 if args.force else args.probe_ttl)
|
||||
failures = 0
|
||||
|
||||
for prof, src, seed in plan:
|
||||
print(f"==> {prof.user} / {src.kind}")
|
||||
stage_dir = args.staging / (src.directory or ".")
|
||||
stage_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Prime the skip-archive from what the archive already holds, so
|
||||
# fetching into an empty staging directory pulls only what is missing.
|
||||
# Done once per source, ever: afterwards the archive DB records
|
||||
# everything gallery-dl has seen and no listing pass is needed.
|
||||
if seed and args.archive_db:
|
||||
try:
|
||||
live = probes.get(src.url, now_ts)
|
||||
if live is None:
|
||||
live = probe_live(src, config_path, args.cookies)
|
||||
probes.put(src.url, live, stamp)
|
||||
probes.save()
|
||||
else:
|
||||
print(f" reusing {len(live)} cached listing items")
|
||||
held = index_existing(index.listing(prof.user))
|
||||
seeded = seed_archive_db(args.archive_db, held, live,
|
||||
src.subcategory)
|
||||
print(f" seeded {seeded} of {len(live)} live items")
|
||||
state.mark_seeded(src.url, stamp)
|
||||
state.save()
|
||||
except subprocess.CalledProcessError as exc:
|
||||
failures += 1
|
||||
print(f" probe FAILED: {exc}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
cmd = gdl_command(src, args.staging, config_path, args.cookies,
|
||||
args.archive_db)
|
||||
result = subprocess.run(cmd)
|
||||
if result.returncode != 0:
|
||||
failures += 1
|
||||
# Keep going: one private or renamed profile must not abort the run.
|
||||
print(f" FAILED (exit {result.returncode})", file=sys.stderr)
|
||||
else:
|
||||
# Recorded even for an empty fetch: the request was still spent.
|
||||
state.mark_fetched(src.url, stamp)
|
||||
state.save()
|
||||
|
||||
# Publish once, at the end, so a partially-fetched profile never reaches
|
||||
# the archive mid-run. Only ever adds -- see rsync_command.
|
||||
print("\n==> publish")
|
||||
if publish(args.staging, args.publish, dry_run=False) != 0:
|
||||
failures += 1
|
||||
|
||||
print(f"\ndone; {failures} step(s) failed")
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,277 @@
|
||||
/**
|
||||
* Generate JDownloader2 .crawljob files for the archives on disk.
|
||||
*
|
||||
* The manual flow is: paste a profile URL into JDownloader, paste the /reels
|
||||
* URL separately (the profile page misses some reels), and set the output
|
||||
* folder by hand — times however many profiles you keep. This emits one
|
||||
* crawljob per source with the folder already pointed at the right directory,
|
||||
* so JDownloader's folder-watch picks the whole batch up at once.
|
||||
*
|
||||
* Profiles and their sidecar directories are derived with the same grouping
|
||||
* logic the server uses, so the output folders always match what the viewer
|
||||
* expects to find.
|
||||
*
|
||||
* Only posts and reels are emitted. Story and highlight URLs can't be rebuilt
|
||||
* from a directory name — highlights need their numeric id and stories expire —
|
||||
* so those stay manual.
|
||||
*
|
||||
* Crawljob format verified against JDownloader's own docs for the extension:
|
||||
* src/org/jdownloader/extensions/folderwatchV2/explain.txt. JDownloader
|
||||
* develops on SVN; read it via the daily mirror at
|
||||
* https://github.com/mycodedoesnotcompile2/jdownloader_mirror (svn_trunk/),
|
||||
* not one of the abandoned GitHub copies — several are a decade stale.
|
||||
*
|
||||
* Entries are separated by `->NEW ENTRY<-` and any property may be omitted.
|
||||
* There is also a `setBeforePackagizerEnabled` companion to
|
||||
* `overwritePackagizerEnabled`, if the Packagizer ever needs to see these
|
||||
* values before they're applied.
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/jd2-sync.ts --archives <dir> [options]
|
||||
*
|
||||
* --archives <dir> Archive root to scan (default: $ARCHIVES_DIR)
|
||||
* --out <dir> JDownloader folder-watch directory to write into
|
||||
* --download-base <dir> Root path as *JDownloader* sees it, when it runs on
|
||||
* a different machine than this script (e.g. a mapped
|
||||
* drive). Defaults to --archives.
|
||||
* --user <name> Only this profile (repeatable)
|
||||
* --skip <name> Never emit jobs for this directory (repeatable).
|
||||
* Also read from a `.jd2ignore` file in the archive
|
||||
* root, one name per line.
|
||||
* --chunks <n> Connections per file (default 1: multi-chunk ranged
|
||||
* requests are the one CDN pattern that doesn't look
|
||||
* like a browser)
|
||||
* --auto-start Start downloads immediately instead of parking them
|
||||
* in the LinkGrabber for review
|
||||
* --all-reels Emit a reels job even where no reels directory
|
||||
* exists yet
|
||||
* --dry-run Print the crawljob instead of writing it
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { groupArchiveDirectories, ArchiveSource } from '../src/lib/archive-grouping.js';
|
||||
|
||||
interface Options {
|
||||
archives: string;
|
||||
out: string | null;
|
||||
downloadBase: string;
|
||||
users: string[];
|
||||
skip: Set<string>;
|
||||
chunks: number;
|
||||
autoStart: boolean;
|
||||
allReels: boolean;
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
const parseArgs = (argv: string[]): Options => {
|
||||
const opts: Options = {
|
||||
archives: process.env.ARCHIVES_DIR ?? '',
|
||||
out: null,
|
||||
downloadBase: '',
|
||||
users: [],
|
||||
skip: new Set(),
|
||||
chunks: 1,
|
||||
autoStart: false,
|
||||
allReels: false,
|
||||
dryRun: false,
|
||||
};
|
||||
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const arg = argv[i];
|
||||
const next = () => argv[++i];
|
||||
switch (arg) {
|
||||
case '--archives': opts.archives = path.resolve(next()); break;
|
||||
case '--out': opts.out = path.resolve(next()); break;
|
||||
case '--download-base': opts.downloadBase = next(); break;
|
||||
case '--user': opts.users.push(next()); break;
|
||||
case '--skip': opts.skip.add(next()); break;
|
||||
case '--chunks': opts.chunks = parseInt(next(), 10); break;
|
||||
case '--auto-start': opts.autoStart = true; break;
|
||||
case '--all-reels': opts.allReels = true; break;
|
||||
case '--dry-run': opts.dryRun = true; break;
|
||||
case '--help': case '-h': printUsage(); process.exit(0);
|
||||
default:
|
||||
console.error(`Unknown argument: ${arg}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (!opts.archives) {
|
||||
console.error('No archive root. Pass --archives <dir> or set ARCHIVES_DIR.');
|
||||
process.exit(1);
|
||||
}
|
||||
if (!opts.downloadBase) opts.downloadBase = opts.archives;
|
||||
if (!opts.out && !opts.dryRun) {
|
||||
console.error('No destination. Pass --out <folder-watch dir>, or --dry-run to preview.');
|
||||
process.exit(1);
|
||||
}
|
||||
return opts;
|
||||
};
|
||||
|
||||
const printUsage = () => {
|
||||
const header = readHeaderComment();
|
||||
console.log(header);
|
||||
};
|
||||
|
||||
/** Print the usage block from this file's own header comment. */
|
||||
const readHeaderComment = () => {
|
||||
try {
|
||||
const self = fs.readFileSync(new URL(import.meta.url), 'utf8');
|
||||
const usage = self.slice(self.indexOf(' * Usage:'), self.indexOf(' */'));
|
||||
return usage.split('\n').map(l => l.replace(/^ \* ?/, '')).join('\n');
|
||||
} catch {
|
||||
return 'See the comment at the top of scripts/jd2-sync.ts';
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* JDownloader escapes nothing in crawljob values, so a stray newline would
|
||||
* silently split a property. Paths with spaces are fine as-is.
|
||||
*/
|
||||
const sanitise = (value: string) => value.replace(/[\r\n]+/g, ' ').trim();
|
||||
|
||||
/**
|
||||
* Instagram usernames are 1–30 characters of letters, digits, dots and
|
||||
* underscores. Archive roots also collect directories that aren't profiles at
|
||||
* all — tool output, exports from other services — and pointing a crawl at
|
||||
* those spends requests on instagram.com to be told the profile doesn't exist.
|
||||
* That's the exact traffic worth not spending.
|
||||
*/
|
||||
const USERNAME_RE = /^[A-Za-z0-9._]{1,30}$/;
|
||||
|
||||
/** Directory names to skip, from `.jd2ignore` in the archive root. */
|
||||
const readIgnoreFile = (archives: string): string[] => {
|
||||
try {
|
||||
return fs.readFileSync(path.join(archives, '.jd2ignore'), 'utf8')
|
||||
.split('\n').map(l => l.trim()).filter(l => l && !l.startsWith('#'));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
interface Job {
|
||||
user: string;
|
||||
kind: 'posts' | 'reels';
|
||||
url: string;
|
||||
packageName: string;
|
||||
downloadFolder: string;
|
||||
fileCount: number | null;
|
||||
}
|
||||
|
||||
const buildJobs = (opts: Options): Job[] => {
|
||||
const dirNames = fs.readdirSync(opts.archives, { withFileTypes: true })
|
||||
.filter(e => e.isDirectory() && !/^[.@_]/.test(e.name))
|
||||
.map(e => e.name);
|
||||
|
||||
const groups = groupArchiveDirectories(dirNames);
|
||||
const jobs: Job[] = [];
|
||||
const skipped: string[] = [];
|
||||
|
||||
for (const name of readIgnoreFile(opts.archives)) opts.skip.add(name);
|
||||
|
||||
const countFiles = (dir: string): number | null => {
|
||||
try {
|
||||
return fs.readdirSync(path.join(opts.archives, dir)).length;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
// JDownloader must be given the path *it* can see, which differs from the
|
||||
// scan path whenever the archive lives on a share.
|
||||
const downloadFolderFor = (dir: string) =>
|
||||
opts.downloadBase.includes('\\')
|
||||
? `${opts.downloadBase.replace(/\\$/, '')}\\${dir}`
|
||||
: path.posix.join(opts.downloadBase, dir);
|
||||
|
||||
for (const [user, sources] of [...groups].sort(([a], [b]) => a.localeCompare(b))) {
|
||||
if (opts.users.length && !opts.users.includes(user)) continue;
|
||||
|
||||
if (opts.skip.has(user)) { skipped.push(`${user} (ignored)`); continue; }
|
||||
if (!USERNAME_RE.test(user)) { skipped.push(`${user} (not a username)`); continue; }
|
||||
|
||||
const has = (kind: ArchiveSource['kind']) => sources.find(s => s.kind === kind);
|
||||
const base = has('posts');
|
||||
if (!base) continue; // sidecar-only group: nothing sensible to point a URL at
|
||||
|
||||
jobs.push({
|
||||
user, kind: 'posts',
|
||||
url: `https://www.instagram.com/${encodeURIComponent(user)}/`,
|
||||
packageName: base.dir,
|
||||
downloadFolder: downloadFolderFor(base.dir),
|
||||
fileCount: countFiles(base.dir),
|
||||
});
|
||||
|
||||
const reels = has('reels');
|
||||
if (reels || opts.allReels) {
|
||||
const dir = reels?.dir ?? `${user} - reels`;
|
||||
jobs.push({
|
||||
user, kind: 'reels',
|
||||
url: `https://www.instagram.com/${encodeURIComponent(user)}/reels/`,
|
||||
packageName: dir,
|
||||
downloadFolder: downloadFolderFor(dir),
|
||||
fileCount: reels ? countFiles(dir) : null,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (skipped.length) {
|
||||
console.error(`Skipped ${skipped.length} director${skipped.length === 1 ? 'y' : 'ies'}:`);
|
||||
for (const s of skipped) console.error(` - ${s}`);
|
||||
console.error('');
|
||||
}
|
||||
|
||||
return jobs;
|
||||
};
|
||||
|
||||
const renderCrawljob = (jobs: Job[], opts: Options): string =>
|
||||
jobs.map(job => [
|
||||
`text=${sanitise(job.url)}`,
|
||||
`packageName=${sanitise(job.packageName)}`,
|
||||
`downloadFolder=${sanitise(job.downloadFolder)}`,
|
||||
`chunks=${opts.chunks}`,
|
||||
// Without this a Packagizer rule can override downloadFolder and scatter
|
||||
// files away from the directory the viewer reads.
|
||||
'overwritePackagizerEnabled=TRUE',
|
||||
`autoStart=${opts.autoStart ? 'TRUE' : 'FALSE'}`,
|
||||
`autoConfirm=${opts.autoStart ? 'TRUE' : 'FALSE'}`,
|
||||
'enabled=TRUE',
|
||||
`comment=instaarchive jd2-sync (${job.kind})`,
|
||||
].join('\n')).join('\n->NEW ENTRY<-\n');
|
||||
|
||||
const main = () => {
|
||||
const opts = parseArgs(process.argv.slice(2));
|
||||
const jobs = buildJobs(opts);
|
||||
|
||||
if (!jobs.length) {
|
||||
console.error('No profiles matched.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.error(`Archive root : ${opts.archives}`);
|
||||
console.error(`JD sees root : ${opts.downloadBase}`);
|
||||
console.error(`Jobs : ${jobs.length} (${new Set(jobs.map(j => j.user)).size} profiles)\n`);
|
||||
for (const job of jobs) {
|
||||
const count = job.fileCount === null ? 'new' : `${job.fileCount} files`;
|
||||
console.error(` ${job.kind.padEnd(5)} ${job.user.padEnd(24)} -> ${job.packageName} (${count})`);
|
||||
}
|
||||
console.error('');
|
||||
|
||||
const body = renderCrawljob(jobs, opts);
|
||||
|
||||
if (opts.dryRun || !opts.out) {
|
||||
console.log(body);
|
||||
return;
|
||||
}
|
||||
|
||||
fs.mkdirSync(opts.out, { recursive: true });
|
||||
const file = path.join(opts.out, `instaarchive-${new Date().toISOString().replace(/[:.]/g, '-')}.crawljob`);
|
||||
fs.writeFileSync(file, body, 'utf8');
|
||||
console.error(`Wrote ${file}`);
|
||||
console.error(opts.autoStart
|
||||
? 'Downloads will start automatically.'
|
||||
: 'Links land in the LinkGrabber for review; start them when ready.');
|
||||
};
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests for the request-budget logic in gdl-sync.py.
|
||||
|
||||
python3 -m unittest discover -s scripts -p 'test_*.py'
|
||||
|
||||
Deliberately stdlib-only, so it runs anywhere the sync itself runs. What is
|
||||
covered here is the part that decides whether to spend a request — the part
|
||||
whose absence got the archive's Instagram account suspended.
|
||||
"""
|
||||
|
||||
import datetime as dt
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
_spec = importlib.util.spec_from_file_location(
|
||||
"gdl_sync", Path(__file__).with_name("gdl-sync.py"))
|
||||
gdl = importlib.util.module_from_spec(_spec)
|
||||
sys.modules["gdl_sync"] = gdl
|
||||
_spec.loader.exec_module(gdl)
|
||||
|
||||
NOW = dt.datetime(2026, 8, 18, 12, 0, tzinfo=dt.timezone.utc)
|
||||
NOW_TS = NOW.timestamp()
|
||||
|
||||
|
||||
def ago(hours: float) -> str:
|
||||
return (NOW - dt.timedelta(hours=hours)).isoformat()
|
||||
|
||||
|
||||
class SourceSelection(unittest.TestCase):
|
||||
def test_only_stories_is_a_single_cheap_source(self):
|
||||
srcs = gdl.Profile("u").sources(gdl.STORIES_ONLY)
|
||||
self.assertEqual([s.kind for s in srcs], ["stories"])
|
||||
self.assertEqual(srcs[0].directory, "story - u")
|
||||
|
||||
def test_full_sync_covers_every_surface(self):
|
||||
srcs = gdl.Profile("u").sources(set(gdl.ALL_KINDS))
|
||||
self.assertEqual([s.kind for s in srcs], list(gdl.ALL_KINDS))
|
||||
|
||||
def test_reels_and_stories_go_to_their_own_directories(self):
|
||||
by_kind = {s.kind: s for s in gdl.Profile("u").sources(set(gdl.ALL_KINDS))}
|
||||
self.assertEqual(by_kind["posts"].directory, "u")
|
||||
self.assertEqual(by_kind["reels"].directory, "u - reels")
|
||||
# Highlights derive their directory from the title mid-extraction.
|
||||
self.assertEqual(by_kind["highlights"].directory, "")
|
||||
|
||||
|
||||
class PlanSource(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.state = gdl.SyncState(Path(self.tmp.name) / "state.json")
|
||||
self.posts = gdl.Profile("u").sources({"posts"})[0]
|
||||
self.stories = gdl.Profile("u").sources({"stories"})[0]
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_first_run_seeds(self):
|
||||
fetch, seed, _ = gdl.plan_source(self.posts, self.state, NOW_TS, 20)
|
||||
self.assertTrue(fetch)
|
||||
self.assertTrue(seed)
|
||||
|
||||
def test_seeding_happens_only_once(self):
|
||||
self.state.mark_seeded(self.posts.url, ago(720))
|
||||
fetch, seed, reason = gdl.plan_source(self.posts, self.state, NOW_TS, 20)
|
||||
self.assertTrue(fetch)
|
||||
self.assertFalse(seed, "a seeded source must never be re-probed")
|
||||
self.assertIn("already seeded", reason)
|
||||
|
||||
def test_stories_never_seed(self):
|
||||
# A story cannot be in the archive before it is fetched, so probing
|
||||
# would double the cost of the cheapest surface for no benefit.
|
||||
_, seed, reason = gdl.plan_source(self.stories, self.state, NOW_TS, 20)
|
||||
self.assertFalse(seed)
|
||||
self.assertIn("no seed", reason)
|
||||
|
||||
def test_recent_fetch_is_refused(self):
|
||||
self.state.mark_fetched(self.posts.url, ago(3))
|
||||
fetch, _, reason = gdl.plan_source(self.posts, self.state, NOW_TS, 20)
|
||||
self.assertFalse(fetch)
|
||||
self.assertIn("under the", reason)
|
||||
|
||||
def test_an_old_fetch_is_allowed_again(self):
|
||||
self.state.mark_fetched(self.posts.url, ago(30))
|
||||
fetch, _, _ = gdl.plan_source(self.posts, self.state, NOW_TS, 20)
|
||||
self.assertTrue(fetch)
|
||||
|
||||
def test_daily_stories_pass_a_20h_floor(self):
|
||||
# The cadence this is built for: once a day, every day.
|
||||
self.state.mark_fetched(self.stories.url, ago(24))
|
||||
fetch, _, _ = gdl.plan_source(self.stories, self.state, NOW_TS, 20)
|
||||
self.assertTrue(fetch)
|
||||
|
||||
def test_force_disables_the_floor(self):
|
||||
self.state.mark_fetched(self.posts.url, ago(1))
|
||||
fetch, _, _ = gdl.plan_source(self.posts, self.state, NOW_TS, 0.0)
|
||||
self.assertTrue(fetch)
|
||||
|
||||
def test_the_aborted_run_scenario(self):
|
||||
"""
|
||||
Yesterday's failure: a run died mid-way and the restart re-enumerated
|
||||
every profile. Seeded-but-not-fetched must not re-probe.
|
||||
"""
|
||||
self.state.mark_seeded(self.posts.url, ago(0.5))
|
||||
fetch, seed, _ = gdl.plan_source(self.posts, self.state, NOW_TS, 20)
|
||||
self.assertTrue(fetch, "the fetch still needs to happen")
|
||||
self.assertFalse(seed, "but the listing pass must not be paid for twice")
|
||||
|
||||
|
||||
class StatePersistence(unittest.TestCase):
|
||||
def test_state_survives_a_reload(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = Path(d) / "state.json"
|
||||
a = gdl.SyncState(path)
|
||||
a.mark_seeded("https://x/", ago(1))
|
||||
a.mark_fetched("https://x/", ago(1))
|
||||
a.save()
|
||||
b = gdl.SyncState(path)
|
||||
self.assertFalse(b.needs_seed("https://x/"))
|
||||
self.assertEqual(b.last_fetch("https://x/"), ago(1))
|
||||
|
||||
def test_a_corrupt_state_file_never_blocks_a_sync(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = Path(d) / "state.json"
|
||||
path.write_text("{ not json")
|
||||
self.assertTrue(gdl.SyncState(path).needs_seed("https://x/"))
|
||||
|
||||
|
||||
class ProbeCaching(unittest.TestCase):
|
||||
def test_fresh_entries_are_reused_and_stale_ones_are_not(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
cache = gdl.ProbeCache(Path(d) / "p.json", ttl_hours=24)
|
||||
cache.put("https://x/", [{"shortcode": "A", "post_shortcode": "A",
|
||||
"num": 1, "media_id": "1"}], ago(1))
|
||||
self.assertEqual(len(cache.get("https://x/", NOW_TS)), 1)
|
||||
|
||||
cache.put("https://y/", [{"shortcode": "B", "post_shortcode": "B",
|
||||
"num": 1, "media_id": "2"}], ago(48))
|
||||
self.assertIsNone(cache.get("https://y/", NOW_TS))
|
||||
|
||||
def test_cache_keeps_only_the_fields_seeding_needs(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = Path(d) / "p.json"
|
||||
cache = gdl.ProbeCache(path, ttl_hours=24)
|
||||
cache.put("https://x/", [{"shortcode": "A", "post_shortcode": "A",
|
||||
"num": 1, "media_id": "1",
|
||||
"description": "x" * 5000}], ago(0))
|
||||
cache.save()
|
||||
self.assertNotIn("description", path.read_text())
|
||||
|
||||
def test_a_miss_is_reported_rather_than_guessed(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
cache = gdl.ProbeCache(Path(d) / "p.json", ttl_hours=24)
|
||||
self.assertIsNone(cache.get("https://never-seen/", NOW_TS))
|
||||
|
||||
|
||||
class Seeding(unittest.TestCase):
|
||||
"""The bug that seeded 5 of 2275: matching the wrong shortcode field."""
|
||||
|
||||
def test_posts_are_keyed_by_post_shortcode(self):
|
||||
item = {"shortcode": "childcode", "post_shortcode": "POSTCODE",
|
||||
"num": 2, "media_id": "9"}
|
||||
self.assertEqual(gdl.live_key(item, "posts"), ("POSTCODE", 2))
|
||||
|
||||
def test_stories_are_keyed_by_the_per_item_shortcode(self):
|
||||
item = {"shortcode": "ITEMCODE", "post_shortcode": "reelid",
|
||||
"num": 3, "media_id": "9"}
|
||||
self.assertEqual(gdl.live_key(item, "stories"), ("ITEMCODE", 1))
|
||||
self.assertEqual(gdl.live_key(item, "highlights"), ("ITEMCODE", 1))
|
||||
|
||||
def test_index_existing_normalises_a_missing_index_to_one(self):
|
||||
held = gdl.index_existing([
|
||||
"u/2023-04-19_u - ABC.mp4",
|
||||
"u/2023-04-12_u - DEF - 3.jpg",
|
||||
"u/2023-04-12_u - DEF.txt", # sidecars are not media
|
||||
"u/2023-04-12_u - DEF.json",
|
||||
])
|
||||
self.assertEqual(held, {("ABC", 1), ("DEF", 3)})
|
||||
|
||||
def test_seeding_marks_only_what_is_already_held(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
db = Path(d) / "a.db"
|
||||
live = [
|
||||
{"post_shortcode": "HELD", "shortcode": "x", "num": 1, "media_id": "11"},
|
||||
{"post_shortcode": "NEW", "shortcode": "y", "num": 1, "media_id": "22"},
|
||||
]
|
||||
n = gdl.seed_archive_db(db, {("HELD", 1)}, live, "posts")
|
||||
self.assertEqual(n, 1)
|
||||
import sqlite3
|
||||
rows = {r[0] for r in sqlite3.connect(db).execute(
|
||||
"SELECT entry FROM archive")}
|
||||
self.assertEqual(rows, {"instagram11"})
|
||||
|
||||
|
||||
class Publishing(unittest.TestCase):
|
||||
def test_publish_only_ever_adds(self):
|
||||
cmd = gdl.rsync_command(Path("/stage"), "host:/archives", dry_run=False)
|
||||
self.assertIn("--ignore-existing", cmd)
|
||||
self.assertNotIn("--delete", cmd)
|
||||
|
||||
def test_tooling_files_are_excluded_from_the_archive(self):
|
||||
cmd = " ".join(gdl.rsync_command(Path("/stage"), "/dest", dry_run=True))
|
||||
for pattern in ("gdl-sync*.json", "*.db"):
|
||||
self.assertIn(pattern, cmd)
|
||||
self.assertIn("--dry-run", cmd)
|
||||
|
||||
|
||||
class UrlsFile(unittest.TestCase):
|
||||
def test_reads_every_form_a_person_might_paste(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p = Path(d) / "urls.txt"
|
||||
p.write_text(
|
||||
"# comment\n"
|
||||
"https://www.instagram.com/a/\n"
|
||||
"https://instagram.com/b\n"
|
||||
"www.instagram.com/c/\n"
|
||||
"d\n"
|
||||
" e # trailing\n"
|
||||
"\n"
|
||||
"https://www.instagram.com/a/\n" # duplicate
|
||||
"https://www.instagram.com/p/ABC123/\n" # a post, not a profile
|
||||
"not a username\n")
|
||||
self.assertEqual(gdl.read_urls_file(p), ["a", "b", "c", "d", "e"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user