#!/usr/bin/env python3
"""
Fetch Instagram profiles into the archive layout using gallery-dl.

The CLI replacement for the JDownloader2 workflow. See docs/gallery-dl.md for
the measurements behind every choice here — especially the safety model, which
is the reason this script exists in this shape rather than a simpler one.

The fetch host needs no copy of the archive. It stages locally and rsyncs
afterwards; what it already holds is learned from a *file listing* alone
(`--index`), which the viewer's own API serves.

Usage:
    ./scripts/gdl-sync.py --index https://instaarchive.ergosteur.com \\
        --staging /var/tmp/gdl --publish user@host:/path/to/archives \\
        --urls-file artms_account_links.txt --dry-run

    # ...then swap --dry-run for --execute. --index also accepts a local path,
    # and --profile / --all work instead of --urls-file.

Always --dry-run first: it prints the plan, and the publish step it reports is
the one that would touch the archive.

Run it from the host whose public IP matches the browser the cookie came from;
using the cookie from elsewhere is what session-hijack detection looks for.

See `--help` for every flag, with worked examples for each way of selecting
what to fetch.
"""

from __future__ import annotations

EXAMPLES = """\
examples:

  Routine incremental sync of the tracked profiles (what gdl-cron.sh runs) --
  --abort 50 stops enumerating each profile once it reaches content already
  held, so a run that has seeded once costs ~40-60 requests, not a full walk:

    ./scripts/gdl-sync.py --index https://instaarchive.ergosteur.com \\
        --staging /var/tmp/gdl --publish user@host:/path/to/archives \\
        --urls-file artms_account_links.txt --archive-db /var/tmp/gdl.db \\
        --abort 50 --dry-run
    # ...then swap --dry-run for --execute once the plan looks right.

  Cheapest possible run -- stories only, the one surface that expires in 24h
  and cannot be backfilled, so it is worth doing often:

    ./scripts/gdl-sync.py --index https://instaarchive.ergosteur.com \\
        --staging /var/tmp/gdl --publish user@host:/path/to/archives \\
        --urls-file artms_account_links.txt --only stories --execute

  One profile, by name, without a urls-file:

    ./scripts/gdl-sync.py --index /path/to/archives --staging /var/tmp/gdl \\
        --publish /path/to/archives --profile some_account --execute

  Every profile the archive already knows about (no urls-file, no --profile):

    ./scripts/gdl-sync.py --index /path/to/archives --staging /var/tmp/gdl \\
        --publish /path/to/archives --all --execute

  An arbitrary single post or reel from an account NOT otherwise tracked --
  e.g. a link someone shared. Filed under its owner like any other post; no
  --index needed, since there is no profile list to plan against:

    ./scripts/gdl-sync.py --staging /var/tmp/gdl --publish user@host:/path \\
        --post-url https://www.instagram.com/p/SHORTCODE/ --execute

  Full sweep -- no --abort, walks every profile to the end. The only run that
  notices a carousel edited after it was archived, and by far the most
  expensive thing here (~420 requests for six profiles). Read docs/gallery-dl.md
  and TOOLING.md before running this one:

    ./scripts/gdl-sync.py --index https://instaarchive.ergosteur.com \\
        --staging /var/tmp/gdl --publish user@host:/path/to/archives \\
        --urls-file artms_account_links.txt --execute

  Hand-paced caution after a scraping warning (roughly double the defaults;
  see docs/gallery-dl.md for where these numbers come from):

    ./scripts/gdl-sync.py ... --sleep-request 12 20 --sleep 5 10 --rate 500K

Always --dry-run first (the default): it prints the plan and the rsync
command that would publish, without spending a single Instagram request.
"""

import argparse
import datetime as dt
import json
import os
import re
import shutil
import subprocess
import sys
from dataclasses import dataclass, field
from pathlib import Path

# --------------------------------------------------------------------------
# Archive layout
# --------------------------------------------------------------------------

# Mirrors src/lib/archive-grouping.ts. Instagram usernames cannot contain
# spaces, which is what makes the username separable from a highlight title.
RE_HIGHLIGHT = re.compile(r"^story highlights - ([^ ]+) - (.+)$")
RE_STORIES = re.compile(r"^story - ([^ ]+)$")
RE_REELS = re.compile(r"^([^ ]+) - reels$")

DATE_FMT = "{date:Olocal/%Y-%m-%d}"
"""Local-time date. JD2 stamped US Eastern, NOT UTC (0/212 mismatches vs 19 for
UTC). `Olocal` is DST-aware per timestamp. The trailing separator must be
omitted or it lands in the strftime format and sanitises to an underscore."""

POST_STEM = DATE_FMT + "_{username} - {post_shortcode}"
ITEM_STEM = DATE_FMT + "_{username} - {shortcode}"


@dataclass
class Source:
    """One gallery-dl invocation: a URL fetched into a specific directory."""

    kind: str                      # posts | reels | stories | highlights
    url: str
    directory: str                 # relative to the archives root
    subcategory: str               # gallery-dl config key
    title: str | None = None       # highlight title, when known


@dataclass
class Profile:
    user: str
    existing: dict[str, str] = field(default_factory=dict)  # kind -> dirname

    def sources(self, kinds: set[str]) -> list[Source]:
        u = self.user
        base = f"https://www.instagram.com/{u}"
        all_sources = [
            Source("posts", f"{base}/posts/", u, "posts"),
            Source("reels", f"{base}/reels/", f"{u} - reels", "reels"),
            # Stories expire after 24h, so these can only ever be captured
            # live. There is no backfill and no re-fetch -- which is why they
            # are the one surface worth visiting daily.
            Source("stories", f"https://www.instagram.com/stories/{u}/",
                   f"story - {u}", "stories"),
            # Highlight directories embed the title, which gallery-dl only
            # learns mid-extraction -- so this one source fans out into many
            # directories and is handled with a directory format string.
            Source("highlights", f"{base}/highlights", "", "highlights"),
        ]
        return [s for s in all_sources if s.kind in kinds]


def scan_archives(root: Path) -> dict[str, Profile]:
    """Group existing directories into profiles, as the server does."""
    profiles: dict[str, Profile] = {}

    def get(user: str) -> Profile:
        return profiles.setdefault(user, Profile(user))

    for entry in sorted(os.listdir(root)):
        if not (root / entry).is_dir() or entry.startswith("."):
            continue
        if m := RE_HIGHLIGHT.match(entry):
            get(m.group(1)).existing.setdefault("highlights", entry)
        elif m := RE_STORIES.match(entry):
            get(m.group(1)).existing["stories"] = entry
        elif m := RE_REELS.match(entry):
            get(m.group(1)).existing["reels"] = entry
        else:
            get(entry).existing["posts"] = entry
    return profiles


RE_PROFILE_URL = re.compile(
    r"^(?:https?://)?(?:www\.)?instagram\.com/(?P<user>[^/?#\s]+)/?", re.I)

# Path segments that are Instagram features, not profiles. A line like
# ".../p/ABC123/" names a post, and treating "p" as a username would silently
# sync nothing under a nonsense directory.
RESERVED_SEGMENTS = {
    "p", "reel", "reels", "stories", "explore", "accounts", "direct",
    "tv", "s", "invites", "challenge", "about", "developer",
}


def read_urls_file(path: Path) -> list[str]:
    """
    Read profile URLs (or bare usernames) from a file, one per line.

    Written for hand-maintained lists: blank lines are skipped, `#` starts a
    comment, and either a full URL or a bare username works. Order is kept and
    duplicates dropped, so a list can be appended to without care.
    """
    users: list[str] = []
    seen: set[str] = set()

    for lineno, raw in enumerate(path.read_text().splitlines(), 1):
        line = raw.split("#", 1)[0].strip()
        if not line:
            continue

        m = RE_PROFILE_URL.match(line)
        user = m.group("user") if m else line.strip("/")

        if not user or "/" in user or " " in user:
            print(f"{path}:{lineno}: cannot read a username from {raw.strip()!r}",
                  file=sys.stderr)
            continue
        if user.lower() in RESERVED_SEGMENTS:
            print(f"{path}:{lineno}: {user!r} is an Instagram path, not a "
                  f"profile — skipping", file=sys.stderr)
            continue
        if user in seen:
            continue

        seen.add(user)
        users.append(user)

    return users


class ArchiveIndex:
    """
    What the archive already holds, as filenames only.

    Deliberately never reads file *contents*, so the fetch host does not need a
    copy of the archive — it can stage locally and rsync afterwards. Backed
    either by a local directory or by the viewer's own API, which already
    serves exactly this listing and is the cheaper option when the archive
    lives on network storage (a full walk there took ~52s).
    """

    def __init__(self, source: str):
        self.remote = source.startswith(("http://", "https://"))
        self.source = source.rstrip("/") if self.remote else None
        self.root = None if self.remote else Path(source)
        if self.root and not self.root.is_dir():
            raise SystemExit(f"archive index not found: {source}")
        self._cache: dict[str, list[str]] = {}

    def _get(self, path: str):
        from urllib.request import urlopen
        with urlopen(f"{self.source}{path}", timeout=60) as resp:
            return json.load(resp)

    def profiles(self) -> set[str]:
        if self.remote:
            return {a["name"] for a in self._get("/api/archives")}
        return set(scan_archives(self.root))

    def listing(self, user: str) -> list[str]:
        """Every filename belonging to a profile, across all its sidecars."""
        if user in self._cache:
            return self._cache[user]

        names: list[str] = []
        if self.remote:
            try:
                data = self._get(f"/api/archives/{user}/files")
            except Exception:
                data = []
            files = data if isinstance(data, list) else data.get("files", [])
            names = [f["path"] for f in files]
        else:
            prof = scan_archives(self.root).get(user)
            for dirname in (prof.existing.values() if prof else ()):
                d = self.root / dirname
                if d.is_dir():
                    names += [f"{dirname}/{n}" for n in os.listdir(d)]

        self._cache[user] = names
        return names


# --------------------------------------------------------------------------
# gallery-dl configuration
# --------------------------------------------------------------------------

def build_config(rate: str, sleep_request: list[float],
                 sleep: list[float], abort: int = 0) -> dict:
    """
    The config is generated rather than checked in so the safety-critical
    options cannot drift out of sync with the docs.

    `api: rest` is the single most important line in this file. The graphql
    backend issues one request PER POST for every video and carousel, which is
    the pattern that got this account banned once already.
    """
    caption_pp = {
        "name": "metadata",
        "event": "post",
        "mode": "custom",
        "content-format": "{description}",
        "extension": "txt",
        # JD2 wrote no .txt when the caption was empty; "empty": false (the
        # default) reproduces that.
    }
    meta_pp = {
        "name": "metadata",
        "event": "post",
        "mode": "json",
        # `include`, NOT `fields` -- `fields` applies to mode:custom and
        # silently does nothing here, dumping audio_user blobs that contain
        # unrelated users' profile picture URLs.
        "include": [
            "post_shortcode", "post_id", "type", "date", "post_date",
            "username", "fullname", "owner_id", "description", "count",
            "likes", "post_url", "sidecar_shortcode",
        ],
    }

    def post_like(stem: str) -> dict:
        """Naming for surfaces whose unit is a post (posts, reels)."""
        skip: dict = {}
        if abort:
            # Stop enumerating once `abort` consecutive files are already in
            # the skip-archive. The listing pass -- not the downloading -- is
            # what costs `instagram.com` requests, and it otherwise walks the
            # whole profile every run to find three new posts.
            #
            # Safe here only because the REST listing is strictly
            # reverse-chronological: the web grid hoists pinned posts to the
            # front, but this endpoint does not (measured 2026-08-20), so old
            # posts never appear before new ones.
            #
            # Counted in FILES, not posts, so it must clear the largest
            # already-held carousel -- 22 media for one real post in this
            # archive. It also means edited carousels (test case 15) stop
            # being noticed, so a full sweep is still worth running
            # occasionally.
            skip["skip"] = f"abort:{abort}"
        return {
            **skip,
            # `sidecar_shortcode` is set only for carousels, so it is the
            # carousel discriminator. First matching condition wins.
            "filename": {
                "sidecar_shortcode and count >= 10":
                    stem + " - {num:02}.{extension}",
                "sidecar_shortcode":
                    stem + " - {num}.{extension}",
                "":
                    stem + ".{extension}",
            },
            "postprocessors": [
                {**caption_pp, "filename": stem + ".txt"},
                {**meta_pp, "filename": stem + ".json"},
            ],
        }

    def item_like(stem: str) -> dict:
        """
        Naming for surfaces whose unit is an item inside a reel (stories,
        highlights). `{shortcode}` is per item; `{post_shortcode}` is the
        reel's id and is shared by every item in it.

        The media filename uses the per-item shortcode, but the sidecar cannot:
        it runs at `event: post`, where the kwdict describes the *reel* and has
        no `shortcode` at all -- which silently formatted as the literal
        "None", producing one "<date>_<user> - None.json" per reel. It is keyed
        by `post_shortcode` instead, and is genuinely reel-level data (the
        reel's own date and item count); per-item dates live in the media
        filenames, which is the more precise source anyway.
        """
        return {
            "filename": stem + ".{extension}",
            "postprocessors": [
                {**meta_pp,
                 "filename": DATE_FMT + "_{username} - {post_shortcode}.json"},
            ],
        }

    return {
        "extractor": {
            "base-directory": ".",
            "instagram": {
                "api": "rest",             # never "graphql" -- see docstring
                "sleep-request": sleep_request,
                "sleep": sleep,
                # The CDN does rate-limit: a first run at 3M/1-3s drew
                # '429 Too Many Requests' from scontent-*.cdninstagram.com and
                # lost two videos. Back off hard rather than retry fast.
                "sleep-429": 120.0,
                "retries": 8,
                "videos": True,
                "include": "",             # never "all"; sources are explicit
                # Directory is forced per-invocation with -D, because a reels
                # tab returns collab reels owned by OTHER accounts and
                # {username} would scatter them into the wrong profile.
                "directory": [],
                "posts": post_like(POST_STEM),
                "reels": post_like(POST_STEM),
                # Ad hoc single-post fetches (--post-url) go through gallery-dl's
                # own post/reel extractor instead of a profile listing, so the
                # owning account is never known ahead of time -- only mid-
                # extraction, same reasoning as highlights below.
                "post": {**post_like(POST_STEM), "directory": ["{username}"]},
                "reel": {**post_like(POST_STEM), "directory": ["{username}"]},
                "stories": item_like(ITEM_STEM),
                "highlights": {
                    **item_like(ITEM_STEM),
                    # The only surface that must derive its own directory,
                    # since the title is not known until extraction.
                    "directory": ["story highlights - {username} - {highlight_title}"],
                },
            },
        },
        # `retries` here is the CDN-side counterpart to sleep-429 above.
        "downloader": {"http": {"rate": rate, "retries": 8}},
        "output": {"mode": "null"},
    }


# --------------------------------------------------------------------------
# Planning and execution
# --------------------------------------------------------------------------

def gdl_command(src: Source, staging: Path, config: Path, cookies: str,
                archive_db: Path | None) -> list[str]:
    cmd = [
        "gallery-dl",
        "--config", str(config),
        "--cookies-from-browser", cookies,
    ]
    if archive_db:
        # Without a seeded skip-archive, staging is empty and every file is
        # re-downloaded; see seed_archive_db.
        cmd += ["--download-archive", str(archive_db)]
    # Forced destination -- never `{username}` -- because a reels tab returns
    # collab reels owned by other accounts, which would otherwise be filed
    # under the wrong profile. Highlights and ad hoc single-post fetches are
    # the exception: their directory is only known mid-extraction (a title, or
    # the post's owner), so the config formats it instead.
    dest = (staging if src.subcategory in ("highlights", "post", "reel")
            else staging / src.directory)
    cmd += ["--destination", str(dest)]
    cmd.append(src.url)
    return cmd


# gallery-dl keys its skip-archive on `archive_prefix + archive_fmt`, which for
# this extractor is the literal "instagram" followed by the per-media numeric
# pk (`instagram.py:25`, `job.py:713-719`). Verified against a real run: a
# 3-image carousel produced 3 rows, one per item.
ARCHIVE_KEY = "instagram{}".format
ARCHIVE_SCHEMA = "CREATE TABLE IF NOT EXISTS archive (entry TEXT PRIMARY KEY)"

RE_ARCHIVED = re.compile(
    r"^(\d{4}-\d{2}-\d{2})_(.+?) - ([A-Za-z0-9_-]+?)(?: - (\d+))?\.(\w+)$")
NON_MEDIA = {"txt", "json"}


def index_existing(listing: list[str]) -> set[tuple[str, int]]:
    """
    Reduce a flat list of filenames to the (shortcode, index) pairs already
    held. Only names matter — never the bytes — which is what lets the sync run
    on a host that has no copy of the archive.
    """
    have: set[tuple[str, int]] = set()
    for name in listing:
        m = RE_ARCHIVED.match(name.rsplit("/", 1)[-1])
        if not m or m.group(5).lower() in NON_MEDIA:
            continue
        # An absent index means a single-media post, which is index 1 — the
        # same normalisation the viewer's EXPORT_RE applies.
        have.add((m.group(3), int(m.group(4) or 1)))
    return have


def live_key(item: dict, kind: str) -> tuple[str, int]:
    """
    The (shortcode, index) a live item *would* be filed under, mirroring the
    filename template exactly.

    The two surfaces disagree about which shortcode identifies a file, and
    getting this wrong silently seeds almost nothing:

      posts/reels       filed under {post_shortcode} — for a carousel, each
                        child item ALSO has its own `shortcode`, which is not
                        what appears in the filename.
      stories/highlights filed under the per-item {shortcode}, because
                        `post_shortcode` there is the containing reel's id and
                        is shared by every item in it.
    """
    if kind in ("stories", "highlights"):
        return (item.get("shortcode"), 1)
    return (item.get("post_shortcode"), item.get("num"))


def seed_archive_db(db: Path, existing: set[tuple[str, int]],
                    live: list[dict], kind: str) -> int:
    """
    Mark everything already held as downloaded, so a fetch into an empty
    directory pulls only what is missing.

    `live` is the metadata of one listing pass — the pass we have to make
    anyway — each entry carrying at least `media_id` plus the shortcode fields
    `live_key` needs. Seeding costs no additional Instagram requests, and needs
    only a *listing* of the archive, never its contents.
    """
    import sqlite3

    db.parent.mkdir(parents=True, exist_ok=True)
    con = sqlite3.connect(db)
    con.execute(ARCHIVE_SCHEMA)
    rows = [
        (ARCHIVE_KEY(item["media_id"]),)
        for item in live
        if live_key(item, kind) in existing
    ]
    con.executemany("INSERT OR IGNORE INTO archive (entry) VALUES (?)", rows)
    con.commit()
    con.close()
    return len(rows)


def probe_live(src: Source, config: Path, cookies: str) -> list[dict]:
    """
    One metadata-only listing pass. `sleep` is forced to 0 because it otherwise
    applies per *file* even with no download — 2275 files at 1-3s each is over
    an hour for a single profile.
    """
    out = subprocess.run(
        ["gallery-dl", "-j", "--config", str(config),
         "--cookies-from-browser", cookies, "-o", "sleep=0", src.url],
        capture_output=True, text=True, check=True,
    )
    items: list[dict] = []

    def walk(node):
        if isinstance(node, dict):
            if "media_id" in node and "shortcode" in node:
                items.append(node)
            for value in node.values():
                walk(value)
        elif isinstance(node, list):
            for value in node:
                walk(value)

    walk(json.loads(out.stdout))
    return items


ALL_KINDS = ("posts", "reels", "stories", "highlights")

# Stories cannot be backfilled and expire in 24h, so a run that only wants
# stories is both cheap and the one worth scheduling daily.
STORIES_ONLY = {"stories"}


class SyncState:
    """
    What has already been spent against `instagram.com`.

    Exists because nothing else in this tool has any memory: every invocation
    used to start from zero and happily re-enumerate profiles it had listed
    minutes earlier. That is what suspended the account — the listing passes,
    not the downloads.

    Two facts are tracked per source:

      seeded   the skip-archive has been primed from the archive listing.
               This is a ONE-TIME bootstrap: afterwards the archive DB records
               every item gallery-dl has seen, so the source never needs
               probing again. This is the single biggest request saving here.
      fetched  when it was last downloaded, so a re-run soon after is refused
               rather than silently repeating the whole pass.
    """

    VERSION = 1

    def __init__(self, path: Path):
        self.path = path
        self.data = {"version": self.VERSION, "sources": {}}
        if path.is_file():
            try:
                loaded = json.loads(path.read_text())
                if loaded.get("version") == self.VERSION:
                    self.data = loaded
            except Exception:
                pass  # a corrupt state file must never block a sync

    def _entry(self, url: str) -> dict:
        return self.data.setdefault("sources", {}).setdefault(url, {})

    def needs_seed(self, url: str) -> bool:
        return not self._entry(url).get("seeded")

    def mark_seeded(self, url: str, stamp: str) -> None:
        self._entry(url)["seeded"] = stamp

    def last_fetch(self, url: str) -> str | None:
        return self._entry(url).get("fetched")

    def mark_fetched(self, url: str, stamp: str) -> None:
        self._entry(url)["fetched"] = stamp

    def save(self) -> None:
        self.path.parent.mkdir(parents=True, exist_ok=True)
        self.path.write_text(json.dumps(self.data, indent=1, sort_keys=True))


def hours_since(stamp: str | None, now: float) -> float:
    """Hours between an ISO stamp and `now`; infinite when never."""
    if not stamp:
        return float("inf")
    try:
        then = dt.datetime.fromisoformat(stamp)
    except ValueError:
        return float("inf")
    if then.tzinfo is None:
        then = then.replace(tzinfo=dt.timezone.utc)
    return (now - then.timestamp()) / 3600.0


def plan_source(src: Source, state: SyncState, now: float,
                min_interval: float) -> tuple[bool, bool, str]:
    """
    Decide what a source needs: (fetch, seed, reason).

    Seeding is skipped once done, and skipped entirely for stories — a story
    cannot exist in the archive before it is fetched, so there is nothing to
    seed from, and probing would double the request cost of the cheapest
    surface we have.
    """
    since = hours_since(state.last_fetch(src.url), now)
    if since < min_interval:
        return (False, False, f"fetched {since:.1f}h ago, under the "
                              f"{min_interval:g}h floor")
    if src.kind == "stories":
        return (True, False, "stories: no seed needed")
    if state.needs_seed(src.url):
        return (True, True, "first run: seeding from the archive listing")
    return (True, False, "already seeded; the skip-archive knows what we hold")


class ProbeCache:
    """
    Listing-pass results, kept so an interrupted run does not pay for them
    twice. Yesterday an aborted sync re-enumerated five profiles on restart.
    """

    def __init__(self, path: Path, ttl_hours: float):
        self.path = path
        self.ttl = ttl_hours
        self.data: dict = {}
        if path.is_file():
            try:
                self.data = json.loads(path.read_text())
            except Exception:
                self.data = {}

    def get(self, url: str, now: float) -> list[dict] | None:
        entry = self.data.get(url)
        if not entry or hours_since(entry.get("at"), now) > self.ttl:
            return None
        return entry.get("items")

    def put(self, url: str, items: list[dict], stamp: str) -> None:
        # Only the fields seeding needs, so the cache stays small.
        self.data[url] = {"at": stamp, "items": [
            {k: i.get(k) for k in ("shortcode", "post_shortcode", "num", "media_id")}
            for i in items
        ]}

    def save(self) -> None:
        self.path.parent.mkdir(parents=True, exist_ok=True)
        self.path.write_text(json.dumps(self.data))


def rsync_command(staging: Path, dest: str, dry_run: bool) -> list[str]:
    """
    Publish a staging tree into the archive.

    `--ignore-existing` is not an optimisation, it is the safety property: the
    archive deliberately outlives Instagram (posts exist here that Instagram no
    longer serves), so publishing must only ever *add*. No `--delete`, and
    nothing already present is overwritten — including sidecars, which get
    rewritten on every run and would otherwise churn the synced share.

    `dest` may be a local path or any rsync destination (`user@host:/path`),
    because the archive usually is not writable from the fetch host.
    """
    cmd = ["rsync", "-a", "--ignore-existing", "--partial", "--info=stats2",
           # Belt and braces: the config lives outside staging, but nothing
           # resembling tooling output should ever reach the archive. Archive
           # sidecars are always "<date>_<user> - <code>.json", so none of
           # these can match real content.
           "--exclude", "gdl-sync*.json",
           "--exclude", "*.gdl-config.json",
           "--exclude", ".gdl-*",
           "--exclude", "*.sqlite", "--exclude", "*.db"]
    if dry_run:
        cmd.append("--dry-run")
    # Trailing slash: copy the *contents* of staging into dest.
    cmd += [f"{staging}/", dest if dest.endswith("/") else dest + "/"]
    return cmd


def publish(staging: Path, dest: str, dry_run: bool) -> int:
    if not any(staging.iterdir()):
        print("    nothing staged; skipping publish")
        return 0
    cmd = rsync_command(staging, dest, dry_run)
    print("    " + " ".join(cmd))
    return subprocess.run(cmd).returncode


def run_post_urls(args) -> int:
    """
    Fetch one or more individual posts/reels by URL -- an ad hoc pull outside
    the tracked profile list, e.g. a link shared from some other account. Each
    is a single request, not a recurring surface, so there is no archive-db
    seeding and no --min-interval floor to plan around.
    """
    config = build_config(args.rate, list(args.sleep_request),
                          list(args.sleep), abort=0)
    args.staging.mkdir(parents=True, exist_ok=True)
    config_path = args.staging.parent / f"{args.staging.name}.gdl-config.json"

    sources = [Source("post", url, "", "post") for url in args.post_url]

    print(f"post-url : {len(sources)} to fetch")
    print(f"pacing   : {args.sleep_request[0]}-{args.sleep_request[1]}s between "
          f"requests, rate cap {args.rate}")
    print(f"staging  : {args.staging}")
    print(f"publish  : {args.publish}")
    print()

    if not args.execute:
        for src in sources:
            print(f"  {src.url}")
        print()
        print("  " + " ".join(rsync_command(args.staging, args.publish, True)))
        print("\ndry run; nothing fetched. pass --execute to run.")
        return 0

    config_path.write_text(json.dumps(config, indent=2))
    failures = 0
    for src in sources:
        print(f"==> {src.url}")
        cmd = gdl_command(src, args.staging, config_path, args.cookies,
                          args.archive_db)
        result = subprocess.run(cmd)
        if result.returncode != 0:
            failures += 1
            print(f"    FAILED (exit {result.returncode})", file=sys.stderr)

    print("\n==> publish")
    if publish(args.staging, args.publish, dry_run=False) != 0:
        failures += 1

    print(f"\ndone; {failures} step(s) failed")
    return 1 if failures else 0


def main() -> int:
    # A sync runs for hours and is normally watched through a redirected log,
    # where Python's block buffering would withhold progress until it happened
    # to flush -- and the gallery-dl subprocesses write to the same descriptor
    # unbuffered, so the log would also interleave out of order.
    sys.stdout.reconfigure(line_buffering=True)
    sys.stderr.reconfigure(line_buffering=True)

    ap = argparse.ArgumentParser(description=__doc__, epilog=EXAMPLES,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--index",
                    help="existing archive listing: a local root, or the "
                         "viewer's base URL (only a FILE LISTING is needed, "
                         "never the contents). Required unless --post-url")
    ap.add_argument("--publish", required=True,
                    help="rsync destination for fetched files; a local path or "
                         "user@host:/path")
    ap.add_argument("--staging", type=Path, required=True,
                    help="local scratch directory gallery-dl writes into")
    g = ap.add_mutually_exclusive_group(required=True)
    g.add_argument("--profile", action="append", default=[],
                   help="profile to sync; repeatable")
    g.add_argument("--all", action="store_true", help="every profile on disk")
    g.add_argument("--urls-file", type=Path,
                   help="file of Instagram profile URLs or usernames, one per "
                        "line; # comments and blank lines allowed")
    g.add_argument("--post-url", action="append", default=[],
                   help="fetch one post or reel by URL (e.g. "
                        "https://www.instagram.com/p/SHORTCODE/), filed under "
                        "its owner's account like any other post; repeatable. "
                        "A one-off fetch outside the tracked profile list: no "
                        "archive-db seeding, no --min-interval floor")
    ap.add_argument("--cookies", default="chrome:/home/matt/.config/google-chrome-devtools",
                    help="gallery-dl --cookies-from-browser value")
    ap.add_argument("--archive-db", type=Path, default=None,
                    help="gallery-dl skip-archive sqlite path")
    ap.add_argument("--rate", default="1M", help="per-download rate cap")
    ap.add_argument("--sleep-request", nargs=2, type=float, default=[6.0, 10.0],
                    metavar=("MIN", "MAX"))
    ap.add_argument("--sleep", nargs=2, type=float, default=[3.0, 6.0],
                    metavar=("MIN", "MAX"))
    ap.add_argument("--only", default=",".join(ALL_KINDS),
                    help="comma-separated surfaces to sync: "
                         "posts,reels,stories,highlights. Use --only stories "
                         "for the cheap daily run.")
    ap.add_argument("--min-interval", type=float, default=20.0, metavar="HOURS",
                    help="refuse to re-fetch a source touched more recently "
                         "than this (default 20h); the guard that makes a "
                         "restart cheap instead of a repeat")
    ap.add_argument("--max-sources", type=int, default=0, metavar="N",
                    help="hard ceiling on sources touched in one run "
                         "(0 = no limit)")
    ap.add_argument("--abort", type=int, default=0, metavar="N",
                    help="stop enumerating posts/reels after N consecutive "
                         "already-archived FILES (0 = walk everything, the "
                         "default). 50 is a safe routine value; it cuts the "
                         "per-run listing cost by roughly 85%%, at the price "
                         "of no longer noticing edited carousels")
    ap.add_argument("--probe-ttl", type=float, default=24.0, metavar="HOURS",
                    help="reuse cached listing results younger than this")
    ap.add_argument("--force", action="store_true",
                    help="ignore --min-interval and the probe cache")
    mode = ap.add_mutually_exclusive_group()
    mode.add_argument("--dry-run", action="store_true", default=True,
                      help="print the plan and the config; default")
    mode.add_argument("--execute", action="store_true",
                      help="actually run gallery-dl")
    args = ap.parse_args()

    if not shutil.which("gallery-dl"):
        print("gallery-dl not on PATH", file=sys.stderr)
        return 2
    if not shutil.which("rsync"):
        print("rsync not on PATH", file=sys.stderr)
        return 2

    if args.post_url:
        return run_post_urls(args)

    if not args.index:
        print("--index is required unless --post-url is given", file=sys.stderr)
        return 2

    index = ArchiveIndex(args.index)
    names = index.profiles()
    if args.urls_file:
        if not args.urls_file.is_file():
            print(f"urls file not found: {args.urls_file}", file=sys.stderr)
            return 2
        wanted = read_urls_file(args.urls_file)
        if not wanted:
            print(f"no usable profiles in {args.urls_file}", file=sys.stderr)
            return 2
        print(f"read {len(wanted)} profile(s) from {args.urls_file}")
        selected = [Profile(p) for p in wanted]
    elif args.profile:
        for p in args.profile:
            if p not in names:
                print(f"note: {p} is not in the index yet; it will be created")
        selected = [Profile(p) for p in args.profile]
    else:
        selected = [Profile(p) for p in sorted(names)]

    config = build_config(args.rate, list(args.sleep_request),
                          list(args.sleep), args.abort)
    args.staging.mkdir(parents=True, exist_ok=True)
    # Deliberately a SIBLING of the staging directory, not inside it: staging is
    # rsynced wholesale into the archive, and a dry run caught this file being
    # published to the archive root.
    config_path = args.staging.parent / f"{args.staging.name}.gdl-config.json"

    kinds = {k.strip() for k in args.only.split(",") if k.strip()}
    unknown = kinds - set(ALL_KINDS)
    if unknown:
        print(f"unknown surface(s): {', '.join(sorted(unknown))}", file=sys.stderr)
        return 2

    state_path = (args.archive_db.with_suffix(".state.json") if args.archive_db
                  else args.staging.parent / f"{args.staging.name}.state.json")
    state = SyncState(state_path)
    now = dt.datetime.now(dt.timezone.utc)
    now_ts, stamp = now.timestamp(), now.isoformat()
    min_interval = 0.0 if args.force else args.min_interval

    plan: list[tuple[Profile, Source, bool]] = []
    skipped = 0
    for prof in selected:
        for src in prof.sources(kinds):
            fetch, seed, reason = plan_source(src, state, now_ts, min_interval)
            if not fetch:
                skipped += 1
                print(f"  skip {prof.user}/{src.kind}: {reason}")
                continue
            if args.max_sources and len(plan) >= args.max_sources:
                skipped += 1
                continue
            plan.append((prof, src, seed))

    print(f"profiles : {len(selected)}")
    print(f"surfaces : {','.join(k for k in ALL_KINDS if k in kinds)}")
    print(f"sources  : {len(plan)} to sync, {skipped} skipped")
    print(f"pacing   : {args.sleep_request[0]}-{args.sleep_request[1]}s between "
          f"requests, rate cap {args.rate}")
    print(f"staging  : {args.staging}")
    print(f"publish  : {args.publish}")
    print()

    if not args.execute:
        for prof, src, seed in plan:
            dest = src.directory or "(per-highlight)"
            note = "  [will seed]" if seed else ""
            print(f"  {prof.user:<20} {src.kind:<11} -> {dest}{note}")
        print()
        print("  " + " ".join(rsync_command(args.staging, args.publish, True)))
        print("\ndry run; nothing fetched. pass --execute to run.")
        return 0

    config_path.write_text(json.dumps(config, indent=2))
    probes = ProbeCache(state_path.with_suffix(".probes.json"),
                        0.0 if args.force else args.probe_ttl)
    failures = 0

    for prof, src, seed in plan:
        print(f"==> {prof.user} / {src.kind}")
        stage_dir = args.staging / (src.directory or ".")
        stage_dir.mkdir(parents=True, exist_ok=True)

        # Prime the skip-archive from what the archive already holds, so
        # fetching into an empty staging directory pulls only what is missing.
        # Done once per source, ever: afterwards the archive DB records
        # everything gallery-dl has seen and no listing pass is needed.
        if seed and args.archive_db:
            try:
                live = probes.get(src.url, now_ts)
                if live is None:
                    live = probe_live(src, config_path, args.cookies)
                    probes.put(src.url, live, stamp)
                    probes.save()
                else:
                    print(f"    reusing {len(live)} cached listing items")
                held = index_existing(index.listing(prof.user))
                seeded = seed_archive_db(args.archive_db, held, live,
                                         src.subcategory)
                print(f"    seeded {seeded} of {len(live)} live items")
                state.mark_seeded(src.url, stamp)
                state.save()
            except subprocess.CalledProcessError as exc:
                failures += 1
                print(f"    probe FAILED: {exc}", file=sys.stderr)
                continue

        cmd = gdl_command(src, args.staging, config_path, args.cookies,
                          args.archive_db)
        result = subprocess.run(cmd)
        if result.returncode != 0:
            failures += 1
            # Keep going: one private or renamed profile must not abort the run.
            print(f"    FAILED (exit {result.returncode})", file=sys.stderr)
        else:
            # Recorded even for an empty fetch: the request was still spent.
            state.mark_fetched(src.url, stamp)
            state.save()

    # Publish once, at the end, so a partially-fetched profile never reaches
    # the archive mid-run. Only ever adds -- see rsync_command.
    print("\n==> publish")
    if publish(args.staging, args.publish, dry_run=False) != 0:
        failures += 1

    print(f"\ndone; {failures} step(s) failed")
    return 1 if failures else 0


if __name__ == "__main__":
    sys.exit(main())