feat: scrape reels by scrolling the real page, since the API is blocked

gallery-dl's dedicated reels extractor POSTs to /api/v1/clips/user/,
which now 302-redirects for this account -- confirmed across multiple
profiles, hours apart, with a freshly-warmed session and a correct
X-IG-WWW-Claim header (ruled out as the cause). The reels tab itself
loads fine in a real, already-signed-in browser, so reels-scrape.py
drives that same Chrome via its loopback CDP port, scrolls the reels
tab like a person would, and scrapes /reel/<code>/ links out of the
rendered page instead of calling the blocked endpoint at all.

It only finds shortcodes -- deduped against the archive via the same
--index gdl-sync.py already uses -- and prints new post URLs. Feeding
many of those into gdl-sync.py needed two small additions: a
--post-urls-file so the list doesn't have to become a giant argv, and
inter-item pacing in run_post_urls (each --post-url was its own
subprocess with nothing pacing the gap between them).

Verified end to end against zindoriyam: 26 reels found, 16 already
archived, 10 new ones fetched and published cleanly.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011qAds5qr7nZRq5R4yAuxUk

scripts/gdl-sync.py

@@ -89,10 +89,12 @@ import argparse
 import datetime as dt
 import json
 import os
+import random
 import re
 import shutil
 import subprocess
 import sys
+import time
 from dataclasses import dataclass, field
 from pathlib import Path
 
@@ -219,6 +221,23 @@ def read_urls_file(path: Path) -> list[str]:
     return users
 
 
+def read_post_urls_file(path: Path) -> list[str]:
+    """
+    Read individual post/reel URLs from a file, one per line -- the output
+    format `reels-scrape.py` writes. Blank lines and `#` comments are skipped;
+    order is kept and duplicates dropped, same conventions as read_urls_file.
+    """
+    urls: list[str] = []
+    seen: set[str] = set()
+    for raw in path.read_text().splitlines():
+        line = raw.split("#", 1)[0].strip()
+        if not line or line in seen:
+            continue
+        seen.add(line)
+        urls.append(line)
+    return urls
+
+
 class ArchiveIndex:
     """
     What the archive already holds, as filenames only.
@@ -736,7 +755,14 @@ def run_post_urls(args) -> int:
 
     config_path.write_text(json.dumps(config, indent=2))
     failures = 0
-    for src in sources:
+    for i, src in enumerate(sources):
+        # gallery-dl's own sleep-request only paces requests INSIDE one
+        # invocation; each URL here is its own subprocess, so back-to-back
+        # items would otherwise fire with no gap at all -- the same pacing
+        # applied here as between requests within a single fetch.
+        if i:
+            pause = random.uniform(*args.sleep_request)
+            time.sleep(pause)
         print(f"==> {src.url}")
         cmd = gdl_command(src, args.staging, config_path, args.cookies,
                           args.archive_db)
@@ -766,7 +792,8 @@ def main() -> int:
     ap.add_argument("--index",
                     help="existing archive listing: a local root, or the "
                          "viewer's base URL (only a FILE LISTING is needed, "
-                         "never the contents). Required unless --post-url")
+                         "never the contents). Required unless --post-url/"
+                         "--post-urls-file")
     ap.add_argument("--publish", required=True,
                     help="rsync destination for fetched files; a local path or "
                          "user@host:/path")
@@ -785,6 +812,11 @@ def main() -> int:
                         "its owner's account like any other post; repeatable. "
                         "A one-off fetch outside the tracked profile list: no "
                         "archive-db seeding, no --min-interval floor")
+    g.add_argument("--post-urls-file", type=Path,
+                   help="file of post/reel URLs, one per line; # comments and "
+                        "blank lines allowed. Same handling as --post-url, "
+                        "paced with --sleep-request between items. This is "
+                        "the format reels-scrape.py writes")
     ap.add_argument("--cookies", default="chrome:/home/matt/.config/google-chrome-devtools",
                     help="gallery-dl --cookies-from-browser value")
     ap.add_argument("--archive-db", type=Path, default=None,
@@ -829,11 +861,20 @@ def main() -> int:
         print("rsync not on PATH", file=sys.stderr)
         return 2
 
+    if args.post_urls_file:
+        if not args.post_urls_file.is_file():
+            print(f"post-urls file not found: {args.post_urls_file}", file=sys.stderr)
+            return 2
+        args.post_url = read_post_urls_file(args.post_urls_file)
+        if not args.post_url:
+            print(f"no usable URLs in {args.post_urls_file}", file=sys.stderr)
+            return 2
     if args.post_url:
         return run_post_urls(args)
 
     if not args.index:
-        print("--index is required unless --post-url is given", file=sys.stderr)
+        print("--index is required unless --post-url/--post-urls-file is given",
+              file=sys.stderr)
         return 2
 
     index = ArchiveIndex(args.index)