tooling: scheduled profiles/full-sweep list posts+reels by browser (chrome-sync per profile/tab, early stop for weekly); GDL_LISTING=api fallback; reels-tab sweep note in MANUAL-RUN

scripts/gdl-cron.sh

@@ -10,6 +10,11 @@
 #                          EDITED after we archived them, and by far the most
 #                          expensive thing here -- see TOOLING.md before running.
 #
+# Posts and reels are LISTED by scrolling the signed-in Chrome (chrome-sync.sh,
+# per profile and tab) rather than by gallery-dl's listing API, which broke on
+# 2026-10-22 (gh#9829) while single-post fetches kept working. gallery-dl still
+# does stories and highlights. GDL_LISTING=api restores the old all-API run.
+#
 # Exits non-zero if the sync does, so cron mails you. Everything is logged.
 set -eu
 
@@ -41,6 +46,15 @@ SLEEP_REQUEST="${GDL_SLEEP_REQUEST:-12 20}"
 SLEEP="${GDL_SLEEP:-5 10}"
 RATE="${GDL_RATE:-500K}"
 
+LISTING="${GDL_LISTING:-browser}"
+case "$LISTING" in
+  browser|api) ;;
+  *) echo "GDL_LISTING must be browser or api, not '$LISTING'" >&2; exit 2 ;;
+esac
+GDL_PYTHON="${GDL_PYTHON:-$HOME/.local/share/pipx/venvs/gallery-dl/bin/python3}"
+# Set per mode below when posts/reels go through the browser: empty = no browser step.
+BROWSER_KNOWN_ROUNDS=""
+
 case "$MODE" in
   # --min-interval 8, not the 20h default. The floor exists to stop an ABORTED
   # RESTART re-enumerating profiles -- a minutes-to-hours concern. At 20h the
@@ -50,10 +64,23 @@ case "$MODE" in
   # request per profile, so the worst case 8h permits is roughly twelve
   # requests in a day instead of six.
   stories)    ARGS="--only stories --min-interval 8" ;;
-  profiles)   ARGS="--only posts,reels,stories,highlights --abort 50" ;;
+  # Browser listing: stop scrolling a grid after 2 rounds of nothing unarchived,
+  # the browser's equivalent of --abort 50.
+  profiles)
+    if [ "$LISTING" = browser ]; then
+      ARGS="--only stories,highlights --abort 50"; BROWSER_KNOWN_ROUNDS=2
+    else
+      ARGS="--only posts,reels,stories,highlights --abort 50"
+    fi ;;
   # No --abort: the whole point of full-sweep is enumerating to the end, so it
-  # is the only run that notices carousels edited after we archived them.
-  full-sweep) ARGS="--only posts,reels,stories,highlights" ;;
+  # is the only run that notices carousels edited after we archived them. The
+  # browser equivalent is scrolling every grid to the bottom (0 = no early stop).
+  full-sweep)
+    if [ "$LISTING" = browser ]; then
+      ARGS="--only stories,highlights"; BROWSER_KNOWN_ROUNDS=0
+    else
+      ARGS="--only posts,reels,stories,highlights"
+    fi ;;
   # Renamed 2026-08-22: "full" was misleading (it is the abort-LIMITED run) and
   # "sweep" did not say it was the exhaustive one. Catch the old names rather
   # than failing with a bare error, in case something still passes them.
@@ -125,6 +152,39 @@ set +e
 status=${PIPESTATUS[0]}
 set -e
 
+# Posts and reels by browser: scrape each profile's grid and Reels tab, fetch only
+# the new shortcodes as single posts. chrome-sync.sh logs and publishes on its own;
+# its output is copied into this run's log so the Healthchecks body shows it too.
+if [ -n "$BROWSER_KNOWN_ROUNDS" ]; then
+  mapfile -t PROFILES < <("$GDL_PYTHON" -c "
+import sys, importlib.util
+from pathlib import Path
+spec = importlib.util.spec_from_file_location('gdl_sync', Path('$GDL_HOME/gdl-sync.py'))
+gdl = importlib.util.module_from_spec(spec)
+sys.modules['gdl_sync'] = gdl
+spec.loader.exec_module(gdl)
+for p in gdl.read_urls_file(Path('$GDL_HOME/artms_account_links.txt')):
+    print(p)
+")
+  browser_failed=0
+  for p in "${PROFILES[@]}"; do
+    for tab in posts reels; do
+      echo "==> browser: $p / $tab" | tee -a "$LOG"
+      set +e
+      GDL_TAB=$tab GDL_MAX_KNOWN_ROUNDS=$BROWSER_KNOWN_ROUNDS \
+        "$GDL_HOME/chrome-sync.sh" "$p" 2>&1 | tee -a "$LOG" >/dev/null
+      s=${PIPESTATUS[0]}
+      set -e
+      if [ "$s" -ne 0 ]; then
+        echo "    browser $p / $tab FAILED (exit $s)" | tee -a "$LOG" >&2
+        browser_failed=$((browser_failed + 1))
+      fi
+    done
+  done
+  echo "browser listing: ${#PROFILES[@]} profile(s), $browser_failed step(s) failed" | tee -a "$LOG"
+  [ "$browser_failed" -gt 0 ] && [ "$status" -eq 0 ] && status=1
+fi
+
 # A stories run that skipped every source is NOT a success. It means the
 # min-interval floor blocked the one surface that cannot be backfilled, and
 # without this it looks identical to a clean run: exit 0, "0 step(s) failed".