tooling: scheduled profiles/full-sweep list posts+reels by browser (chrome-sync per profile/tab, early stop for weekly); GDL_LISTING=api fallback; reels-tab sweep note in MANUAL-RUN

scripts/chrome-sync.sh

@@ -9,7 +9,8 @@
 # Usage: GDL_TAB=reels|posts ./chrome-sync.sh <username-or-profile-url>
 # Normally called through reels-sync.sh / posts-sync.sh, which set GDL_TAB.
 #
-# Deliberately NOT wired into gdl-cron.sh or the timers -- see TOOLING.md.
+# Also run by gdl-cron.sh (profiles, full-sweep) once per profile and tab since
+# 2026-10-23, with GDL_MAX_KNOWN_ROUNDS set -- see TOOLING.md "Posts grid by browser".
 # Exits non-zero if either step does. Everything is logged.
 set -eu
 
@@ -29,6 +30,8 @@ SLEEP="${GDL_SLEEP:-5 10}"
 RATE="${GDL_RATE:-500K}"
 SCROLL_PAUSE="${GDL_SCROLL_PAUSE:-2.0 3.5}"
 MAX_IDLE_ROUNDS="${GDL_MAX_IDLE_ROUNDS:-3}"
+# Empty = instagram-chrome-scrape.py's own default (2 for posts, 0 = scroll to the end for reels).
+MAX_KNOWN_ROUNDS="${GDL_MAX_KNOWN_ROUNDS:-}"
 
 PATH="$HOME/.local/bin:$PATH"; export PATH
 
@@ -72,6 +75,7 @@ set +e
     --index "$INDEX" \
     --scroll-pause $SCROLL_PAUSE \
     --max-idle-rounds "$MAX_IDLE_ROUNDS" \
+    ${MAX_KNOWN_ROUNDS:+--max-known-rounds "$MAX_KNOWN_ROUNDS"} \
     --out "$URLS_FILE" 2>&1 | tee -a "$LOG"
 scrape_status=${PIPESTATUS[0]}
 set -e

scripts/gdl-cron.sh

@@ -10,6 +10,11 @@
 #                          EDITED after we archived them, and by far the most
 #                          expensive thing here -- see TOOLING.md before running.
 #
+# Posts and reels are LISTED by scrolling the signed-in Chrome (chrome-sync.sh,
+# per profile and tab) rather than by gallery-dl's listing API, which broke on
+# 2026-10-22 (gh#9829) while single-post fetches kept working. gallery-dl still
+# does stories and highlights. GDL_LISTING=api restores the old all-API run.
+#
 # Exits non-zero if the sync does, so cron mails you. Everything is logged.
 set -eu
 
@@ -41,6 +46,15 @@ SLEEP_REQUEST="${GDL_SLEEP_REQUEST:-12 20}"
 SLEEP="${GDL_SLEEP:-5 10}"
 RATE="${GDL_RATE:-500K}"
 
+LISTING="${GDL_LISTING:-browser}"
+case "$LISTING" in
+  browser|api) ;;
+  *) echo "GDL_LISTING must be browser or api, not '$LISTING'" >&2; exit 2 ;;
+esac
+GDL_PYTHON="${GDL_PYTHON:-$HOME/.local/share/pipx/venvs/gallery-dl/bin/python3}"
+# Set per mode below when posts/reels go through the browser: empty = no browser step.
+BROWSER_KNOWN_ROUNDS=""
+
 case "$MODE" in
   # --min-interval 8, not the 20h default. The floor exists to stop an ABORTED
   # RESTART re-enumerating profiles -- a minutes-to-hours concern. At 20h the
@@ -50,10 +64,23 @@ case "$MODE" in
   # request per profile, so the worst case 8h permits is roughly twelve
   # requests in a day instead of six.
   stories)    ARGS="--only stories --min-interval 8" ;;
-  profiles)   ARGS="--only posts,reels,stories,highlights --abort 50" ;;
+  # Browser listing: stop scrolling a grid after 2 rounds of nothing unarchived,
+  # the browser's equivalent of --abort 50.
+  profiles)
+    if [ "$LISTING" = browser ]; then
+      ARGS="--only stories,highlights --abort 50"; BROWSER_KNOWN_ROUNDS=2
+    else
+      ARGS="--only posts,reels,stories,highlights --abort 50"
+    fi ;;
   # No --abort: the whole point of full-sweep is enumerating to the end, so it
-  # is the only run that notices carousels edited after we archived them.
-  full-sweep) ARGS="--only posts,reels,stories,highlights" ;;
+  # is the only run that notices carousels edited after we archived them. The
+  # browser equivalent is scrolling every grid to the bottom (0 = no early stop).
+  full-sweep)
+    if [ "$LISTING" = browser ]; then
+      ARGS="--only stories,highlights"; BROWSER_KNOWN_ROUNDS=0
+    else
+      ARGS="--only posts,reels,stories,highlights"
+    fi ;;
   # Renamed 2026-08-22: "full" was misleading (it is the abort-LIMITED run) and
   # "sweep" did not say it was the exhaustive one. Catch the old names rather
   # than failing with a bare error, in case something still passes them.
@@ -125,6 +152,39 @@ set +e
 status=${PIPESTATUS[0]}
 set -e
 
+# Posts and reels by browser: scrape each profile's grid and Reels tab, fetch only
+# the new shortcodes as single posts. chrome-sync.sh logs and publishes on its own;
+# its output is copied into this run's log so the Healthchecks body shows it too.
+if [ -n "$BROWSER_KNOWN_ROUNDS" ]; then
+  mapfile -t PROFILES < <("$GDL_PYTHON" -c "
+import sys, importlib.util
+from pathlib import Path
+spec = importlib.util.spec_from_file_location('gdl_sync', Path('$GDL_HOME/gdl-sync.py'))
+gdl = importlib.util.module_from_spec(spec)
+sys.modules['gdl_sync'] = gdl
+spec.loader.exec_module(gdl)
+for p in gdl.read_urls_file(Path('$GDL_HOME/artms_account_links.txt')):
+    print(p)
+")
+  browser_failed=0
+  for p in "${PROFILES[@]}"; do
+    for tab in posts reels; do
+      echo "==> browser: $p / $tab" | tee -a "$LOG"
+      set +e
+      GDL_TAB=$tab GDL_MAX_KNOWN_ROUNDS=$BROWSER_KNOWN_ROUNDS \
+        "$GDL_HOME/chrome-sync.sh" "$p" 2>&1 | tee -a "$LOG" >/dev/null
+      s=${PIPESTATUS[0]}
+      set -e
+      if [ "$s" -ne 0 ]; then
+        echo "    browser $p / $tab FAILED (exit $s)" | tee -a "$LOG" >&2
+        browser_failed=$((browser_failed + 1))
+      fi
+    done
+  done
+  echo "browser listing: ${#PROFILES[@]} profile(s), $browser_failed step(s) failed" | tee -a "$LOG"
+  [ "$browser_failed" -gt 0 ] && [ "$status" -eq 0 ] && status=1
+fi
+
 # A stories run that skipped every source is NOT a success. It means the
 # min-interval floor blocked the one surface that cannot be backfilled, and
 # without this it looks identical to a clean run: exit 0, "0 step(s) failed".

scripts/run-reels-all.sh

@@ -5,9 +5,9 @@
 # version of running reels-sync.sh six times by hand, which is what the
 # 2026-09-03 run was before this script existed.
 #
-# Deliberately not wired into gdl-cron.sh or a timer -- same reasoning as
-# reels-sync.sh itself: it drives your actual browser session and is slower
-# by design, both good reasons to run it deliberately, not unattended.
+# A by-hand full sweep of every Reels tab (no early stop). The weekly
+# gdl-cron.sh profiles run lists reels through chrome-sync.sh itself, with an
+# early stop, since 2026-10-23 -- see TOOLING.md "Posts grid by browser".
 #
 # Usage: ./reels-all.sh
 #   GDL_ARTMS_LINKS=/path/to/list ./reels-all.sh   # override the profile list