tooling: instagram-chrome-scrape.py (was reels-scrape.py) gains --tab posts with early stop; chrome-sync.sh + reels/posts-sync.sh wrappers; MANUAL-RUN.md; atomic-deploy note

2026-10-22 23:53:35 -04:00
parent 30c9fbe611
commit 1b67765d8b
10 changed files with 457 additions and 193 deletions
@@ -0,0 +1,116 @@
+#!/bin/bash
+# The full scrape-then-fetch pipeline for one profile: scroll the real page in
+# the signed-in Chrome (instagram-chrome-scrape.py), then fetch and publish
+# whatever's new as single posts. See TOOLING.md ("Reels: the API is blocked,
+# scrape by scrolling instead", and 2026-10-22 for the posts grid) for why this
+# exists at all -- it drives the actual signed-in browser session instead of
+# the listing API, and is slower by design.
+#
+# Usage: GDL_TAB=reels|posts ./chrome-sync.sh <username-or-profile-url>
+# Normally called through reels-sync.sh / posts-sync.sh, which set GDL_TAB.
+#
+# Deliberately NOT wired into gdl-cron.sh or the timers -- see TOOLING.md.
+# Exits non-zero if either step does. Everything is logged.
+set -eu
+
+RAW="${1:?usage: GDL_TAB=reels|posts chrome-sync.sh <username-or-profile-url>}"
+TAB="${GDL_TAB:?set GDL_TAB=reels or posts (or use reels-sync.sh / posts-sync.sh)}"
+
+GDL_HOME="${GDL_HOME:-$HOME/gdl}"
+GDL_PYTHON="${GDL_PYTHON:-$HOME/.local/share/pipx/venvs/gallery-dl/bin/python3}"
+INDEX="${GDL_INDEX:-https://instaarchive.ergosteur.com}"
+PUBLISH="${GDL_PUBLISH:-agentapi@10.20.28.200:/volume1/rslsync/sync/Instagram-archive/archives/}"
+
+# Same hand-paced pacing gdl-cron.sh uses -- see its comment for why. Override
+# per-run with GDL_SLEEP_REQUEST etc. if you ever need to, but raise them
+# rather than lower them.
+SLEEP_REQUEST="${GDL_SLEEP_REQUEST:-12 20}"
+SLEEP="${GDL_SLEEP:-5 10}"
+RATE="${GDL_RATE:-500K}"
+SCROLL_PAUSE="${GDL_SCROLL_PAUSE:-2.0 3.5}"
+MAX_IDLE_ROUNDS="${GDL_MAX_IDLE_ROUNDS:-3}"
+
+PATH="$HOME/.local/bin:$PATH"; export PATH
+
+if [ ! -x "$GDL_PYTHON" ]; then
+  echo "instagram-chrome-scrape.py needs gallery-dl's own pipx venv python (websocket-client" >&2
+  echo "was injected there, not into the system python): $GDL_PYTHON not found" >&2
+  exit 2
+fi
+
+# Same username-from-URL parsing gdl-sync.py already does for --urls-file,
+# reused rather than re-implemented so the two never drift apart.
+PROFILE=$("$GDL_PYTHON" -c "
+import re, sys, importlib.util
+from pathlib import Path
+spec = importlib.util.spec_from_file_location('gdl_sync', Path('$GDL_HOME/gdl-sync.py'))
+gdl = importlib.util.module_from_spec(spec)
+sys.modules['gdl_sync'] = gdl
+spec.loader.exec_module(gdl)
+raw = '$RAW'
+m = gdl.RE_PROFILE_URL.match(raw)
+user = m.group('user') if m else raw.strip('/')
+if not user or '/' in user or ' ' in user:
+    print(f'cannot read a username from {raw!r}', file=sys.stderr)
+    sys.exit(1)
+print(user)
+")
+
+mkdir -p "$GDL_HOME/logs"
+LOG="$GDL_HOME/logs/$TAB-$PROFILE-$(date +%Y%m%d-%H%M%S)-$$.log"
+URLS_FILE="$GDL_HOME/$PROFILE-$TAB.txt"
+STAGING="$GDL_HOME/staging-$TAB-$PROFILE"
+
+echo "=== $TAB-sync $PROFILE $(date -Is) ===" | tee -a "$LOG"
+
+# The exit status has to survive the pipe into tee -- see gdl-cron.sh's
+# comment on PIPESTATUS for why this needs to be bash, not sh.
+set +e
+"$GDL_PYTHON" "$GDL_HOME/instagram-chrome-scrape.py" \
+    --profile "$PROFILE" \
+    --tab "$TAB" \
+    --index "$INDEX" \
+    --scroll-pause $SCROLL_PAUSE \
+    --max-idle-rounds "$MAX_IDLE_ROUNDS" \
+    --out "$URLS_FILE" 2>&1 | tee -a "$LOG"
+scrape_status=${PIPESTATUS[0]}
+set -e
+
+if [ "$scrape_status" -ne 0 ]; then
+  echo "=== exit $scrape_status (scrape failed) at $(date -Is) ===" | tee -a "$LOG"
+  exit "$scrape_status"
+fi
+
+if [ ! -s "$URLS_FILE" ]; then
+  echo "no new $TAB for $PROFILE; nothing to fetch" | tee -a "$LOG"
+  echo "=== exit 0 at $(date -Is) ===" | tee -a "$LOG"
+  exit 0
+fi
+
+# Staging is wiped every run on purpose -- same reasoning as gdl-cron.sh: what
+# we already hold is decided by the archive dedupe in instagram-chrome-scrape.py, not by
+# what happens to be sitting in staging.
+rm -rf "$STAGING"
+
+set +e
+# shellcheck disable=SC2086
+"$GDL_HOME/gdl-sync.py" \
+    --publish "$PUBLISH" \
+    --staging "$STAGING" \
+    --post-urls-file "$URLS_FILE" \
+    --sleep-request $SLEEP_REQUEST \
+    --sleep $SLEEP \
+    --rate "$RATE" \
+    --execute 2>&1 | tee -a "$LOG"
+status=${PIPESTATUS[0]}
+set -e
+
+echo "=== exit $status at $(date -Is) ===" | tee -a "$LOG"
+
+# Keep the log directory from growing without bound -- scoped to this
+# script's own logs so it never touches gdl-cron.sh's rotation.
+ls -1t "$GDL_HOME/logs" | grep "^$TAB-[^0-9]" | tail -n +30 | while read -r old; do
+  rm -f "$GDL_HOME/logs/$old"
+done
+
+exit "$status"