121 lines
4.5 KiB
Bash
Executable File
121 lines
4.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# The full scrape-then-fetch pipeline for one profile: scroll the real page in
|
|
# the signed-in Chrome (instagram-chrome-scrape.py), then fetch and publish
|
|
# whatever's new as single posts. See TOOLING.md ("Reels: the API is blocked,
|
|
# scrape by scrolling instead", and 2026-10-22 for the posts grid) for why this
|
|
# exists at all -- it drives the actual signed-in browser session instead of
|
|
# the listing API, and is slower by design.
|
|
#
|
|
# Usage: GDL_TAB=reels|posts ./chrome-sync.sh <username-or-profile-url>
|
|
# Normally called through reels-sync.sh / posts-sync.sh, which set GDL_TAB.
|
|
#
|
|
# Also run by gdl-cron.sh (profiles, full-sweep) once per profile and tab since
|
|
# 2026-10-23, with GDL_MAX_KNOWN_ROUNDS set -- see TOOLING.md "Posts grid by browser".
|
|
# Exits non-zero if either step does. Everything is logged.
|
|
set -eu
|
|
|
|
RAW="${1:?usage: GDL_TAB=reels|posts chrome-sync.sh <username-or-profile-url>}"
|
|
TAB="${GDL_TAB:?set GDL_TAB=reels or posts (or use reels-sync.sh / posts-sync.sh)}"
|
|
|
|
GDL_HOME="${GDL_HOME:-$HOME/gdl}"
|
|
GDL_PYTHON="${GDL_PYTHON:-$HOME/.local/share/pipx/venvs/gallery-dl/bin/python3}"
|
|
INDEX="${GDL_INDEX:-https://instaarchive.ergosteur.com}"
|
|
PUBLISH="${GDL_PUBLISH:-agentapi@10.20.28.200:/volume1/rslsync/sync/Instagram-archive/archives/}"
|
|
|
|
# Same hand-paced pacing gdl-cron.sh uses -- see its comment for why. Override
|
|
# per-run with GDL_SLEEP_REQUEST etc. if you ever need to, but raise them
|
|
# rather than lower them.
|
|
SLEEP_REQUEST="${GDL_SLEEP_REQUEST:-12 20}"
|
|
SLEEP="${GDL_SLEEP:-5 10}"
|
|
RATE="${GDL_RATE:-500K}"
|
|
SCROLL_PAUSE="${GDL_SCROLL_PAUSE:-2.0 3.5}"
|
|
MAX_IDLE_ROUNDS="${GDL_MAX_IDLE_ROUNDS:-3}"
|
|
# Empty = instagram-chrome-scrape.py's own default (2 for posts, 0 = scroll to the end for reels).
|
|
MAX_KNOWN_ROUNDS="${GDL_MAX_KNOWN_ROUNDS:-}"
|
|
|
|
PATH="$HOME/.local/bin:$PATH"; export PATH
|
|
|
|
if [ ! -x "$GDL_PYTHON" ]; then
|
|
echo "instagram-chrome-scrape.py needs gallery-dl's own pipx venv python (websocket-client" >&2
|
|
echo "was injected there, not into the system python): $GDL_PYTHON not found" >&2
|
|
exit 2
|
|
fi
|
|
|
|
# Same username-from-URL parsing gdl-sync.py already does for --urls-file,
|
|
# reused rather than re-implemented so the two never drift apart.
|
|
PROFILE=$("$GDL_PYTHON" -c "
|
|
import re, sys, importlib.util
|
|
from pathlib import Path
|
|
spec = importlib.util.spec_from_file_location('gdl_sync', Path('$GDL_HOME/gdl-sync.py'))
|
|
gdl = importlib.util.module_from_spec(spec)
|
|
sys.modules['gdl_sync'] = gdl
|
|
spec.loader.exec_module(gdl)
|
|
raw = '$RAW'
|
|
m = gdl.RE_PROFILE_URL.match(raw)
|
|
user = m.group('user') if m else raw.strip('/')
|
|
if not user or '/' in user or ' ' in user:
|
|
print(f'cannot read a username from {raw!r}', file=sys.stderr)
|
|
sys.exit(1)
|
|
print(user)
|
|
")
|
|
|
|
mkdir -p "$GDL_HOME/logs"
|
|
LOG="$GDL_HOME/logs/$TAB-$PROFILE-$(date +%Y%m%d-%H%M%S)-$$.log"
|
|
URLS_FILE="$GDL_HOME/$PROFILE-$TAB.txt"
|
|
STAGING="$GDL_HOME/staging-$TAB-$PROFILE"
|
|
|
|
echo "=== $TAB-sync $PROFILE $(date -Is) ===" | tee -a "$LOG"
|
|
|
|
# The exit status has to survive the pipe into tee -- see gdl-cron.sh's
|
|
# comment on PIPESTATUS for why this needs to be bash, not sh.
|
|
set +e
|
|
"$GDL_PYTHON" "$GDL_HOME/instagram-chrome-scrape.py" \
|
|
--profile "$PROFILE" \
|
|
--tab "$TAB" \
|
|
--index "$INDEX" \
|
|
--scroll-pause $SCROLL_PAUSE \
|
|
--max-idle-rounds "$MAX_IDLE_ROUNDS" \
|
|
${MAX_KNOWN_ROUNDS:+--max-known-rounds "$MAX_KNOWN_ROUNDS"} \
|
|
--out "$URLS_FILE" 2>&1 | tee -a "$LOG"
|
|
scrape_status=${PIPESTATUS[0]}
|
|
set -e
|
|
|
|
if [ "$scrape_status" -ne 0 ]; then
|
|
echo "=== exit $scrape_status (scrape failed) at $(date -Is) ===" | tee -a "$LOG"
|
|
exit "$scrape_status"
|
|
fi
|
|
|
|
if [ ! -s "$URLS_FILE" ]; then
|
|
echo "no new $TAB for $PROFILE; nothing to fetch" | tee -a "$LOG"
|
|
echo "=== exit 0 at $(date -Is) ===" | tee -a "$LOG"
|
|
exit 0
|
|
fi
|
|
|
|
# Staging is wiped every run on purpose -- same reasoning as gdl-cron.sh: what
|
|
# we already hold is decided by the archive dedupe in instagram-chrome-scrape.py, not by
|
|
# what happens to be sitting in staging.
|
|
rm -rf "$STAGING"
|
|
|
|
set +e
|
|
# shellcheck disable=SC2086
|
|
"$GDL_HOME/gdl-sync.py" \
|
|
--publish "$PUBLISH" \
|
|
--staging "$STAGING" \
|
|
--post-urls-file "$URLS_FILE" \
|
|
--sleep-request $SLEEP_REQUEST \
|
|
--sleep $SLEEP \
|
|
--rate "$RATE" \
|
|
--execute 2>&1 | tee -a "$LOG"
|
|
status=${PIPESTATUS[0]}
|
|
set -e
|
|
|
|
echo "=== exit $status at $(date -Is) ===" | tee -a "$LOG"
|
|
|
|
# Keep the log directory from growing without bound -- scoped to this
|
|
# script's own logs so it never touches gdl-cron.sh's rotation.
|
|
ls -1t "$GDL_HOME/logs" | grep "^$TAB-[^0-9]" | tail -n +30 | while read -r old; do
|
|
rm -f "$GDL_HOME/logs/$old"
|
|
done
|
|
|
|
exit "$status"
|