226 lines
10 KiB
Bash
Executable File
226 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
|
# Unattended wrapper around gdl-sync.py. One argument: the run mode.
|
|
#
|
|
# stories daily ~6 requests; the only surface that cannot be backfilled
|
|
# profiles weekly every surface, --abort 50: stops enumerating a profile
|
|
# once it reaches content already held, so it costs
|
|
# ~40-60 requests and catches everything NEW
|
|
# full-sweep rarely every surface, no abort: walks each profile to the end
|
|
# for ~420 requests. The only run that notices posts
|
|
# EDITED after we archived them, and by far the most
|
|
# expensive thing here -- see TOOLING.md before running.
|
|
#
|
|
# Posts and reels are LISTED by scrolling the signed-in Chrome (chrome-sync.sh,
|
|
# per profile and tab) rather than by gallery-dl's listing API, which broke on
|
|
# 2026-10-22 (gh#9829) while single-post fetches kept working. gallery-dl still
|
|
# does stories and highlights. GDL_LISTING=api restores the old all-API run.
|
|
#
|
|
# Exits non-zero if the sync does, so cron mails you. Everything is logged.
|
|
set -eu
|
|
|
|
MODE="${1:?usage: gdl-cron.sh stories|profiles|full-sweep}"
|
|
|
|
# healthchecks.io (or compatible) ping URL, e.g. https://hc-ping.com/<uuid>.
|
|
# Set by the run-*.sh wrapper for whichever mode has a check configured;
|
|
# unset here means no ping, silently. A ping failure (network blip, check
|
|
# deleted) must never fail the sync itself -- hence `|| true` throughout, and
|
|
# a short timeout (-m) so a hung ping can't hang the run.
|
|
HC_URL="${HC_URL:-}"
|
|
|
|
GDL_HOME="${GDL_HOME:-$HOME/gdl}"
|
|
INDEX="${GDL_INDEX:-https://instaarchive.ergosteur.com}"
|
|
PUBLISH="${GDL_PUBLISH:-agentapi@10.20.28.200:/volume1/rslsync/sync/Instagram-archive/archives/}"
|
|
STAGING="$GDL_HOME/staging-$MODE"
|
|
# $$ in the name because two runs in the same SECOND would otherwise share a
|
|
# log file, and `tee -a` appends -- which made a test see the previous run.
|
|
LOG="$GDL_HOME/logs/$MODE-$(date +%Y%m%d-%H%M%S)-$$.log"
|
|
|
|
PATH="$HOME/.local/bin:$PATH"; export PATH
|
|
|
|
# Pacing. These are the values the 2026-08-22 runs used by hand, after the
|
|
# scraping warning -- roughly double the caution of gdl-sync.py's own defaults
|
|
# (6-10s / 3-6s / 1M). An archive sync has no deadline; being slow is free and
|
|
# being restricted is not. Override per-run with GDL_SLEEP_REQUEST etc. if you
|
|
# ever need to, but raise them rather than lower them.
|
|
SLEEP_REQUEST="${GDL_SLEEP_REQUEST:-12 20}"
|
|
SLEEP="${GDL_SLEEP:-5 10}"
|
|
RATE="${GDL_RATE:-500K}"
|
|
|
|
LISTING="${GDL_LISTING:-browser}"
|
|
case "$LISTING" in
|
|
browser|api) ;;
|
|
*) echo "GDL_LISTING must be browser or api, not '$LISTING'" >&2; exit 2 ;;
|
|
esac
|
|
GDL_PYTHON="${GDL_PYTHON:-$HOME/.local/share/pipx/venvs/gallery-dl/bin/python3}"
|
|
# Set per mode below when posts/reels go through the browser: empty = no browser step.
|
|
BROWSER_KNOWN_ROUNDS=""
|
|
|
|
case "$MODE" in
|
|
# --min-interval 8, not the 20h default. The floor exists to stop an ABORTED
|
|
# RESTART re-enumerating profiles -- a minutes-to-hours concern. At 20h the
|
|
# daily timer silently did nothing whenever a manual run had happened the
|
|
# previous afternoon, which is exactly what happened on 2026-08-21: it fired,
|
|
# skipped all six sources and reported success. A stories fetch is one
|
|
# request per profile, so the worst case 8h permits is roughly twelve
|
|
# requests in a day instead of six.
|
|
stories) ARGS="--only stories --min-interval 8" ;;
|
|
# Browser listing: stop scrolling a grid after 2 rounds of nothing unarchived,
|
|
# the browser's equivalent of --abort 50.
|
|
profiles)
|
|
if [ "$LISTING" = browser ]; then
|
|
ARGS="--only stories,highlights --abort 50"; BROWSER_KNOWN_ROUNDS=2
|
|
else
|
|
ARGS="--only posts,reels,stories,highlights --abort 50"
|
|
fi ;;
|
|
# No --abort: the whole point of full-sweep is enumerating to the end, so it
|
|
# is the only run that notices carousels edited after we archived them. The
|
|
# browser equivalent is scrolling every grid to the bottom (0 = no early stop).
|
|
full-sweep)
|
|
if [ "$LISTING" = browser ]; then
|
|
ARGS="--only stories,highlights"; BROWSER_KNOWN_ROUNDS=0
|
|
else
|
|
ARGS="--only posts,reels,stories,highlights"
|
|
fi ;;
|
|
# Renamed 2026-08-22: "full" was misleading (it is the abort-LIMITED run) and
|
|
# "sweep" did not say it was the exhaustive one. Catch the old names rather
|
|
# than failing with a bare error, in case something still passes them.
|
|
full) echo "mode 'full' was renamed to 'profiles'" >&2; exit 2 ;;
|
|
sweep) echo "mode 'sweep' was renamed to 'full-sweep'" >&2; exit 2 ;;
|
|
*) echo "unknown mode: $MODE (want stories|profiles|full-sweep)" >&2; exit 2 ;;
|
|
esac
|
|
|
|
mkdir -p "$GDL_HOME/logs"
|
|
|
|
[ -n "$HC_URL" ] && curl -fsS -m 10 --retry 3 -o /dev/null "$HC_URL/start" || true
|
|
|
|
# Preflight: refuse to scrape logged out. gdl reads the Chrome profile's cookie DB, and the
|
|
# session in it can die silently -- from 2026-10-07 every stories run fetched "No results"
|
|
# for all six profiles and still exited 0, so nothing alerted and a week of stories (which
|
|
# cannot be backfilled) was lost. An open Chrome tab keeps showing its logged-in UI after
|
|
# that, so only the cookie DB tells the truth. Exit 76 turns the Healthchecks check red.
|
|
SESSION_CHECK="$GDL_HOME/check-session.py"
|
|
if [ -x "$SESSION_CHECK" ]; then
|
|
if ! SESSION_MSG=$("$SESSION_CHECK" 2>&1); then
|
|
echo "=== $MODE run $(date -Is) ===" | tee -a "$LOG"
|
|
echo "ABORT (no requests sent): $SESSION_MSG" | tee -a "$LOG" >&2
|
|
echo "=== exit 76 at $(date -Is) ===" | tee -a "$LOG"
|
|
[ -n "$HC_URL" ] && curl -fsS -m 10 --retry 3 --data-binary "@$LOG" \
|
|
-o /dev/null "$HC_URL/76" || true
|
|
exit 76
|
|
fi
|
|
echo "preflight: $SESSION_MSG" | tee -a "$LOG"
|
|
fi
|
|
|
|
# Upgrade gallery-dl/yt-dlp first (PyPI only, no Instagram requests; never fatal). Instagram
|
|
# endpoints change and gallery-dl follows within days -- on 1.32.14 the posts listing silently
|
|
# returned "No results" after 1.32.17 had already fixed the query. See gdl-upgrade.sh.
|
|
UPGRADE="$GDL_HOME/gdl-upgrade.sh"
|
|
if [ -x "$UPGRADE" ]; then
|
|
"$UPGRADE" 2>&1 | tee -a "$LOG" || true
|
|
fi
|
|
|
|
# Staging is wiped every run ON PURPOSE. What we already hold is decided by the
|
|
# skip-archive (--download-archive), never by which files happen to be sitting
|
|
# in staging, so starting empty is correct -- and it keeps the publish rsync
|
|
# to just the new files instead of re-walking gigabytes each time.
|
|
rm -rf "$STAGING"
|
|
|
|
echo "=== $MODE run $(date -Is) ===" | tee -a "$LOG"
|
|
|
|
# The exit status has to survive the pipe into tee. The left-hand side of a
|
|
# pipeline runs in a SUBSHELL, so an `exit` in there sets the subshell's status
|
|
# and the script goes on to return tee's, which is always 0. An earlier version
|
|
# of this file did exactly that and reported success no matter what the sync
|
|
# did -- which is why the shebang is bash: PIPESTATUS is the fix.
|
|
# This run's output only. The check below must never see a previous run's
|
|
# lines, so it reads this rather than the (appended-to) log.
|
|
RUNOUT=$(mktemp)
|
|
trap 'rm -f "$RUNOUT"' EXIT
|
|
|
|
set +e
|
|
# shellcheck disable=SC2086
|
|
"$GDL_HOME/gdl-sync.py" \
|
|
--index "$INDEX" \
|
|
--staging "$STAGING" \
|
|
--publish "$PUBLISH" \
|
|
--archive-db "$GDL_HOME/artms.db" \
|
|
--urls-file "$GDL_HOME/artms_account_links.txt" \
|
|
--sleep-request $SLEEP_REQUEST \
|
|
--sleep $SLEEP \
|
|
--rate "$RATE" \
|
|
$ARGS --execute 2>&1 | tee -a "$LOG" "$RUNOUT"
|
|
status=${PIPESTATUS[0]}
|
|
set -e
|
|
|
|
# Posts and reels by browser: scrape each profile's grid and Reels tab, fetch only
|
|
# the new shortcodes as single posts. chrome-sync.sh logs and publishes on its own;
|
|
# its output is copied into this run's log so the Healthchecks body shows it too.
|
|
if [ -n "$BROWSER_KNOWN_ROUNDS" ]; then
|
|
mapfile -t PROFILES < <("$GDL_PYTHON" -c "
|
|
import sys, importlib.util
|
|
from pathlib import Path
|
|
spec = importlib.util.spec_from_file_location('gdl_sync', Path('$GDL_HOME/gdl-sync.py'))
|
|
gdl = importlib.util.module_from_spec(spec)
|
|
sys.modules['gdl_sync'] = gdl
|
|
spec.loader.exec_module(gdl)
|
|
for p in gdl.read_urls_file(Path('$GDL_HOME/artms_account_links.txt')):
|
|
print(p)
|
|
")
|
|
browser_failed=0
|
|
for p in "${PROFILES[@]}"; do
|
|
for tab in posts reels; do
|
|
echo "==> browser: $p / $tab" | tee -a "$LOG"
|
|
set +e
|
|
GDL_TAB=$tab GDL_MAX_KNOWN_ROUNDS=$BROWSER_KNOWN_ROUNDS \
|
|
"$GDL_HOME/chrome-sync.sh" "$p" 2>&1 | tee -a "$LOG" >/dev/null
|
|
s=${PIPESTATUS[0]}
|
|
set -e
|
|
if [ "$s" -ne 0 ]; then
|
|
echo " browser $p / $tab FAILED (exit $s)" | tee -a "$LOG" >&2
|
|
browser_failed=$((browser_failed + 1))
|
|
fi
|
|
done
|
|
done
|
|
echo "browser listing: ${#PROFILES[@]} profile(s), $browser_failed step(s) failed" | tee -a "$LOG"
|
|
[ "$browser_failed" -gt 0 ] && [ "$status" -eq 0 ] && status=1
|
|
fi
|
|
|
|
# A stories run that skipped every source is NOT a success. It means the
|
|
# min-interval floor blocked the one surface that cannot be backfilled, and
|
|
# without this it looks identical to a clean run: exit 0, "0 step(s) failed".
|
|
# Note this is not the same as "no stories today" -- that shows up as sources
|
|
# being fetched and returning no results, which is normal and stays quiet.
|
|
if [ "$MODE" = "stories" ] && grep -q "sources : 0 to sync" "$RUNOUT"; then
|
|
echo "WARNING: every stories source was skipped by --min-interval." | tee -a "$LOG" >&2
|
|
echo " Nothing was fetched. Stories expire in 24h and cannot be" | tee -a "$LOG" >&2
|
|
echo " backfilled, so this is a real loss, not a quiet no-op." | tee -a "$LOG" >&2
|
|
[ "$status" -eq 0 ] && status=75
|
|
fi
|
|
|
|
# Every stories source empty for several days running = a session Instagram stopped
|
|
# honouring while the cookie is still in the jar (the preflight cannot see that). Stories-only:
|
|
# other modes legitimately return few items per source. See stories-streak.py.
|
|
if [ "$MODE" = "stories" ] && [ "$status" -eq 0 ] && [ -x "$GDL_HOME/stories-streak.py" ]; then
|
|
if ! STREAK_MSG=$("$GDL_HOME/stories-streak.py" "$RUNOUT" "$GDL_HOME/stories-empty-streak" 2>&1); then
|
|
echo "WARNING: $STREAK_MSG" | tee -a "$LOG" >&2
|
|
status=77
|
|
fi
|
|
fi
|
|
|
|
echo "=== exit $status at $(date -Is) ===" | tee -a "$LOG"
|
|
|
|
# The log becomes the ping body -- healthchecks.io keeps the last ping's body
|
|
# and shows it in the dashboard, so a failure is diagnosable without SSHing
|
|
# in. The trailing URL segment is the exit code by convention: 0 means
|
|
# success, anything else means fail (this is how the "0 skipped" -> 75
|
|
# remap above also reaches the dashboard as a failure, not a quiet success).
|
|
[ -n "$HC_URL" ] && curl -fsS -m 10 --retry 3 --data-binary "@$LOG" \
|
|
-o /dev/null "$HC_URL/$status" || true
|
|
|
|
# Keep the log directory from growing without bound.
|
|
ls -1t "$GDL_HOME/logs" | tail -n +30 | while read -r old; do
|
|
rm -f "$GDL_HOME/logs/$old"
|
|
done
|
|
|
|
exit $status
|