#!/bin/bash
# Unattended wrapper around gdl-sync.py. One argument: the run mode.
#
#   stories     daily      ~6 requests; the only surface that cannot be backfilled
#   profiles    weekly     every surface, --abort 50: stops enumerating a profile
#                          once it reaches content already held, so it costs
#                          ~40-60 requests and catches everything NEW
#   full-sweep  rarely     every surface, no abort: walks each profile to the end
#                          for ~420 requests. The only run that notices posts
#                          EDITED after we archived them, and by far the most
#                          expensive thing here -- see TOOLING.md before running.
#
# Posts and reels are LISTED by scrolling the signed-in Chrome (chrome-sync.sh,
# per profile and tab) rather than by gallery-dl's listing API, which broke on
# 2026-10-22 (gh#9829) while single-post fetches kept working. gallery-dl still
# does stories and highlights. GDL_LISTING=api restores the old all-API run.
#
# Exits non-zero if the sync does, so cron mails you. Everything is logged.
set -eu

MODE="${1:?usage: gdl-cron.sh stories|profiles|full-sweep}"

# healthchecks.io (or compatible) ping URL, e.g. https://hc-ping.com/<uuid>.
# Set by the run-*.sh wrapper for whichever mode has a check configured;
# unset here means no ping, silently. A ping failure (network blip, check
# deleted) must never fail the sync itself -- hence `|| true` throughout, and
# a short timeout (-m) so a hung ping can't hang the run.
HC_URL="${HC_URL:-}"

GDL_HOME="${GDL_HOME:-$HOME/gdl}"
INDEX="${GDL_INDEX:-https://instaarchive.ergosteur.com}"
PUBLISH="${GDL_PUBLISH:-agentapi@10.20.28.200:/volume1/rslsync/sync/Instagram-archive/archives/}"
STAGING="$GDL_HOME/staging-$MODE"
# $$ in the name because two runs in the same SECOND would otherwise share a
# log file, and `tee -a` appends -- which made a test see the previous run.
LOG="$GDL_HOME/logs/$MODE-$(date +%Y%m%d-%H%M%S)-$$.log"

PATH="$HOME/.local/bin:$PATH"; export PATH

# Pacing. These are the values the 2026-08-22 runs used by hand, after the
# scraping warning -- roughly double the caution of gdl-sync.py's own defaults
# (6-10s / 3-6s / 1M). An archive sync has no deadline; being slow is free and
# being restricted is not. Override per-run with GDL_SLEEP_REQUEST etc. if you
# ever need to, but raise them rather than lower them.
SLEEP_REQUEST="${GDL_SLEEP_REQUEST:-12 20}"
SLEEP="${GDL_SLEEP:-5 10}"
RATE="${GDL_RATE:-500K}"

LISTING="${GDL_LISTING:-browser}"
case "$LISTING" in
  browser|api) ;;
  *) echo "GDL_LISTING must be browser or api, not '$LISTING'" >&2; exit 2 ;;
esac
GDL_PYTHON="${GDL_PYTHON:-$HOME/.local/share/pipx/venvs/gallery-dl/bin/python3}"
# Set per mode below when posts/reels go through the browser: empty = no browser step.
BROWSER_KNOWN_ROUNDS=""

case "$MODE" in
  # --min-interval 8, not the 20h default. The floor exists to stop an ABORTED
  # RESTART re-enumerating profiles -- a minutes-to-hours concern. At 20h the
  # daily timer silently did nothing whenever a manual run had happened the
  # previous afternoon, which is exactly what happened on 2026-08-21: it fired,
  # skipped all six sources and reported success. A stories fetch is one
  # request per profile, so the worst case 8h permits is roughly twelve
  # requests in a day instead of six.
  stories)    ARGS="--only stories --min-interval 8" ;;
  # Browser listing: stop scrolling a grid after 2 rounds of nothing unarchived,
  # the browser's equivalent of --abort 50.
  profiles)
    if [ "$LISTING" = browser ]; then
      ARGS="--only stories,highlights --abort 50"; BROWSER_KNOWN_ROUNDS=2
    else
      ARGS="--only posts,reels,stories,highlights --abort 50"
    fi ;;
  # No --abort: the whole point of full-sweep is enumerating to the end, so it
  # is the only run that notices carousels edited after we archived them. The
  # browser equivalent is scrolling every grid to the bottom (0 = no early stop).
  full-sweep)
    if [ "$LISTING" = browser ]; then
      ARGS="--only stories,highlights"; BROWSER_KNOWN_ROUNDS=0
    else
      ARGS="--only posts,reels,stories,highlights"
    fi ;;
  # Renamed 2026-08-22: "full" was misleading (it is the abort-LIMITED run) and
  # "sweep" did not say it was the exhaustive one. Catch the old names rather
  # than failing with a bare error, in case something still passes them.
  full)  echo "mode 'full' was renamed to 'profiles'" >&2; exit 2 ;;
  sweep) echo "mode 'sweep' was renamed to 'full-sweep'" >&2; exit 2 ;;
  *) echo "unknown mode: $MODE (want stories|profiles|full-sweep)" >&2; exit 2 ;;
esac

mkdir -p "$GDL_HOME/logs"

[ -n "$HC_URL" ] && curl -fsS -m 10 --retry 3 -o /dev/null "$HC_URL/start" || true

# Preflight: refuse to scrape logged out. gdl reads the Chrome profile's cookie DB, and the
# session in it can die silently -- from 2026-10-07 every stories run fetched "No results"
# for all six profiles and still exited 0, so nothing alerted and a week of stories (which
# cannot be backfilled) was lost. An open Chrome tab keeps showing its logged-in UI after
# that, so only the cookie DB tells the truth. Exit 76 turns the Healthchecks check red.
SESSION_CHECK="$GDL_HOME/check-session.py"
if [ -x "$SESSION_CHECK" ]; then
  if ! SESSION_MSG=$("$SESSION_CHECK" 2>&1); then
    echo "=== $MODE run $(date -Is) ===" | tee -a "$LOG"
    echo "ABORT (no requests sent): $SESSION_MSG" | tee -a "$LOG" >&2
    echo "=== exit 76 at $(date -Is) ===" | tee -a "$LOG"
    [ -n "$HC_URL" ] && curl -fsS -m 10 --retry 3 --data-binary "@$LOG" \
      -o /dev/null "$HC_URL/76" || true
    exit 76
  fi
  echo "preflight: $SESSION_MSG" | tee -a "$LOG"
fi

# Upgrade gallery-dl/yt-dlp first (PyPI only, no Instagram requests; never fatal). Instagram
# endpoints change and gallery-dl follows within days -- on 1.32.14 the posts listing silently
# returned "No results" after 1.32.17 had already fixed the query. See gdl-upgrade.sh.
UPGRADE="$GDL_HOME/gdl-upgrade.sh"
if [ -x "$UPGRADE" ]; then
  "$UPGRADE" 2>&1 | tee -a "$LOG" || true
fi

# Staging is wiped every run ON PURPOSE. What we already hold is decided by the
# skip-archive (--download-archive), never by which files happen to be sitting
# in staging, so starting empty is correct -- and it keeps the publish rsync
# to just the new files instead of re-walking gigabytes each time.
rm -rf "$STAGING"

echo "=== $MODE run $(date -Is) ===" | tee -a "$LOG"

# The exit status has to survive the pipe into tee. The left-hand side of a
# pipeline runs in a SUBSHELL, so an `exit` in there sets the subshell's status
# and the script goes on to return tee's, which is always 0. An earlier version
# of this file did exactly that and reported success no matter what the sync
# did -- which is why the shebang is bash: PIPESTATUS is the fix.
# This run's output only. The check below must never see a previous run's
# lines, so it reads this rather than the (appended-to) log.
RUNOUT=$(mktemp)
trap 'rm -f "$RUNOUT"' EXIT

set +e
# shellcheck disable=SC2086
"$GDL_HOME/gdl-sync.py" \
    --index "$INDEX" \
    --staging "$STAGING" \
    --publish "$PUBLISH" \
    --archive-db "$GDL_HOME/artms.db" \
    --urls-file "$GDL_HOME/artms_account_links.txt" \
    --sleep-request $SLEEP_REQUEST \
    --sleep $SLEEP \
    --rate "$RATE" \
    $ARGS --execute 2>&1 | tee -a "$LOG" "$RUNOUT"
status=${PIPESTATUS[0]}
set -e

# Posts and reels by browser: scrape each profile's grid and Reels tab, fetch only
# the new shortcodes as single posts. chrome-sync.sh logs and publishes on its own;
# its output is copied into this run's log so the Healthchecks body shows it too.
if [ -n "$BROWSER_KNOWN_ROUNDS" ]; then
  mapfile -t PROFILES < <("$GDL_PYTHON" -c "
import sys, importlib.util
from pathlib import Path
spec = importlib.util.spec_from_file_location('gdl_sync', Path('$GDL_HOME/gdl-sync.py'))
gdl = importlib.util.module_from_spec(spec)
sys.modules['gdl_sync'] = gdl
spec.loader.exec_module(gdl)
for p in gdl.read_urls_file(Path('$GDL_HOME/artms_account_links.txt')):
    print(p)
")
  browser_failed=0
  for p in "${PROFILES[@]}"; do
    for tab in posts reels; do
      echo "==> browser: $p / $tab" | tee -a "$LOG"
      set +e
      GDL_TAB=$tab GDL_MAX_KNOWN_ROUNDS=$BROWSER_KNOWN_ROUNDS \
        "$GDL_HOME/chrome-sync.sh" "$p" 2>&1 | tee -a "$LOG" >/dev/null
      s=${PIPESTATUS[0]}
      set -e
      if [ "$s" -ne 0 ]; then
        echo "    browser $p / $tab FAILED (exit $s)" | tee -a "$LOG" >&2
        browser_failed=$((browser_failed + 1))
      fi
    done
  done
  echo "browser listing: ${#PROFILES[@]} profile(s), $browser_failed step(s) failed" | tee -a "$LOG"
  [ "$browser_failed" -gt 0 ] && [ "$status" -eq 0 ] && status=1
fi

# A stories run that skipped every source is NOT a success. It means the
# min-interval floor blocked the one surface that cannot be backfilled, and
# without this it looks identical to a clean run: exit 0, "0 step(s) failed".
# Note this is not the same as "no stories today" -- that shows up as sources
# being fetched and returning no results, which is normal and stays quiet.
if [ "$MODE" = "stories" ] && grep -q "sources  : 0 to sync" "$RUNOUT"; then
  echo "WARNING: every stories source was skipped by --min-interval." | tee -a "$LOG" >&2
  echo "         Nothing was fetched. Stories expire in 24h and cannot be" | tee -a "$LOG" >&2
  echo "         backfilled, so this is a real loss, not a quiet no-op." | tee -a "$LOG" >&2
  [ "$status" -eq 0 ] && status=75
fi

# Every stories source empty for several days running = a session Instagram stopped
# honouring while the cookie is still in the jar (the preflight cannot see that). Stories-only:
# other modes legitimately return few items per source. See stories-streak.py.
if [ "$MODE" = "stories" ] && [ "$status" -eq 0 ] && [ -x "$GDL_HOME/stories-streak.py" ]; then
  if ! STREAK_MSG=$("$GDL_HOME/stories-streak.py" "$RUNOUT" "$GDL_HOME/stories-empty-streak" 2>&1); then
    echo "WARNING: $STREAK_MSG" | tee -a "$LOG" >&2
    status=77
  fi
fi

echo "=== exit $status at $(date -Is) ===" | tee -a "$LOG"

# The log becomes the ping body -- healthchecks.io keeps the last ping's body
# and shows it in the dashboard, so a failure is diagnosable without SSHing
# in. The trailing URL segment is the exit code by convention: 0 means
# success, anything else means fail (this is how the "0 skipped" -> 75
# remap above also reaches the dashboard as a failure, not a quiet success).
[ -n "$HC_URL" ] && curl -fsS -m 10 --retry 3 --data-binary "@$LOG" \
  -o /dev/null "$HC_URL/$status" || true

# Keep the log directory from growing without bound.
ls -1t "$GDL_HOME/logs" | tail -n +30 | while read -r old; do
  rm -f "$GDL_HOME/logs/$old"
done

exit $status