feat: guard against a dead Instagram session, show progress, fix missing pings

- check-session.py: preflight in gdl-cron.sh. No unexpired sessionid in the
  Chrome profile gdl reads -> abort before any request, ping /76 (red check).
- stories-streak.py: postflight. Every fetched stories source empty for 3
  consecutive days -> exit 77, for a session Instagram stopped honouring while
  the cookie is still present. The 2026-10-07 session death went unnoticed for
  a week because every run still exited 0.
- gdl-sync.py: [n/N] source headers, a 60s heartbeat with new-file counts, and
  per-source timing, so a healthy paced run no longer looks hung.
- gdl-sync@.service now runs run-%i.sh instead of gdl-cron.sh: HC_URL is set in
  the wrappers, so scheduled runs had never pinged Healthchecks.
- Tests for both guards (offline) and the progress helpers.
This commit is contained in:
2026-10-14 21:02:58 -04:00
parent fcfb8e5fd3
commit 4a2bd7aa19
7 changed files with 433 additions and 6 deletions
+79 -5
View File
@@ -94,6 +94,7 @@ import re
import shutil
import subprocess
import sys
import threading
import time
from dataclasses import dataclass, field
from pathlib import Path
@@ -460,6 +461,71 @@ def build_config(rate: str, sleep_request: list[float],
# Planning and execution
# --------------------------------------------------------------------------
def _fmt_elapsed(sec: float) -> str:
sec = int(sec)
h, rest = divmod(sec, 3600)
m, s = divmod(rest, 60)
return f"{h}h{m:02d}m" if h else (f"{m}m{s:02d}s" if m else f"{s}s")
def _count_new(root: Path, since: float) -> tuple[int, str | None]:
"""Media files under `root` written since `since`, and the newest one's name."""
n, newest, newest_t = 0, None, 0.0
for d, _dirs, files in os.walk(root):
for f in files:
if f.endswith((".txt", ".json", ".part")):
continue
try:
mt = os.stat(os.path.join(d, f)).st_mtime
except OSError:
continue
if mt >= since:
n += 1
if mt > newest_t:
newest, newest_t = f, mt
return n, newest
class Heartbeat:
"""
Prints a progress line every `every` seconds while a quiet step runs.
A healthy run looks hung otherwise: pacing is 12-20s between requests, the
generated config sets `output: null` (no per-file lines), and the live
listing probe captures gallery-dl output until it finishes. With `watch`
set, the line also reports how many files have landed there so far.
"""
def __init__(self, label: str, watch: Path | None = None, every: float = 60.0):
self.label, self.watch, self.every = label, watch, every
self._stop = threading.Event()
self._thread = threading.Thread(target=self._run, daemon=True)
self.t0 = time.time()
def _run(self) -> None:
while not self._stop.wait(self.every):
msg = f" ... {self.label}: {_fmt_elapsed(time.time() - self.t0)} elapsed"
if self.watch is not None:
n, newest = _count_new(self.watch, self.t0)
msg += f", {n} new file(s) so far"
if newest:
msg += f" (latest: {newest})"
print(msg, flush=True)
def __enter__(self) -> "Heartbeat":
self.t0 = time.time()
self._thread.start()
return self
def __exit__(self, *exc) -> None:
self._stop.set()
self._thread.join()
@property
def elapsed(self) -> float:
return time.time() - self.t0
def gdl_command(src: Source, staging: Path, config: Path, cookies: str,
archive_db: Path | None) -> list[str]:
cmd = [
@@ -1003,8 +1069,9 @@ def main() -> int:
0.0 if args.force else args.probe_ttl)
failures = 0
for prof, src, seed in plan:
print(f"==> {prof.user} / {src.kind}")
for n_src, (prof, src, seed) in enumerate(plan, 1):
print(f"==> [{n_src}/{len(plan)}] {prof.user} / {src.kind} "
f"({dt.datetime.now().strftime('%H:%M:%S')})")
stage_dir = args.staging / (src.directory or ".")
stage_dir.mkdir(parents=True, exist_ok=True)
@@ -1016,7 +1083,10 @@ def main() -> int:
try:
live = probes.get(src.url, now_ts)
if live is None:
live = probe_live(src, config_path, args.cookies)
print(" listing live items (metadata only; quiet until done)...")
with Heartbeat("listing") as hb:
live = probe_live(src, config_path, args.cookies)
print(f" listed {len(live)} item(s) in {_fmt_elapsed(hb.elapsed)}")
probes.put(src.url, live, stamp)
probes.save()
else:
@@ -1034,12 +1104,16 @@ def main() -> int:
cmd = gdl_command(src, args.staging, config_path, args.cookies,
args.archive_db)
result = subprocess.run(cmd)
with Heartbeat("fetching", watch=stage_dir) as hb:
result = subprocess.run(cmd)
n_new, _ = _count_new(stage_dir, hb.t0)
if result.returncode != 0:
failures += 1
# Keep going: one private or renamed profile must not abort the run.
print(f" FAILED (exit {result.returncode})", file=sys.stderr)
print(f" FAILED (exit {result.returncode}) after "
f"{_fmt_elapsed(hb.elapsed)}", file=sys.stderr)
else:
print(f" done in {_fmt_elapsed(hb.elapsed)}, {n_new} new file(s)")
# Recorded even for an empty fetch: the request was still spent.
state.mark_fetched(src.url, stamp)
state.save()