feat: guard against a dead Instagram session, show progress, fix missing pings
- check-session.py: preflight in gdl-cron.sh. No unexpired sessionid in the Chrome profile gdl reads -> abort before any request, ping /76 (red check). - stories-streak.py: postflight. Every fetched stories source empty for 3 consecutive days -> exit 77, for a session Instagram stopped honouring while the cookie is still present. The 2026-10-07 session death went unnoticed for a week because every run still exited 0. - gdl-sync.py: [n/N] source headers, a 60s heartbeat with new-file counts, and per-source timing, so a healthy paced run no longer looks hung. - gdl-sync@.service now runs run-%i.sh instead of gdl-cron.sh: HC_URL is set in the wrappers, so scheduled runs had never pinged Healthchecks. - Tests for both guards (offline) and the progress helpers.
This commit is contained in:
+79
-5
@@ -94,6 +94,7 @@ import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
@@ -460,6 +461,71 @@ def build_config(rate: str, sleep_request: list[float],
|
||||
# Planning and execution
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _fmt_elapsed(sec: float) -> str:
|
||||
sec = int(sec)
|
||||
h, rest = divmod(sec, 3600)
|
||||
m, s = divmod(rest, 60)
|
||||
return f"{h}h{m:02d}m" if h else (f"{m}m{s:02d}s" if m else f"{s}s")
|
||||
|
||||
|
||||
def _count_new(root: Path, since: float) -> tuple[int, str | None]:
|
||||
"""Media files under `root` written since `since`, and the newest one's name."""
|
||||
n, newest, newest_t = 0, None, 0.0
|
||||
for d, _dirs, files in os.walk(root):
|
||||
for f in files:
|
||||
if f.endswith((".txt", ".json", ".part")):
|
||||
continue
|
||||
try:
|
||||
mt = os.stat(os.path.join(d, f)).st_mtime
|
||||
except OSError:
|
||||
continue
|
||||
if mt >= since:
|
||||
n += 1
|
||||
if mt > newest_t:
|
||||
newest, newest_t = f, mt
|
||||
return n, newest
|
||||
|
||||
|
||||
class Heartbeat:
|
||||
"""
|
||||
Prints a progress line every `every` seconds while a quiet step runs.
|
||||
|
||||
A healthy run looks hung otherwise: pacing is 12-20s between requests, the
|
||||
generated config sets `output: null` (no per-file lines), and the live
|
||||
listing probe captures gallery-dl output until it finishes. With `watch`
|
||||
set, the line also reports how many files have landed there so far.
|
||||
"""
|
||||
|
||||
def __init__(self, label: str, watch: Path | None = None, every: float = 60.0):
|
||||
self.label, self.watch, self.every = label, watch, every
|
||||
self._stop = threading.Event()
|
||||
self._thread = threading.Thread(target=self._run, daemon=True)
|
||||
self.t0 = time.time()
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self._stop.wait(self.every):
|
||||
msg = f" ... {self.label}: {_fmt_elapsed(time.time() - self.t0)} elapsed"
|
||||
if self.watch is not None:
|
||||
n, newest = _count_new(self.watch, self.t0)
|
||||
msg += f", {n} new file(s) so far"
|
||||
if newest:
|
||||
msg += f" (latest: {newest})"
|
||||
print(msg, flush=True)
|
||||
|
||||
def __enter__(self) -> "Heartbeat":
|
||||
self.t0 = time.time()
|
||||
self._thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc) -> None:
|
||||
self._stop.set()
|
||||
self._thread.join()
|
||||
|
||||
@property
|
||||
def elapsed(self) -> float:
|
||||
return time.time() - self.t0
|
||||
|
||||
|
||||
def gdl_command(src: Source, staging: Path, config: Path, cookies: str,
|
||||
archive_db: Path | None) -> list[str]:
|
||||
cmd = [
|
||||
@@ -1003,8 +1069,9 @@ def main() -> int:
|
||||
0.0 if args.force else args.probe_ttl)
|
||||
failures = 0
|
||||
|
||||
for prof, src, seed in plan:
|
||||
print(f"==> {prof.user} / {src.kind}")
|
||||
for n_src, (prof, src, seed) in enumerate(plan, 1):
|
||||
print(f"==> [{n_src}/{len(plan)}] {prof.user} / {src.kind} "
|
||||
f"({dt.datetime.now().strftime('%H:%M:%S')})")
|
||||
stage_dir = args.staging / (src.directory or ".")
|
||||
stage_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -1016,7 +1083,10 @@ def main() -> int:
|
||||
try:
|
||||
live = probes.get(src.url, now_ts)
|
||||
if live is None:
|
||||
live = probe_live(src, config_path, args.cookies)
|
||||
print(" listing live items (metadata only; quiet until done)...")
|
||||
with Heartbeat("listing") as hb:
|
||||
live = probe_live(src, config_path, args.cookies)
|
||||
print(f" listed {len(live)} item(s) in {_fmt_elapsed(hb.elapsed)}")
|
||||
probes.put(src.url, live, stamp)
|
||||
probes.save()
|
||||
else:
|
||||
@@ -1034,12 +1104,16 @@ def main() -> int:
|
||||
|
||||
cmd = gdl_command(src, args.staging, config_path, args.cookies,
|
||||
args.archive_db)
|
||||
result = subprocess.run(cmd)
|
||||
with Heartbeat("fetching", watch=stage_dir) as hb:
|
||||
result = subprocess.run(cmd)
|
||||
n_new, _ = _count_new(stage_dir, hb.t0)
|
||||
if result.returncode != 0:
|
||||
failures += 1
|
||||
# Keep going: one private or renamed profile must not abort the run.
|
||||
print(f" FAILED (exit {result.returncode})", file=sys.stderr)
|
||||
print(f" FAILED (exit {result.returncode}) after "
|
||||
f"{_fmt_elapsed(hb.elapsed)}", file=sys.stderr)
|
||||
else:
|
||||
print(f" done in {_fmt_elapsed(hb.elapsed)}, {n_new} new file(s)")
|
||||
# Recorded even for an empty fetch: the request was still spent.
|
||||
state.mark_fetched(src.url, stamp)
|
||||
state.save()
|
||||
|
||||
Reference in New Issue
Block a user