diff --git a/scripts/gdl-sync.py b/scripts/gdl-sync.py index 0ae6dec..38288a4 100755 --- a/scripts/gdl-sync.py +++ b/scripts/gdl-sync.py @@ -331,6 +331,12 @@ def build_config(rate: str, sleep_request: list[float], "directory": [], "posts": post_like(POST_STEM), "reels": post_like(POST_STEM), + # Ad hoc single-post fetches (--post-url) go through gallery-dl's + # own post/reel extractor instead of a profile listing, so the + # owning account is never known ahead of time -- only mid- + # extraction, same reasoning as highlights below. + "post": {**post_like(POST_STEM), "directory": ["{username}"]}, + "reel": {**post_like(POST_STEM), "directory": ["{username}"]}, "stories": item_like(ITEM_STEM), "highlights": { **item_like(ITEM_STEM), @@ -363,9 +369,11 @@ def gdl_command(src: Source, staging: Path, config: Path, cookies: str, cmd += ["--download-archive", str(archive_db)] # Forced destination -- never `{username}` -- because a reels tab returns # collab reels owned by other accounts, which would otherwise be filed - # under the wrong profile. Highlights are the exception: their directory - # embeds a title only known mid-extraction, so the config formats it. - dest = staging if src.subcategory == "highlights" else staging / src.directory + # under the wrong profile. Highlights and ad hoc single-post fetches are + # the exception: their directory is only known mid-extraction (a title, or + # the post's owner), so the config formats it instead. + dest = (staging if src.subcategory in ("highlights", "post", "reel") + else staging / src.directory) cmd += ["--destination", str(dest)] cmd.append(src.url) return cmd @@ -639,6 +647,54 @@ def publish(staging: Path, dest: str, dry_run: bool) -> int: return subprocess.run(cmd).returncode +def run_post_urls(args) -> int: + """ + Fetch one or more individual posts/reels by URL -- an ad hoc pull outside + the tracked profile list, e.g. a link shared from some other account. Each + is a single request, not a recurring surface, so there is no archive-db + seeding and no --min-interval floor to plan around. + """ + config = build_config(args.rate, list(args.sleep_request), + list(args.sleep), abort=0) + args.staging.mkdir(parents=True, exist_ok=True) + config_path = args.staging.parent / f"{args.staging.name}.gdl-config.json" + + sources = [Source("post", url, "", "post") for url in args.post_url] + + print(f"post-url : {len(sources)} to fetch") + print(f"pacing : {args.sleep_request[0]}-{args.sleep_request[1]}s between " + f"requests, rate cap {args.rate}") + print(f"staging : {args.staging}") + print(f"publish : {args.publish}") + print() + + if not args.execute: + for src in sources: + print(f" {src.url}") + print() + print(" " + " ".join(rsync_command(args.staging, args.publish, True))) + print("\ndry run; nothing fetched. pass --execute to run.") + return 0 + + config_path.write_text(json.dumps(config, indent=2)) + failures = 0 + for src in sources: + print(f"==> {src.url}") + cmd = gdl_command(src, args.staging, config_path, args.cookies, + args.archive_db) + result = subprocess.run(cmd) + if result.returncode != 0: + failures += 1 + print(f" FAILED (exit {result.returncode})", file=sys.stderr) + + print("\n==> publish") + if publish(args.staging, args.publish, dry_run=False) != 0: + failures += 1 + + print(f"\ndone; {failures} step(s) failed") + return 1 if failures else 0 + + def main() -> int: # A sync runs for hours and is normally watched through a redirected log, # where Python's block buffering would withhold progress until it happened @@ -649,10 +705,10 @@ def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--index", required=True, + ap.add_argument("--index", help="existing archive listing: a local root, or the " "viewer's base URL (only a FILE LISTING is needed, " - "never the contents)") + "never the contents). Required unless --post-url") ap.add_argument("--publish", required=True, help="rsync destination for fetched files; a local path or " "user@host:/path") @@ -665,6 +721,12 @@ def main() -> int: g.add_argument("--urls-file", type=Path, help="file of Instagram profile URLs or usernames, one per " "line; # comments and blank lines allowed") + g.add_argument("--post-url", action="append", default=[], + help="fetch one post or reel by URL (e.g. " + "https://www.instagram.com/p/SHORTCODE/), filed under " + "its owner's account like any other post; repeatable. " + "A one-off fetch outside the tracked profile list: no " + "archive-db seeding, no --min-interval floor") ap.add_argument("--cookies", default="chrome:/home/matt/.config/google-chrome-devtools", help="gallery-dl --cookies-from-browser value") ap.add_argument("--archive-db", type=Path, default=None, @@ -709,6 +771,13 @@ def main() -> int: print("rsync not on PATH", file=sys.stderr) return 2 + if args.post_url: + return run_post_urls(args) + + if not args.index: + print("--index is required unless --post-url is given", file=sys.stderr) + return 2 + index = ArchiveIndex(args.index) names = index.profiles() if args.urls_file: diff --git a/scripts/test_gdl_sync.py b/scripts/test_gdl_sync.py index 7b412c5..4e251fe 100644 --- a/scripts/test_gdl_sync.py +++ b/scripts/test_gdl_sync.py @@ -209,6 +209,29 @@ class Publishing(unittest.TestCase): self.assertIn("--dry-run", cmd) +class PostUrl(unittest.TestCase): + """--post-url: a one-off fetch outside the tracked profile list, whose + owning account is only known mid-extraction -- same reasoning as + highlights, so it must be exempted from the same forced-destination rule.""" + + def test_config_keys_a_username_directory(self): + config = gdl.build_config("1M", [6.0, 10.0], [3.0, 6.0]) + for kind in ("post", "reel"): + self.assertEqual( + config["extractor"]["instagram"][kind]["directory"], + ["{username}"]) + + def test_destination_is_not_forced_like_posts_and_reels(self): + staging = Path("/stage") + for subcategory in ("post", "reel"): + src = gdl.Source(subcategory, "https://www.instagram.com/p/ABC/", + "", subcategory) + cmd = gdl.gdl_command(src, staging, Path("/cfg.json"), "chrome:x", + None) + self.assertIn(str(staging), cmd) + self.assertNotIn(str(staging / subcategory), cmd) + + class UrlsFile(unittest.TestCase): def test_reads_every_form_a_person_might_paste(self): with tempfile.TemporaryDirectory() as d: