From 84dd76e11ef38d515679f9481a778403edb7e11f Mon Sep 17 00:00:00 2001 From: ergosteur Date: Tue, 1 Sep 2026 23:21:22 -0400 Subject: [PATCH] feat: capture coauthors and per-item width/height/tagged_users Two additions to the metadata sidecars, following up on the 2026-09-01 collab/dedup investigation already documented here: - coauthors added to the post-level JSON's include list -- a direct, native signal for collab detection instead of inferring it from the filename/directory identity mismatch also documented that day. - width, height, width_original, height_original and tagged_users get their own per-file sidecar (event: "file", one JSON per downloaded media item, named ".json"), since a carousel's items can each have different dimensions and tags -- data the existing post-level JSON has no way to represent. owner is deliberately left out, same reasoning as audio_user's existing exclusion. Verified against a live re-fetch of an already-archived carousel: correct per-item data came back, zero media re-downloaded (skip-archive still applies; only the new sidecars are new files). 3 new tests. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011qAds5qr7nZRq5R4yAuxUk --- docs/gallery-dl.md | 35 +++++++++++++++++++++++++---------- scripts/gdl-sync.py | 27 +++++++++++++++++++++++++++ scripts/test_gdl_sync.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/docs/gallery-dl.md b/docs/gallery-dl.md index 862e58a..b389ee5 100644 --- a/docs/gallery-dl.md +++ b/docs/gallery-dl.md @@ -301,16 +301,31 @@ participant's page shows the post, not just the one whose directory JD2/ gallery-dl happened to land the JSON in. See that repo's `NOTES.md`, 2026-09-01 entries, for the full fix. -**`coauthors` is a native, richer signal for this that we don't currently -capture.** The raw API response carries a `coauthors` array -(`{"full_name", "id", "username"}` per collaborator) that **excludes the -post's own owner** — confirmed on both saved examples, including one where -the owner (`dailyfashion_news`) is an external account with no ARTMS members -in her own name, present only via `coauthors`. It is not in the metadata -sidecar's `include` list today. Adding it would give collab detection a -direct field instead of inferring it from filename/directory identity -mismatches, and it never needs the "is this the owner" branch `coauthors` -already excludes for us. +**`coauthors` is a native, richer signal for this, and is now captured.** +The raw API response carries a `coauthors` array (`{"full_name", "id", +"username"}` per collaborator) that **excludes the post's own owner** — +confirmed on both saved examples, including one where the owner +(`dailyfashion_news`) is an external account with no ARTMS members in her +own name, present only via `coauthors`. Added to the post-level metadata +JSON's `include` list on 2026-09-01: a direct field instead of inferring a +collab from filename/directory identity mismatches, and it never needs the +"is this the owner" branch `coauthors` already excludes for us. + +**Per-carousel-item fields got their own sidecar, added the same day.** +`width`, `height`, `width_original`, `height_original` and `tagged_users` +live on the per-FILE kwdict, not the per-post one the metadata JSON above +reads — a carousel's items can each have different dimensions and tags, +which one post-level JSON can't represent. `gdl-sync.py`'s `media_pp` is a +second `metadata` postprocessor, `event: "file"` (gallery-dl's default when +omitted), so it runs once per downloaded file and writes `.json` +alongside it — e.g. `... - 01.jpg.json` next to `... - 01.jpg`, never +colliding with the post-level `....json`, which has no per-item number. +`owner` — a full user object (profile pic URLs, privacy flags) for whoever +posted that specific item — is deliberately left out, the same reasoning as +`audio_user` above. Verified against the same two saved examples: correct +per-item `width`/`height` and `tagged_users` came back on a live re-fetch of +an already-archived carousel, with zero media re-downloaded (the existing +skip-archive still applies; only the new sidecars are new files). ### The shared archive-db dedups media across profiles too, not just within one diff --git a/scripts/gdl-sync.py b/scripts/gdl-sync.py index 6638c38..358080f 100755 --- a/scripts/gdl-sync.py +++ b/scripts/gdl-sync.py @@ -325,6 +325,32 @@ def build_config(rate: str, sleep_request: list[float], "post_shortcode", "post_id", "type", "date", "post_date", "username", "fullname", "owner_id", "description", "count", "likes", "post_url", "sidecar_shortcode", + # A Collab's `username`/`fullname`/`owner_id` above are always the + # original poster's, never the scraped account's -- see + # docs/gallery-dl.md. `coauthors` is the API's own list of every + # OTHER collaborator (it excludes the post's owner), a direct + # signal instead of inferring a collab from identity mismatches. + "coauthors", + ], + } + # Per-carousel-item fields (`width`/`height`/`tagged_users`/`owner`) live + # on the per-FILE kwdict, not the per-post one `meta_pp` reads -- a + # carousel's items can each have different dimensions and tags, which one + # post-level JSON can't represent. `owner` is deliberately left out: it is + # a full user object (profile pic URLs, privacy flags) for whoever posted + # that specific item, the same reason `audio_user` is excluded above. + media_pp = { + "name": "metadata", + "mode": "json", + # event defaults to "file" when omitted -- runs once per downloaded + # file, appending ".json" to that file's own full name, e.g. + # "... - 01.jpg" gets "... - 01.jpg.json" alongside it. Never + # collides with the post-level "....json" above, which has no + # per-item number. + "include": [ + "media_id", "shortcode", "num", + "width", "height", "width_original", "height_original", + "tagged_users", ], } @@ -363,6 +389,7 @@ def build_config(rate: str, sleep_request: list[float], "postprocessors": [ {**caption_pp, "filename": stem + ".txt"}, {**meta_pp, "filename": stem + ".json"}, + media_pp, ], } diff --git a/scripts/test_gdl_sync.py b/scripts/test_gdl_sync.py index 7dec9cb..5534eda 100644 --- a/scripts/test_gdl_sync.py +++ b/scripts/test_gdl_sync.py @@ -232,6 +232,37 @@ class PostUrl(unittest.TestCase): self.assertNotIn(str(staging / subcategory), cmd) +class MetadataFields(unittest.TestCase): + """Extra fields captured from the raw API response, verified against two + saved real examples on 2026-09-01 (see docs/gallery-dl.md).""" + + def test_post_level_json_captures_coauthors(self): + config = gdl.build_config("1M", [6.0, 10.0], [3.0, 6.0]) + pps = config["extractor"]["instagram"]["posts"]["postprocessors"] + post_json = next(pp for pp in pps if pp.get("filename", "").endswith(".json")) + self.assertIn("coauthors", post_json["include"]) + + def test_per_item_dimensions_and_tags_get_their_own_sidecar(self): + config = gdl.build_config("1M", [6.0, 10.0], [3.0, 6.0]) + pps = config["extractor"]["instagram"]["posts"]["postprocessors"] + # No "filename" of its own -- unlike the other two, which are keyed + # by POST_STEM -- because it must vary per carousel item, not per post. + media_pp = next(pp for pp in pps if "filename" not in pp) + for field in ("width", "height", "width_original", "height_original", + "tagged_users", "shortcode", "num"): + self.assertIn(field, media_pp["include"]) + # `owner` is a full user object (profile pic URLs, privacy flags) for + # whoever posted that item -- deliberately excluded, same reasoning + # as `audio_user` on the post-level json. + self.assertNotIn("owner", media_pp["include"]) + + def test_reels_get_the_same_capture_as_posts(self): + config = gdl.build_config("1M", [6.0, 10.0], [3.0, 6.0]) + posts_pps = config["extractor"]["instagram"]["posts"]["postprocessors"] + reels_pps = config["extractor"]["instagram"]["reels"]["postprocessors"] + self.assertEqual(posts_pps, reels_pps) + + class UrlsFile(unittest.TestCase): def test_reads_every_form_a_person_might_paste(self): with tempfile.TemporaryDirectory() as d: