diff --git a/CLAUDE.md b/CLAUDE.md index c5fd10d..c52ad24 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -64,6 +64,18 @@ story highlights - 4utumn07 - Sunstory -> highlight "Sunstory" Results are cached to IndexedDB. Media records store a stable `path`; **`url` is not persistable** for local archives because blob URLs die with the document. +Three different JSON shapes turn up as `.json`, so they are told apart structurally, not by filename (`src/lib/gallery-dl-sidecar.ts`): + +| shape | marker | +|---|---| +| Instagram export manifest | top-level `media` array | +| Instaloader `.json.xz` | GraphQL node under `node` / `__typename` | +| gallery-dl sidecar | flat, `post_shortcode` + `type`, none of the above | + +The gallery-dl sidecar is the only source that states what a post *is*: its `type` (`post` / `reel` / `story` / `highlight`) is Instagram's own classification, so `post.isReel` set from it beats every fallback in `post-tabs.ts`. This matters — of the 781 items in `official_band - reels`, the sidecars say only **360 are reels**; the other 421 are ordinary feed videos the clips endpoint returns via `include_feed_video`. Directory-based classification counted all 781. + +**Dates: a real date always beats an mtime.** Only JDownloader highlights lack a date in the filename, and `parseArchiveFilename` marks those with `dateFromMtime` so the scanner can upgrade them when the same item also appears under a dated name. Without it the date depended on which file the scan happened to reach first. + ### Cache and local-archive persistence (`src/lib/archive-cache.ts`) IndexedDB keys are namespaced (`archive:`, `thumb:`, `handle:`) so listing archives does not deserialize every cached thumbnail blob, and thumbnails are scoped per archive to avoid cross-archive collisions. diff --git a/src/hooks/useArchiveScanner.ts b/src/hooks/useArchiveScanner.ts index 8f61b79..aa5a3e2 100644 --- a/src/hooks/useArchiveScanner.ts +++ b/src/hooks/useArchiveScanner.ts @@ -4,6 +4,7 @@ import { XzReadableStream } from 'xz-decompress'; import { ArchiveFile, CacheData, Post, ServerArchive } from '../types'; import { setCachedArchive, getDirectoryHandle } from '../lib/archive-cache'; import { parseArchiveFilename, scopedPostId, EXPORT_RE, INSTALOADER_RE } from '../lib/archive-patterns'; +import { isGalleryDlSidecar, sidecarDate, sidecarIsReel } from '../lib/gallery-dl-sidecar'; const hasDirectoryHandle = async (name: string) => Boolean(await getDirectoryHandle(name)); @@ -142,6 +143,8 @@ export const useArchiveScanner = ( try { const postsMap = new Map>(); + // Posts whose date is only a file mtime, so a real date can replace it. + const weakDates = new Set(); const mediaFilesMap = new Map(); const discoveredProfilePics: { name: string, url: string }[] = []; const allImageFiles: ArchiveFile[] = []; @@ -297,16 +300,38 @@ export const useArchiveScanner = ( if (!post) { post = { id: postId, date, username: user, caption: '', media: [], isStory, source: kind, highlightTitle: file.source?.title }; postsMap.set(postId, post); + if (parsed.dateFromMtime) weakDates.add(postId); } else if (isStory) post.isStory = true; + // A JDownloader highlight filename carries no date, so it is dated + // by mtime — but the same item is usually also present under a + // gallery-dl name that does carry one. Let the real date win + // regardless of which file the scan happened to reach first. + if (!parsed.dateFromMtime && date && weakDates.has(postId)) { + post.date = date; + weakDates.delete(postId); + } + const lowerExt = ext.toLowerCase(); if (lowerExt === 'txt') { try { post.caption = await file.text(); } catch(e) {} } else if (lowerExt === 'json' || lowerName.endsWith('.json.xz')) { try { const data = lowerName.endsWith('.xz') ? await parseXZFile(file) : JSON.parse(await file.text()); - if (data) { + if (isGalleryDlSidecar(data)) { + // The only format that states what a post is rather than + // leaving it to be inferred from filenames. + if (data.description) post.caption = data.description; + const reel = sidecarIsReel(data); + if (reel !== undefined) post.isReel = reel; + if (data.type === 'story') post.isStory = true; + const day = sidecarDate(data); + if (day && weakDates.has(postId)) { + post.date = day; + weakDates.delete(postId); + } + } else if (data) { const node = data.node || data; const iphone = node.iphone_struct || {}; const captionText = node.edge_media_to_caption?.edges?.[0]?.node?.text || node.caption?.text || iphone.caption?.text || ''; if (captionText) post.caption = captionText; diff --git a/src/lib/archive-patterns.test.ts b/src/lib/archive-patterns.test.ts index 9b58466..47ca1c0 100644 --- a/src/lib/archive-patterns.test.ts +++ b/src/lib/archive-patterns.test.ts @@ -10,6 +10,7 @@ describe('parseArchiveFilename — Instagram export format', () => { index: 1, ext: 'mp4', isStory: false, + dateFromMtime: false, }); }); @@ -156,3 +157,35 @@ describe('gallery-dl / JDownloader naming interop', () => { expect(dated.date).toBe('2024-08-04'); }); }); + +/** + * Highlights are the only files with no date in the name, so they fall back to + * mtime — which is when the file was written, not when it was posted. Callers + * need to know the difference to let a real date win. + */ +describe('dateFromMtime', () => { + const mtime = Date.parse('2026-08-17T00:00:00Z'); + + it('flags an undated highlight name as mtime-dated', () => { + const p = parseArchiveFilename('4utumn07 - C-IImhvpFuk.jpg', 'highlight', mtime)!; + expect(p.date).toBe('2026-08-17'); + expect(p.dateFromMtime).toBe(true); + }); + + it('does not flag a highlight that carries its own date', () => { + const p = parseArchiveFilename('2024-08-04_4utumn07 - C-IImhvpFuk.jpg', 'highlight', mtime)!; + expect(p.date).toBe('2024-08-04'); + expect(p.dateFromMtime).toBe(false); + }); + + it('never flags ordinary post or Instaloader names', () => { + expect(parseArchiveFilename('2023-04-19_u - ABC.mp4', 'posts', mtime)!.dateFromMtime).toBe(false); + expect(parseArchiveFilename('2024-01-01_12-00-00_UTC.jpg', 'posts', mtime)!.dateFromMtime).toBe(false); + }); + + it('leaves the date empty rather than guessing when no mtime is given', () => { + const p = parseArchiveFilename('4utumn07 - C-IImhvpFuk.jpg', 'highlight')!; + expect(p.date).toBe(''); + expect(p.dateFromMtime).toBe(false); + }); +}); diff --git a/src/lib/archive-patterns.ts b/src/lib/archive-patterns.ts index 0f65c8c..b6ddb98 100644 --- a/src/lib/archive-patterns.ts +++ b/src/lib/archive-patterns.ts @@ -30,6 +30,15 @@ export interface ParsedFilename { index: number; ext: string; isStory: boolean; + /** + * True when `date` is the file's mtime rather than anything Instagram said. + * + * Only highlights fetched by JDownloader lack a date in the filename, and + * their mtime is just when the file was written. Callers should let any real + * date win over this one — the same item is often also present under a + * gallery-dl name that does carry the date. + */ + dateFromMtime: boolean; } /** @@ -54,6 +63,7 @@ export const parseArchiveFilename = ( index: indexStr ? parseInt(indexStr, 10) : 1, ext, isStory: Boolean(story), + dateFromMtime: false, }; } @@ -67,6 +77,7 @@ export const parseArchiveFilename = ( index: indexStr ? parseInt(indexStr, 10) : 1, ext, isStory: Boolean(story), + dateFromMtime: false, }; } @@ -81,6 +92,7 @@ export const parseArchiveFilename = ( index: 1, ext, isStory: false, + dateFromMtime: Boolean(mtime), }; } } diff --git a/src/lib/gallery-dl-sidecar.test.ts b/src/lib/gallery-dl-sidecar.test.ts new file mode 100644 index 0000000..84c7e9a --- /dev/null +++ b/src/lib/gallery-dl-sidecar.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from 'vitest'; +import { + GalleryDlSidecar, isGalleryDlSidecar, sidecarDate, sidecarIsReel, sidecarSource, +} from './gallery-dl-sidecar'; + +// Trimmed from real files published to the archive on 2026-08-16. +const REEL: GalleryDlSidecar = { + post_shortcode: 'Db-lNCoib9m', post_id: '3962768346034323302', type: 'reel', + date: '2026-08-13 11:00:44', post_date: '2026-08-13 11:00:44', + username: 'official_band', fullname: 'Official ARTMS', + description: 'Dancing in the spotlight', count: 1, likes: 22914, +}; +const FEED_VIDEO: GalleryDlSidecar = { ...REEL, post_shortcode: 'DbdG9L9jU4m', type: 'post', count: 2 }; +const HIGHLIGHT: GalleryDlSidecar = { + post_shortcode: 'BATVdRZi_3', post_id: '18099435932626935', type: 'highlight', + date: '2026-08-08 16:22:09', username: 'official_band', count: 154, +}; + +describe('isGalleryDlSidecar', () => { + it('accepts a real sidecar', () => { + expect(isGalleryDlSidecar(REEL)).toBe(true); + expect(isGalleryDlSidecar(HIGHLIGHT)).toBe(true); + }); + + it('rejects an Instaloader GraphQL payload', () => { + expect(isGalleryDlSidecar({ node: { __typename: 'GraphVideo', shortcode: 'x' } })).toBe(false); + expect(isGalleryDlSidecar({ __typename: 'GraphImage', post_shortcode: 'x', type: 'post' })).toBe(false); + }); + + it('rejects an Instagram export manifest', () => { + expect(isGalleryDlSidecar({ media: [{ uri: 'a.jpg' }] })).toBe(false); + expect(isGalleryDlSidecar([{ media: [] }])).toBe(false); + }); + + it('rejects junk', () => { + for (const v of [null, undefined, 0, '', 'string', {}, { post_shortcode: 'x' }]) { + expect(isGalleryDlSidecar(v)).toBe(false); + } + }); +}); + +describe('sidecarDate', () => { + it('takes the day from the timestamp', () => { + expect(sidecarDate(REEL)).toBe('2026-08-13'); + }); + + it('falls back to post_date', () => { + expect(sidecarDate({ post_shortcode: 'x', post_date: '2024-01-02 03:04:05' })).toBe('2024-01-02'); + }); + + it('returns empty when there is no usable date', () => { + expect(sidecarDate({ post_shortcode: 'x' })).toBe(''); + expect(sidecarDate({ post_shortcode: 'x', date: 'not a date' })).toBe(''); + }); +}); + +describe('sidecarIsReel', () => { + it('distinguishes a reel from an ordinary feed video', () => { + // Both are single mp4s -- the lone-video heuristic cannot tell them apart. + expect(sidecarIsReel(REEL)).toBe(true); + expect(sidecarIsReel(FEED_VIDEO)).toBe(false); + }); + + it('declines to answer for stories and highlights', () => { + expect(sidecarIsReel(HIGHLIGHT)).toBeUndefined(); + expect(sidecarIsReel({ post_shortcode: 'x', type: 'story' as const })).toBeUndefined(); + expect(sidecarIsReel({ post_shortcode: 'x' })).toBeUndefined(); + }); +}); + +describe('sidecarSource', () => { + it('maps type onto the archive source kinds', () => { + expect(sidecarSource(REEL)).toBe('reels'); + expect(sidecarSource(FEED_VIDEO)).toBe('posts'); + expect(sidecarSource(HIGHLIGHT)).toBe('highlight'); + expect(sidecarSource({ post_shortcode: 'x', type: 'story' as const })).toBe('stories'); + }); + + it('is undefined for an unknown type', () => { + expect(sidecarSource({ post_shortcode: 'x' })).toBeUndefined(); + }); +}); diff --git a/src/lib/gallery-dl-sidecar.ts b/src/lib/gallery-dl-sidecar.ts new file mode 100644 index 0000000..c1e27a5 --- /dev/null +++ b/src/lib/gallery-dl-sidecar.ts @@ -0,0 +1,87 @@ +import { SourceKind } from '../types'; + +/** + * gallery-dl `.json` metadata sidecars. + * + * Written one per post next to the media (see docs/gallery-dl.md). This is the + * only source in any archive format that states outright what a post *is* — + * `type` is Instagram's own classification, the `product_type: "clips"` signal + * carried through the listing response. Everything else the viewer knows about + * reels is guesswork from filenames and directory names. + * + * Deliberately separate from the two older JSON shapes the scanner reads: + * + * Instagram export `posts_1.json`, an array of entries with `media` + * Instaloader `.json.xz`, a GraphQL node under `node` + * gallery-dl this — flat, no wrapper + */ +export interface GalleryDlSidecar { + post_shortcode: string; + post_id?: string; + /** Instagram's own classification of the post. */ + type?: 'post' | 'reel' | 'story' | 'highlight'; + /** Local-time "YYYY-MM-DD HH:MM:SS" — gallery-dl is configured to emit local. */ + date?: string; + post_date?: string; + username?: string; + fullname?: string; + description?: string; + count?: number; + likes?: number; + post_url?: string; +} + +/** + * Recognise a gallery-dl sidecar. + * + * Checked structurally rather than by filename, because the older formats are + * also plain `.json`. `node` and `__typename` are what an Instaloader or + * export payload carries, and their absence is what makes this shape + * unambiguous. + */ +export const isGalleryDlSidecar = (data: unknown): data is GalleryDlSidecar => { + if (!data || typeof data !== 'object' || Array.isArray(data)) return false; + const o = data as Record; + return typeof o.post_shortcode === 'string' + && typeof o.type === 'string' + && o.node === undefined + && o.__typename === undefined + && o.media === undefined; +}; + +/** The ISO date (YYYY-MM-DD) a sidecar reports, or '' if it carries none. */ +export const sidecarDate = (s: GalleryDlSidecar): string => { + const raw = s.date || s.post_date || ''; + const day = raw.slice(0, 10); + return /^\d{4}-\d{2}-\d{2}$/.test(day) ? day : ''; +}; + +/** + * Whether the sidecar says this post is a reel. + * + * Returns undefined rather than false for stories and highlights: those are + * neither reels nor grid posts, and answering "no" would let them be counted + * as ordinary posts. + */ +export const sidecarIsReel = (s: GalleryDlSidecar): boolean | undefined => { + if (s.type === 'reel') return true; + if (s.type === 'post') return false; + return undefined; +}; + +/** + * Which source kind the sidecar implies, for cross-checking the directory. + * + * A reel shared to the profile grid legitimately appears under `posts`, so a + * disagreement is not an error — the directory says where the file was + * fetched from, `type` says what Instagram considers it. + */ +export const sidecarSource = (s: GalleryDlSidecar): SourceKind | undefined => { + switch (s.type) { + case 'reel': return 'reels'; + case 'post': return 'posts'; + case 'story': return 'stories'; + case 'highlight': return 'highlight'; + default: return undefined; + } +}; diff --git a/src/lib/post-tabs.test.ts b/src/lib/post-tabs.test.ts index 9f0a780..7253d50 100644 --- a/src/lib/post-tabs.test.ts +++ b/src/lib/post-tabs.test.ts @@ -98,3 +98,44 @@ describe('postsForTab', () => { expect(postsForTab([post('A')], 'saved')).toEqual([]); }); }); + +/** + * Once an archive carries gallery-dl sidecars, the guesswork above is replaced + * by Instagram's own classification. These are the cases the heuristic got + * wrong (see docs/gallery-dl.md). + */ +describe('explicit isReel from a sidecar', () => { + it('beats the lone-video heuristic for an ordinary feed video', () => { + // A single mp4 that Instagram calls a post, not a reel — indistinguishable + // by shape alone. + const posts = [video('DbdG9L9jU4m', { isReel: false })]; + expect(postsForTab(posts, 'reels')).toEqual([]); + expect(postsForTab(posts, 'posts')).toHaveLength(1); + }); + + it('recognises a reel that lives in the profile grid', () => { + // Shared to feed, so it sits in the base directory with source 'posts'. + const posts = [post('A'), video('C8FHM6EJl15', { source: 'posts', isReel: true })]; + expect(postsForTab(posts, 'reels').map(p => p.id)).toEqual(['C8FHM6EJl15']); + expect(postsForTab(posts, 'posts')).toHaveLength(2); + }); + + it('beats the directory when both are present', () => { + const posts = [ + video('u - reels/A', { source: 'reels', isReel: false }), + video('u - reels/B', { source: 'reels' }), + ]; + // A is a feed video that the reels tab happened to return; B is unlabelled + // and falls back to its directory. + expect(postsForTab(posts, 'reels').map(p => p.id)).toEqual(['u - reels/B']); + }); + + it('falls back per post, so a mixed archive still works', () => { + const posts = [ + video('labelled', { isReel: true }), + video('unlabelled'), + carousel('C'), + ]; + expect(postsForTab(posts, 'reels').map(p => p.id)).toEqual(['labelled', 'unlabelled']); + }); +}); diff --git a/src/lib/post-tabs.ts b/src/lib/post-tabs.ts index 013c740..5c9da41 100644 --- a/src/lib/post-tabs.ts +++ b/src/lib/post-tabs.ts @@ -41,11 +41,16 @@ export const hasReelSource = (posts: Post[]): boolean => posts.some(p => p.sourc * or an old IGTV upload, all three of which are plain `GraphVideo` nodes * distinguished only by `product_type`. */ -export const makeIsReel = (posts: Post[]): ((post: Post) => boolean) => ( - hasReelSource(posts) +export const makeIsReel = (posts: Post[]): ((post: Post) => boolean) => { + const guess = hasReelSource(posts) ? (post: Post) => post.source === 'reels' - : (post: Post) => post.media.length === 1 && post.media[0]?.type === 'video' -); + : (post: Post) => post.media.length === 1 && post.media[0]?.type === 'video'; + + // `isReel` comes from a gallery-dl sidecar and is Instagram's own answer, so + // it beats both fallbacks — per post, since an archive is usually a mix of + // files fetched before and after sidecars existed. + return (post: Post) => post.isReel ?? guess(post); +}; /** Preference order when the same post was fetched into more than one directory. */ const SOURCE_RANK: Record = { reels: 0, posts: 1, stories: 2, highlight: 3 }; diff --git a/src/types/index.ts b/src/types/index.ts index b49211e..000324a 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -37,6 +37,12 @@ export interface Post { isStory?: boolean; /** Defaults to 'posts' for archives without sidecar directories. */ source?: SourceKind; + /** + * Instagram's own answer to "is this a reel", from a gallery-dl `.json` + * sidecar. Undefined when the archive carries no such sidecar, which is when + * the viewer has to fall back to guessing — see src/lib/post-tabs.ts. + */ + isReel?: boolean; /** Highlight this post belongs to, for source === 'highlight'. */ highlightTitle?: string; }