fix: rank date sources instead of letting scan order decide
The previous commit had this backwards: the sidecar date was only consulted when the existing date came from an mtime, so a filename date silently outranked what Instagram itself reported. The order is sidecar, then filename, then mtime -- metadata first, mtime last, since mtime is when the file hit disk and says nothing about when the post was made. Ties keep the incumbent so two equally authoritative files cannot flip a post's date by scan order. Extracted to src/lib/post-dates.ts rather than left inline, because the rule is easy to state and easy to get wrong -- the tests include an order-independence case that would have caught the original mistake. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -74,7 +74,7 @@ Three different JSON shapes turn up as `.json`, so they are told apart structura
|
||||
|
||||
The gallery-dl sidecar is the only source that states what a post *is*: its `type` (`post` / `reel` / `story` / `highlight`) is Instagram's own classification, so `post.isReel` set from it beats every fallback in `post-tabs.ts`. This matters — of the 781 items in `official_band - reels`, the sidecars say only **360 are reels**; the other 421 are ordinary feed videos the clips endpoint returns via `include_feed_video`. Directory-based classification counted all 781.
|
||||
|
||||
**Dates: a real date always beats an mtime.** Only JDownloader highlights lack a date in the filename, and `parseArchiveFilename` marks those with `dateFromMtime` so the scanner can upgrade them when the same item also appears under a dated name. Without it the date depended on which file the scan happened to reach first.
|
||||
**Dates are ranked, not last-write-wins** (`src/lib/post-dates.ts`): sidecar (what Instagram reported) beats filename (what the fetcher wrote) beats mtime (when the file hit disk, and unrelated to when it was posted). Ties keep the incumbent. Several files describe one post and they are scanned in directory order, not in order of trustworthiness, so without the ranking the date was decided by whichever file came first. Only JDownloader highlights fall to mtime at all — `parseArchiveFilename` flags those via `dateFromMtime`.
|
||||
|
||||
### Cache and local-archive persistence (`src/lib/archive-cache.ts`)
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import { ArchiveFile, CacheData, Post, ServerArchive } from '../types';
|
||||
import { setCachedArchive, getDirectoryHandle } from '../lib/archive-cache';
|
||||
import { parseArchiveFilename, scopedPostId, EXPORT_RE, INSTALOADER_RE } from '../lib/archive-patterns';
|
||||
import { isGalleryDlSidecar, sidecarDate, sidecarIsReel } from '../lib/gallery-dl-sidecar';
|
||||
import { DateSource, shouldReplaceDate } from '../lib/post-dates';
|
||||
|
||||
const hasDirectoryHandle = async (name: string) => Boolean(await getDirectoryHandle(name));
|
||||
|
||||
@@ -143,8 +144,17 @@ export const useArchiveScanner = (
|
||||
|
||||
try {
|
||||
const postsMap = new Map<string, Partial<Post>>();
|
||||
// Posts whose date is only a file mtime, so a real date can replace it.
|
||||
const weakDates = new Set<string>();
|
||||
/**
|
||||
* Which source supplied each post's date, so a better one can replace it.
|
||||
* Sidecar beats filename beats mtime — see src/lib/post-dates.ts.
|
||||
*/
|
||||
const dateSources = new Map<string, DateSource>();
|
||||
const applyDate = (postId: string, post: Partial<Post>, date: string, source: DateSource) => {
|
||||
const current = post.date ? { date: post.date, source: dateSources.get(postId) ?? 'mtime' } : undefined;
|
||||
if (!shouldReplaceDate(current, { date, source })) return;
|
||||
post.date = date;
|
||||
dateSources.set(postId, source);
|
||||
};
|
||||
const mediaFilesMap = new Map<string, ArchiveFile>();
|
||||
const discoveredProfilePics: { name: string, url: string }[] = [];
|
||||
const allImageFiles: ArchiveFile[] = [];
|
||||
@@ -300,18 +310,13 @@ export const useArchiveScanner = (
|
||||
if (!post) {
|
||||
post = { id: postId, date, username: user, caption: '', media: [], isStory, source: kind, highlightTitle: file.source?.title };
|
||||
postsMap.set(postId, post);
|
||||
if (parsed.dateFromMtime) weakDates.add(postId);
|
||||
}
|
||||
else if (isStory) post.isStory = true;
|
||||
|
||||
// A JDownloader highlight filename carries no date, so it is dated
|
||||
// by mtime — but the same item is usually also present under a
|
||||
// gallery-dl name that does carry one. Let the real date win
|
||||
// regardless of which file the scan happened to reach first.
|
||||
if (!parsed.dateFromMtime && date && weakDates.has(postId)) {
|
||||
post.date = date;
|
||||
weakDates.delete(postId);
|
||||
}
|
||||
// Files describing one post are scanned in directory order, not in
|
||||
// order of trustworthiness, so every date goes through the ranking
|
||||
// in post-dates.ts rather than last-write-wins.
|
||||
applyDate(postId, post, date, parsed.dateFromMtime ? 'mtime' : 'filename');
|
||||
|
||||
const lowerExt = ext.toLowerCase();
|
||||
if (lowerExt === 'txt') {
|
||||
@@ -326,11 +331,7 @@ export const useArchiveScanner = (
|
||||
const reel = sidecarIsReel(data);
|
||||
if (reel !== undefined) post.isReel = reel;
|
||||
if (data.type === 'story') post.isStory = true;
|
||||
const day = sidecarDate(data);
|
||||
if (day && weakDates.has(postId)) {
|
||||
post.date = day;
|
||||
weakDates.delete(postId);
|
||||
}
|
||||
applyDate(postId, post, sidecarDate(data), 'sidecar');
|
||||
} else if (data) {
|
||||
const node = data.node || data; const iphone = node.iphone_struct || {};
|
||||
const captionText = node.edge_media_to_caption?.edges?.[0]?.node?.text || node.caption?.text || iphone.caption?.text || '';
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { DatedValue, preferDate, shouldReplaceDate } from './post-dates';
|
||||
|
||||
const sidecar: DatedValue = { date: '2024-04-07', source: 'sidecar' };
|
||||
const filename: DatedValue = { date: '2024-04-08', source: 'filename' };
|
||||
const mtime: DatedValue = { date: '2026-08-17', source: 'mtime' };
|
||||
|
||||
describe('date precedence', () => {
|
||||
it('ranks sidecar above filename above mtime', () => {
|
||||
expect(preferDate(mtime, filename)).toEqual(filename);
|
||||
expect(preferDate(filename, sidecar)).toEqual(sidecar);
|
||||
expect(preferDate(mtime, sidecar)).toEqual(sidecar);
|
||||
});
|
||||
|
||||
it('never lets a weaker source overwrite a stronger one', () => {
|
||||
expect(preferDate(sidecar, filename)).toEqual(sidecar);
|
||||
expect(preferDate(sidecar, mtime)).toEqual(sidecar);
|
||||
expect(preferDate(filename, mtime)).toEqual(filename);
|
||||
});
|
||||
|
||||
it('keeps the incumbent on a tie, so scan order cannot flip the date', () => {
|
||||
const other: DatedValue = { date: '2020-01-01', source: 'filename' };
|
||||
expect(preferDate(filename, other)).toEqual(filename);
|
||||
expect(preferDate(other, filename)).toEqual(other);
|
||||
});
|
||||
|
||||
it('accepts anything when nothing is held yet', () => {
|
||||
expect(preferDate(undefined, mtime)).toEqual(mtime);
|
||||
expect(shouldReplaceDate(undefined, mtime)).toBe(true);
|
||||
});
|
||||
|
||||
it('ignores an empty date regardless of source', () => {
|
||||
const empty: DatedValue = { date: '', source: 'sidecar' };
|
||||
expect(shouldReplaceDate(filename, empty)).toBe(false);
|
||||
expect(preferDate(filename, empty)).toEqual(filename);
|
||||
});
|
||||
|
||||
it('replaces a held-but-empty date', () => {
|
||||
const empty: DatedValue = { date: '', source: 'filename' };
|
||||
expect(preferDate(empty, mtime)).toEqual(mtime);
|
||||
});
|
||||
|
||||
it('is order-independent for the full three-source case', () => {
|
||||
const orders = [
|
||||
[mtime, filename, sidecar],
|
||||
[sidecar, mtime, filename],
|
||||
[filename, sidecar, mtime],
|
||||
[mtime, sidecar, filename],
|
||||
];
|
||||
for (const order of orders) {
|
||||
const won = order.reduce<DatedValue | undefined>(
|
||||
(acc, next) => preferDate(acc, next), undefined);
|
||||
expect(won).toEqual(sidecar);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* Where a post's date came from, and which source wins.
|
||||
*
|
||||
* A post is usually described by several files — media, a caption `.txt`, a
|
||||
* `.json` sidecar, sometimes the same item under two naming conventions — and
|
||||
* they are scanned in directory order, not in order of trustworthiness. Without
|
||||
* an explicit ranking the date is decided by whichever file happened to be
|
||||
* reached first.
|
||||
*
|
||||
* Ranked best to worst:
|
||||
*
|
||||
* sidecar what Instagram reported, straight from a gallery-dl `.json`
|
||||
* filename a date the fetcher wrote into the name; correct, but derived
|
||||
* mtime when the file was written to disk — unrelated to when it was
|
||||
* posted, and only ever a last resort for JDownloader highlights,
|
||||
* whose filenames carry no date at all
|
||||
*/
|
||||
export type DateSource = 'sidecar' | 'filename' | 'mtime';
|
||||
|
||||
const RANK: Record<DateSource, number> = { sidecar: 0, filename: 1, mtime: 2 };
|
||||
|
||||
export interface DatedValue {
|
||||
date: string;
|
||||
source: DateSource;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `next` should replace the date currently held.
|
||||
*
|
||||
* Ties keep the incumbent, so scanning stays stable: two files of equal
|
||||
* authority cannot flip a post's date back and forth by scan order.
|
||||
*/
|
||||
export const shouldReplaceDate = (
|
||||
current: DatedValue | undefined,
|
||||
next: DatedValue,
|
||||
): boolean => {
|
||||
if (!next.date) return false;
|
||||
if (!current || !current.date) return true;
|
||||
return RANK[next.source] < RANK[current.source];
|
||||
};
|
||||
|
||||
/** Apply `next` if it outranks `current`, otherwise keep what we have. */
|
||||
export const preferDate = (
|
||||
current: DatedValue | undefined,
|
||||
next: DatedValue,
|
||||
): DatedValue => (shouldReplaceDate(current, next) ? next : (current ?? next));
|
||||
Reference in New Issue
Block a user