From 4fee8b1dfe53239f25d033873dbf2955ddcacee8 Mon Sep 17 00:00:00 2001 From: ergosteur Date: Sun, 16 Aug 2026 21:10:37 -0400 Subject: [PATCH] feat: seed gallery-dl's skip-archive so the fetcher needs no archive copy gallery-dl skips already-held media either by file existence -- which requires the archive mounted where it writes -- or by a sqlite skip-archive, which requires nothing on disk. Using the latter lets the fetch host write to local disk and rsync afterwards, avoiding tens of thousands of small writes over CIFS and keeping a mid-sync failure from leaving partial files on the live Resilio share. The key is archive_prefix + archive_fmt: the literal "instagram" plus the per-media numeric pk. Verified against a real run -- a 3-image carousel produced 3 rows and a re-run skipped every media file. media_id is absent from our filenames, so the DB cannot be built from names alone, but the listing pass we already make maps every live item to its media_id, and a file listing says which we hold. Seeding therefore costs no extra Instagram requests and no archive content -- the listing GET /api/archives/:name/files already serves is enough. Measured on 0ct0ber19: 2275 live items, 2248 seeded, 27 left to fetch -- exactly the media of the two posts added since the last crawl. The trap worth the comment it carries: posts and reels are filed under post_shortcode, while stories and highlights use the per-item shortcode (post_shortcode there is the containing reel's id, shared by every item). Matching on the wrong field seeded 5 of 2275 rather than failing loudly. Co-Authored-By: Claude Opus 5 --- docs/gallery-dl.md | 47 ++++++++ scripts/__pycache__/gdl-sync.cpython-314.pyc | Bin 0 -> 20665 bytes scripts/gdl-sync.py | 110 +++++++++++++++++-- 3 files changed, 146 insertions(+), 11 deletions(-) create mode 100644 scripts/__pycache__/gdl-sync.cpython-314.pyc diff --git a/docs/gallery-dl.md b/docs/gallery-dl.md index 915b157..57908d7 100644 --- a/docs/gallery-dl.md +++ b/docs/gallery-dl.md @@ -204,6 +204,53 @@ viewer retire the lone-video heuristic in `src/lib/post-tabs.ts` — see `/p//` URL leaves it `null`. Sync always uses listing URLs, so this only matters when testing by hand. +## Incremental sync — why the fetch host needs no copy of the archive + +gallery-dl can skip already-held media two ways, and the difference decides +whether the fetcher needs the archive mounted: + +- **By file existence** (default). Needs the destination to already contain the + files, so it only works if the archive is mounted where gallery-dl writes. +- **By skip-archive** (`--download-archive`). A sqlite DB of ids. Needs nothing + on disk. + +We use the second, so the fetch host can write to **local disk and rsync +afterwards**. That avoids writing tens of thousands of small files over CIFS, +and keeps a mid-sync failure from leaving partial files on the live Resilio +share. + +The key is `archive_prefix + archive_fmt`, which for this extractor is the +literal `instagram` plus the per-media numeric pk (`instagram.py:25`, +`job.py:713-719`). Verified: a 3-image carousel produced + +``` +instagram3079387627521318672 +instagram3079387627521429433 +instagram3079387627529716672 +``` + +and a second run skipped every media file, rewriting only the idempotent +`.txt`/`.json` sidecars. + +**Seeding.** `media_id` is not in our filenames, so the DB cannot be built from +names alone — but one listing pass (the pass we make anyway) maps every live +item to its `media_id`, and the archive's *file listing* says which we already +hold. No extra Instagram requests, and no archive content — a listing is +enough, which `GET /api/archives/:name/files` already serves. + +Measured on `0ct0ber19`: 2275 live media items, 2248 seeded from the existing +listing, **27 left to download** — precisely the media of the two posts added +since the last crawl. + +The one trap, which silently seeds almost nothing if you get it backwards: + +| surface | filed under | why | +|---|---|---| +| posts, reels | `post_shortcode` | carousel children each have their own `shortcode`, which never appears in a filename | +| stories, highlights | `shortcode` (per item) | `post_shortcode` is the containing reel's id, shared by every item | + +`live_key()` encodes this. Matching on the wrong field seeded 5 of 2275. + ## Known quirks - **`count` is not the emitted file count.** For 135 of 214 posts it was exactly diff --git a/scripts/__pycache__/gdl-sync.cpython-314.pyc b/scripts/__pycache__/gdl-sync.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e90de1c24053d0d15742bf6c115e6c20f25578ca GIT binary patch literal 20665 zcmb_^eQ+DswcjqV_y!4(1jP?g;_^chL6M*+$)ZHjmSm9xuENJ@b!8IqvW2!MH41 zgumZx;<#B(;*#7&PT~jnepAv^PjS&MUSJKt*PP$n=!azxXX|lAQyI>vOZhSs}Z{Q@$eqMV{dLNp{n#)u6&^)$Wo^r{a ztk}w3sE{0O`A>04AKo}Oys5n4s^riHR~09@_hT&R+OOTXkV3Dnq=S#cQMr#Cp=%i?9abr+GXZA=ZQ!%G$9U5Daqg!IXEdMqaU>EE!|{;PQBcujm`XY6o0E~CGA*;36!B7IB03coBhTT3 zVrU{B4MoI>h#V3B=j?aHh?0neqG-)E_BkbZIg-34PR7HLSerOA5e-c^qY6usQD=M# zQ9LUlIhsg{k>{gIlDewDod_l(q8vmEsPVxmF(@k0$pm^T#-}3a>B#7@(Q_l+qH;A7 zizMSy0g?J1icejRVoV}a5fqN%BSEwmV=*;FV--YGLvgAmiP1z#GNL3S;ls|S+uI}0 zM?%xd$kQSkg(gQYQ4JAv<_d0Mqw8o%C63 zE5Rt{k##mkGp5fR+I>z5UWs%&MO=Xnt@9ZAaI9UqHWdmau8HmKdS*rJ2}k9}@It2q zaZf0@2hHj{gm+;~IeqGnstA+wrxq>N27Sn!PHB2dqydrRldR7ZaddtLVMI(!U&3gK z1HsAD-$vNfr{G2mM>itFYNYHEU??(oGQ~ZUI21iCk_OvL462-N*z zs%BGqJ}Fel4zS4S;_picXSoYplE>~~y0|}SZpIFg6eQCHi)7x3R2CC*F-tCHWieaQ zD%p|O!JeE+n`CR^{4Uix5}%esk<{_CQ;`D3fMxX@xEy-vy z7I};F+g1DcIE~o&xau4q2OF4mrF1#YUTJi*kdvp8V@Fua`X}wtb(*sq}|6Kr=IS3E+}_Q#3v&iptod4Z#*AgWf3}v^azMGHZV>8kvd6#nz&1LzOI`eM@OZ@G|#X;mloU$=3(64)@pl zv!3#=4`s`ID}pauT{qh|*S#jx$yU~tTU`A8QwR&D|A;pI7cl)aofjVb|RTfDBT?$Gcz**wk$~Y2w;=!P%Xsgl$5Q5-SN!WB%ay5BV`9K zWKq?oPkXBI;nnF`I#eDdHJ05js?I#yPPsbr4=U$3%QVew`Ki`mA`zJi`z@-VL}Hia zdZeiQG{`KdL{u}RA%#YD=T2;g8Y$Tn#tIeHV^ER)f)UVPA@eMkb$f3NTpyU@vt_#% zk1g)Ycsk~U58dVSUFl@{^1{hXUE5-CscOZ2=+1_hOJz&urGtx8x2~+XPpn(a9_O6- z6FcYLx^Cr4yf;o=KlPpGvfi4^wj=NGx0H7pGFyAI<+bU~bklt4g5$o$RK5En&Q#L2 z?k>n)x8Pf_E?muQ-@EkqE%$vhUwwlAh~rC6@^?Mub54z^5ThINJ7o{f|ohQ*WRyhUK(Ak@1E_Sn_3aNKM{;Y zrWK}g{~UfHlcu5yay*`tY0BgV1gY{oYtpC7U=wm9(hGQm+(d;ac#0CXB4C_=_hIex zzO(>N!22={``Dw#3JlhUyd1G~EN;vFAAvyxg;}nHiwhE8g@w5fj19ci9`h#n^SLtT zbBtt|26xld0A^w=O5^V3`uG!QcEa?esm+xyFihXxbQua!j!9HACSe+OY|4m7Fc!U4 zx!mYU9XEK04c^!kuqb5W7uvrF)pDijBO(H z5t6bY2S8JUb{A0s53E^C+vnZm*_ze3G3}&m$!ZLQn2Mh-x<=I|B@~<*Hrm&;P~l;;CBmG%kA@ z7xuqhwdDKpwiQoT#@@yHqw)c{74^z36wrc{Y0mv_)v7?Ph=f&BTv2T#Hes4oYg8Gh zJ0$I(oK$9_$qCgh^^Nxroa{f1zfsk~v@6weg=j=${WLnN6S+o4&q@P*BdQ&7sju(! zh~FYVLZ9rR92Q8&@->y8ln>)k+eWdA9@RQrvZ1f?AhIyNYr%k{k9V7P%=UjGV7Azl zePrdzt6w+0;ry<1VgE{X%UYR#&XV=|{*`;p)4c3yUi0jH*RykR$69Oma%=akeQT{p z-fcbdzT0=lTes$IT=q6DJhI~TXI%cd&i7pY_uZa({-*6!Tblp2^P6Ya+|A4G<~4W6 zvb$r+eJk>wyZ26c-CB9$aygYN_h(A{b5Fci;?M5wnmhhr$%Us@+#PqyYi|Z#4J=fy zl<&%v?4ta;vU?sWn*YgFcl+Jldyr=N=BYJz{j$4$!M5sdxmU@Rx2|vD+`iefdGa(! zP!~4&`!u(INPw0sk|06c>tO5K*chQ!X>xpt5{pC<<8tKLX`mD-7n7lZ*#wgpR0~Vf z7UDj9Mwh9JE<|m|q`BS08uSVG6wh1lv2b2lXZOX!+C5v{v~H&4Y)jic7SZ~LeI|F& zmdkcfD|#HX1g!u$b(H`;LVqN?lmbj(hy-RWjIBC~ezC(yxCBS}TmNDx-lT@nj zxMqU<@P=owAML?{R3?>66_QV?l(tA!W4u%i5xhpKg+RMiszYtt#>@@e`CP+{T}s;i zkoWat0`lt&0AQp(W0Kf`-M?7MabBvE8UlO=f1K}dc9^6_sYz-c<2vw0+JWBgM9d6U z(h__el`;5DhoFEW=_nMm>_Q+5YIo==)4WARgY`6|tQ|lY6x(3T8BdU=t^g4s3`7je z(aTAZ0KfQUkx(L$)<0wLgxdjAdODbhc0&|bk_^3KtRFi>4CoyJF~H5hQ-Xj3NzKk- zYy>W9s2N25#Kf~P=cOQ820SaOC?Gik+E6K@K5@8D5{J)@j50J)!~dR(h9hx?e}#hJ zI*`kCm=EO{beMnw4%7rK4JO4a=m`V-f}v1+8g03R-eDNxQz5`(F+hUhYXPS!5HO(H zCnL!qAwJ+v&w)Rw0;wRXKwzS34NXIRm`s&oq?1T&*YwC#$qV6#2B=~5UQ`{C=aH%) zu9}n2qwq7p995gHsQ&5O;ae7cr2;veWGnUaEkm{O(sVQy9@prj_;&bh&~k1&EVDT8 zaR}1pb-FDG>?zN047w+NgYD3A|0{^}n6%Y625GSnj)z1B<#fXvE8f@&-*lKIdj~dC zP(6>d4FEq|0o;g5Pul?XQ(C~p-~dDYXduKAMbOSV;*15YDHT08x&{Va+n_@TxNuWNUOhkx*?! ztB5GXBX`c~f`Wsp8jnSHCQJE_85kk7-6g2~=70V4#=3DW880@-qYf(m2gxj#flbF=R6t}_3NlLz>95?bZqK^wGW91G zb|F~oUP|7&^6$U0Tz?|7{Um6H$i=La3uCA=Y@z45i)v%j7Y(aI@>(JS`~*Unsx4s_wHsdr zlR^#J9)&MOo73@2zh3#&tg_gboErQqp&~rJ6c90!|Hc?3AeP>k1074e^+gLN@ zAsPu9A^8dgS1E{4z~-Hn#!f=Gj_R_xyO6QS-=K{6Q+|)&_n6}?oWnEk%-TKkTe9}j zd3)Ago$komE9TFwyX@uG*;DJjjXA9AcFs}$J5KO9*FA2Jb@t>Zb;c(2$xbX+wshC4 zIfoBKlMZFwU{#g#fV%1zz=x{Rm053fx*5njBPz&n;LN zLAjYNfvnv-e;|DzQ{K3~uhwl{=V~q1JV()mJI;;a0Bh#&Ytrw-@oYmJ*Th%J)nSET z=hilRvm%EyYXDQ62)KeA!idRJvt^2??J%m=I6s`yXVV zO{o7}{3!A6uu}!hUvJ^0C`H;Hh|-t=qM3rhD5X_vo$-1vS;Utcb9o zXu;q>PJrB=$GP$tkz^V(2P_H8m_UjtU11=IK#E|TD#$Mw@^iq{R)9I795NpCS^tq?hT*q6@u(58Z##&U?&18uStL*LL+lAbnQ#@Fi6V~~LL zQ)k{Z{;9paU7Pn$w@{2smr6SsKHGJhW~v;d1 zR0sHg6{{v|FaOT*wDsGAS-ZHfYuzTeth2{IadP%@N_J{YtLn{s48PDO{_QTiBca5YU)MPxnZ`*gH zq+omb)Jv!4lp8bGXI6!(tgCeP6eGoyE4M+s@M7xlajEavXrDNG?D5lm;=l=U=^dwzu0m<{KEbh+Y#>7L;pBTy+@eI z_e;mxzZh&!?P)(W-hKh^T8?yM>p+p+e*cm4ftK?#unEg>+u&pZGae9x*cuVp=slcW zXkv!O9vDyqsaJl5>TE#psSmi)my*+PHP9?>aN@Y|*^B-jismrwQr>%yLy#U=r(34= zG4_~eamdUr!iv#m%l!mf*ZGxYpvwu!^Iq=6za>mMum2V^M2(q>trLj{7Obyn$<#)6 z8T~TWN&;rE9JlbZHEu43VPDL-Bn{KPzv(jCd!7~ry8vzLrpLh`$BpvEaB85}Gy=+OJV4nUZDtAhe@HDBi zxNDPKvq-sTXI5T8^q3Vo1MXdht( z`i=$p6BEb%!$0BjJ7E3?)ae3bcuYFpKQPwUEB7H4CS3*YFG=|XJy_sD6p>ZO(AlB! zGkv`S#~}HI;VuAT_Y3l$Qx;(&h&DfgsC){6A|S|H=}OrUj2nh%`FluZYk(NuEccO{ zYuxdJ%Gurzyj63;9e355d)u;mTRQy4)z_{rHZ4B8vb}Su@%HvdR@?{X_`mab=a1by z`Rd8^v2UMR^EEH~npb=~-}bNi_5orfMptsfe%(IbaC6tIyV4EcZhbwp(D1`u-`};^ z@PpPLb}V!(@hjW+X3F-xXWxIXge$9tnC$k>4(1memKVE-|AIi^c}RM0QBoa#%k&jAi@|0^ z)4+Lp1};MWBwF9`9lmOx{P z0a;=nNzTOcphP$dS1jmfV(=1yYuY->txl+H*nQv=gJp&_4QK*pm8q~3AU}3+`W!7e zIgY48xCK4}drKrt3z?yAfbQTdcct427cJ5P6wPc-{#v-GLcJ3Y_!F^ZP54O=5TJpq zMS}33)5>lP;Sv}FWQQJ$$4-x&WzJ|(Fad}l;?ueAJe|kp$W^UqO^kt%p*;w8#60n| z!6r5oKnYzfF0j&X=mxP8sKo88DFwQ68zwCjga%dgqfc+toS&w7gqI!$m+B*k53Y1H z;2SjQqE&r{cC9*cqai;Bx>wD>k6`X;L7WL_Fl6;0j#mrxiWaD*Z12>hEIX0o_|?cY z`AsCT^+$>?M6B(c-JS8YtlC;&?6_%v)t=tE;@z?EY{uiy*!|>Y(0{!@Q@LZsy>n6c zsr@bcQss|58P~y#aPUFqLJ}ZjE+h|r7IVp??O0AOjmen8mo(FQ9V(CsUQz{W`ln8& zeUPdB!Y1}3zfM{IfHjX95w;M^A?8IYNi|=2j{>r|IZ1PHm?^3~6q8lph6HDuE}gMKS{I7RE_u@*G!siPNg;1L zH&`6%GI_ak)@f6Kn>L*Qg(R%!b2`qGAww1#`zbE5{jJMFk3>=OjP>?h= z8N3>C2B)sg1h26?#34mZV*uNfuTcd|A__tjdwMbwjs}6&JPmK)X%wZ(yZ~PX%KIp4vLNrLY}BD+;j-{SJz)dRJOE30rN* zMzS1nYGQI|H!L|rqi>l2Mgbwjqwb$K9=}TV5Uus8Jp)4{ebT6SRuTtJ4xN>tIt`7U zHAFaFWAck*$4;N?8xdQM_+e7EDbGS;i|kXaTFo#PUnOfYfD4#lLP^!8JMyZfx~w-o za=gFq%rVC2RG0A*s+a20vqICnMhM8{WHbrPD+X6z)r@V>Z$9|GovVWM41wc4Nc>)9@;?-mjNIs@_$4Y#&w{&fJfZl!dWWr3f7kgUK)7$ z>`Q0Uy^H&2&#nr4vd*#_9oIY3jVsP=v&TsX9eiouaA5FL326J}oT(P&j@2g(7a2Cfpha!OEEq3})#&_b^ z!m4ma*KWUYnj&b#HwN#xO6QW_Jd_c9zyDnsiYTPI{K8%S_#xpJO;rdtXqVJw#@lJr z`8>Rx*@hjAuWOh9aZUMU3JBTJfB~viGeRv=?6cT-C$Vv@0mR13h>e%cpCI;Kw$RIN zOZ(XERyRx2csJUeyGVD#v*rFOB0(W&nJuux7DIn<5QN!?&r_?}mB(txTZk;j;QS<5 zZ@K~w>~ZwZ-lgvz33C8GTVukQ1@>bH*^lFv5qQm>r*UA{Fzun6F6Kp)(<}3d21hjR z^_stpbNUwYILVRP;7;vqO&9Ve7zLOio-r~=Y$3>W;7ofAi3iJDgcul+XQB{)AX>#p zoFuzJD+z$D#BHZEb1ry`!Cx6S+q<{x5hh+ipiGLL?fVpwF*tD1I2Z>QAC$Ujnc3Nb z5J8!_SNB#Zxxc+(-9}`ps%78L@D*W8478lQ5%UEt7qk z5iJU_i-sLG8U8*Q{K>$W2N>XElhZ4V4QrQPEPRZ#3Ha1+LtAqM@<3aCn&$I)YNT4>oca@V zQ~oyyhTxTolPwI~f-zgu?nrinog)~7A)TZz1qw0D4`jc#u(zs(Pl=(4Qz7b@BA?J>(az6Ze@0zROT~|ZaQB*y_NqvGB6$$Mg*1&pa8DQ8-Z5>S-}6v^cUZ*SRDKD)~uspp=+^s$$x9) zZ=d{|C)drq;{<=VYU@JtV)!Es-~{`w**e?zkAmG&ch3i={w?2)+E;4lkKa7?>Zuh+ z{i+~lUEbN#56a*~)QnG)2>mcV-N7?L*p4h3_x)T#^l6$t@<@@)L~7xaet6&19g z_z1x72*(6JyXKmUrt`GzvI`zcTLUKPxK!7r?{*Ju`L+$^woCP&vz&;LY1lL}Mdrg` zlnES8C^{F7sK*4Kroy=>p0gP_cWnA%5zQs{Mc~21VDg)<;Xk0M7#qix-NcOZT8p%^ zcwKZ<<2?MOwQINll{Vfd6@zQ#7FqIsaHT69enH1O=XG2sw+jINbpgB-I!=Y(cTTVlcrE!c`} zt{oevrC^L($1FI$vU_h1paabc!#naVF=Atu^G2gkv(bKIL&fIXp=0Bp)mF6Z+0ije zp_Ytxq6e&Y3T8TRo+L1KH)hvefo<=cyVMP?D-&mCJYY^^~Lpk4DYc@S;7jM2OD zjg71Ne2~@AXZ5NU_TN;Zzo;7N@N3hM68xBnssAm1^JJe#jSbwn<5 ziPA9|f&Wu50{xq72f}rjc>{n~z~E4Hrzk#hg{x_Jut>dx3HBH>g&8_ysbNl}cZeMzl-{6l?TF}}f#p^(8kM1>p{+(?QHag-h1vMWVD@84j zZ`eOdjrFD-9=!;8nJ7+29xc($F>}I|Z~~9)1qA;U^iTC@uDvjSj^SOlJLH3 zxi-Np3j@?u^mG*kWL65F);&s6dqNZNDT#C!9A(C#O!{$yjw|svT)W$~4Dcvrx~Q>newYQs>)TrP7if{N~N%-)hQNyDc7G~xP>DWrBL zU`o_}wt;>#f@RgsP#F2c_+1EgqMC~6IB2R0^FPgg^a4t8!I}G1jdAixBTw?=zYGWF zaJNW5T|n@U2GVRP7<0-5!Wcd_+MlxGA<`|T{QA#J6wvmyNb)d2OEXdYo&zkRA$ZT@ z%uuQ_kozGCjaHcFG6=09BWcoaDY~XZXFQD${F$5~_Pubx0ITblf z$y%>k6u)>mNETLjT$;}e3?Vr71%I1r2Y~?t2UcBrOuoW?7+{zlWf}Mur*4&kGrjpU z1pVL&C)gQvFd)?p=pWFuR*HeIY8}0roYv+aP7F8#uaJq59m2xzRM1%+)yY0cC4uN0 zG$GUGjsw<2Zt%3YrU55t2LP0b>0~sfT9`eW9k_x;KtJD4B8_(N?w1t%FBCwO;WoRE zcH*7t#1AUy>p~_hQ~ntRPf?O3JUy9EG_NyKo0v6Fb=O{&>&a&fVDo+mbn$<_OaKHmyWC%B+SbFfmg{vIgZ843&C>8kz6~XTbDELJKg%nRBg$tjCt|ec_zH7yPVD?y6u)I9<($M_jh39XTXR9Bf+!d6&>#m;C zwa}9}aAtOBRT#?ZRS#zTU&_`zo~yZ+H$FP_wr?q!t$8e$bTpULzYxxJ_LFsEK(8m` z+p+k>&(FOx^6RE|PG#%hE~~e+&8Ti*;fc4;-5PnP=~nUdp~aqT&6!-ev;19G$(pNm z+10w@YMbrN3idUja#^UHzq;7=j*#7YoU(4A?)9Nv1&P_M`*TSLP|1x)uRr?DBeT7C zT^@?Ax$2i)^%^R9$6k{0`WKsj+WuDiVtuCc&@auua{j`3+dlZd$9uCrbXymn#5vwOBbYjeH)<(I!a`{nHR#y4W$ zjjfa)eJz$QTY7#uaO7s^eCKPiIa{Xu=m%xhbI0zuVbh>9_nB=cGUX>%-Thg&Pd~)R zzuC3e`GfBF+<|)*uDk}WL9KzG*8Qk%rQ*O_bqmMRPuv`wAAGBB&Yr0_FlYa;qAG2F z!}FSFrP{wz(TW!(HRE3jk1EZ8X*G9}13doTQoTxzRJ%gE{$D_{KA@Jd@tC$gh^RuG)9gD;wZq{B#2P zvE}BYuRgj~@kj>$n;(IR4Fmu9eDdbCSFbHp%*Pj=SWtfW{P&+mzSPc-P*M(9L##Wv;7}b z)?9bY32-RAapd}ubZphtnk^~2apC%f^axSN?UL3tcjvOZ^H%e(eHnM>s(a{8X-&H6 zjTc{gaq;Tk_WVsxws!j)9j|rZw-r3_RQ=FiGB3=J-hArSr{)LK%EG=M9{m2nh1N`I z`>H)~r)=xZ?SH;~!Msw|JZH_i%GO+U%dWZw{ypIAZeN=Jvomx3AG*u1CKoQR?d)3K z*_GLG@D`sb@BU@QuWEl$yXHQPW1?GXKdPfi>!1I^{He_Lj*N59s?hm&-s-u|j8LBr zBKZC9U~H*4#{d3zcFx!J`}?JsX@yX&Uzga=H44A3wIcklO=ahH3BTE~|6H4ueprW? z-1xYEm$q-=_jN8~7lNcYE<@|&@W0HG21T3TzbAP(WIlwkxr% zqmdqePQhPOaFc@XQt&+r-lTx6ybSece4G8ag(}eY&P int: - """ - TODO: populate the skip-archive from filenames already on disk. +# gallery-dl keys its skip-archive on `archive_prefix + archive_fmt`, which for +# this extractor is the literal "instagram" followed by the per-media numeric +# pk (`instagram.py:25`, `job.py:713-719`). Verified against a real run: a +# 3-image carousel produced 3 rows, one per item. +ARCHIVE_KEY = "instagram{}".format +ARCHIVE_SCHEMA = "CREATE TABLE IF NOT EXISTS archive (entry TEXT PRIMARY KEY)" - gallery-dl keys its archive on an extractor-specific id string, so this - needs the exact format `InstagramExtractor.archive_fmt` produces. Until - it is implemented, the first run of any profile re-downloads everything — - bandwidth on the CDN (the tolerant surface), but hours of it. +RE_ARCHIVED = re.compile( + r"^(\d{4}-\d{2}-\d{2})_(.+?) - ([A-Za-z0-9_-]+?)(?: - (\d+))?\.(\w+)$") +NON_MEDIA = {"txt", "json"} + + +def index_existing(listing: list[str]) -> set[tuple[str, int]]: """ - raise NotImplementedError( - "seed_archive_db is unimplemented; run without --archive-db and accept " - "a full re-download, or implement this first") + Reduce a flat list of filenames to the (shortcode, index) pairs already + held. Only names matter — never the bytes — which is what lets the sync run + on a host that has no copy of the archive. + """ + have: set[tuple[str, int]] = set() + for name in listing: + m = RE_ARCHIVED.match(name.rsplit("/", 1)[-1]) + if not m or m.group(5).lower() in NON_MEDIA: + continue + # An absent index means a single-media post, which is index 1 — the + # same normalisation the viewer's EXPORT_RE applies. + have.add((m.group(3), int(m.group(4) or 1))) + return have + + +def live_key(item: dict, kind: str) -> tuple[str, int]: + """ + The (shortcode, index) a live item *would* be filed under, mirroring the + filename template exactly. + + The two surfaces disagree about which shortcode identifies a file, and + getting this wrong silently seeds almost nothing: + + posts/reels filed under {post_shortcode} — for a carousel, each + child item ALSO has its own `shortcode`, which is not + what appears in the filename. + stories/highlights filed under the per-item {shortcode}, because + `post_shortcode` there is the containing reel's id and + is shared by every item in it. + """ + if kind in ("stories", "highlights"): + return (item.get("shortcode"), 1) + return (item.get("post_shortcode"), item.get("num")) + + +def seed_archive_db(db: Path, existing: set[tuple[str, int]], + live: list[dict], kind: str) -> int: + """ + Mark everything already held as downloaded, so a fetch into an empty + directory pulls only what is missing. + + `live` is the metadata of one listing pass — the pass we have to make + anyway — each entry carrying at least `media_id` plus the shortcode fields + `live_key` needs. Seeding costs no additional Instagram requests, and needs + only a *listing* of the archive, never its contents. + """ + import sqlite3 + + db.parent.mkdir(parents=True, exist_ok=True) + con = sqlite3.connect(db) + con.execute(ARCHIVE_SCHEMA) + rows = [ + (ARCHIVE_KEY(item["media_id"]),) + for item in live + if live_key(item, kind) in existing + ] + con.executemany("INSERT OR IGNORE INTO archive (entry) VALUES (?)", rows) + con.commit() + con.close() + return len(rows) + + +def probe_live(src: Source, config: Path, cookies: str) -> list[dict]: + """ + One metadata-only listing pass. `sleep` is forced to 0 because it otherwise + applies per *file* even with no download — 2275 files at 1-3s each is over + an hour for a single profile. + """ + out = subprocess.run( + ["gallery-dl", "-j", "--config", str(config), + "--cookies-from-browser", cookies, "-o", "sleep=0", src.url], + capture_output=True, text=True, check=True, + ) + items: list[dict] = [] + + def walk(node): + if isinstance(node, dict): + if "media_id" in node and "shortcode" in node: + items.append(node) + for value in node.values(): + walk(value) + elif isinstance(node, list): + for value in node: + walk(value) + + walk(json.loads(out.stdout)) + return items def main() -> int: