From 4ce5cf048e85011d957100a75ec63424160e800a Mon Sep 17 00:00:00 2001 From: ergosteur Date: Mon, 17 Aug 2026 13:53:18 -0400 Subject: [PATCH] feat: give the sync a memory, so it stops paying for the same listing twice Nothing in this tool had any memory: every invocation started from zero and would happily re-enumerate a profile it had listed minutes earlier. That is what suspended the account -- the listing passes, not the downloads -- and an aborted run re-enumerating five profiles on restart was a large part of the bill. Three changes, in order of how much they save: - Seeding is now a one-time bootstrap per source. After the first successful sync the archive DB records everything gallery-dl has seen, so the source is never probed again. A second full sync costs roughly half what the first did. - Stories never seed at all. A story cannot be in the archive before it is fetched, so there is nothing to seed from, and probing would double the cost of the cheapest surface we have. - A source fetched within --min-interval (20h) is refused, and listing results are cached for --probe-ttl (24h), so a restart mid-run is free rather than a repeat. --force overrides both. --only replaces --no-stories and takes any subset of the surfaces, which is what makes a daily stories-only run possible: one source per profile, no seeding, a handful of requests. Everything else stays monthly. Tested with stdlib unittest -- no new dependencies, and it runs anywhere the sync does. The cases include the aborted-restart scenario, which now plans zero work. Co-Authored-By: Claude Opus 5 --- docs/gallery-dl.md | 42 ++++ scripts/__pycache__/gdl-sync.cpython-314.pyc | Bin 32606 -> 45694 bytes .../__pycache__/test_gdl_sync.cpython-314.pyc | Bin 0 -> 20167 bytes scripts/gdl-sync.py | 229 +++++++++++++++-- scripts/test_gdl_sync.py | 231 ++++++++++++++++++ 5 files changed, 476 insertions(+), 26 deletions(-) create mode 100644 scripts/__pycache__/test_gdl_sync.cpython-314.pyc create mode 100644 scripts/test_gdl_sync.py diff --git a/docs/gallery-dl.md b/docs/gallery-dl.md index e35469d..09af9ac 100644 --- a/docs/gallery-dl.md +++ b/docs/gallery-dl.md @@ -270,6 +270,48 @@ viewer retire the lone-video heuristic in `src/lib/post-tabs.ts` — see `/p//` URL leaves it `null`. Sync always uses listing URLs, so this only matters when testing by hand. +## Cadence, and the budget that enforces it + +**Monthly for everything, daily for stories only.** Stories expire in 24h and +cannot be backfilled, so they are the one surface where missing a day means +losing the content permanently. Everything else can wait — the skip-archive +means an infrequent full sync costs barely more than a frequent one, because it +only fetches what is new. + +``` +# monthly, everything +gdl-sync.py --index --staging ~/gdl/staging \ + --publish @: --archive-db ~/gdl/artms.db \ + --urls-file artms_account_links.txt --execute + +# daily, stories only -- one request per profile +gdl-sync.py ... --only stories --execute +``` + +A stories-only run is one source per profile and **never seeds**, because a +story cannot be in the archive before it is fetched; probing would double the +cost of the cheapest surface for no benefit. Six profiles is a handful of +requests. + +When scheduling it, **randomise the minute and avoid the hour boundary**. A job +that fires at exactly 09:00 every day is a machine; one that fires somewhere in +a window looks like someone opening the app. + +The tool now refuses to repeat itself: + +| flag | default | what it prevents | +|---|---|---| +| `--min-interval` | 20h | re-fetching a source touched recently — the aborted-restart case that re-enumerated five profiles | +| `--probe-ttl` | 24h | paying for a listing pass twice within a run cycle | +| `--max-sources` | off | a runaway list touching more than intended | +| `--force` | off | (escape hatch: ignores both guards) | + +State lives beside the archive DB as `.state.json`, recording per source +when it was seeded and last fetched. **Seeding is a one-time bootstrap**: after +the first successful sync the archive DB records everything gallery-dl has +seen, so the source is never probed again. That is the single biggest saving +here — a second full sync costs roughly half what the first did. + ## Incremental sync — why the fetch host needs no copy of the archive gallery-dl can skip already-held media two ways, and the difference decides diff --git a/scripts/__pycache__/gdl-sync.cpython-314.pyc b/scripts/__pycache__/gdl-sync.cpython-314.pyc index 00b9e937aad3bc011ceda938d37e6013671f17e0..da3ff0e388580eee49c16e28cf154409245201d3 100644 GIT binary patch delta 18744 zcmbV!2~-@{m3CF{8#K_(BGzIN`yvq95L%Ex0xf_V)Us?O)S?S$3%Xm?jTWqN<9JRi z#r`93B@xbeexbzw2>tPl@zG?Woa7|7Xa3nw1_jw7Z6)z!oa7{#oS=~u*`Ar?|L&_U z8f4j;nYOs~>aF+Pckg}o-TTTv`W^qf_ce(Xi3t`Czsz0FJ~{gAL}CWNFi}!8(N$EX zX|seahdGy~%^J4isco~hCWI4O6T^vl9Nu+p_SU3uQXY56)L9_EX?G^M^z}UT#c|;j zHFM%0t$N4e%YrrHbeY1bS=^yim$_1@th7i&j%8U+`k{nWE}wH*(>a%|UZYg5kF^PB z#7YxTn)tEO6)rpSlhV1Nq;Mt;uMQ>uo!w?|B_nIC%qKbIi@eE`y^3?K8|K3~E(aPp zS)*JwbSmD{)Xu|su5`R-uyVn*0_jYa&Ua;XHmd`~pW#ARcDP8Ld&HHqNS&3gTvy&I zO}N-4pp$&oNr|ff=|Yw+brm7KlBLUB#YmU1bh&GNxWbjPiVs&Z^h!}uwuoMps~qnY ztY)>V66q?Iu3>dss*zE%Sb3Fe72a!Eg<4k~(yLjz&Q*`}8kSytsNS`9{bL;G&E&Xj z?vSuEylql63ihTU~OdtyDg0 z6N;*JL7iKd#O>FQXsWpp%?@;Me1E5owT%J!^|rJux4vDodj<8Ei)ulAAz&U$`Q-Dq zOHEm*EEx&Vh-HSTE+h{1O9Eb^{76)H)Dse;R?pB-ZzwP#^@*V<9|{>zT3jsPhbayw zs zJhgT@Yc_w)jC8GW&epn+ZcDeEwcgI;vP50qNQSBr~IWv!LF{DQX^9y+p=|69;!rjQXmkP^i-SVax{hdCXT2SI;B9r zZ%91Ex-R3TLX@jKLG_DN&?|S?i}>`bVS9_VmU@-gf|w{pohryE%=PhJ-+|2+{z32~1r=$F^vt8;1hv<61Ma`}6iwfsjfW<6uriGih7`MIpMwy41q42phlnMvx9|2nIvme`h& zCPV~h49$2NFVRFv^muzmq@hqREv0k}nPXS8vWsZ~+j2_zSFRq*$<~y!2^bXNkl4C? z;%dE~m&?B<&*o*Lhu-G&t2)8Y^W}0VzhJF|-bYP?;czh2P*YQ8mRbl0gCq_I!XlpF zDj!x4nF^WMDm2Ugke`=8naY9|V?pod7h#O-g3n-#{3jY$F`nw6cAttHRR{unL$($I zy{n~#2EOnN}6;$hR&zPc7RV33qs@D_zRKflFHQTt8NpREM*TVT|Tn%_Nov{rOyz6 zX6G1xNkkPeJK-1QJql5vqNse6{7i8s|84o@;>iYv%mZ@PNtGUj)|;}w1k85z#S)h` zn-QRVv00*Yg3=0FFIMEYkc`C9Vmc>kIN=!@fiyHz*){^(0id=RNeQK>-tQTPmeqMZ zVUP5@++R__XUH#B6!33e{d&dY{L%O*j0XZ}LwfIFYR^dnoFu^LL{chUAdp6YCc8xI z4bv-uSLEYWWd>3Q(y%;L)yn6|zo@FKV2ej)%coL_6x$Ew9n~8^rLpSM8ve8LN2?y= zBXU=5TOAV>T_`LLE6n)lF6Hmlmhiume_Kmb&#c?Y^Of?^)ioxzP+DnPera_+$n&k$ zHK_Hg)gdtb*cPW8tSu{%EN)mXnqE&i0ipUFR3w=QLxk~08t zeOa=;hEJ9ctZ&0em)7U;Tje*`m+>#lzgl0OSxMEc1WE{$5@3q7Lay1+#(!Bpz2Q0j zRhe(d-T7rI`y&D`6L_7#Ul4eW02`9_#Aa!z)xdLh2o6HIj_y7*YlD z?3T>@N1+ywty^hRT($L}CPEVSi{Il1rAU$QA1DQ>6b{@5dM}kA5Vg+E(M_T}6a(#0-CfnPb ztZjY^okg|&kzr^tg>{etMM=5@S-+sMXuw5%k95>DgdMi`n0Q+HHD1T$Ki|0%xc0sS{;cCuaE#J;qg|Qv6zQT57Qz*R0 zb}yzdZd2Z)(0)z!x7PygsrJ`-`#f5VYO0=;en3s5SC4dd@ddx7^sfNg%k*wXS_mvf~%_9+5Ml-Pg9u<3u*wu|gfj-YrXmj;R&(N`N%LOh!C_drJ-ycEC zsGj!8P}C3#djlikGJ_&ai*gjzB3DX4pAvN(v!;s|1>F)KNJ>S8*JNYYaD)_cAO049 z8YRWwA=_b$JH&-~7Z=vJ_^{Tc3F{8?VZBR>Ly(Rgf($M_(opL-Ng7>7q)jYsa+#5~ zu(Vk@qq4Nc#ZIIwZN(`xq8>wSuH8V9*m5 z20bCcGejoTY2m0S`h^e-udv`5@c8_puyBN+XF&1{S3`*$skT^JNZ>-K+vgbxiGn{6 z9)$P_;t*bZejywg^o4|QATXqK?LRFH!#0vmHwYr6`84XBz>BF==k|=~FPv0^078E5xF((@=7C|71L2g7~%|pk0!K#>1%_zZ$8kY*=G!B5oz#pRt4w-copZxD?Y{Al

sBd5(B7$us@i?p}klu5GS`3PkNOS8K!+nF8Kh_1DhDAIH?i=!;qDpPi%aHNV zx=SPA#z4TY%#}zXiT%VgOJ@eYK<~ibgUT@f{n=s0H9-+PhM@_&lY;>uM++?P$=i5@ z>8}pRqM~g-MMFi?7Uib8x7XtbZ-5PZdw)jSK(%_!`qbMTz+^Xj!0N@3A=7-P9F@kn z14;fxMd0GxT&G6fm+nld;c7H)&L2|^9Ovfc-`iIju=i$jS@Q0le4U%`On}EIQyw~K zG6XelO{X0#9T86T>QG;cT`wk>`vjUkn>!%HyLV{{;aF4t1g`v{55?AOtC8$M1IR=|r)pG2(7@&u;Rq5uYyV(*{COEhIEoqc{^I7Z`BC>-PFof$8f=Tmd1beC zwUk+B<5?Xtt@iVc=Niv%{@msTjXoju){2~qnP1MFYQEflseP`fes0B@al^c0#f656 zhDqO?qjb9YwLMq%+;FU)7mB~sc3aCiH{3RIw#*OU>Tu@V<@kiuXp(aQ&X3fOLMa&i zYLjJ~mU~NU-Ik(#E1BP>NzuxGo4DyHVO~Oi|-sXJcn#y!Zcs7rX&5r#)*xSp*ct4O-IRfN6EBl&f=)OVXvJv)q*q^x375eUjj=;Y-OOS3^RT5uN+7Sc$i7@_?NP9%6rqATZDmY=ttvyN|{Gi6L>%$W)l40%re?3RXyF_4r1tR6{` zj|iZx;2O|G7^=c#fvXRn?$Rjb6{I9{IKUK8gUsftE@lwSEyqC!3!gZKjO zYJe#%F+}C`cJ`QQW|=BqplSHu%198ft>^%=G#iA0s?eO32i_ppXF|A zlJus9BF>&Pw)372hSt3_+BWYQussl{0LaqUP_pvAAic1rgqaThRAwszI(lNZd?bk3EIj<30OO+z^qAJO$IMEWsqkHlh2$rj z6Zh=Gu-K_Mbx#_wk?GZgT;=EDRlX_YvbdgDQ;u66EK=&q{!M9e;y%;*pr%{XN!s~= zUw;L4jxtiA5B#WWIrH&M`KQULb`6O&{eqf=~iNr2c7 z<|Cw&%*Th#VnmaJU3bx>)QvW@;(S3z$M8uyX~JS}>*}D>-*Awf`Ho;Uj%*Zs5jf6Z z1Crs6YV0&pO`f5Cira-pkq8X1@r75n!lSwy6NqZO;iw6&STT$fRy2W5SiVp|3GziP zZt~`~NK!zO+Nrit5`#mYJ~3*d>Z90qquP;hU(`&mp$K6F%a!p7ty)xvV}yU1-5RwH z(qwvZ^63+$$54r}#u?x>#{Jxybj$7>-~arXSyR@6Myt=hWltuxf74!i-CjDKI%lsr zt6MN|CfjpQJ@eG}jhVD9oH=*qmAcCtFKyg+dCR3OXV1*pSD)3*+w2#rC#vUC3f{IA zO!dEQt(?_WesI^!+0tOGQpaqZ?_(S-UOx&Y1A{N-Z8LLknbWovXy3}`0p;KC+!Qgm z;6tcEU=cK9aHzrE(hBNsXO;fyPWY<;FaTJgutB;hM z6b*Nb3$16=xzDBWozT(*$K4C;In|~-XIhl!d~V@_mMYAbZobVDiahDqx+{PbG8C@< z&iky}I1A!11uIYo#jay9_k~j98n;FbMF({Uwy%e2uduF*$A22P&ZSL&2ga>+>G3qw z=~qKvxaBu{(~^zsNQHs2;fg^fFE=$U?_6$6N{qP7I9^#mTFTlyb$dDJ9jU2_dPk<#fBEf?KdWi6b zz%ggpF=~p9ER1TaYx_r&2jN)=R0<>5BI!Uc3Q>Mw)HoOoGp z&@+sm?>f1Jj7j~?jMD2FrPGer(yye?Wz^2Bp0lkRYneCNZWy!XlTyb=rcBeR)1j_u z>+I_0`TVMB&#bU&#t3bYw*gjPQUidw;XR|}xl_-a8sBikm_2V_H)~pl9&Q@buN%`R zwHJ*q8E+be?;C}A`>t8ju6b+6tgZtF*|waLi0YG8QQb0JmH zLaT=slRob@2Qb;aK#ysWy(t*1%VWEF$Xv?BX)^%=mn~| z=of|)*Ozxqg)cvS z>FK$Wb#vM4$8}#a-_|pSk3n(xtW3z8&(0m!T`-f^Cr{ZxzgE_i!M&N0(p0T|vx*0l zzp}mQ!4vA@MyYI#$?Pr!T`f5zoDm**miJ=LbixP~u*ux#bozqZ1jcce$73p$J!UH! zSuU7C-qxK4htOM8pWQRUCWjNNHAm~#sy3yUnSDJJj$uEymbSvwxI-kq57O?Inza54 z*a^X7*xYdXSE|duPq#!Mx!S146K-b#5X9t|`Oh4CO!xHzfAxw1Q-KOOFlvanbWz7K zCrdPO$x`C30h{=CRKqGJIWL$e%oh?S5+?iRk_yi1e{6TYY^|JXx!iWCZQA?Qj+w5x zj1AYCzuWrF)*BhkKX9ba+nwZ=noOUw=g*q*p{S;+8gX8RE~?Q1Z_@)_7A1sp8^b9k z9Z~$zXG>b3O|(Fz4A1+VUT?ikfEEakDQH$aX5gs8gD0-Nu=!uoM> z-C7s#*5FE*4wu5T>{3_}%x{<=$~CbXx+aFB)bagp9sD!jO3#iQ)$GP`Nxd3|i(+us zus3Z`AJma6`C-0bH_yB}@cS-aB~wzYm5-M1NlcQFryVa8WRr-!q<dt}q+wmpHI8@ByFNX`5=s=dHzGp%QJbgCUM zd_M8RsBQm;ceE%7(FL+|<|do%#mpvS%Bx1C-Zbg~<@UU_&C(W{E{-gpc&CLHy_ptW z_3mr*aHat$?u2^q+L4tj^nZ;m6dC;Q`wHaWl$zwf&?H7+elWj4yR@Cq+5xcST+a^4 zQTr0Pp<_M6jf4<_^FKP)&*jv;V_HpI8X+aTc&3*~o*#Wbg~@-A(EY?SyoX8&&{iK3 zfGeHhwR+P%3--2qPUyCKHb_d!{X~_xOlZ(XzErhQe(5n=7dxKJp{Ww0xkV{Dbw-U| z>2xp50jZcqDWH7v-hAqsLb$j#jhxiI3UTJ9?q6zfm8s(#tn%o<+4UK z-JX(IzJXdgfi2H}3CRGlit59ATEh#sIf7^VlqaO_PoG#}sDyjovr>Mu zUAX$oC!W{vC*(gnTqwVE*vV2W%gQ=&fA+y&4;fkPr+QahyhGI8)TukFtq{c zjV_1k5zK={)|p4nmBgU7M~L;$DHqv`T#YJj>OMY@OIvc!O0}YElU(V^%i6r?ljIIV zt$)0uEsyMID|M9fU`Oh>umCzs)kCv9c0`XGM-+sdLaI_87qW7$6Jn@UsiGEtcGjS7 z!HXWFJ&t!1Rh5&Drpgr_;bX(rs*PDKx*w0Ty8IcBV@VI&miDmjLB!)DZSPsV*qcEm z1N^wnvG%L^HFA5OVA%VI$XH8rF|%+hd;9#+in4fDEp8(k?x@qM-Q;n*!4$OPvf4j6 zTBvlU+gsSPK`!r8Ui0~l^0mI)#GWmSh-f>vs?=Pby-m%Qf7F-YXjU7P!zmH4cQvYJm=C-Na z=Y_|28P~VDc6F|h&-CS3Z7NRnygcJ6b3DKYm`^ujPb?E-&u(?}__Ol>&8~2_+SA_E z|1)xyx^_Q6&V-(W>M-sEwfkVAJUfu9>FkxiGmzsrqE?7M<)kN_WDFRBcDKD#1ZgvM zNDNTI6if~#0lh&?DnGJkuNCPh<4x66Fr}IcIu3I|r`wdo?YIgvGS!{hIjok)pIs|e zyigAp8iQ#)0kzbf=1z1cxl`(OO3q2FZ!kTW5nSO;r=?+{K2Xmn>paX?E^#RbolA7o zUJM;1hBLU@-D#Z?B{N+e?iDt!Zc!p3N)S6tCrs-VfrW|{k4vsYMU!f%2*#i5zm4OH z?ctd0eh&RQIw|ztWM|-syHb0rzLuEq>m#teG2>!EM%LM=Dpj zxVLc}njDCgIOYE<+WE)iH~P}#S<${kmmWloX41x44@c>7l%JyVgk|OO*8Ws}6zi3l zJpdkqfH}ZL`#FwrN@_4In1S^^BaaQPtau`r>CRXbXr?J#J$rfAp9!lk22wBaK2$m{BhF#hk%t-q5tSnt0iZ4UQ=|;! z(*va&<15k^AMY@H8*WSIGitl|v*$Bvy7Le5c~a_1^I?t4tLo@@4?SQ<<>yXNR8NAa zmI4bV*+xaO&$7+nB5y1YW&?lfbyj1wziJceYy{f5Cph>byaZn_&^NVV2ddhouC9nKHY7SK5LwSa+ z2}wvd;hm~i>IVyZrqq_5e?ok3FI=QVk-MmKnp9F1w;t-(yO!yQoCmgwoF%gluol=K zG_IgL`(*y!CHr}cW;c%UK>lm$fbr*lZPLUB)#wHm(Q=*PIc2 z&);1jmwF1fEg=j1t=YkXEd`j@9o*GnX&=8BUOm~3=xE}Q&oA~K9qI2EB}6`OM*aJ% z&kxn}1%HLnrE@r?NuLA2Juym;nyadU%9m!6DF5?tJ^wj5&A-a684LJS4*CWLhwv8;RSWxZ-B{^R zK)jX(gjg($;zlg!BzQgS{_u#udNi@B3Zcv@HIx~{seWrz-`cU?)iwI8a!v_=WQ-E*pH@uSZ6(G7#td7tRRVhbeFr2A~eda zYU()_mJv?lzVv{4$Cjo{i;DnapEz3P8}J9DVp@u&1ZJU1S_yF1(OAFIDS2(Mvv3~# zFq*dH5;BE!Fo({j?p6sF1t*=BuLQru-;uW-PmPQ^NBk6i!`)tmLrOzs4Z^6VQW&*X zBXScKnad_N6ZCAvJ#0iy1qtUBPJ+$Ajap+Y9um;fy2xn_0t@+|2K&)A0xOiiLow<` zz5sU`#<)l~w*%h7`BU7O9;R%vqG2c!vPSFUSrkNZM!|Vry-Etu zItpm3gg1mS#W`wj91#8Dsi3qunm&x1_VMo&77Km`m&l$_I+}j)3!pM9G+$|(I(|+3 zX6EJOuOxr1?L+p}#Y5XP3*zP(jyBDS+tv$YE``i|;0ZQ6`VLX9A8r6xAIrA|ykd@`E) zAm_lbG#rxGNz_t#mz|zKj}?aH(IuDdqA``M1)(OeL1~8LDG$ z+Owy3cUyaNm-JIKl-onukzVS74mMFe`|^i<%NjMuzx!mNrejo#C!&*S95u$)UTUKJ zSpr`s@Oc8Xdn=!H%~9$@0v{0gEI?F8%1Y{?_d@^??FXVAl;POPrwZjHOFtF=$YI|6cT1vby?6?`IMMG&xztf`bSv>5c9to$Z*afX5 zxjme*>WB1B-aqMC0*wG=#fSt2r9qG& z^P|R?_~J`vc5|MUEJnto28ecVIHW{nSeRT#0r99YcFjSO2^R{d93W-YWbme=nR?vB2^N#UU@d%o^2-r}~&TmReG*SrJK!j>yM%IOVY& zdi5`7@w;dp38wV_Fb(FNypHwx;OeH47x+l)RNv*nOM|bTo-5w4pwVR3FK9Wv^DaJp zU9(`~^v35pp6Qr%7QMRWTG8Jcslukk3Y!-zjCZ_RICK1s!UYYVSsp9b=qqq3NaC%PYRn;M#5(Y#o)++(>DhPtUq=X5!4v^z!TJdAZzr_v@7q$xpPaMhkG0&g*v?m+t2keCuIA-}ixn?bT&#JiX3kPRwte22eqry# zUb$nc>++#XhrZf-wqY|nc&wPVeZ`9%BJ&UceC zaBm71vl}NGr}XbQiqCGpRZunEH?!ti@f($M1s&IId1qV3>nGEG?8ul;N}sI#qHWxG zD=Br-h%XF4zR9Pi^QX7J)_SFNx?#q3%`kInHf`&Tq^2JvW#FFHO}e<7H*3#(W&5=5 zt8FvRo3%~XYnyHqHO)uV&Zwp>>HO|lxM4PtRp+ zp3f`1IC^n(a`YwbbojNWuRMK6%O`Fb+wmT*W46AZm3Pj5R(G~=EV>9up|b>p-+FE-xFS~=A>z45!YZ`$Uv zTE?5_otYPQP3)QsT|D{H$!XoRc*9vwdAlcee{t{o&J~k;UUf`&e{BU_bZMQ0hUtO?3Jjb+dKO0I!={ zogOggZrGiGKg(-22dtXkSW|-z{cnVlV4{8@QIB-g)Z0rMr?+=A`PZbYq+H2A9@T_T z;5$uN!cjf0w&06I3P?u{h}?*M;iwU2#1KqVgme7H6WDn}_#ZaBex}H+``FjbXjZmX za#CwcZ*TtyLOb}FNrIulNqmSrbc2J8p0=o!& zoxt}9{1X8q0os1qNA*#@O*%|3eFWJ4OuM+U8MD2nkY4|jz<&_<34!+su#M{1l+w}I zY%lu?Qjw^wtGmP1*3#A6(Z1)PvIVnC33P>u?S?FzrR<%g!P#}?s1BFm!pu5T0*(kL zA%5359Po||iJPT)RK!0nbO8{TL3sYQE`!(ItLON_f8p%!a;fif)^|DcyK?f=&pMs_ z#sv=Gj+WC{7c@xStCpvqes;&gS}tMzm>G&bIrHpOW48N-TAnVY6TCObfhl zH0qelJ9NFe_3GBGs#|q$-OJa%*L?CdpI()2Gjs6#Zt2lG_U$^Ep3DEu$%?Wx-2f+O zy7ed5FDTM>o4YKsrOPT?OE@6wx@}$Qa(W53-QX{#XZoBcPh0dE$wr1?Si++~j*~N! z#N|TT95G8U%8mkVyF)Nlt9&Z66Da0yQL?vNXdPdm&4G6>Uy$h%>@H5oaBYt7 ze1F?K-DXkQBNWJ`$$`g&!a19~LXl8hr;*Eq5)k5MLKX-fz)KlkE_eYiV|azI0Pu2# zFBB>OU&wHu-~+sp;gv!a;MEMT+FmWxEWefG!ucFm$Za?IZ)f_@zwfhawpY=wHAVcQ zsY2~WJ->q9Vf+O@N`0nXyp5hP74Yfw1=9|o+srk54qX9oEsdDpYPUyXgVB+&7?k8V ziHMS-+Zm5X6$26F5fU4q2YN$t6i**y^^;Cwl2c>yyh-36cgtF@zEcF&6ge6`dqTcaah#@*`kk zND>uod_MKiY~6%#4@lC${xt(|WDld`$a=k+T*|8#r6W#QNjwAKH5% zBkz3Csf{qJW`IGgN-V^DkOjq=p+^Vab##TLToyT5y%6ZCgYR~!NhSO_a zwO-Nkxoi2W9B*64U$SQ%Gc&I+&-VJv#E!~mAQ;O;n1#TJKnFn4g(HJ9Da3X?f@}mh zE2>>h*PR2E&XO& z!}?-qDmoI6%LF||JODF4lOsd^}65kbx&P4_(h40C*Kz?r$?4U2$%lMqB5A2=V zdX!4oG+_WRp{mwue0^UEUFTTHhiSs$h1h+_v9->}_xd!7SrX;2xGOXgm05qEhDlY7 z2~{mg!eA>u1an*9ECT^;PH!#-p~BaaN*sD20t`4(P7gYJ49v65^gU-`%TlDa08sQ| zOeO~ueweI)rW!I3F>M6ht=Oc&P%IcCgF}(MqC_x&c1)FJr}Lod;+!?~z2TLsRW~L= z@=(f+|6^jQ1#V2{)%WcNS6bZZ`iZu@sim%KS_5;QkJc5`Lz3?-IBbAi;mBJ0 zS-~zY(p_ z3_LS+zNk=B$@Di!Y!x7E&VF#Fa@@tmr*Sw309*?U`b9GcCMEVFAi2Xe6>-OS-bXKZ zN|(ieanv{@%L%D@(ITITv;zSbpy9YI!W%pkNsv82RGr5rW(}WHZy~JDJyu z{eNga@f)i3COuaH5jgdEg`h2D#{1^Fr4xd|zZIsnp<&#C!XY_C#_8KtRlJLSURBD!Ih9>~JHInE42uDrn{Tg~sU;Jm zk4OlZqcCv^MUagEt(~`GMzD$C8Tvtu&wwigi9A@_#TV1=+J-8eL9!UZjl6_Q-6!<# zYMa1Imue4a_#^c8`dj&fbgI6)0sRh}kSq?Xfp~L~w6>vw|Aqz{(CecO>v+D3zPWgj zk*T01Pt)s*J>Z>!C5wQoXNd$cKeNqA&)54Z$Wu4d^%-EuqTZ#rrhH&`PHkIyL1XzT z_WBuu=V_p$nYqFNdzwgqgJL4o5+i;o(p$OfCYYkG|D<6jWKaiW*KK^Ff%6 zwhWeTEbN$49$MFy@41<33A(2Zx5^W3w`s;O%Rc)C9#|JK+ZBt@8C)&2cXbCkI|#1& zxI!kP5t-n2Pl8A>_4f4!H+FV(uO=aEYa=lk7A{mo(G7+6isT6L;?|%pT&W!)5(|s_ zgW`Uu`XaI2ME@!9B@e3d*+Stf2tYm8F0KvzzT^LCJ1}%aE&Mo=KZ;Zme8{8uvAfG|= zFm-mV16}*OHaMBCn($6_+hAM!Z|G}XZZ%9UqqLjN#ltkMy9Lzi=$_K@pV2QjEZOv* z*pCVO6JpE(R{&GA5SvU#ESUC5>VC^&IJw{2B~c6~ONAgPhNafSQ+I9D^1UonSlD6c zt6`@{jK$uZhsl2-?PhA9rssO=`J7k!&hYjRp~I-A<__{}?D^5uS^s9f^mD}j0HD{W zC&#hDW|O7Bec1dlT`KtCm;$I7lvHw|q}nM4(bu&t(fD8}Dz(<`3q|+5PLBvNaQKYi z*>Vur6+Lc~l46kLFcjE6gPKdJG^#_HCXmSlO9p77lahIEP7Opdfa^5fy?L0=d9b#H z{=+`&X67cAn9MmBP#8K#Nehz$K{65}ZX|gS^dh*KtKQ{l-u^n6;`RIgsDX-P>A~B; znu&w1u^bqE)KCklUQ-3h5Cs!d&GZFncdC3;OU!a+mY7mmgag-$oA~X=Ko=Unpb>KKky_ z_OwdCIH8JW-{Z}$2E0aaG=e;Q;LyqW^XO$)flb?w)e3w3g?R0u5o+mI_qZ&0zGRQE zh&c6WkDbi~F zZqy~J@(|Q??061+f5PN?ibb9c?17489|S#l zEnxNc(z_m9kky_@pJzh{xPx9^;4llFHcnUzTmro_*HN+#T3vy3E7vd=DUWQ8cHq(4D#^8sNNK!0f*bds}W# z_V)Kf9%nb31~kI@tCJQ;xu|DTV)0nBdK0rYVJ*B?qs+WHRIypx7#fi1INOj6;#tp0wdc2*^b^Gsh> ze`M~f)<2NMQ|~}lAQQB@gC)!T4;i;P=2_PpaDZKX^CB%X=^3E=8}!Lm3DBnx1fBIN*}1V?<-R8up1vqq|E?6D<$IeqG}Izt8HqZc0|2G*~fZu&{ouu!GG z^^;}`qt3RCk8f7F7Shn;CGFOlnpnJM_U?JKLD~~ZxMy2#I9$1hBD;s8@RKF?!bDtx z7gC8RMkRMV7Cqo2PI}i9e*Qyx?ukOfW)3vYrqfS6()izij%KjxO+7Q#;w27Pf~&%) zC3!>WZg!7a)Qc_24GsG5f4j}Bal1#gEe&;K6Ahkf&s7X~DvHUYx|J(i;fs5bn_ynk zKc4D`zi%iqIHZMlR1C`E zewnCce|Tg#A(3Aq8`kaWx#d%AQUn|i$4z2I6l+N(t34ErieW#A4~ml1Nl2WqlQb(b zNeQs{T#6=?Q25>C3FxKh&|jnlMc+?<^HiRQM+(KFx;n`2Rwq#UImVobhGGOuaK*SQ z6p6xANQz04L%YQwmbZUmOhaNc3^M?SGBK=Z#r=bv zV;vV9u18x=ww&(xdEc4759``bw0z{~IMzO`=gjH%SuQ&|X7rrNdC^%26uoJNgML@7 z^taFDrvJ5evg2{niG%0O-Vb$UwEFpxoV=pr-6%5W(Um7xe(YF!9{$Rf(tm#5YdfE{ z>LY6#2+J?NYM@^}zsY*Ro^hz__t|;JO~>GWDKD&ZPn$Vw#>A?LRmWCMYfQGhe{p$E zc|oxaXEyxGwfcmPE`M=j+EpuO&!6j|`=8`Epl&QZ|ToJPBFI)A1E5=|DJsb?S;wnzMFiLny(a3x8j#Y#l3GIXvCA(Qt z3^0r0psb{c`xDVP5g{>Rdm$nvxHzC&UiNBhiG!YeIlFi<1`I|hX#=2WASauV#11su z>A9C1#$fZ{2$UKyCW12v-a+s#0(Kn79Ag)n3g1n(VPgk^-3TyU)r`A{SSf<*2);z{ z34%)qST1~q7#{o8B*9#2Qu7T1i~ajY(ZThL%nos>J4(gaG8_+&M8#Gj0wer!Qi%Tc zw2NLpeaxA|JEl1RSG1hQGOYpZS_6If%u&a+W%<1QTC;(#yq1LrsOf$k9eAZshli^F E2OUkI#{d8T diff --git a/scripts/__pycache__/test_gdl_sync.cpython-314.pyc b/scripts/__pycache__/test_gdl_sync.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7cb63cc34fe166a80431468883af203d42efeaca GIT binary patch literal 20167 zcmdUXdvp|6dS_L?RO)WM&we$;-nZ8l7C1SS-TU7<{nkqy_s?{qm;BVhqcSVU zUF3Y6!kyxL{61c>uy3nkW8Zeg&b}RrgMH^IdF)$I1bkceU zl|I&weAbiB72L^!R$jZ(y-dH8{`T4QyR8=Os#3JT6IP^K=|?Av^wvtr$&!o zjGD_>%>vXcd`8XXtY#5v7C)nA4Xas#nv0%M)5B_(qGs7MYOY{4%Tcr988ug)to2p4 zy~c4N&}KD<+IzKG$%HGoQx@Oi%_(ZE!5XM#pZcoAUR-n@6=g;C^u;8PG9Y>+@oWMY z8haC=eo^s6V*TNuCmi+kha!#gg=nxTexb?f^q|BqCGnWlZb{D1+6@xupErN zAxfS`*&_#~a2!21#yzX)roRDgS38}ZVkF!vN&!WTT=2+B2xpD4Xyn2=j~w%a6_1pN z%AP>Xkv?{}$#Z-lpm@TvlRZofd05r3%xcF2l7a(N2#LXPNR&P3 z39p9ti+SX@7!7#>`fGar>EaKJM$U5sFIUaha3K7KxLWq?i^@u% zUkVI*0>NM`5mh{Lf^{f{nw(wrylU$XCT8rFZ9P|hdRJXwiJVNo8+r-KWgMo*%vK2NoRw$;3DDb^Q`h^WM-SytUW zM-Th(-mRNhHB;rC zte;#uD{M1rjFycC{=9Oos$r(8Vf?^dq3u@>t=y7knMmUkOW7`)U`~h57pFsGHD^$y zfXC((og(#bMwzahN;oy4OcNT)rpyyXdOy!Oh7`ZnWN!xjStQaQyXpmLDZZs@0;Da1 zXN%xM?FSF~5A5se?2(AMNFKUli-f_?SK=_6nrazE^d%yi=I(!# zwZ)W-p5AB;J?aR6IsLrfar4l`q2!va$)cB^N`@PonR`p&z;?q7Xg<<+Rg zRkn-Erk0ch5FY*U?~Gc&t@wn+OWX>$K3ihOt&Z`%mIEakzlvMpJUA2yy_|9kd@PGY zkqpQw;JgNfEZB?XnY~z_qZivVd$Ff`@i`1`$%9+wF=|!q@fZ|F)h>x*1WYplhRF_9 z=RmlBAc7C>nh>=_R)<7p1EXY(rdQy!-YOB3l3>AcYF_UJKk?CQ>fXSpn>_|fM#x~8 z{V~6S{{b;9`D5pzex3J%ovXV{iT!K!LH3ONeE`sYUHyA!Zk(B{P1bFitJ^+PxBX-J zvzDK4`(#^k$C24OUvg2;Or3A6nW~n_ zo>^hb0=HhD725CXta!9?tolw>(_B^COjX;|%Dck0Up>s{N{{j~t&*RXcW|}|%yqsh z`2EqqAT%%4>Gu!DLJ5cvKj~y7no@U){QkbMBrB0{RE)-;q}8DhiC$77g-9*;V`2kJ zIOvB;D200yn00hg;`aw55Okq%Pzj1)F#ZW%Ey%ctW>~4*PA1o+O6rdQV^SoX|L@zF zz=syAeal0q#lDAsBG~QQA8-InBVv_FLK%g6qw6NaW_rn{Ulq!HGN~Z&ljVC*!$~z>zw-K6X|nvKx$?%D^2YIVP&a)?yas9bXO z%`0z?wInN7&sBP7D!r3cldfdtD^o}ARPGqI{i$0sEc8`_fpGMSR?_^pvSES3=xROwKQX5^ouW3?L`0=IquNqu6C_mlegrOmxDr-%xwR7lGS`@dUg+X!Y z7RQVgz-$k*8Ww#GX?k6~2+kD;V@WCS8d;HZMs>i*^dId}oto)zR7!{w zHb!bN*hM8cKt_WYQhC2j1BJPevKPuU4UZVOA3hR4)soektU1Rm#^KWywxbVOBZ4oh zs;}2vs~Io5xnyF=hd+wKa>o`NTG zAd$b~d8x}BXQmi4tjM%on$}EC7+lCy=Q8D|=A%4e=wqgoVZzyRD8qz*lFpYNE1pYE zf}G;B9Lvzx$S$q7g{QG>hgJtXgj#sDWM*@E1{3hCMM;zQ^pb;){4>2`Xs9p{k;Fg< zPBd1%tKKCs3Udtxq%({>#1JDq)tZnZ%u}J+89t0f>LG9xKsRFd1R}Di3L07Ni`HvI zDIKGF$EhAXO9AlU9I{-_$dZNtyVXAsh{MAs`Dy3g;!~ zoUnXGSU##;AG$Vl{oB{R{k}Ze^5gTfLfei;ahFo}M^8wd(I0{<`6_zT3UY zrjQWz#i#XN7%qv*1#}bVc2`GyDygY8NU+nUG3 z&xn#yn!0ixn&;KoPu=YVxX3+ra7&k8_g?dkue!NzV%?PO@0@?_{A|zdUCAYfFBJ{* zBNf1rj(zdV-t;x1B=QRqsW9K7A@Yn5h+Xwg zjeWQo`(UCd#XqEzsA|eT<`EC$7L0;^4{dUYry=MCq3g*OU6v`)7F#4#&u{t`sCBAPTx2^x$2{a4;nu1yVaYl*mub_ zY#%;5vVv%YDM@#Q<-cnRS^WP2g)GLr!#|NhAx!=}l|rN_VT|~b;VjqOo1s8B!&@GQ zssexchUNi3B3&^_ZOSE!r`IR3E|V^Fk%PbGGSfFwjKqDzJ?Ckj@ic!}JK6h_)w9CZ zG-X6?L?$~w+W*1+&m6Zc$%+I28Y#mtUjAhz6x3WB<`0F8spsY$TF{cr)qd6(pi3GA zG(r3_#tsmH3ySdW$${Pod{PisDU`3tMfpNCd(r(fnLtWvm@nJnc20>zOA7$D;%*!)$(fLgz35ZfCZ~>wAKCJYFByRNtDB_R20+>UC z;ZWnbzy;(6gjir+LVb$$!y*!+h+b1lLCDD>9v+xT7Y-V_l|(G(IYgwy^r}~hWNCe+eE`~OAo36N#w5J1X6Qqq3P|CYI&U>BRFmFS z4gp-`9yq!D0*(1CWNbaJ-(b3CCw$jwoeJhw_g&!#6O*tmnA?slROY*5UZt-3Vres7 zX(O{FM!uaqHWQFm` z&6iWsuIQI}KY~~ZtKV==$N3ZF)@CKG&1E~2J;huZcwgs4y&D2ek9)AkI%pF*%Jv9>^vzaCX8ttDx zCq4`PwF1jj;828&^l)pr)!GHFADFJ@!bn|MhP(7fHD0zZixq7uyku!oKq6C#(CL>fEXjRClph~ubE#^{(q&%$%66LFuEsa9{- zoRoY>wZ$pTi4UlJFAGBve@ZziH4~3H)hX)y1`?-8N0UUPB8Ak(Tr$`47#65bD_s6h z0C1}pl+P8c`MhAwTtUNZLBn|9%hJkG{+-REC+?Ki4|jf9Tza+OO2KF-SzJ5Ze$QQS z`Ou|9qwC(cj<^5VG3)lym99%&qp!SQI)3EG6|?SUy0Yiep6~6=ZCX6%uAXsMj~=;x z;@XMvb;)I$XWec0-19GgTsn2+Z_w4wCna3rT8s<-pBFTIB~$vq4?7AvHrhYk=<0Oj zXD#JC++<5x(+vm}S~1VK)XDqTUCPHSX4y|^onfK8nrFB(3>ILzkd3>#c>-d!(q9I7 z!{7M~@{lTK!ST%80g<((kN{ro5oe=IVp_nGetbFFxO@V(SHXm^g#1l zIF|kpAlt@h)GSY=PDoNBuCQ1r^h~G`DMwdtBo;iwQgjeXQn#BEbKp7Kf1939_;H!CchAFjakzySMUpe9|aEy$Fd_{@)S&pnk>_g3$2cSX+VIjy1l)`%EIW}&Ds-@EEg z1zY5y)OO;1Csw6fElMMtGwEEr3>+G{x3FzR%x6XdL>9LhBIKbc61?{^% zW99Ev-KZLW<>QLvik-=2?YGv?x_94y{ywEoH&dSnwLX{4xR;HJ*TdJs;-wtZ$Jmn;x7y7;q&;5FCS~zwa$1x$9e5p z9LGo{Nv}6^ZCzJP7)LU=59TOUeEH^_NN<?R2Oywuw4mAlverk;bHJfihFcWjZOl@3N@|a&* zrZJeU*p%k{+MUvupgC*ncdT`C11;N4PhGb6>X5iL)%us;dSJJ(^=rZU{U53IYqwzi zK5%dai!SfKwEuet(>n9X@y?rF6J5!5ugto)Jt;C#P<9Esn(aSpc6IRCi!N>WzWAcE zVLDj(jR;A?Qy9tH07qoA*f3nfF%b}~l`KBuHf51Sb2?FmE zc$vT}1aiGfe@Yh#d>0>?rqsf4!-M4Xar?86{wAEl3ns{)EEc9q$E4zG=6KI^Zd zy%2*L`*kt|j$Yvy6kdHb`1185tOo(F<79kX&Bs6^A=1zoH&=!3!#zhk4tMTO^L3Tq z!u*Pqrbfvc5+#2@lOk=4@o`m%zy|@2dtt(&IWTN65eH>j0(B{;MI9q_me$M^DpZC( zg+$~Wov8pkA=p+k*sQ+jNKLXFsd=fg_f6+feY(z3>VHcE}+E!I(QZI9();n`SO8Rmxfni z-_hNN7#>MTQ(h6a1cuT`m41TDnv-}vjtIAZLs#A)@F9V}0?0yPGee=ikA}2j1BoPI z<6u{W_6!BTEen}nqa|^NRM?Qv`mO2uEmLjJM(e(0)uySUSz)i?A2%?5Fj=)_YR6sS zzcU%oA)_8^v*r(%2 zSej{{4xV_nFFRZPjO74aLbRVSo)<^B#dK78zCz@p6BVJDyBktFwj5#Zxg8J1L$*Lu zs8^F)w!OO#c50@{f5b!5j83(zUyog#+`QQ+LD`kzvt6$DIV|Wwhl}MDiapjqPCeV?4|Yktd&xfE%_T zr#lvnVzZ)}C!QCB*zN&+k(QS9765j__R@B=$hi(k8kiyOz+PRdieZ>_&|dfInvy7>45yS1!v zxbv~Vm6XjD*Ul8zj_n#>I$kiTOdU@a@1|ga;rburXWc8ma4)&Hvi?2q4e#X2cEqo2NjgL6KtO@&?Ya2(8j0CQBj&!n<;ZC?Z%9;%4#^}+pLt{H;-7QZl zp{mXlH2mt}BHVvcswuA16h&B$_xxddTZha3vx<@qhw!u2jMseaWd8_hz3T{RfqZU(4XttEwou;xS8}?de!OK?@ES_Z=#sIXyF&eZ>1Nmw zKV^OUehuC7gn_Db1WS`)Izn~3Iu7finA=mvobRUSJH8@~-~$z!lz47L3o7rC{t@pn zWcR|Qq#TvVJvx*RG)fm5fn=zxLxuW}KQu5+XUNt7flzN(ZKq@5D8sUJAFa~3)x%T& z9aq&n#8{CH6U{csH8od>#ju61pM|D0N3VE37)ij;^Y=-yL6#nsB0ac~jFqHe{cAM- zFBphSsTeR|3QH~zTpIX+JXg79rgF{uW#a=sUOrWMd)0L1nq=i+MiO6Ct{?se1*jbeq7>@sWXL-T1cJ z8=ingt)UQN5;WRp!-yoy+`8VTU z7$NI7w0NBqVh(r^P?2bVAb!=wE`1kME&UOI$_F*gh!?S=qJu7X5}^5#{+K}8?NG?P z!!!YGnd)o%E!0x^c=oE+ehGpa=o|Lw7e|mb0kahwZs114o@zgG%oJ?l*XdAO&Ck{7 z4j!7yn4h0WQCv21Q$?Ow z3hjsCRwJk}6RJbop9hf{?)T%DZt@}WdSh}}xlsR-^bhQA2&sYen=aBKIxP&-K2dCg z?rlmSzYo9hqMw5~W5V(C3m0UDUsce5+K4e^>~8O6*j8<Ii^-D{(6AB=GwLn8T9kcZ_?o=_TisYNLr` zc>&sdQvQl6yrKb3*fww&>MpljT zqZ>x0vGwD=o3Bs2e$zkUpLJ|Q`+WE1WtW!yNzFya|8d|ah8_z={PvN7xr)^@6{{a} zIC;G4W&D)V$l|$@l`|zPA9Hl}_}}ow{C+pc45k2N`^7tU{FyJ)8UO TLw|5+w0+LD9KzA2ZGruNcl^LF literal 0 HcmV?d00001 diff --git a/scripts/gdl-sync.py b/scripts/gdl-sync.py index 4f4831a..36e385a 100755 --- a/scripts/gdl-sync.py +++ b/scripts/gdl-sync.py @@ -28,6 +28,7 @@ using the cookie from elsewhere is what session-hijack detection looks for. from __future__ import annotations import argparse +import datetime as dt import json import os import re @@ -72,25 +73,23 @@ class Profile: user: str existing: dict[str, str] = field(default_factory=dict) # kind -> dirname - def sources(self, include_stories: bool) -> list[Source]: + def sources(self, kinds: set[str]) -> list[Source]: u = self.user base = f"https://www.instagram.com/{u}" - out = [ + all_sources = [ Source("posts", f"{base}/posts/", u, "posts"), Source("reels", f"{base}/reels/", f"{u} - reels", "reels"), - ] - if include_stories: # Stories expire after 24h, so these can only ever be captured - # live. There is no backfill and no re-fetch. - out.append(Source( - "stories", f"https://www.instagram.com/stories/{u}/", - f"story - {u}", "stories")) + # live. There is no backfill and no re-fetch -- which is why they + # are the one surface worth visiting daily. + Source("stories", f"https://www.instagram.com/stories/{u}/", + f"story - {u}", "stories"), # Highlight directories embed the title, which gallery-dl only # learns mid-extraction -- so this one source fans out into many # directories and is handled with a directory format string. - out.append(Source( - "highlights", f"{base}/highlights", "", "highlights")) - return out + Source("highlights", f"{base}/highlights", "", "highlights"), + ] + return [s for s in all_sources if s.kind in kinds] def scan_archives(root: Path) -> dict[str, Profile]: @@ -456,6 +455,133 @@ def probe_live(src: Source, config: Path, cookies: str) -> list[dict]: return items +ALL_KINDS = ("posts", "reels", "stories", "highlights") + +# Stories cannot be backfilled and expire in 24h, so a run that only wants +# stories is both cheap and the one worth scheduling daily. +STORIES_ONLY = {"stories"} + + +class SyncState: + """ + What has already been spent against `instagram.com`. + + Exists because nothing else in this tool has any memory: every invocation + used to start from zero and happily re-enumerate profiles it had listed + minutes earlier. That is what suspended the account — the listing passes, + not the downloads. + + Two facts are tracked per source: + + seeded the skip-archive has been primed from the archive listing. + This is a ONE-TIME bootstrap: afterwards the archive DB records + every item gallery-dl has seen, so the source never needs + probing again. This is the single biggest request saving here. + fetched when it was last downloaded, so a re-run soon after is refused + rather than silently repeating the whole pass. + """ + + VERSION = 1 + + def __init__(self, path: Path): + self.path = path + self.data = {"version": self.VERSION, "sources": {}} + if path.is_file(): + try: + loaded = json.loads(path.read_text()) + if loaded.get("version") == self.VERSION: + self.data = loaded + except Exception: + pass # a corrupt state file must never block a sync + + def _entry(self, url: str) -> dict: + return self.data.setdefault("sources", {}).setdefault(url, {}) + + def needs_seed(self, url: str) -> bool: + return not self._entry(url).get("seeded") + + def mark_seeded(self, url: str, stamp: str) -> None: + self._entry(url)["seeded"] = stamp + + def last_fetch(self, url: str) -> str | None: + return self._entry(url).get("fetched") + + def mark_fetched(self, url: str, stamp: str) -> None: + self._entry(url)["fetched"] = stamp + + def save(self) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(json.dumps(self.data, indent=1, sort_keys=True)) + + +def hours_since(stamp: str | None, now: float) -> float: + """Hours between an ISO stamp and `now`; infinite when never.""" + if not stamp: + return float("inf") + try: + then = dt.datetime.fromisoformat(stamp) + except ValueError: + return float("inf") + if then.tzinfo is None: + then = then.replace(tzinfo=dt.timezone.utc) + return (now - then.timestamp()) / 3600.0 + + +def plan_source(src: Source, state: SyncState, now: float, + min_interval: float) -> tuple[bool, bool, str]: + """ + Decide what a source needs: (fetch, seed, reason). + + Seeding is skipped once done, and skipped entirely for stories — a story + cannot exist in the archive before it is fetched, so there is nothing to + seed from, and probing would double the request cost of the cheapest + surface we have. + """ + since = hours_since(state.last_fetch(src.url), now) + if since < min_interval: + return (False, False, f"fetched {since:.1f}h ago, under the " + f"{min_interval:g}h floor") + if src.kind == "stories": + return (True, False, "stories: no seed needed") + if state.needs_seed(src.url): + return (True, True, "first run: seeding from the archive listing") + return (True, False, "already seeded; the skip-archive knows what we hold") + + +class ProbeCache: + """ + Listing-pass results, kept so an interrupted run does not pay for them + twice. Yesterday an aborted sync re-enumerated five profiles on restart. + """ + + def __init__(self, path: Path, ttl_hours: float): + self.path = path + self.ttl = ttl_hours + self.data: dict = {} + if path.is_file(): + try: + self.data = json.loads(path.read_text()) + except Exception: + self.data = {} + + def get(self, url: str, now: float) -> list[dict] | None: + entry = self.data.get(url) + if not entry or hours_since(entry.get("at"), now) > self.ttl: + return None + return entry.get("items") + + def put(self, url: str, items: list[dict], stamp: str) -> None: + # Only the fields seeding needs, so the cache stays small. + self.data[url] = {"at": stamp, "items": [ + {k: i.get(k) for k in ("shortcode", "post_shortcode", "num", "media_id")} + for i in items + ]} + + def save(self) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(json.dumps(self.data)) + + def rsync_command(staging: Path, dest: str, dry_run: bool) -> list[str]: """ Publish a staging tree into the archive. @@ -529,8 +655,21 @@ def main() -> int: metavar=("MIN", "MAX")) ap.add_argument("--sleep", nargs=2, type=float, default=[3.0, 6.0], metavar=("MIN", "MAX")) - ap.add_argument("--no-stories", action="store_true", - help="skip stories and highlights (posts and reels only)") + ap.add_argument("--only", default=",".join(ALL_KINDS), + help="comma-separated surfaces to sync: " + "posts,reels,stories,highlights. Use --only stories " + "for the cheap daily run.") + ap.add_argument("--min-interval", type=float, default=20.0, metavar="HOURS", + help="refuse to re-fetch a source touched more recently " + "than this (default 20h); the guard that makes a " + "restart cheap instead of a repeat") + ap.add_argument("--max-sources", type=int, default=0, metavar="N", + help="hard ceiling on sources touched in one run " + "(0 = no limit)") + ap.add_argument("--probe-ttl", type=float, default=24.0, metavar="HOURS", + help="reuse cached listing results younger than this") + ap.add_argument("--force", action="store_true", + help="ignore --min-interval and the probe cache") mode = ap.add_mutually_exclusive_group() mode.add_argument("--dry-run", action="store_true", default=True, help="print the plan and the config; default") @@ -572,14 +711,36 @@ def main() -> int: # published to the archive root. config_path = args.staging.parent / f"{args.staging.name}.gdl-config.json" - plan: list[tuple[Profile, Source]] = [ - (prof, src) - for prof in selected - for src in prof.sources(include_stories=not args.no_stories) - ] + kinds = {k.strip() for k in args.only.split(",") if k.strip()} + unknown = kinds - set(ALL_KINDS) + if unknown: + print(f"unknown surface(s): {', '.join(sorted(unknown))}", file=sys.stderr) + return 2 + + state_path = (args.archive_db.with_suffix(".state.json") if args.archive_db + else args.staging.parent / f"{args.staging.name}.state.json") + state = SyncState(state_path) + now = dt.datetime.now(dt.timezone.utc) + now_ts, stamp = now.timestamp(), now.isoformat() + min_interval = 0.0 if args.force else args.min_interval + + plan: list[tuple[Profile, Source, bool]] = [] + skipped = 0 + for prof in selected: + for src in prof.sources(kinds): + fetch, seed, reason = plan_source(src, state, now_ts, min_interval) + if not fetch: + skipped += 1 + print(f" skip {prof.user}/{src.kind}: {reason}") + continue + if args.max_sources and len(plan) >= args.max_sources: + skipped += 1 + continue + plan.append((prof, src, seed)) print(f"profiles : {len(selected)}") - print(f"sources : {len(plan)}") + print(f"surfaces : {','.join(k for k in ALL_KINDS if k in kinds)}") + print(f"sources : {len(plan)} to sync, {skipped} skipped") print(f"pacing : {args.sleep_request[0]}-{args.sleep_request[1]}s between " f"requests, rate cap {args.rate}") print(f"staging : {args.staging}") @@ -587,32 +748,44 @@ def main() -> int: print() if not args.execute: - for prof, src in plan: + for prof, src, seed in plan: dest = src.directory or "(per-highlight)" - print(f" {prof.user:<20} {src.kind:<11} -> {dest}") + note = " [will seed]" if seed else "" + print(f" {prof.user:<20} {src.kind:<11} -> {dest}{note}") print() print(" " + " ".join(rsync_command(args.staging, args.publish, True))) print("\ndry run; nothing fetched. pass --execute to run.") return 0 config_path.write_text(json.dumps(config, indent=2)) + probes = ProbeCache(state_path.with_suffix(".probes.json"), + 0.0 if args.force else args.probe_ttl) failures = 0 - for prof, src in plan: + for prof, src, seed in plan: print(f"==> {prof.user} / {src.kind}") stage_dir = args.staging / (src.directory or ".") stage_dir.mkdir(parents=True, exist_ok=True) - # Seed the skip-archive from what the archive already holds, so + # Prime the skip-archive from what the archive already holds, so # fetching into an empty staging directory pulls only what is missing. - # The listing pass this needs is one we have to make anyway. - if args.archive_db: + # Done once per source, ever: afterwards the archive DB records + # everything gallery-dl has seen and no listing pass is needed. + if seed and args.archive_db: try: - live = probe_live(src, config_path, args.cookies) + live = probes.get(src.url, now_ts) + if live is None: + live = probe_live(src, config_path, args.cookies) + probes.put(src.url, live, stamp) + probes.save() + else: + print(f" reusing {len(live)} cached listing items") held = index_existing(index.listing(prof.user)) seeded = seed_archive_db(args.archive_db, held, live, src.subcategory) print(f" seeded {seeded} of {len(live)} live items") + state.mark_seeded(src.url, stamp) + state.save() except subprocess.CalledProcessError as exc: failures += 1 print(f" probe FAILED: {exc}", file=sys.stderr) @@ -625,6 +798,10 @@ def main() -> int: failures += 1 # Keep going: one private or renamed profile must not abort the run. print(f" FAILED (exit {result.returncode})", file=sys.stderr) + else: + # Recorded even for an empty fetch: the request was still spent. + state.mark_fetched(src.url, stamp) + state.save() # Publish once, at the end, so a partially-fetched profile never reaches # the archive mid-run. Only ever adds -- see rsync_command. diff --git a/scripts/test_gdl_sync.py b/scripts/test_gdl_sync.py new file mode 100644 index 0000000..7b412c5 --- /dev/null +++ b/scripts/test_gdl_sync.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +""" +Tests for the request-budget logic in gdl-sync.py. + + python3 -m unittest discover -s scripts -p 'test_*.py' + +Deliberately stdlib-only, so it runs anywhere the sync itself runs. What is +covered here is the part that decides whether to spend a request — the part +whose absence got the archive's Instagram account suspended. +""" + +import datetime as dt +import importlib.util +import json +import sys +import tempfile +import unittest +from pathlib import Path + +_spec = importlib.util.spec_from_file_location( + "gdl_sync", Path(__file__).with_name("gdl-sync.py")) +gdl = importlib.util.module_from_spec(_spec) +sys.modules["gdl_sync"] = gdl +_spec.loader.exec_module(gdl) + +NOW = dt.datetime(2026, 8, 18, 12, 0, tzinfo=dt.timezone.utc) +NOW_TS = NOW.timestamp() + + +def ago(hours: float) -> str: + return (NOW - dt.timedelta(hours=hours)).isoformat() + + +class SourceSelection(unittest.TestCase): + def test_only_stories_is_a_single_cheap_source(self): + srcs = gdl.Profile("u").sources(gdl.STORIES_ONLY) + self.assertEqual([s.kind for s in srcs], ["stories"]) + self.assertEqual(srcs[0].directory, "story - u") + + def test_full_sync_covers_every_surface(self): + srcs = gdl.Profile("u").sources(set(gdl.ALL_KINDS)) + self.assertEqual([s.kind for s in srcs], list(gdl.ALL_KINDS)) + + def test_reels_and_stories_go_to_their_own_directories(self): + by_kind = {s.kind: s for s in gdl.Profile("u").sources(set(gdl.ALL_KINDS))} + self.assertEqual(by_kind["posts"].directory, "u") + self.assertEqual(by_kind["reels"].directory, "u - reels") + # Highlights derive their directory from the title mid-extraction. + self.assertEqual(by_kind["highlights"].directory, "") + + +class PlanSource(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.state = gdl.SyncState(Path(self.tmp.name) / "state.json") + self.posts = gdl.Profile("u").sources({"posts"})[0] + self.stories = gdl.Profile("u").sources({"stories"})[0] + + def tearDown(self): + self.tmp.cleanup() + + def test_first_run_seeds(self): + fetch, seed, _ = gdl.plan_source(self.posts, self.state, NOW_TS, 20) + self.assertTrue(fetch) + self.assertTrue(seed) + + def test_seeding_happens_only_once(self): + self.state.mark_seeded(self.posts.url, ago(720)) + fetch, seed, reason = gdl.plan_source(self.posts, self.state, NOW_TS, 20) + self.assertTrue(fetch) + self.assertFalse(seed, "a seeded source must never be re-probed") + self.assertIn("already seeded", reason) + + def test_stories_never_seed(self): + # A story cannot be in the archive before it is fetched, so probing + # would double the cost of the cheapest surface for no benefit. + _, seed, reason = gdl.plan_source(self.stories, self.state, NOW_TS, 20) + self.assertFalse(seed) + self.assertIn("no seed", reason) + + def test_recent_fetch_is_refused(self): + self.state.mark_fetched(self.posts.url, ago(3)) + fetch, _, reason = gdl.plan_source(self.posts, self.state, NOW_TS, 20) + self.assertFalse(fetch) + self.assertIn("under the", reason) + + def test_an_old_fetch_is_allowed_again(self): + self.state.mark_fetched(self.posts.url, ago(30)) + fetch, _, _ = gdl.plan_source(self.posts, self.state, NOW_TS, 20) + self.assertTrue(fetch) + + def test_daily_stories_pass_a_20h_floor(self): + # The cadence this is built for: once a day, every day. + self.state.mark_fetched(self.stories.url, ago(24)) + fetch, _, _ = gdl.plan_source(self.stories, self.state, NOW_TS, 20) + self.assertTrue(fetch) + + def test_force_disables_the_floor(self): + self.state.mark_fetched(self.posts.url, ago(1)) + fetch, _, _ = gdl.plan_source(self.posts, self.state, NOW_TS, 0.0) + self.assertTrue(fetch) + + def test_the_aborted_run_scenario(self): + """ + Yesterday's failure: a run died mid-way and the restart re-enumerated + every profile. Seeded-but-not-fetched must not re-probe. + """ + self.state.mark_seeded(self.posts.url, ago(0.5)) + fetch, seed, _ = gdl.plan_source(self.posts, self.state, NOW_TS, 20) + self.assertTrue(fetch, "the fetch still needs to happen") + self.assertFalse(seed, "but the listing pass must not be paid for twice") + + +class StatePersistence(unittest.TestCase): + def test_state_survives_a_reload(self): + with tempfile.TemporaryDirectory() as d: + path = Path(d) / "state.json" + a = gdl.SyncState(path) + a.mark_seeded("https://x/", ago(1)) + a.mark_fetched("https://x/", ago(1)) + a.save() + b = gdl.SyncState(path) + self.assertFalse(b.needs_seed("https://x/")) + self.assertEqual(b.last_fetch("https://x/"), ago(1)) + + def test_a_corrupt_state_file_never_blocks_a_sync(self): + with tempfile.TemporaryDirectory() as d: + path = Path(d) / "state.json" + path.write_text("{ not json") + self.assertTrue(gdl.SyncState(path).needs_seed("https://x/")) + + +class ProbeCaching(unittest.TestCase): + def test_fresh_entries_are_reused_and_stale_ones_are_not(self): + with tempfile.TemporaryDirectory() as d: + cache = gdl.ProbeCache(Path(d) / "p.json", ttl_hours=24) + cache.put("https://x/", [{"shortcode": "A", "post_shortcode": "A", + "num": 1, "media_id": "1"}], ago(1)) + self.assertEqual(len(cache.get("https://x/", NOW_TS)), 1) + + cache.put("https://y/", [{"shortcode": "B", "post_shortcode": "B", + "num": 1, "media_id": "2"}], ago(48)) + self.assertIsNone(cache.get("https://y/", NOW_TS)) + + def test_cache_keeps_only_the_fields_seeding_needs(self): + with tempfile.TemporaryDirectory() as d: + path = Path(d) / "p.json" + cache = gdl.ProbeCache(path, ttl_hours=24) + cache.put("https://x/", [{"shortcode": "A", "post_shortcode": "A", + "num": 1, "media_id": "1", + "description": "x" * 5000}], ago(0)) + cache.save() + self.assertNotIn("description", path.read_text()) + + def test_a_miss_is_reported_rather_than_guessed(self): + with tempfile.TemporaryDirectory() as d: + cache = gdl.ProbeCache(Path(d) / "p.json", ttl_hours=24) + self.assertIsNone(cache.get("https://never-seen/", NOW_TS)) + + +class Seeding(unittest.TestCase): + """The bug that seeded 5 of 2275: matching the wrong shortcode field.""" + + def test_posts_are_keyed_by_post_shortcode(self): + item = {"shortcode": "childcode", "post_shortcode": "POSTCODE", + "num": 2, "media_id": "9"} + self.assertEqual(gdl.live_key(item, "posts"), ("POSTCODE", 2)) + + def test_stories_are_keyed_by_the_per_item_shortcode(self): + item = {"shortcode": "ITEMCODE", "post_shortcode": "reelid", + "num": 3, "media_id": "9"} + self.assertEqual(gdl.live_key(item, "stories"), ("ITEMCODE", 1)) + self.assertEqual(gdl.live_key(item, "highlights"), ("ITEMCODE", 1)) + + def test_index_existing_normalises_a_missing_index_to_one(self): + held = gdl.index_existing([ + "u/2023-04-19_u - ABC.mp4", + "u/2023-04-12_u - DEF - 3.jpg", + "u/2023-04-12_u - DEF.txt", # sidecars are not media + "u/2023-04-12_u - DEF.json", + ]) + self.assertEqual(held, {("ABC", 1), ("DEF", 3)}) + + def test_seeding_marks_only_what_is_already_held(self): + with tempfile.TemporaryDirectory() as d: + db = Path(d) / "a.db" + live = [ + {"post_shortcode": "HELD", "shortcode": "x", "num": 1, "media_id": "11"}, + {"post_shortcode": "NEW", "shortcode": "y", "num": 1, "media_id": "22"}, + ] + n = gdl.seed_archive_db(db, {("HELD", 1)}, live, "posts") + self.assertEqual(n, 1) + import sqlite3 + rows = {r[0] for r in sqlite3.connect(db).execute( + "SELECT entry FROM archive")} + self.assertEqual(rows, {"instagram11"}) + + +class Publishing(unittest.TestCase): + def test_publish_only_ever_adds(self): + cmd = gdl.rsync_command(Path("/stage"), "host:/archives", dry_run=False) + self.assertIn("--ignore-existing", cmd) + self.assertNotIn("--delete", cmd) + + def test_tooling_files_are_excluded_from_the_archive(self): + cmd = " ".join(gdl.rsync_command(Path("/stage"), "/dest", dry_run=True)) + for pattern in ("gdl-sync*.json", "*.db"): + self.assertIn(pattern, cmd) + self.assertIn("--dry-run", cmd) + + +class UrlsFile(unittest.TestCase): + def test_reads_every_form_a_person_might_paste(self): + with tempfile.TemporaryDirectory() as d: + p = Path(d) / "urls.txt" + p.write_text( + "# comment\n" + "https://www.instagram.com/a/\n" + "https://instagram.com/b\n" + "www.instagram.com/c/\n" + "d\n" + " e # trailing\n" + "\n" + "https://www.instagram.com/a/\n" # duplicate + "https://www.instagram.com/p/ABC123/\n" # a post, not a profile + "not a username\n") + self.assertEqual(gdl.read_urls_file(p), ["a", "b", "c", "d", "e"]) + + +if __name__ == "__main__": + unittest.main(verbosity=2)