From ee66b87bff66ced82a476a7b406caf6a55a86ae0 Mon Sep 17 00:00:00 2001 From: ergosteur Date: Sun, 16 Aug 2026 22:32:06 -0400 Subject: [PATCH] feat: drive gdl-sync from a file of profile URLs Adds --urls-file so a run can reference a hand-maintained list rather than repeating --profile, which is how this actually gets used: the ARTMS accounts now live in artms_account_links.txt at the archive root. The parser takes what a person would paste. Full URLs, scheme-less URLs and bare usernames all work; blank lines and # comments are ignored and duplicates dropped, so the list can be appended to carelessly. Lines that are not profiles are rejected loudly rather than silently syncing nothing: an Instagram post URL yields the segment "p", which would otherwise be treated as a username and create a directory called "p". Co-Authored-By: Claude Opus 5 --- scripts/__pycache__/gdl-sync.cpython-314.pyc | Bin 20665 -> 31998 bytes scripts/gdl-sync.py | 76 +++++++++++++++++-- 2 files changed, 70 insertions(+), 6 deletions(-) diff --git a/scripts/__pycache__/gdl-sync.cpython-314.pyc b/scripts/__pycache__/gdl-sync.cpython-314.pyc index e90de1c24053d0d15742bf6c115e6c20f25578ca..d0ad8f06144bc6b9c7bc7acbf9fcbad035e27400 100644 GIT binary patch delta 14310 zcma)j33MChnb-_4xIuy-!27^JJi!ByC`!CZQ9NXdhu{$fSurIFB1R-E5`bqw(S)-C z+N~-&v%-kjsXA z4$0fNT=xMo)Q%)<=kmh4V8UpAEiKoCi?{-=aJM2{%-NtjOS+eEcF6A{`BJV3^2H=y z#+5+6l;q2~GRT*cyn}NRW#61e}J77c_({%${!x{iu~mGX?DshvS<15$QbWq zy#XH^nmjY^7x60`WW!@TJMKTvvtD6j4Bq4PDL6-9&LKUUc&e*$FL)HL^IpLfo(Q={ zedBC(b>c+v=!FA#0*$VaH$3JF2VIFV#T~;;3gc0+n(V|YgeSyd@5o4SG7ugf_Xp02 z&hRr~7}O_BR|}H?J(<3Ubvm7JR{>U>@`ko**|TsH)z$nn{K#Y&?mcl1a4uqyg;DuX z817L%d=}SKl^8CHkN_(ef@B#k$iwxpQ$gVzi-!T1zy{AkDXdjqD%^AX_>^~Agu=wy z08RwhEZ$p4@CRTj1gFrrH&8|HnS9C624KL4u%W<*f~h5|T);5k5qpdg)6-KSe`xz+!Urdy>4nkVU=TSv6jG(B+Z4|E3<@cN;_MAJRe z&y2U}ky|gM{76ZUO1FIP^q~%n3TO$FWHh>^U8+=VoO$|96Z3LP| z1@FWVLXfMx&E1Tk(fZ^mS6k7Lc(T&rYI`ZALx&e4r~t2}T6Ry-^N7e~bIMMe z?_;uH-N4tPgM3LBiu0;sK+yQ?T_3WgGDJ&bgz#A@Z}tp=y4DqIOAwUz7|45>V-vc#yPFnm7MTSrkt z#^gk9Q!=;p_NbC{Dx~(Plf8ZE*xRFao26egTBJOOPTim8QMnD|v&o@*ETr*hHe18E z`4p%0Xc}PPQ7!pmdZWTf)u@K7$%gJsoXeuA zOE3l-)q97cJQ{aSvde)XvUTi$^ZFeB>`YpZ28ZW#HR?wvh;Eypi|&PsIxdjW7#s~s zfWLVZ?gE@;=i|{NPtHbpXt!-=Eq-lp0VBC6N=Fq!+ak)L_h!OWdXA3*{w9(87}qDV z5*oYm0$pLACL-TGPkbFEh=s1H16j3y?lY5{W zE(j*-L5_hqE|XQ{lF&DKgme`E zS?0wB-^@uSLjZL##n?`7p)j$Dd7 zRMII$zc5ngM4tk@CGMZ=lgiJWC5%_IB2{J@K zz%!qrPEui-qr!?)OJOBPhZ#-*BBGLrh$@bOyo%)2oEq{PlGkur$m>X6%jv^9&QL@} z9mcq^U6$#3P}G^6N5XI%IW02L1b5vjZvlz|hlEQd^Z_Ws_%u+0PJY~fhJY=IZvh^p zK!Pc9OSWPp7yyYdELK#pfQmRaih|KGSrqgI0ap<>P$neAAT*7F#r8G)5^x1114LiM zDG=`-l@-}7QPIgB1isGu^a)CG21Lq)eB<*A!0UqoOdv#AsK80iKOzGr1;m7)sRu*= zk&xw|@s5OHe<13D%qfePM23M+90sIkFH*-~hzJuPa|E{UL60&t$e97yXBdU3a@a~x zz|E8=Xx)@|{2X=xLKy~8guSq@Mg-wmx?9(x2P6(o3M0IrmFBFrx_Hj^n{y5i6Z!;n zV|e%$K#7Q44}}D^%5@);1(Cd@H_CH#x(6r~%upWbk~t%jW0K6oLj|B*ek7YcQ##kI z)5+|Z1~~zEVh){b+fQ=}Xs>*AwO zW~lYdX#QGe*>YxCth{X{^S~_gqG7#i_Zz)$^e*+zpS?VBX=1TsHM9KoAgRe`ul8Q; zUFf~k`TF3^((epGp$>^v!(78`!@}Uz6IV{G6;v-5RIe4d-Y;<7OV5wl>ThS>*2W6j zSJDs0j0e}NoU_ay8)V9ZARxu-8q~-J2(+$N)3X%m2dC5H!bPC#0 zlm>nP3c3fkAu~grqBy#TdW;S!Kw?qY;m)MT?bcc&h$MKOrwhv`Yhq;hym zVpR^QaZG&NTFJiG8uciX5HP8nRB%fCt4#o-jVhv8s*>7-Ay`*F<%gLIC>#0XltomN zvVii_4(1aa)j`KKz^;NI6<{{(pWs0v7;vcKsZb2G%_;CtPx1f>&p>P8Tolqc^OP72 z05=SJeGW#xb#i3bB+dxD7!n!CmI7~-L4X+^1r~4KdHqmCPPPe}&QSkNTk%1Pb%D8R zxMG<9gT=wc?pQ|UO>Qlv{%%VB?aq6atji5AH7x8Jl-fR5QmGA40smG~>VGR5V1VnH z?ON5lT0?t|4v5ACaI;ZGOk;TtRMF<4Yp-l=fq>Q#p<^$Z#AgAqb-i7ce=7NiQadkqN|$(_R_v>Wfd6l(mBvac}?L+@8(6l+rFeOiQB&fLgOF)X z=}=%ovYesHxR1#~#|FL@d&rk`RhLti^5A^{*cs}0nj0EF`Va*HG?jem9bJ~yv*V7QL~9UA>}q2tsK%zsrzyaaOe5dpt9fWQA$9KcX3K5 zmDCACjE8~CR!u6L$;&>*F;ag^mPHX#^?zrJPJ?2Z2Fg+{43SZ=N&a`}d3gkw)Y9Ya zR;j5ee?M>(bsE(@Oi`fVLH9>@j!I1x0(s&IX+2tZRzf37lh@trq^LoK^t-0QD9{*f z60%^;y2P4+DLs7|jzQ(o0pU>&D0*Q?)Qte{bX5ST$tR@f^~yEJ+23~&9=a5odVt@s z)E0SCQ$C=4`IIe+cL5}(i|$MDCx}e~yjA`rpz8q8HIcS1ke8-6g(}cbz+y#gNT8f- z>I)!vjR-*z6bn#_{Gc_7ejg7ii|CA`?!XjRMD053kAihBu0lagB=S7rghDDT*^w#; z@Y+J8xUTCNv;u(zB(C>^4Fx2uzzC0WJ`lWSY`ckNLOy)P)#ttAKn)2K6b@j6c1Rq0 znFu0`$D=ADAjTP=H|!N&z#{r=oI%`+Yd{La+Boft(*Y69xJV@BM<$h;gnSjZ?x2>5 zQi4j=G-Am|2GixHOHHeW!gZjHkXs1fwG_vU(c+Kt3T9QCc)g)ersS??G`-k* zv-oD_&GRvP>q=JJJoA~FDkxhFzdpE@U-N!`&85!U%sbk*w0E-pV%JK2|7_O><<%=z z=MB~3p84~aPt2dVp_=WAS)H?8A6rm!k?Fvi#kFj4-DFlQ`(`^o+#hXR=)Bs0rGG7d z_j3O3)%=<}iudym&TH1K*_V4R^(<=cTFd6?b&K_K{iXWLjW0K@TXWu4yrcht{?4Aa zQ&tMP-%7c;XHmCgUfQ!%v{LAND`mN$`>wV7rLOxLssNM)Jyo!4&T!E%-?^;KSvOf{ zjlcOUhq4xYM$x#@!#NVO0qRHY|N2x-Ww^fh)JPQ`qD3?&yp_>qpx$jZciEU9+e}@> z+MgIwy7HAj$)h0!PPPRdNLPh((zC|A43sAY5fegtk*o@m^tv&-!H-!?mf$C{ko+Na zU!hfLA6h}~e3%AO!b3f9n$Pt~qDPpG;FnUZ-#YdS?6eyaq>95dt-epvd2xfhmzw1F zdUnx<606tCtL~GG%&`(IANo_k*$Kx|JxG_d88vi;)RfU>&O4>$^1&!?Hl>y%wcv`R<&3;a8&~Y4U33TSGP)F830KOMc_}o)1;m z)-;U5Cp@PP#5DpBqAD1#+X^P=SETP2ywHkw;!wnO?juJBhmQ_+^~ZHRkdmxJBfJ9D z!evZwP~m%+{23;%WAa@|C@eDk6=caR>1k=PaN4+`rOf#Y$JWzw7WPOvwqv_XvU5{6 zRC$^DjYdkBF@Jthy~Hf}Z<}MbrdVe4x;AZo|HA%Q`mT*O)~Mc~Sm}!Gj4rN0Jq|1% zFR={@0l8%H$wPqAVm(ZUu#irh?qf3V*}&JLkGyVnlGmNR=@z*Z+SIJ(~~9kK692bQGbR9jY3k6L=#o-N(7e{Vn|{0w>v zKgT47$ucJILlRexO!$Dbfj}q%xd-Y6Pg2S=fWdzl4O(E05i_-aGm3$-7P5Oy@eInLYB&ql?x*IT16I-DS#VI;8xfZ>tw!1mRif zcg9YQAYpz$$}4VDjM2A-if0unA_YmGl-e8ua84?&8#ppB+~3vN(=K@NTL~69z-jCm zXU4q1X`YkzmX)c01qCq1>9P`m<;}{@ACmT9e*#eoBt1%~fW$!)4gf`ek3~92 zcB(a~)efKIr-e_TYFbiPU8nz6`d(ED{R_NP`Wv^tr~*3liS*xh*V_IE4uKp*_yfTzuYCM?m+~EGF=xl*f09X-Zr!Q*6>VCD zl8B=EDd8?w{mrdEt367We2V#BLjuyga1lTDVp4_4A7S!!NThxHD{Xq1Ii3w1>I@HF zmh-%`nHU+wRwwG^|3QlGpQoRd(&{qu$ilWj0?C)4gWdw5$bmYWDh={WlCSQ9RA{Y| z&eoXA2@sJDtYc%`0I)$}dKla{0*ggtI|L8Le+1n{yaB|6T*L!N7HT)}wFra}UN;BH z>$acNeVG;};9m9M-;iN|@F|bbA*DxA2H*H3%O38+Bqg5e*e!O(w!U+TAym z@J*D_dO+*PlS@8M#SW24=|OVHOtL^z_3ngM$4e|W#4{uC;DiYzT#5;u9IQvD!6h{s z2nswnsbsUtX|$k5iM9~94Uk!Mj4GzUWgNMgQQh@?j;~pxUsGsrH{{-S=qX;o|c2)KZM{^{BN#oFs^%e9u9mbIdN?-%X6QhU4h zPSJmAj2XJ`GTk#B>xR^s!;;b3iqxg}_fQvstc!)oWXeLQ@#&AYQx-K{0X9LpVjgG- zycR9wwG<|=+e%XRWy*q<7{XCMBOBSqIngeVaH)BcvQ(1XjHaSiw59rUo5lkh*oMF; zfMlLL$7zouX~_Wv6$u%6I3PXfgRI&z{cNX1$q8-L5b6`;aT_186}e#*k3K-Kl#yc8 zJ(6fj49vc!P2Wrs=OYZZ5*~*U99nRg1g7H}1QpO`gdr#t@Q%SM6!ZrKBSJ+Ae(R$k zl>sV_1i=N9jpxrmTa*_PnofeE2rSf*`|&{i40X?tIy*HNy%+_r$C|!pGQyUxHFfe)o{snQ|%GI42~nM3C$1y{kP*5NI|W0Ov&W@{>;xC)y5Q)7j3C|#}r zTcj1>D}%0<{W~ul0-GDsv%|OgI4Kcll@DhCoxg(38U}YZjBkOB>0Km zJMwUQw+~kL=u)@Z)ebq5eLU(U5`rwTZ?k{fL1Uu_EIY?E2@?$5QAjfc6Uxt)s~U_k zX|l;Bqz!5MpNAo;q!Z0~QFmg;+Yvs|8!Sd(ZwP6QPc{HRCqfSl5PGn+#Qh!(EU0eF z?gjpQ5JHJV9A-$aW9$9sTBH9Ti!B;*B?s7wYj|{^|MxgU=sqS7Y`!LumDUd!>3!14 z)DLr?L|X ziUDOW2(x&Wy&wkTJCqVKgpAwHziAtYXr-UF+Ebdhm84*ass%Q%U$V6osai?y5aik* zR|D&{O8=@YQ_US9d&JMXty!ve`06H=cUv>1D{U!zACniiJx#*==&J;PQefd-iHMfB zJx2Fo$#tN_axj#-ZLuA&?LIt1U1ti#bwR5Ea-!Yvb;y${{pkT#`j*qG<~q18_v6y? z0ZYo!|ixpO|b&uzmWCVK+_Q1z01;aLK9O0TwMGMmTIw*xZLJZWwMQkDcr z$?E_oj?(=Vz)dzij~Y6eSRHlzFQNDp5~OP26!?V9LtZ%two=ADlFVG7{2AC}U_W*~T5fD@pTYB&P^vdNSQ1>Dhxh)v!1c$kPd% zH=Uw)=t`z-3x1sRfQwObHA>_S4ciDVgztb`5HV|8kE6-iU<-$mOW6s3J%ik_y2Le{ zqUxtW=K~fFoa0|wiN|AtBRamFR(ed5qoW|I*|b`JIj0O6Lk2$uhVZRpc`CW8tv49Q zYjFi3>)^k%2KS&9eS$;T$)0YY@RFg~9Nl&*1z?H`S=+2#ie6wqa^4S)ql8=U1PJZ_ z1^UNL5S%pOg@9CHcx2owiV+Q}4G`=w7=h55q!V8bf=D{`8`)&IfDjBqP!fb6q2_Wv znv4@T17WiyFsYoVe_j^jZ`i(`qk}yIhfuAkC12ATI9_BDfdmXdn8zWdk2+cuCA4JgbVW;( z^%29i0MN)uTp$K-Y$?(ScN?A;=)3Y-iEm(MJXsT41cyYzz%Y2qw=;BX93z)}aH(W* zu!duu=QL&V8l?9+8*)f_q5xK_uC69QR0=RrcJ1DseP9$e1yAt-2NMU^gfJ?~Lt&S= zW`YlUAzl}OgVNexeKrP!AcqaS~o9AG2P2vWG zHZ1xpOkTm{>zG`|WC0VRPCtxd_A^Y-t}mcq76((#Fb3^_k)7yk!sl28;-(x5CtEZN zVPCQ?s3(X;V)%$lAg=OFPJ~2aLl)3TD4R2Xh5aQ=(D*4BF=wP<{&;!^BB{?U1VBQ)l#rQYe!aG;SI&7Aa; z!#zvPhC-3++E7v|>t_(Kup8VDD$NT=zIJ53cd2+oLFZN`>J+M~4fST-YrdO<>p4A% zI$G5${ko^arg~xEYXkEwuZ5vw4mi{Bu68eI;vXBc7pzxvujH;6i)Xsljpn(5ivugh ze8_7IFI;%x!pwzrlV$G0#S3eul4VoLnyKu4QyD}L#;lH|;y0W(oH0j3EWPo^miKbs z&An^v`@mwoXSOXe*Hf>hu0+kQS@nme4AQ=0srHT58?CX5##nmOkBi=`c(>xNu^-!O zQ|47Kk1Ui*=jKnwcI~~{`Tc=62V#3WVyT_0TJT>wS1hg@>cu@TA6jUC4k7c zj_Nmz-!_7e^M-MAG$W3-Z&v$pxpQgq^^_IM{u?Qql_^-c|E0dN7DoF*-`D!qn9OA+ zb0NIsi!qt2%wD)o@Y_~hs#;4gSxzs39X8)+UaM?gu56A~w8UEaW90*P(+8xzeQ)fU z?S;S?;}zrlg_yo{l_^8R$CrzqKLkGEx#qd%*=9-Af4J&J4IIy`Z~o}zCoVnl&55PX zH~Mb$-7bEo>aD83U{(tIS270frVgxXk3ci&NPk)M*Jfm6;#BeB#oH`Q8O_ zaqso|YxRp2vDE5StrOk2eV6(cJFfR%>t8Fbj}_P7JbKsKFsobFn&(UxO^eJ@c-FM4 ztp%5lF>S7Gu5Gq$L!nE_xtC>MEP@N_d$VsPt9_oimsPUJU!S-(0cAaJ_N zrrd?1t6#nH)uktKi|e_CS6x?J&=bOCl^>h)E*HL3xTst)7ei>R!MtY3UpC|~(qLg) zH(D3yZ}iVfngKQ%GxpuzM`h-I)=a7Nv#wPp_nszoHaz#$i(iF^vY{(OH}|b-8g4(n zs%b?BU;8)Uo1!dj+#ix48y8nw` zTK0eOAf2gD|K>A}rmTf)mfg#i-GJaJIqPZJ3(@v}oieWlIE-aiEbUz?iDmBobMgC; zHzRk`4*p$Q_CIZCpy6*tq{%<6v!1GD{yN)ws)70IX6va|CRS2AtWtSJc>)iM;6y6{$CEz$dNH5*5EOS>wF;hVKkZGa0(V2ML#@zb`s2h z5S}T3a|GB4l4x`j3kR@XA@D*y77@eYN$H(|-soRq?N2e$V1gWs1Y*SL9^py+IE@Jk zHiQ|<{0tqA!Y)j{z~ncW{2eAAVL~{~CzvCghOi*CTFT4}$+2L9+BgGTTN*o(m*ayX z_(zFeNWqUs)AyM?nt9kLee=lm_KgZE-Fi9qrQCI<^Uj&~eDC@I&eHqnnbc2>$_zz} zGCwrJ|4jM(F}j9^cqd363J~2#s~`5EyLO{SdfeS^26xuOMle7>Oh+^1#sDK-akKvy Dmz7on delta 4831 zcmai24Qw07mEKu$Ns6RMNz}h+aj73mltfarW&Qu!mhFU=B5f?Dj`d@jTuHPkYCXG@ zEyazJHmK!ZTFafgS}qQ?+!~3TpiLSDxD>tqrfAX&+LI$WR$1dNMhqNqJ>aCKxr1@Q z<=!ksOTG(m1Mr(S^WMz7c{B6&J^uGY^y3@IQf)~$GEn+QzH<6MI6D zGmjCn@|b9bQXtrX&Q|F{A*ZjN%0FHdyO68wx-wrdn^%XDETve;7Yb@w#UT{J@FI0M zFW79V16p{(&s3==M77&|kR)_3&rg=x#l43#;V zI+a1*W^=UVag%2GcWe`ymCuM#QJIW+_+#TDADWDfM#lItF(%@m5}Azg!C07=qQO`! z5*y=#n2JnDFhW#0iYO~$xW#zV?H13Ap|~QRdwhR6A5TAqw0^pRL78V`%)oF{?CrznIdgeh16Tr3`<^eFl3`WE&T(6jy}?NiW0rWz(t z4|&jV5d}%KvDnc=#d;}FHXfsJhJr|8l-z18HN8Yh_{k3$a<_4s3^ol^7ugCe8(eO# zal^x;<;+hl>Xx(%&#&Zq)(si+^$YbYStT2L3(dL>rjXokI>q6AFabH*+(iB*w`S-= zI$I+EJ+usD-J~!d*r3&7om#z_tyaI#sMWf?VnYMW^_qQ~v`!u#2Q#FK4hT3r^l2qA zv`pol13$2v`hfr^aD!WZ1~mAqmJ;&wmi(-q?8g}a1l_+mtixY}c~oWbMM}+4xCD^U zgeJl=U1c{dJkFz?CU{39;0PwbJK%ZfQFrz=vfA3*yskH1oPBY2p=M3Ld);7~x8AfZ z9($wbR?nUCrd4P2inZmAq2-e1;}q-r1uNF-JBI2x4QXk6O$VNg!NFGZBL{~x+42Kh%NKA2Q@wcdmc04#G;(l_pt3vl(=z#gByUGz_m#^<{)u1VI zue%s6lK<*Cr;IeHzT zKe9~wQMi6G^)~dj4_>%~OTEhv;CoAIdP>MM&-U5})sHU~5;TIg zL7NnJFaf47oj4C1;x_kkgq>SIC2pu_{wgFw728ccK=dP$=@S!U4hFz){J9U5{~#1;f>UQ{O7&Ihc!g8je&kg1Z6i zNekEwJf>uN=Jma7uf`9rncCD3&quYbSu#qd9X9n0HYLZw8R!WYa;AHC^q6T6R|_iE zk{3fZ5)EZ=zO7a%w};n_*CtxmpeL5pF1We?{ zsceJL2#uydI=LU_D~0CN8NoGXN|``f-{I6Ax9g-%hLld^zL;aPs4D#X4i%2jrOI0x z>9-B;41C+*^=wa)qEuxsw-mE|e(Dwu!_#X`?b5*v_N3&VS^lEY&yc>}l*D!oD-mC% zt+0-|W+;+%hZ)*bGoX=dtMmDUw|H>kDW8hDhcB3X8<)1i)nby%U3{AAp@cgMr`Zzk{54)Lyl1zza>$6ly~T#W3R z} zCu1TH5x5vqCh;^ciI`8tz*00W^Kv9ChJsjjxikp_7SHCyK{kxf({;4tVpMV={2@bL zKXZUxWyr=EuYL_!80hat&*<-L(HRp>1#}4B>VU7G&h4hDMS|T+PEFL>GdjnB55?Z; z(k8fA5Rb{~DlmlnaH5Fa=petE@SA4R9%Th2D|WPerr=ThukME?AfuDLwGFDVHqXVAU%+I*^WTvvcZSr|6-#nA!d6Za?(#aDQFkT1Qju=yBtd(FG_jqJ% zJPJR>HN*1!jHY!@Z36M+? zaOcJ&p>gb|a{8?^!_8EU0t&yT@Gb@LAq=K3XDe1FH_qnFzC+t@QAh?2;rN6kW9m0C zq@c;DCt*B?BZ@epi02h-roC@cprPV+Y>EGYHa`GJ7_q3taV#_$7G0SMHYg>~7#^S; zt$Lg-ltfZOKN0DdA-ow&7-Yy>Ly%dju~Wj7%v?vLpfaA&(mkmmEs&7ACbVJ5Qxk^K zU?dvHqKq%o>2yHFqlye4eVk4Hgw3YUp<9hw-qI3QuFN{RRQGmxy`XDQX~jK4JAG5G&r&A3N?qwup2toDUt*N3hRty-&JII^xcUA%Dd!V4GHi%YIYzY|@} z>bw?R$b9qsZBNHl-@NZybWXpL)%nNF{JE}s=FIs+3%-TYm0gEcvJS7AkF1-s@0uOA z&5lKMvth~i{pOF%o=uL)D%j95Rh~EOuh~~^^|$PcT?_qJ{qz1?_BliDimiUm@W7V0 zV7P9*W?jv9t=g*Q45VtR!km?R)qTaiQqpuMtBHJb>W$Ku4%0ohUT&Rhg`K?|zI^)Y zrx$XU(3+)c%}`Cd^dxrfN%G)a#q1w*3NEG1Y1d7b%N>_G7NTpWs`d2D%R`rj7M`Zt zy^~&b*X+A(_PtZ~e)fvlw`T6WXDL`Hz5eBEUtT)%^VT1?t``Q(^YG&D-Zrzl5*I=^WHrW@^ zN96PKRiVusCb#e}c7`*~9hv{q{F5ui-W6l*n%4Kv8ToU*6|G|-2=MUFA;j3a(8E6) znCym!pIWq4IyoP%!u#olffDWeg*w3BmSzrAXg?^g8>rDG3?m~G@c+cfh^vB}oyjY{ z55nqWz@0!PSS;dyqh0F#{#WwKj9>T)ZGV-5o`U+$==)1PG4)-P(B|6|mML7N@Er;? zH^Db4(A)w4p2Fu02|rg+V1S5&1|4!4AfzD(ZE7=-_OspO=5w!FKHEc6|Ia#U>bT({ SFJI`=(>Q!XV#z;U;Qt3tO~eQQ diff --git a/scripts/gdl-sync.py b/scripts/gdl-sync.py index bee0323..98424d4 100755 --- a/scripts/gdl-sync.py +++ b/scripts/gdl-sync.py @@ -6,9 +6,8 @@ The CLI replacement for the JDownloader2 workflow. See docs/gallery-dl.md for the measurements behind every choice here — especially the safety model, which is the reason this script exists in this shape rather than a simpler one. -STATUS: exercised end-to-end against `withaseul` (posts, reels, stories, -highlights) publishing to a scratch directory. It has never written to the live -archive. +STATUS: in use. `withaseul` was fetched and published to the live archive +(322 files added, nothing overwritten or deleted). The fetch host needs no copy of the archive. It stages locally and rsyncs afterwards; what it already holds is learned from a *file listing* alone @@ -17,9 +16,13 @@ afterwards; what it already holds is learned from a *file listing* alone Usage: ./scripts/gdl-sync.py --index https://instaarchive.ergosteur.com \\ --staging /var/tmp/gdl --publish user@host:/path/to/archives \\ - --profile 0ct0ber19 --dry-run + --urls-file artms_account_links.txt --dry-run - # ...then swap --dry-run for --execute. --index also accepts a local path. + # ...then swap --dry-run for --execute. --index also accepts a local path, + # and --profile / --all work instead of --urls-file. + +Always --dry-run first: it prints the plan, and the publish step it reports is +the one that would touch the archive. Run it from the host whose public IP matches the browser the cookie came from; using the cookie from elsewhere is what session-hijack detection looks for. @@ -114,6 +117,54 @@ def scan_archives(root: Path) -> dict[str, Profile]: return profiles +RE_PROFILE_URL = re.compile( + r"^(?:https?://)?(?:www\.)?instagram\.com/(?P[^/?#\s]+)/?", re.I) + +# Path segments that are Instagram features, not profiles. A line like +# ".../p/ABC123/" names a post, and treating "p" as a username would silently +# sync nothing under a nonsense directory. +RESERVED_SEGMENTS = { + "p", "reel", "reels", "stories", "explore", "accounts", "direct", + "tv", "s", "invites", "challenge", "about", "developer", +} + + +def read_urls_file(path: Path) -> list[str]: + """ + Read profile URLs (or bare usernames) from a file, one per line. + + Written for hand-maintained lists: blank lines are skipped, `#` starts a + comment, and either a full URL or a bare username works. Order is kept and + duplicates dropped, so a list can be appended to without care. + """ + users: list[str] = [] + seen: set[str] = set() + + for lineno, raw in enumerate(path.read_text().splitlines(), 1): + line = raw.split("#", 1)[0].strip() + if not line: + continue + + m = RE_PROFILE_URL.match(line) + user = m.group("user") if m else line.strip("/") + + if not user or "/" in user or " " in user: + print(f"{path}:{lineno}: cannot read a username from {raw.strip()!r}", + file=sys.stderr) + continue + if user.lower() in RESERVED_SEGMENTS: + print(f"{path}:{lineno}: {user!r} is an Instagram path, not a " + f"profile — skipping", file=sys.stderr) + continue + if user in seen: + continue + + seen.add(user) + users.append(user) + + return users + + class ArchiveIndex: """ What the archive already holds, as filenames only. @@ -451,6 +502,9 @@ def main() -> int: g.add_argument("--profile", action="append", default=[], help="profile to sync; repeatable") g.add_argument("--all", action="store_true", help="every profile on disk") + g.add_argument("--urls-file", type=Path, + help="file of Instagram profile URLs or usernames, one per " + "line; # comments and blank lines allowed") ap.add_argument("--cookies", default="chrome:/home/matt/.config/google-chrome-devtools", help="gallery-dl --cookies-from-browser value") ap.add_argument("--archive-db", type=Path, default=None, @@ -478,7 +532,17 @@ def main() -> int: index = ArchiveIndex(args.index) names = index.profiles() - if args.profile: + if args.urls_file: + if not args.urls_file.is_file(): + print(f"urls file not found: {args.urls_file}", file=sys.stderr) + return 2 + wanted = read_urls_file(args.urls_file) + if not wanted: + print(f"no usable profiles in {args.urls_file}", file=sys.stderr) + return 2 + print(f"read {len(wanted)} profile(s) from {args.urls_file}") + selected = [Profile(p) for p in wanted] + elif args.profile: for p in args.profile: if p not in names: print(f"note: {p} is not in the index yet; it will be created")