From b7b91b37bf3ce96170ac53a581aeb2392c624ffa Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Fri, 11 Jul 2025 14:40:37 +0530 Subject: [PATCH] Add unit tests for log_broken and update_contribution in fetch_updates.py - Added tests for log_broken() and update_contribution() in tests/test_fetch_updates.py - Ensured both functions behave correctly with expected contribution input - Removed unused mocks and patch decorators - Verified tests pass using pytest --- scripts/__init__.py | 0 scripts/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 155 bytes .../__pycache__/fetch_updates.cpython-312.pyc | Bin 0 -> 5051 bytes ...nd_validate_properties_txt.cpython-312.pyc | Bin 0 -> 6896 bytes scripts/fetch_updates.py | 183 +++++++++--------- tests/__init__.py | 0 tests/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 153 bytes ...fetch_updates.cpython-312-pytest-8.4.1.pyc | Bin 0 -> 5367 bytes tests/test_fetch_updates.py | 42 ++++ 9 files changed, 133 insertions(+), 92 deletions(-) create mode 100644 scripts/__init__.py create mode 100644 scripts/__pycache__/__init__.cpython-312.pyc create mode 100644 scripts/__pycache__/fetch_updates.cpython-312.pyc create mode 100644 scripts/__pycache__/parse_and_validate_properties_txt.cpython-312.pyc create mode 100644 tests/__init__.py create mode 100644 tests/__pycache__/__init__.cpython-312.pyc create mode 100644 tests/__pycache__/test_fetch_updates.cpython-312-pytest-8.4.1.pyc create mode 100644 tests/test_fetch_updates.py diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/__pycache__/__init__.cpython-312.pyc b/scripts/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9d7d71e60e062f6b5cceb0ab1a5d22dc60c92010 GIT binary patch literal 155 zcmX@j%ge<81XETQWPs?$AOanHW&w&!XQ*V*Wb|9fP{ah}eFmxdW#wiS<6n@PSDas3 zl$;t2 cKczG$)vkyYXcQw57lRldnHd=wirL`-WG%xHfEsLN6 zAqY^VF%bDBB9ItRi##Yu3+PdKOPnG#kUVfjxuUlYs-S(SpDNoyojO3#f0kU4QiGxo zU4k>S-~RK@KQsUT%|G8*t!4zz#9Hk7W+_5{!7ut_$v2**D1;UekGzN{crrwI38<+M z8794CnDSCY4yU~|;B?3kF7cLVm?2~gGhRk(mxN4Vv)3H9cr9Vp%K~P67kRBb17#_1 zg0f7o@#Y&H2+b2>ltcm%Y}QRF)Su;>-g2Ql=q})Ti?@76MadUp7mA= z&R|#Jo8A)HpU1tGU>6CV|`P)65gz7QkA0 zo6!6b;_Y`S{TqP*XZZ5pR_F*=(I&J7PZvhlTS6Jgx(j-B8wSu@FzOi0!dvO>5IO{~ z0drnI@;n7PJ>h#1u)OtB8Q#ZS@v)6-^QUfyaIxpC)s-ry((BJ}>xr zi3>zH`MSW3MpixH(HBabYXUHU(gpI&1$WuvllMoG)2oMS*xW?fUG!^>lE%-*` zZ@MM|C5a$7>CQ)YG;sV=2+dGQiZ^J1cQ{ETHSOXa85}?N=vc5!M+!AxqK_4{7f`*$ zZv})dl|)Hx7PpV$8Rgsk`UqRdFpYRdn|Tw=$%N;`Z zKxHIZoY0)I#2vz^^@Y})Bon2rtFmbSMjbn80ZsnDHE!z+zS*XwKasSSJOLhYT} z4_$MHti?7zylUaHWsdpLUnL*4cP))BANX`^z5BS*ef$e5(><`>eM;#*b^oPI_nA!l z*?BfKx@v3LXz$Ql&F&?`ue;`kQ_fYkakC7SH*A(8raT4Kv}Id%FEdKZE9u^`l_U2V z}bLN^a7x}Z|2urw)Q2I4r8!+;rNeAmS24Iv`p^a74KBprid7G;eU z^RFq!`o*eMV@I~h1Cza?Fbx|-A~u7H z@nYi{0JO#U7od1C@M$_j%#i3gqb<2O(Ry!W)w0O(a6e8}Js)&?Y2r z(IcQG#iTf6fUjF*C2x^Q(x6j_Ue6elWY7r;cRaj>M@1(li|x_2)tDPUnX`+PtMraQ z=*B`1pgVGcyntGetYx%VsuhXcOi2;xlDsil0{I4$q&`A->7BjD$*Vw#GbL}AT*%i` zgbZ16?g4F;1lp#}r0FTjyK+`wJr~dv(ogRHs>A%h=9*Fo* z6WgHhuo};b4h&%a6Y++L_<&XIL!fyiDi2}3gQ6G}wIEPUgSW?I<0UX_YP zV;aMG7?lplLRd8b0SmWsk!99$fgIVg^7Vk^%g;eAEy^N9|CCAyVm;`Suu{y$_{jyM zBTvKINfkU8;VfL*50L|5Sqo8CH9I(W;Z53TazCoyv+P>AuIxRTsef&5WYb0i=4ksQ zxZIFwe>LOin;YCTBa?HTsaBZkRQ(dY#&l-k60T7!b*bd4rSlQnxX3OEYwZ3F zfV4zwtdq8%6|HRMYftE*|(R2%VX)b6Kib$PT3dI%dmQ1lBFjc z=u5k!zaHIlnEGp}d-cu}hp2n|2;4r@2<2BbxcybVtN#e~)nNikmGSw){y@a%OBlPm zwe40Ze<+kFFFyBjiK$A6s&yNjsJbBgA$^2gv~a+3(FrbL-;vn%^gM>N89@SS%1a8; zV@#h3^I0)EDZ<&@%c;cf1j%(g<}pt7?43xEZJtC`Uno?pemokEg#=mH#=dX^0WyJf zbQp>^U?|MMZ^4l;LnH|vVfHKVu)LL`>O6-JEQz>XlO#SQ76{1i8)!65{EnI;AeSZs zJWi>p0A%3UPchW9MF8bDyW;21sm&a_@bNi7|EUGaJGHIEU1KYu5>uWo)#&r_(Q%(oW&Gl#mXzmW9EZ1R zjQQp3p};lOBF6n;A*9_JM7#oOSvVe&1BJa1ahk4r@<~olF1?4Fg>Ngl2ic+ts#&JuwU!LHIiaWWo;JIPsmV-VGnoRKQ)=XoF4tbNn(6YwsnG%mIE zgF6UJ04u}12g-0~P^DUdV*J<_6X-Af>=jES&wDjkE1k6IcC`(wU%Br5WP2q~vYh=8dgWVJb#qhSkW)d-ZE zGl7qXW1k}Jw9io(b0i$)<00Xg7=RVQ9xVL>%1w$Oh$kdT(9ak|*#3-~|Ab1vL$2=- z_nf8)@(F_CX=w#9Nj&W`60J`ijYQ+-0D)+BE|H;XvewqQE9=%)#oC&&w$Bb_Y4bW= z1w5ECt{@s1RUCWQ9fuUhq4eQ1isQ`e>4%oG_g|hJ_}*ll3#TrunVJ^QXUi%+n0{|MRsR0W z9Q~X^=H|^3RNIh>Dm6W6x)NtXhIih6Xsbv&+A}scWTg)0oarHBP1_*1T%1bx98Z_N zy2kWl zR+Bn+r*z)*(B@2;7hhTITspTDUz%J#zf$wXfzKQ7S17Lx-ycsOJfA*)F-ViO zFX>>>5?szQ><6azO!LRntuLp`53W?fY_yd~o0>LH!v54guZDGjw-?VHFmBq)y?9=m tMoHvOXRZTt|{ z_9PjttU++UG%Mk?g@^h!k!|OiP85OL|jnpB^RjsAM3x~dkqICzw=<{<_*XYLGb7!B& zhYmphg%|Yq;GVgQ&-ZI>l@)b%f_+*$?)e*}b^KhY^mA;#omywDje{R>ChZ6GLyN_KCK52?&K9_?!bdJszrX0??(90e&8$A z57AAr_``rqxJ=jPFZa0ShCW$K19^?uyHbA=a;3HC1iMt%#cny#oyiX>B* z7*VV%#)6?tM`Ku-CdoLUV$o|vC5yefLU0VLI#yI5oK_=--Y=^OO}VMYqMB;3p|eI_j+v($bQ6CLh=dQxzG#^Rq_ zO~Uk%y5iBqTli+wByh#mg=(WQl8mQRBYG`n-kN?PsvCDXDuL9gI+fyZlkibvvE+OZ zy77WR)S!<#!ePc`I85bm7>Lp_R(FTPH_~dXj1hE$P+vH#CK5>lmPjV_aF{rO9}-}I zc`VY%*b|}&gY06>-3+udz&sswv-M+Pkb01yyJ!pg(07-|hOS;r;e?({lL)>F$`!%7 z9!<;~X4*lb*V5)%SM>;qrVRZmoRN-!i>ln=a7`h@ppXNpMdD#=GSCD+Jpvwy_{~J=1DeZa3u*pDwnKWJXtxXGc~~7Q~Tn8#_zh=52v*viaB8mUd`~1ih?m-2+gsB_Orn+3!`Yc}T<(c+-lG2pZ@EU@P|7jvZJs^~9!`+5xgG zV`-W%_nF!O8sn;?GxY@4acfRZzUQjW<$*r0=GHvl>!W=j=>ghMll0LA5Hz?mDqC{S z^w&Q?_v(hDEz*YLEOjv3)(B8&-1lBMqF4`kq3qL*N(&h#RK9=Y6W zudTlJ@ejUk+?BoY$+4C3M~zLUoCN)350H>91{4NNJ!5A{_A;=KftuRcS(CA99V$~c zvY!DL1Ftf0fB{nm8S^Aq0jyt70r&}e*w~W0Rc!3fys>h*AoiOIa{pjaIa$+@lO<0> z_SKw_J-!+)h#jTw-pqLRcy8pUAn(1Fyq8)#wnKXYPk2#k z-V%`QG-p;VW*8q4_*F~gZnj#Wt6m{$Zex)Luhw+Ia<#Q1pTP6xkxT@oXHTzoX3reM zQCNg9a2$R*J4Ub*{>G0kTHGf)xmzi0gd?#so`1eG735;mR#e>rD!as4jvtNX~}7TyBM4`;;|sV zBv0v>93Gwl*F|l^kqAy1kD16a-I;|r3EUO68__sUrj5s60%FOngV#vw5c6nQS!yT; zIao8}7I2D6&_*2u-$=7yh6*#-pe}0&I1CNbz!=0?6G>w%nNDb~^XJE~} z>dwA@=k4EKD>{ZV!;kEazg+(Da!&Z2z5Q!%*X{m0H-58xclk~-fAV7f(wiHXrq(Y_ zeL0_3-Y$6G$vfU*ntp8cSZ3T5s~z&f6x)=Se}&4Y=#NQB#a_)-90!c5l!}ErgVwk; zJ|aX>gntiA;#H8hWey~Yt4Yl*-l$4O#qg7TAPdZ5JX5r)mDEgkGRGY2Go`Y20_5ov z{^JuOS{BSTh0DMNha$o<=)gP*3KNv;w#=-Z@^dLF#gY(KQwajuxK3pVY7%hm5DuA^ zj!Jqe2D&DJKZ&LwBCWq>qRvy{I$op_grI3meRU$BL4c|=1%kUT8*JGM9hkh3DZu;) zetH5x25s9=d*?6ve%5#URH5yajke?KZO4mkLtw~u+rDiZT0Orql5O4;kkftd{QdM7 zkz)Vpm8rFNSKrMw7CfCfb={+EczV`7Jw;F7m(sdpywoJJwp<_@Q(+yCs6_!<=CRDcCS~{M(%aci1n##44-QfcRl|f-#jsVDeK>rr} zbmnX_=-&d}8-Z8W1FzhFy$~45oPXqM-f$`FF6EAqcPRzek-T{1d9C=99tL?EID^fb zLE;stlMn;!3A|%-?&{_e;9A0feut4%_SLl!7+4Ps-1&<_;PwAk`!b!{|FEy7`hA7V zPgbJB-k1Tb?(e`x;K+L5$o+63F#5mUAM;A45dR%i>aCek^?5J!Y8;CdwQ4wf6zmm< zb=AiKvRpL5U>4IXFY;S8#n}0%*C2(ap^ByB&?4$aFC(3s2itT7aqzcTP7egd3F>*{ z+~nD*)8UI#lb5C@DW@*km;&f0!AFVk_)kzi3H}WyZbk{b(IoR3PQ^K$HJIM$;0Hb6 z`ci2K56ve#<)(KCmkbIRwkSv(wxhvM{~>@3`r7IF_}bds>RhgC-5b2qU33oI-&=Ax z*X*nIY|m#+zij(i+wJi}>nr(|SM&ZOMaR+q5Rg2$;T%|Z4t%FG3;v6~VW05#K54kc zzQf2=qmwKU{b}y!7WzM&yqQ8E>@1dSW=wrDC=v`!GR?pY1G4~FNFLW`NDS&bj@4f~ zRx2ncEVgAsbfy-6>~M39IVtNcI=Xh8D3iEfG@lT@kcK5Ydo0J*XaXL{qOk~xAS()) zHH_4fZwlfVcph7Mm?34!Gaz4vfH4J6qk(~er6Goo>NOHb{1}Qpn@qZ|=B{QZAC&OkFk|t^-YAh0FafrwObA*(KxSWX#wZZ)WmsVm6aK$J`@TV^3+VJ$sD0b!<80Y0 z+Xz0}`}|yUZqGJ?&vp;THRVoiBltXVAl|<2-jlmgaPNKShR5$;jhB3l+Y-+=Ke5SN z!SL%H7u_pM1)+7*hP($>-r4Z%U-#@UczQBp zB~jiGcdv`PbGD+`RclE_aZkw=*l=~MyE+Q4J()9)-2Sz5tLHMKCE2^yx7wE(DY?9B zr&dqpdWx>S4;`MY@3xRj-=4^OUoAS0Jan|=+oy_-w@coRyrbh=(YZ33?anr@94?A2 zo1!TBAGY@2xm;*{dBqD)U`fuqJ9CMA=i$6OP!tCt7~HjgqxsN!^PxK<_nQjM$1)R} zyj}8_T>k92-?=&;Huc?YyFXC~jue{Ctd4%$r2MMw&O~AF4+>2uRz^47D6p&4&7p2)WXU8;`mJN+e)jeqELjQP<&3j4--(F4CUwqJPA bERS^w54y~HuQYZ_c+l?|drf%o8ms>|`U_v- literal 0 HcmV?d00001 diff --git a/scripts/fetch_updates.py b/scripts/fetch_updates.py index 971c346..57e4e3c 100644 --- a/scripts/fetch_updates.py +++ b/scripts/fetch_updates.py @@ -1,121 +1,120 @@ """ Reads in the contributions.yaml file, and updates the entries by hitting the 'source' url. """ + import argparse from datetime import datetime, UTC import pathlib from ruamel.yaml import YAML from multiprocessing import Pool -from parse_and_validate_properties_txt import read_properties_txt, parse_text, validate_existing +from scripts.parse_and_validate_properties_txt import ( + read_properties_txt, + parse_text, + validate_existing +) def update_contribution(contribution, props): - datetime_today = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%S%z') - contribution['lastUpdated'] = datetime_today - if 'previousVersions' not in contribution: - contribution['previousVersions'] = [] - contribution['previousVersions'].append(contribution['prettyVersion']) - - # update from online - for field in props.keys(): - # process category list - if field == 'categories': - if props[field]: - contribution[field] = sorted(props[field].strip('"').split(',')) - else: - contribution[field] = [] - else: - contribution[field] = props[field] + datetime_today = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%S%z') + contribution['lastUpdated'] = datetime_today + + if 'previousVersions' not in contribution: + contribution['previousVersions'] = [] + + contribution['previousVersions'].append(contribution['prettyVersion']) + + for field in props.keys(): + if field == 'categories': + if props[field]: + contribution[field] = sorted(props[field].strip('"').split(',')) + else: + contribution[field] = [] + else: + contribution[field] = props[field] + + if 'download' not in contribution: + contribution['download'] = contribution['source'][:contribution['source'].rfind('.')] + '.zip' - if 'download' not in contribution: - contribution['download'] = contribution['source'][:contribution['source'].rfind('.')] + '.zip' - def log_broken(contribution, msg): - if contribution['status'] == 'VALID': - contribution['status'] = 'BROKEN' + if contribution['status'] == 'VALID': + contribution['status'] = 'BROKEN' + if 'log' not in contribution: - contribution['log'] = [] + contribution['log'] = [] + contribution['log'].append(msg) + def process_contribution(item): - index, contribution = item + index, contribution = item - date_today = datetime.now(UTC).strftime('%Y-%m-%d') - this_version = '0' + date_today = datetime.now(UTC).strftime('%Y-%m-%d') + this_version = '0' - if contribution['status'] != 'DEPRECATED': - # compare version to what is at url. If has changed, update contribution to - # what is online - if 'version' in contribution: - this_version = contribution['version'] + if contribution['status'] != 'DEPRECATED': + if 'version' in contribution: + this_version = contribution['version'] - try: - properties_raw = read_properties_txt(contribution['source']) - except FileNotFoundError as e: - log_broken(contribution, f'file not found, {e}, {date_today}') - return index, contribution - except Exception: - log_broken(contribution, f'url timeout, {date_today}') - return index, contribution + try: + properties_raw = read_properties_txt(contribution['source']) + except FileNotFoundError as e: + log_broken(contribution, f'file not found, {e}, {date_today}') + return index, contribution + except Exception: + log_broken(contribution, f'url timeout, {date_today}') + return index, contribution - try: - props = validate_existing(parse_text(properties_raw)) - except Exception: - log_broken(contribution, f'invalid file, {date_today}') - return index, contribution + try: + props = validate_existing(parse_text(properties_raw)) + except Exception: + log_broken(contribution, f'invalid file, {date_today}') + return index, contribution - # some library files have field lastUpdated. This also exists in the database, but is defined - # by our scripts, so remove this field. - contribution.pop('lastUpdated', None) + contribution.pop('lastUpdated', None) + contribution['status'] = 'VALID' - contribution['status'] = 'VALID' + if props['version'] != this_version: + update_contribution(contribution, props) - if props['version'] != this_version: - # update from online - update_contribution(contribution, props) - return index, contribution + return index, contribution if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('--index') - args = parser.parse_args() - - index = 'all' - if args.index: - index = args.index - - database_file = pathlib.Path(__file__).parent.parent / 'contributions.yaml' - - # read in database yaml file - yaml = YAML() - with open(database_file, 'r') as db: - data = yaml.load(db) - - contributions_list = data['contributions'] - - if index == 'all': - total = len(contributions_list) - completed = 0 - print(f"Starting processing of {total} contributions...") - - with Pool(processes=256) as pool: - for index, contribution in pool.imap_unordered(process_contribution, enumerate(contributions_list)): - contributions_list[index] = contribution - completed += 1 - print(f"Progress: {completed}/{total} ({(completed/total*100):.1f}%)") - - print("All processing complete") - else: - # update only contribution with id==index - contribution = next((x for x in contributions_list if x['id'] == int(index)), None) - print(contribution) - process_contribution((index, contribution)) - print(contribution) - - # write all contributions to database file - yaml = YAML() - with open(database_file, 'w') as outfile: - yaml.dump({"contributions": contributions_list}, outfile) + parser = argparse.ArgumentParser() + parser.add_argument('--index') + args = parser.parse_args() + + index = 'all' + if args.index: + index = args.index + + database_file = pathlib.Path(__file__).parent.parent / 'contributions.yaml' + yaml = YAML() + + with open(database_file, 'r') as db: + data = yaml.load(db) + + contributions_list = data['contributions'] + + if index == 'all': + total = len(contributions_list) + completed = 0 + print(f"Starting processing of {total} contributions...") + + with Pool(processes=256) as pool: + for index, contribution in pool.imap_unordered(process_contribution, enumerate(contributions_list)): + contributions_list[index] = contribution + completed += 1 + print(f"Progress: {completed}/{total} ({(completed/total*100):.1f}%)") + + print("All processing complete") + else: + contribution = next((x for x in contributions_list if x['id'] == int(index)), None) + print(contribution) + process_contribution((index, contribution)) + print(contribution) + + with open(database_file, 'w') as outfile: + yaml.dump({"contributions": contributions_list}, outfile) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__pycache__/__init__.cpython-312.pyc b/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0a4e571523d4596f2bf33b38540246ea17791c61 GIT binary patch literal 153 zcmX@j%ge<81ZyuAWPs?$AOanHW&w&!XQ*V*Wb|9fP{ah}eFmxdW#MKO<6n@PSDas3 zl$;tpP83g5+AQuP7&-72ZGYl3YpDpKU2|Y+9-sF_LLp)=%qNbuH8_k{U`Iv_dL4AXc1}v_oFc&&=ag%YMSEF_5-4mBHBj`>n_K7PQ{UU!A=iZrDZP5J zdh_0!H*aPhXW#oqf9>o{3h+$rl~(_#2*N*5;7`y4!t;}&AUqI=K*WNuC%T@Ykh5=b3`!JLv2rRi+1}8%TsL&&b`8NkoUJ+v-2_R z4Id&smKeu1Tn_dp7rNF{iz(ohd%|Kji2?1|6f^3-aL{~3%4Rd7-??nM`PE#xL=4w- z^2{Hj8Z>+;q=i#Re{LuWq>5&9%>>{YEo6KN*|AcBv)mp z;Kum4HI;Mimc**^vPe`Mvn)M^9~z&?m^jl!X;tKbIKocic0_w5Rbcj-LgFM*RV*=( zVy@2pMm%+JUAwKuxF_EJ9uC_bu_nTwT$4z$Cc+Mc+icHYM7$lgXFb?+2afpednTQ3 zim&Xj1Nu1Zye&87xuUUdX5zFTSES=G7J-XV z;c`@%U;>+UDk{uGg~_Ne?cm3f4xbluKf7}4{gPR9>@v-pw@TE`n~q}@SI$QR>Diii1jv=({<(a!&~0i zkM@+SPxVs|<8}RFMdFg6}?)nb1Iiu`LP%?%!;6bGFqD42(K}rN-*|8Bdw?hA(?VlgNNLG+Eatn+m@B zT*JghAToXz_@U5@X&Bv8CZ9vJgu_eRUmzW`5=UgBJd$eyi4m1(Bo4fIRLO(F6P7}# zwh2g1(y>#1GD!J4jyQJ8Zq#^~hhxHbfJ`yU{KE7EHkgEzt$Xs_x1-(>r*N^4lP|L3X6 zspBkWr9A7Pi|{>Yg}N^s%OkDW1eK9;x9 zi~(6AQA+|)JUd^}5HFm!NJYMsy;vDpb=}gH3m42g#(Jq>W&vnjC|FC>pqp8MBPGCq zmg!_<0DZ*oEK$>SHy2FmAh=Q)Y^lf-;HZD-=hP^zT6t&udeH(L1{OmV{ns#A`g+-2 zwJA8!ZihpYdAL=s*c5#Gsu6Vidcknq-vswfLNQTNyk!S~jN%=d;`Nvg0P(wFA-65N z?1W2kD5hhIsE6VWjbbIEcoZ9=(Bg>iU?cOh1WjZ$RVeW7`{beY&aOyB{i;)CeTT z?<&*V8;p%oFhN9`sOv+XGPY#`@${iB6Xs|t`0j@aCN%vK&7-+iuOVj~b4zYBaOG-H~U4SHrke`4z+PnkfV|506^Xe#*b zhYBV&0*Udv3fMC?O2Gs@ChPie0K