From d81e8bf38b6c1dd3ed49516f490c7fc3f1b7a516 Mon Sep 17 00:00:00 2001 From: Xavier Dumetz Date: Wed, 18 Jan 2023 15:36:45 +0100 Subject: [PATCH 01/89] =?UTF-8?q?Initialisation=20des=20d=C3=A9veloppement?= =?UTF-8?q?s=20sur=20la=20librairie=20Raster.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 6 ++++++ README.md | 12 ++++++------ rok4/Raster.py | 7 +++++++ tests/test_Raster.py | 7 +++++++ 4 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 rok4/Raster.py create mode 100644 tests/test_Raster.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b9559b5..8abc4c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ Prise en charge de plusieurs clusters S3 de stockage. * Librairie d'abstraction du stockage : * Prise en charge de plusieurs clusters S3. Les variables d'environnement pour le stockage S3 précisent plusieurs valeurs séparées par des virgules, et les noms des buckets peuvent être suffixés par "@{S3 cluster host}". Par défaut, le premier cluster défini est utilisé. L'hôte du cluster n'est jamais écrit dans le descripteur de pyramide ou le fichier liste (puisque stockés sur le cluster, on sait sur lequel sont les objets). Les objets symboliques ne le précisent pas non plus et ne peuvent être qu'au sein d'un cluster S3 +* Librairie de lecture de données raster : + * Squelette minimal avec test unitaire d'invocation du constructeur + +### [Changed] + +* Appels explicites à python3 dans le README, pour tenir compte des distributions où l'exécutable `python` correspond par défaut à Python 2.7. |7}VLmo@q z?jFW|Vuu#CV$B>IP_}Za=#OJa?{g=;dg3|BnN&CN`>^s(FL0j@L9|9j5`3Hwj=U_x4sN&B&2-gsYyAASV)=1M(|t zXrS;#Y1S_}=MH0UuxLVOvNZ5I^Qtu1BquST?#F*t%Hk@9{A6~^BxatiuZSy){J!@g zx0322q7-(L0QR@uyoJ;60$2!l;LcdJCY7q@#yW5F96yOKgcp7;>CCO^A|8YHvE-~uT#~_4OQZB3 z$T`sMj2K;rBG*9|J(aH|vGCsKUjzC1#kx}~Pda5%4YNC@sqPSaD`kSO2JcNj7~JU3 z_s;LW&E@|(qyQmM&gauUSNH+p$Y-mQuGKYSehz}7h0ZvupKsL2rYV(k<0Zm}QF$b) z%Rf>J>bXtnNp1&EIB`5Cpo9pG<8;*eTExi|=MN-z;Z2>WvJbFaF_CHw;HJUzblS)D z0oLF`m&pzxJ%{0yI10mvb%@Yun(dTIg>1GuA?k7pFly}gac~}1%6wEv%g9sDYm8kE zbq2HPF`lt8o6Q}Vpc{-c#xon?taE^uRMYz*DjH_X72{v-RQ>GeUO8C}U5o9jufHi5RBV7A0{l83}b4dkPz?1n1kd#OVyysx#Evl0Y!0A9D&=7c2 zQiy~H$(xpIFcTsZobCEj;%3-NZ<-_*XhFU!JV5v}a(FIOhrvbvp58pC#pwkQj(7|n z3A~Q!H+xalSH3`Ydy%wzJ~vz!GDu*&Muu{?v&|%i@GgnO=&76vUfuRcdX9Lwu!jSb zEMtX`@D~!l?LFv~NisIJBcBjr)mOrsPQT0|fE0E<6DpiK+)b_za3BllBu2-yGI>$t zw91i`m8*&wmom|nwv;U@=^a`cWK9E zJfPylKs;L`vr%%_Z-`yuv=E;+8E2RQc<|DEGKS=CTfX>h7t0Touzqjbdpc)ea}nf# zv;iE1qu@~0`;&4fi7+Hj0S-q37X?0w#jq1(E9sk1Bp!Z{rJpA-5OWw%!+Ny8n1lS4PKxA zE&*0xV^HG-W#|k2rg1A)6ovJk(VGW zNv9agvO+1GCBVn>g3(`tycidW3{`OY?Ott0%zuKrrI8kK2gYILZs*x80hcua_oCq6 zItL#1oK5ogrbj>%s;A-5Q%l}Eu(Tm>eW}nxBq+%Cp0|cj-)>Tq2>| zpqg`O_iALGpD1T2`y*-j?_+&rqJ#KrN(bh@lz+@v|>gdYD7e^LPNPNi?UFoj;}*4-f{< z>9QEAMTG$~ktvIL@HxcFV#tM%yA&o?ErP?$NfHNu^f@N@k+E)`4D&OJMM}bB-|xMQ z>@hJTSf=||?%{k5!bgSIIf|G?jd~CX8{Ua|>GdoH1sx{O1<*8@^9J|L?kKYcpmNTg zI~C%8S;gp=Pe?>LNR7ObN%$(-Y2-K7Qmy+;wH^x21-JRPyI>Xe0|<#CFDWJn6C4Ln z4$?4^N*TR5riZO|MxqB%a9H^@LeFQIx#N%XvVFmL4FgNj=d9vDNK1muh=ZE+gXBNE zs#V!RS_oeN93f)Xh(Q~NR>a>*5>?n)_GgD6?xQf2;MO_~Av_$)?iD%zmV#H^&feCl z!~??0_r2hq{)6rJwj&c6QwDzULTF2{6$wc?LbsO00@3j`ujSXQj5s(gFqQ&>0uPe$ z#(2fl@2E@*Jka@ayae#|cg*Nwx~0_}IO9mVK%jb6Sob$YFhnW-$PGU`oK4SzsA2%< zRRACmFX63h+Gv^$i}ABi@oYmwqV~uy){Ar_kt#bZ)0G&dSfo>rD1rbuQh-2yL;~im zEE0$>i)Ff#Qx%h-@d1#>mgUJ@w07|KT;*US|C%~)tVHM|4dBBLoM#H*0{$9Ju(@p5BpByEw1AxIf}qdT|vvBiph3C{y)gj|O!hy#(3QJjqUbf=?C&3UMM zJUD?8dcEUZ!Ss!epp70kBxIt8(_#bU0Qrb-W_MI^E!7&RUP2ZK3#8eY?T0+^mar!P zSBB0JEi4IW8h?~x#xD{J@GmZvtb(8m5Tgi?yL19#I{d^+#j}tLY%L3ygPEd^NJsZe z2pOIsZXk-m=;9IRhZq!kACQsmvJ>_UO2~u6!w`NvC9J|4(*y)%-EcP`V;MnB$|ujD z`rLcPe3pmV`;qlBsYDS7A~kCf^OnhSU_S!@*pA4WoY^ljxet-E6v*s$HnonwV8{Qc z8+@(fk@VxKo#Atpg=xotZwIC*StTV_nr#izmg^cs*9+N9OhT1MS3~-q&M>R}k3_*aw3;T0f-a(!9mDiY>QrkIGhF`;}CY)-aUvwg4 z>$aG6UeqEzsX$@npGi9JysPu>G-3c35C@NtM#e}_X$-?SxI0~RY6*%n?{l3s10Wd4SVLw=CBVu?T(ILE zKCU$s9BeB-?MH=`^ZdN~4te&_=53yh7@Bw{dY7J+nI?lCIVMSxA4hHo#6$O%o>@*7-+&4*IBDdiFAwU@ zEGvN@9YL%OL&w9Lz-LYzp~hd0#)b0o`8!x#RNg6w8qa|4(?^MRuUC=~tfGy0dxqrX z0)>#?>fdx}enr}1)mIcA9cj3MtC3EA27WyRR1g8(^3!$aqG*^5B^Po zNIKhqgB>l1z^W)$@@?=H0F_`trn1C+&S)=l$l`E$x3i2mgfF6|0{mz`d>u6ddt>5yO*N3KU3)Xse{zxdv`5O_j&3d^@_;!P+LyVp-pgq28X)ewa0@`EKWAR9zr-^M(Y}+$FZF}1@xqZvFcv{ZNF!$)p_|k^< zj!lnF#MeGu@Ox&@(i)hovm6KCy=7}0?tgOd8?I2@8&AuLf7~-Ov#0X2gv6&$E*w^n zcqyX&%!1^w<@sYr`wkvkSP?mX+Q@nH6{i$YTYAsWE1*u?yt$HQym9FbikqK2e#1$H z)Svv-)5XmdQa^okIk?>lB%e|Ud-pZR4={Zz<{!oT;EIEb%zruF0?+wj z&u12A-?w=9<%@mG=VCF}6({j?3&pqM-Un}5xcWHjZemdrD$g_&|8t76^SUKeZ!{?? z&Wq?eK|%!-0UmoR4ew3;#YV?Q|*|4d`-%z?RN;nYHtES~;s z2E)3g(O7K1xrwWnGCrC_b%Z%66-`Qc5H zb;}U1OoU@kzH85^BMT?5zwQ`$6U#_wceb8W`FZ9h#DRoul;>>w);QH+1QNcS z8o=My)luxmmzqV_Duny#5OR6?05MC2*rt@5gs`o9j$cJ+p71-jAV{&E@a`Z9!3|MqylD5E65va(*i6EuuV6ZpKum z(1b(_IjZHaB4~?_>SPDWp4M{eytC|#$eNCr;2`shWuiozDZm|@NKCHQ%?rjuSd-p? zdk~XLZP#(_VC7oenJ;AZ*X47-h^FdHK3)uSZGw_i0-)@OFg?*`E|AYaW~jLg6gors z;OYI`DR?Nz#MhhAwfhStoX6#WsA#sHu4gBA%&GiOdUavgnW0hL9AzFe++8YRI}n}3 z5SAT?EfwL_kt;6j2d-5LD{zOGuj{Wm4H5!)bXt`~6Jf!hAU}lCwCI4-o^bw%dYbdY zby}&0MLg`z;q0!P!5f=OSpS*e+y0x9L%3CCTjM*3t6{onvfe~9x$A>Uo+b~OskUy9 zYC8EHWG!T(=b3Y7d+XYwUArTAAl!F{%dzI2U&5{k8#3<)u%&X@o>D0@mK)gS)Ym$r zA&pR{$BG?Ye1YwL=u&}}@};jd!_&<*Y_jK`O)`xBtx2`wmRU^Xl*94;q~q+P&mbpZ z?hK(9tqrO;5Ptc;CLi;i2i$gz6>yvp8EoWb6b$#QC>59qA% z*;>CWjx@Gy`@v{MN=i;Y-x~-)b48f(f#IJ3rzlBFrMr2aTmfOwgD1})wMItrTBSwcMEIq9 zT|NgU+8H>kh%ero$Z2a!@g!D5>X4kBI`~ZR3Wt&iZIi@x#U`q_8x%Cjtanoc9P}h& z$E0ttJ;9XCaLP9>%z+Njram|KAWx)P_-$Ur*{oZFNX#QU%I?_o#893|lA}e9?Vb(dlhZsfh9i?z@ylo zMGh)-H@@O~?f16fg3@ipYP~@*?ARG6BTfPDQZbwa{|VMoHqLEeAiP(+9oGjn zGCv@mu_cRdM4jVv=|X9YaA>wr$jNuj&fi6E^w{mpn}KsCdT-O^G_xdE>4J}3eWM$k zOMlnD8$OMXkkyy#xYlC0;vnVK5>sKGP?> zNK;&L1(+=E$ZT~i>@ShmG}~yf`zTG)3Gf@n6}^)HIIZb9X?4x6E7A`fGf~x;_yKp0 zh5kx%ORBeREz_SngS%4?Wq>El2B~l+b|yZjr6zQln()8e;QPt5;hA)8yd3Rh?M0SH zbe_aAIA1+iEXb9PST=W*rAtYu2}W#cI3k+@XOga%D3f`^_N!z2#fY_Zs$%(@(}$Qik=fd{cho@$ZQ2El6Hcp!FEA{^wf6gmwTYW>8b zbPFz6LW05>^Rtt-bL>^FbDNt*ZyC?-bgn=sXQCg*K-gtKIFcMqr2)eF%HMmzxwgBy zVL}d}5r$){0Yl{D*UKRL@1#17FkM!NWb1<}(rH>2BW2hNScz82>Wb z0&xKq|4}0*PykpC$e}s2HUd#_?eN%8FwQ%b5Tk&1V~Si}7Av_1cCe^h+*bfV9qkKC zq7E2sCOC0$RLL+)h{|`r<5q{b3!yF*G85e8zS2Deuwg>v%D|eXG$7f)f=n>GIEAjM zljfko0^Sn6R_$V?0rL3hcVm6%mtp#`j;`d&TIIH6@PW*osXG&_o@u2*?{Rpf>f3YuyjA67nCaBDJaeeg_DscpSF}H<#n+bfldJ%;}U`;>Njsi zV?wbdQIx}1nWZq!lyvxn@+nLH01H6QqJ@y|%gJ9xzz$6+W zbwtQfy1(!|R2G%^69P~O6G+_W{-b+8SrDv>SsGXrf+0H;ohiUX_(NM{%>|17p-$A5 zd)zO%^W3dR(2#;xaSOT_s?qFm3MmCF6t*|HMdCwjmWsg}*ez7k3~)TpFhrbbH5jtK zLV)AZuE)GPl3KWg4X`F+gM7r8A`7Dz%4Z{Z^~Hf`K*T}jC6Az3(pp%Slph_KN;ElC zdN8D&+CVCsfW}H$#E|ITQU?0_8{m*~b>K|AgU~zDQUDQr_=6UB#v6jQ8-Gw#+36|L zGywo*q6w5ond%?aC_G~+HPh|EP>lt9TuFMoz@0NJ#t__hFCo&ew-YbP#c~dC6HR91~y{FR~yny8!dF4gY_jWVUKN)#+qMNmooYF(i&}XaVh1!7 z>F4_Ub(lgWhg8fG>*W`cG=)xgxo=EVxJ8l>cJd+bm8tucJ7_pjq?H861WCyRl|+$Q zDxmXh+Nppl;LWQOA4m?N`IsK9hm?Czln^d__n+?XB<9#7ZAwv{v`0uxyBXVFEMvMz z&we4TV5Ijc{<`ni#hBqtpN|>y3tD(rF}$OF#F?%;5|5JxYt}?mB(nfDtCxsl9R^Sr zDS(352M};b$PaGh=cU~VCerJ~2*8aJz6uChR=2hx(!t*AP9(_=#GpjQV@+x|A$|dq z=vQcz15#Ar70$UOXzo&})G+sPtFuhKQ3bRO&HQy1zYm zB=y}cj2PuTN;fUYwrP~fi2i=+mFb6=AO%wB&=tPS78TpYl2X7>_Kf16DF32hcE_Dk z+304C1m`~CpM~7H5UgoPT~Q7mx2QlBnI-uHn-l(tfeD`hS?P_#qQHk%EQl)r;HV{zqB77Hi8jp~QR>~4Kw;U!t58Vb! z%I@H)Fl&|OL+@&011shV!%l=PTLxIXR5qGcB#dD<5Q%V8)Te_0+kpq(9+3qKp$UGR zC%{$-pjro10@>(e`q`Q|41kNWpj$H(+a?vX#Pie~AMP|Q}uF?FXjGZ0@C%18VvWi9PYlN_}l44y7F$o!+2 zh_LhiXu|K0lwr_=!4Q%YNMdd@qRsLk_6zcG;{3(yz&gD~I*V`tu@6b2ooi(rLUfiuBkbWYp$748v>J{PMhkSbFeQtd6=K}LvOak> z#|MAqd2CG-h7CcB(_7pbumi7!NM)ar`P~eScE5@L(goiMoehA7fOGw4U3gN zOcJGHqV#LhH9z)lL8i+s04v}Su>Pf|3N+tzKj4nyS29dSX_cEJYs!?GEYch74?Xy` zaEteG{cY}9w<4oJ2Z`{!!!_ndV+ijqj~z$>`>gi^?=Wyc%K*7KBNJ9SB@R~wF%RMF z1FGtKqg~yW$QI>B6EFPd^!wXx_r}YUq&rMXt+z}J^59i6R=&`|BMBgfqkY35SUIS} zfuqaIJDjlyM+XP~8`#Ky>2)OMojMzTM|kjzi{%9aQ@Vi_kcfzxAywo^IdS9xU@gzE z3!nq|al}GqIb>Zk9ra5Cw#~f#Hdg${xKnf z;O2GrxKSH|Clj5G^S>@Ke}j~w6#jTRg7Bf>*f^ zP-1Ki03a^ZPL4MkrszQUS^W_8CPRDh06bTwD?fpBw4g`Bk2MBd!ay0~Ftmq=0XzIl z7a576H1PwEC!15Kb&Fe{gHh`ui5j$Mhx#_e;pNxJF2tgX)d-1RC=}KKm1ct4m%O4F znB}G)oMDr|40fZ-%s6^G-C*Vol%ODF!x6F~GQ89yc>fSc#mkg62de*_Tak!|xA0rx zPrM-$D%upsx$}e2Q-nnpeKpqYE^KLrRnrVVO{enq$Gkhi8oWUx$UAg7pjj@D7-QfF zKnGx?MDH?suXq5Tusf7k(2XNeLKWBXd3m@tJSiRk6uDO50#$Wb{7)qmTFeWtfZ@TO z)mT}MdM9d_Q^WWzFcC@vZQ~8bQ3L_M8Xqy5igqo{Ue!vRAgv!rrs2T&xx~wYB6^D% zX43dzaSrW>b-l?Rz=K)iSo=Q9&o+?8J!$(JLO)E?E`3Axt zb-w^#G6t;4%CX$QDfb((!t;DHtwkgmOzZWP?}Wi?+s|~6>;u!{K+^q1MxGM507>@W z=}KqPSN65#u}Khvh4>f5eGSb)sL9D9ss`6#x9pfg_CNxr1LWvF7JSnE+r%TQO<66Y zbI^-CDpNNDaA*dN;{qZVWf{J6fZC6_svDNWP4 z>yO2BxQI(2UT#*sT0JJMvdIW&iie3vU{R#|3kCXhoPs-sq7+nqLU(~*vM!z@Xd>I8 z53;`csmzl;t54++0UC%{e#=g)=F{0s+d7(=WOL{Ul3P>CHMV#axyM*$_`Jm`(ACIA zfE1vDb1baPfheE^D5U8*epP-( z1;EsJi3>=ztRjpEgb8}0arphs-T=PxMol|H(+f*n zl*43^52h?LyFr6(i>!%_1928lRf{%6rm8g672xFT=#>7JIwaiw@w4`j!)JJ$CSo;`n`;-XKf}LqO4+vW+?zW@`iy6cBDNdk~1It z(+Z@ovslO;z(D1>gNJ|=P>z`w5h$P{_Kqp{C2vXs90n(}vWcgvTte@24GR5~bxJ@aYQU(u^YZ6uG$AZYk^Z6%<}!ZYjE@d}!>_#fCGSB$ z!|e%*aui04eCphE2sEuqG6ccB zenF}Y--mxiZ&yFv<>DH?(>p^wLK zol+Xgf7>S)0Gz#K+B5RIM<$$b87_YbtAeDj1&m0hG zrqFbiatN@UP6#FBr*i!ZRYkt+W&OOY&dgBnIh{TAdc8E(-bGrYkLV??YK!j{=8 z%w;#AARNUaF(9R$BC!BUuol&%KIqtmTH-Xl{sadcm2Fw{+c$dP@g0 zPF+U@0wa4+y2X{HLF=f+Qz+npwZ>deK1EruVv{l_;I%SCe7XhLIT~P`O@4kS`yECf zVlTh~i?Ehf)bZTMiwiZKd!+T6605Du1yI{*uBcoRc3u^Lg7Ftf-f;VV9+r!nY; z&bi-wKwI6$Qk=54Z3A)2t#}hZ3XZZn3_jf@R#KS27pzz|hO=BUobEIYR%N-sN1Iz$o;QYr4G%N-!o#9vs9G}}yY^T*tocF`I*?rer2gAUr*y>wONwaFR1 z{{OgB-Po^cY@njdPDa=K%Fa0ehMPis*3gb^(Y1rPze3e1fkKj=D@L{i$Se21$YTr30am#?cT=|@+CVG}#Hru+a z&n~OO8@}sL#Z1y1IY3nk*Jz}+imRyx1JH5fZxvQI(uJj8T(of^6WtegMkNVz&&UU`9r@(tq!RcRFLkJ zxWgywjH2hPaw!ehrkQ3h2kZFSK`J~F1?)Ep`Yck1Pq8}I6Qp_o%mI7VC9PMr*{i_P z2fMBb+s1p#HEQ~lfD~mZ1jK2Y{Bzt&UuEq4L+;nyxitgP*f0`BDCCzJ8p1@@FU2v4 z-7Pmy!rVNcaD&I%9_+fG=%@ni=(gs*B#fJuNK*X~#yI*x3892$iOPezQZrP}XL?^m zUtdy5z;}d*6e@1V@S#zIqwYIW%H?N7hJnQ#W?4uBYrD_AjW~&PY>uq`;yDh?5Z4SI zWM7In_z+$qCP7H-8{aAOTIo|%Pq8q7C>K;SG2@Ds4zR{_fX?6_g1fqlJ{@;;b~6Uh zJ+17GcFW$_6udU%BIP5r!=tb0M;Vc-3_-B0AGoK7z%FzXWHm4PqnN#3-g;BP-gIY@ zVs5kzAON@|CS+x@La{cdL7o&a(FodNO;)JLY zu||Za6>qJy5tbx&$}7ZyKDp*Mt{MJ1F-q@_cH{J@n`Tqw*3o5>`Qw#y3ng^TP! zcmTJe{x=d=hs%;sNKR{5@)Ai`1fX3;!Db;$!e; z7$2~hd^yBj!7I7ca-48k0HQLC_@iVkpb8-6cFagQ1PWZK*GfAW_Zb>vOoBeapYRT` zW~_OghOA}0?$RG+N8RZunTcJ&$tFe>ah871!WxlQoK^G^QWIG)%aJuO;Eh0b%WAPh>pPUX40?jyEjuDL>@{>8kie*A@ipXj# z2e4<@Ac7JcL&i>BHjq>?SptAgCmZ94iBVQqj<8~7B#Dv#C-cbWz-S=>yF@ap%Rg5+ z(Ga$*kO%Z5Lh!%clfVe{BMTQ+3c1otnH%*Q-b=7FxCXKpJ|d%0EEK|!4re5Ize zb%*H+LT-J<`(JTniY3MtkKwww-cd|Tu$TjTyIYk%PP`!)%T@#*DczPlLwjsAnjgC4 z1gR$udhHMJ#SV#bb+g1^WPB6U$CvJcUFJ39BtIDQ&x>39Q*vZ|XFXJ)g;(l zDo{;ZHLor=V>AY%1-Y4_iR2{+0bRh}i5-c&g=6x$nBvI0rBZM)UNOE0$s>Xe3V#<- z;pGZ&hWJRXY`i+mI;@QbJ)+`8P(c4f)m z)@{uWv=hrMYNu)Vx>Rd8JxROUEs6$JSCss%DDK-eJwCoQF8SNF+<|swMc;B6-=2}> zdOZ5&PEXr&e|qKJIK5=!ZnY~b?{=*yxBF=|c6;{jorp^bpHObMd+X#D1;zJtrK|JfUk{=X=28VyILuWjsGxMtxbt#ubEe7g49 zB{XOi3mH_GD~VJT^+e!n->s`RtpGvXd-dq)6UQq5zp8KYDqdfLKgsCxu?E`fR_UKN zncP5YtRS7J{Bbha`P$@}wqXQI4Y`+Q^a~ntodJ{fL z#1IZ!1^tlC@`qsx;7P&@WaTv$u91X6(uBD>HkoPh2xIaCQ5|@i)O#eid0|i|5o`q5 z^gU7zp@brWU!aHbs%%FRan~wJLFu#m$ZF-`Xb`V*XVCljIZva*D5zp`WP;~9Q>dma zQDFmVG;-!_myZV>BK;@HhZ1d_-*)L}#quzm5t&34v-#49>95Qf!qaE1G~{_|*0->X zGMSxEQ{O_$;%pKJjq@4Y{CJp9hF3Yxj$~4Lv>&v$JF{~vqeHn)scv_z$zbLUaV2Xy z@JndFaY;t`9N7q)fu7ZF+>-dZ(%zEmYwcYqJo2K@szx3e>_Ail8byRvjCzcLus}u` z?ht$l>YOJ}0|NwrR;VJYTz#`YfJT?2+9CKcl?g-RBPzTgl@kT$PVcMkP=UcuTmj%T zhv8#xRLcvP|KYyf!!qOz_}#8CvuGTADkrulE4YzRM=F=0A$>1(zo$?_)^ciWMVy5& zhX=AQX&Rip^bYf;rF}p)opfla5DX@3ZIem_0J&JRsh!RCwj1qTQ^|FycHWhtz3sr? zCFyXJ>5#Yfr({(sI?TDn8lf~*DT=~$P-Y^mrUnwug?4Jw_IEp`6q_8gO^&$}OPi$i z5u02W4EzNJO=)%iajxRL@Y_p=C8VBYFO*ls3g2*R!QHmtvu?2O{mj_sl*Tf~Q_C}WRjNfkC-^&#)& zqfT3OHDA$ZR?6rn-Ts;VQ6ZmA6+5{KILP++CXH7|{qRD7f^|R{16hZ79ms|wu(BPu zt`u?lSMp%KnwJ+$)oZUXkc=8gbQXC)lBpc$EP~q;gI!1a)buGoCqGMo%@(Tw4Cs~#KH*L9kznuv7t2atHe0inHuQ5 zks(-OOj#-`F|kThvTN*6z+WXK{;B(0-s9~m^HjFbN`I}mSs5jK7=ng=!t!SR73$$y zWY7!4pt>EWiv+$fJ=i}&f~-tNcqaZl5{T92kEntvSZ{cD16vXoAhY}om0d6J$#E)O`UNR@fMgT$lo=dznZOke=m%ecZ z$BK4|GYWvgf-=E>#bdd`kDhyjJ0$P*7Jxu!;b7H~p$?6_;k;Co(vfO;+2 zb!kQnfZ&=Q9flkBxUKF8@?8AaZ(9RvS`++!I8X`J0`49)L}n-qp9W6S%(#~s_Rhu;a~8?(OS*ij(@o{qXM)YIch9&u^FSujSmU}q!Bz*}Loj+X zJ_9_h-;R0SQ0on$y@AGTr7tAUB{+1^!%_eNkcq-z?-naGT$AXQjsc5SToHDVrv};V zT45FlTB4{&uLT=fD3~Idu3+d-4MenGKn>>X>ZgezhUdT8Td^z#EkY#e(P(umJ090yxRtEjP;XOp?M}Xz2x42i)N<@@6%ov`p#{Q*Etph9Afqk0h zZNb1J?S3-b%Op$2>hNr(zVJ`KrZz$9YEGNW7Dh>9ZYYtKCs>@0?{oBO z216(q=1y&vbrD75K@d0zC(?5oh9Ur5Nlx6OM^nVrBf?g8xMT|~4_HRdd_POnj41F3 z=Lw3GIa;)ECCk!KGeUIfN>c$z)#N>5uyQ*|PMD0;(fz+PWtJeJ|>Z`a5>nbUa;h`609>8%sU9;Wsj^25|r?~*+m#xK#0s) zq@}p)O;?;B7z%EjotG?>wfE4h^M2C#~)myOWFj(xSUZ(4Qml0FdVGu6L5^zBA z#gCgSZj9(R0DzbdSsJd^3*-CumFk5ePTl+{N$6oin?l@WqHo23>u){0%O3uDm&v^! zPT!eQ{UDl*8V++@vUaeO1h%M9&QA7|OJ-SE+2a24D@iz}s-1A2)Nw?wWyw?tEs++_ z-#g@aB-u`F|x@&v{WC@~f&D2FgqGie5DnQ~{Wn8t>< zV&M;{{{ND=-+fX{?8C{;=mXeI(aQ{mgYK3kd~z8zoT(fR?(sfIpGo)){TS6XlDy4Y zp%&z4$8AL{QC>|{H9FoJq_vfyT1ywKaAQ8rD6Z-0=Jc%b>6zx_ zsuj-7$+6wzBXJY6$?@GIO9QXAPwt%_iQ9B--?f`Jn`?6Wo;|y_#q+nO_w1cs`gCSw z*TmAbGh?hgc4O|^K0UrQZXosqN9eBI0q9x@|n%u;gp3KyNS^i>k<}v7-!M0MYSoqZt*yOHy=&c ztLJdfDJEVWyY6`7*p0_dUUl-=vE`vVYBzQAc;CefCtkA9nt*i@&t2;gPq?Dl{^ki+ z&~XWW-i-^_()`I<7CqG_xyU?oyN?+hj-Gt^Q>=!Ie!An5-c8TGdI7)K!pX+$@soW# z3TNSw<3~>~o<7xDHdmRSlu2667e2*Rba>iz_0@|vEHo}%xa#`TPxmk`Te>b(Z^E;9^1L}-^ozQ5;pBA-M;H2@573UW{HNRkE>#ebJ8flDo|o^V`T8#@c1;k7OF(3#!5Nb@r-_sOW#^wc&X{@kY5 z_bE4r>-DKuEcAUcXf;cUmU36OCVw?I_|%sVhnt$?zn0GbC$EO2jF2mIi%!FGcCOLT=aWx@wZiF&NUw zX=YLfyDvM8Uc(%fVfw|GZep4$9V%K5krDGpJS*8tVQ?{Bz#;OT#A``Pr8ZfP2>(q> ziKwIz4as^51NF)#JgRt|9Dk)P=!eRZi`J8qfh0r|e1nJaD5pj6koWuYCQ`LbL3PyY zlaDmJfU>GUaoi(l6Md)SZFb-gSzFl=e4umSjGJK=b-+Yo(|Q$~&#hXo+%Z*ap<=MP z4`>4Q+Q8X@nfZtO>?WC-jm7wmSg+aGdTy6J_g2Pmw`a~IF?C}n!-}c*!&Zzd6Aa8-lBMupy30MK7)Nv!4XRm{zg`vXknff4S0Tu!3k^GqEQ^5nc9nBn_6M*&O zJsBu&N4v?@nJ((Hz$2!RtUC|zmjt(DQZwk^tU_jN$O$riH+aBZf7T_n7=uP`2z}?a zrMgf*Fjiz#%Sh2TXu6C-MOO`c+QE$4ONN=J5(C$7O99++aA#)VP`XQ7Asde=>&ksf zq@r9K_uOFjsH1v(PrCu`mzjA#z!H*6gXyuDNZ!g25*)zzr*l+1sJK14H$JZ|wp1Y^+XE5bEIstk+}o3Q^9SW1 z!}3;tFM6`04@xgpmTg{-W6)X|ovThxg?)xwiBFX`fg;HeG5(byD61VClr{nBeyY*4 zJ+xyd1Bvvf5)OfEcDwTP)0;6XdP>`6GN7yK2)C1`jUuXy#OVFatj#7|1B2^ZJzMOV z6vow*=t4-w|RcAaZ12 zeX)c~s49`O0KZi25*mDiPq@crx%LKAPQF0j0gLJ%W^egu#JL!&wa&f7LwE*Cs*>Rt z;H)+#3P=o)0gLs{0g;6Au=ltp+pst@JOW8t_8t9Y036vYs*eQJ@EQ3g>B%8SHFz(J zPs%%`NT8g`rls=pZ2q$M@~VDR_EMU5ZZxQf4|j75r!t09x4FTG(*tLcZdW=?c9Zy) z|AbpN(~$1Rw6_|di4e%?^eTuUUMFT#lMgdReD_gUuB$mVw|r9~*vJ zu0D>_ah^@XHD!MR);p&9mXuQ&8sf-6u&w^|#2>PRN{nGq#WTwllj3ubn?#>xYp3xR zb<;cy>ZY~R>(dYU=ZFs|4@W6m)d}ol?hzXnhS8~h2Tlw8WZ)neuW|2?mi!6=D(?DB z@T-@Pc7vU2ECf@*>3y4f83}awRT)xLqnm&*U+j@qKs*#nl&dir-%UOblF{iVIeB}r zdv%;*T-`s)d&+Ho2CtfOX+G+n^X_)xXqL<){3cO2SE7X5S>ty@d7OHgSvEMpN%_r0 ze-Pu+widR_$G^5h$Gh*RzL>fz_p2MAnvWsS<-3<58lchdd2s2f|&B09_K?Rd@YI5hLUPQbqaqwue7FO@@P@@ z20C(RWBm=@es;n{l-C=x3h9CaWOeMlC4km*#dqWJ<@4ipCCTB-6PYMf$2kbHwNh<6@8CToaPp9) zX*SY0I4X6xAd!_Doy*is2uBEv5FnME%M?czP2)QJ5bROT*8t(i2#8| z5|^xs?J!N{c@S*2@NC3@)F~~!hcDIKK_V1kh&YAcNPe&;Eu55esm-rV-V)3enezaO z+8iBWH|;5-Up7`c*dm#3Bbj}{Isehr84v#Gm#D#Ap0A=AqO z4UghOjfFUfl;)ynyJ8V`Utz!Oozz4>&nXrB#& z=EA8U_SNegbV|?~R2yv^Y2+KjRJ{OP9NHnQ1NsAmO`7Vl&@IqI3+h1>!9+@m?t|Ez z5*45ngh+9e`i{UA=GhVJJn28iQhL&xQDj(4Ewyb9xz6b;(#z}P(#QD0r_DnLzkV=wYd%^N@VagdK<3SbO z${OvhPAT4>CZx%FRAWpwJXH{Q8~F(|MS`e;0v4`t7LqC#fdxLbaBvj$kkN#RvIe1{lSBb#bgt#N?HYHm`BR0J%-wk zfM}e9Dsr(upbT+gMN2DEMk>478Yw96h}Z#`a`O;~9VxjWN7C$FV65e}USIH}A; z-FB!EABm)}w|WFO!b*#$kZv`cm--Na^I?T4q`^H{cW`TUjAD~0*G$zDvk)C*34F^( zT8%_I0Gfj_0!4HvsaS13_yXPs$s?gy3Q<)}#*xZ0X$#-p0(Y#ERf6wy4V+AMttqZr UTMhd`Eud;Di_jA!U6c8L03Lc80ssI2 literal 0 HcmV?d00001 diff --git a/tests/test_Pyramid.py b/tests/test_Pyramid.py index cfd86eb..7a888a6 100644 --- a/tests/test_Pyramid.py +++ b/tests/test_Pyramid.py @@ -158,7 +158,7 @@ def test_vector_ok(mocked_tms_class, mocked_get_data_str): @mock.patch.dict(os.environ, {}, clear=True) @mock.patch('rok4.Pyramid.TileMatrixSet') -def test_tile_read(mocked_tms_class): +def test_tile_read_raster(mocked_tms_class): tms_instance = MagicMock() tms_instance.name = "UTM20W84MART_1M_MNT" @@ -166,8 +166,6 @@ def test_tile_read(mocked_tms_class): tm_instance = MagicMock() tm_instance.id = "8" - tm_instance.resolution = 1 - tm_instance.point_to_indices.return_value = (0,0,128,157) tm_instance.tile_size = (256,256) tms_instance.get_level.return_value = tm_instance @@ -181,8 +179,37 @@ def test_tile_read(mocked_tms_class): assert data.shape == (256,256,1) assert data[128][128][0] == 447.25 except Exception as exc: - assert False, f"Pyramid creation raises an exception: {exc}" + assert False, f"Pyramid raster tile read raises an exception: {exc}" + + + +@mock.patch.dict(os.environ, {}, clear=True) +@mock.patch('rok4.Pyramid.TileMatrixSet') +def test_tile_read_vector(mocked_tms_class): + tms_instance = MagicMock() + tms_instance.name = "PM" + tms_instance.srs = "EPSG:3857" + + tm_instance = MagicMock() + tm_instance.id = "4" + tm_instance.tile_size = (256,256) + + tms_instance.get_level.return_value = tm_instance + + mocked_tms_class.return_value = tms_instance + + try: + pyramid = Pyramid.from_descriptor("file://tests/fixtures/TIFF_PBF_MVT.json") + + data = pyramid.get_tile_data_vector("4", 5, 5) + assert data is None + + data = pyramid.get_tile_data_vector("4", 8, 5) + assert type(data) is dict + assert "ecoregions_3857" in data + except Exception as exc: + assert False, f"Pyramid vector tile read raises an exception: {exc}" def test_b36_path_decode(): assert b36_path_decode("3E/42/01.tif") == (4032, 18217,) diff --git a/tests/test_Storage.py b/tests/test_Storage.py index 175568c..4c1d05b 100644 --- a/tests/test_Storage.py +++ b/tests/test_Storage.py @@ -64,7 +64,7 @@ def test_s3_invalid_endpoint(mocked_s3_client): @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("builtins.open", side_effect=FileNotFoundError("not_found")) def test_file_read_error(mock_file): - with pytest.raises(StorageError): + with pytest.raises(FileNotFoundError): data = get_data_str("file:///path/to/file.ext") mock_file.assert_called_with("/path/to/file.ext", "rb") From ce0f152a4d9f38dc103c22570622436ce05f53d4 Mon Sep 17 00:00:00 2001 From: CorentinPeutin Date: Fri, 17 Mar 2023 14:36:53 +0100 Subject: [PATCH 08/89] Gestion des fichiers lourds --- src/rok4/Storage.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/rok4/Storage.py b/src/rok4/Storage.py index e8a8a7f..a46e13f 100644 --- a/src/rok4/Storage.py +++ b/src/rok4/Storage.py @@ -324,7 +324,7 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: try: - reponse = requests.get(path) + reponse = requests.get(path, stream=True) data = reponse.content print(reponse.content) if reponse.status_code == 404 : @@ -435,7 +435,7 @@ def get_size(path: str) -> int: elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: try: - reponse = requests.get(path) + reponse = requests.get(path, stream=True) return reponse.content.__sizeof__() except Exception as e: raise StorageError("HTTP", e) @@ -492,7 +492,7 @@ def exists(path: str) -> bool: elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: try: - reponse = requests.get(path) + reponse = requests.get(path, stream=True) if reponse.status_code == 200 : return True else : @@ -809,8 +809,9 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: offset = 0 for chunk in reponse.iter_content(chunk_size=65536) : if chunk: + size = len(chunk) to_ioctx.write(to_base_name, chunk, offset) - offset += 65536 + offset += size except Exception as e: raise StorageError(f"HTTP(S) and CEPH", f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}") From 96b33245eb276d3f257ef37093b07c4d32e89e6b Mon Sep 17 00:00:00 2001 From: Theo Satabin Date: Fri, 17 Mar 2023 14:40:32 +0100 Subject: [PATCH 09/89] =?UTF-8?q?Ajout=20d'un=20cache=20=C3=A0=20la=20cr?= =?UTF-8?q?=C3=A9ation=20des=20spatial=20reference?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 ++ src/rok4/Utils.py | 24 +++++++++++++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8ecadd..a7e242b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ Fonction de lecture d'une tuile vecteur décodée. * Pyramid * Décodage d'une tuile vecteur avec `get_tile_data_vector` (le pendant vecteur de `get_tile_data_raster`) : le résultat est un "dictionnaire GeoJSON", et les coordonnées sont en relatif à la tuile (souvent entre 0 et 4096) +* Utils + * Ajout d'un cache pour la création de spatial reference (via la fonction `srs_to_spatialreference`) ### [Changed] diff --git a/src/rok4/Utils.py b/src/rok4/Utils.py index 58f8a13..7f8260b 100644 --- a/src/rok4/Utils.py +++ b/src/rok4/Utils.py @@ -8,9 +8,12 @@ ogr.UseExceptions() osr.UseExceptions() +__SR_BOOK = dict() def srs_to_spatialreference(srs: str) -> 'osgeo.osr.SpatialReference': """Convert coordinates system as string to OSR spatial reference + Using a cache, to instanciate a Spatial Reference from a string only once. + Args: srs (str): coordinates system PROJ4 compliant, with authority and code, like EPSG:3857 or IGNF:LAMB93 @@ -21,15 +24,22 @@ def srs_to_spatialreference(srs: str) -> 'osgeo.osr.SpatialReference': osgeo.osr.SpatialReference: Corresponding OSR spatial reference """ - authority, code = srs.split(':', 1) + global __SR_BOOK + + if srs.upper() not in __SR_BOOK: + + authority, code = srs.split(':', 1) + + sr = osr.SpatialReference() + if authority.upper() == "EPSG": + sr.ImportFromEPSG(int(code)) + else: + sr.ImportFromProj4(f"+init={srs.upper()} +wktext") + + __SR_BOOK[srs.upper()] = sr - sr = osr.SpatialReference() - if authority.upper() == "EPSG": - sr.ImportFromEPSG(int(code)) - else: - sr.ImportFromProj4(f"+init={srs.upper()} +wktext") - return sr + return __SR_BOOK[srs.upper()] def bbox_to_geometry(bbox: Tuple[float, float, float, float], densification: int = 0) -> 'osgeo.ogr.Geometry': """Convert bbox coordinates to OGR geometry From b56e594273eb85b9263e968a317c7dd49e9830ee Mon Sep 17 00:00:00 2001 From: Theo Satabin Date: Mon, 20 Mar 2023 14:42:00 +0100 Subject: [PATCH 10/89] Ajout d'un gestionnaire de chemin GDAL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cela permet d'ouvrir un objet S3 avec GDAL ou OGR sans avoir à le télécharger entièrement, en utilisant les systèmes de fichiers virtuel. Pour un fichier, seule l'extension est enlevée. + Correction de la lecture S3 --- .github/workflows/build-and-release.yaml | 11 +- CHANGELOG.md | 21 +-- README.md | 3 + src/rok4/Exceptions.py | 8 -- src/rok4/Storage.py | 157 ++++++++++++++--------- tests/test_Storage.py | 30 ++++- 6 files changed, 150 insertions(+), 80 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 2802b78..40fb13e 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -73,7 +73,7 @@ jobs: echo "/usr/lib/python3/dist-packages/" >.venv/lib/python${{ matrix.python-version }}/site-packages/system.pth - name: Run unit tests - if: "(${{ matrix.os }}=='ubuntu-20.04')&&(${{ matrix.python-version }}=='3.8')" + if: "matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8'" run: | source .venv/bin/activate pip install -e .[test] @@ -81,20 +81,20 @@ jobs: coverage report -m - name: Build unit tests report - if: "(${{ matrix.os }}=='ubuntu-20.04')&&(${{ matrix.python-version }}=='3.8')" + if: "matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8'" run: | source .venv/bin/activate coverage html -d dist/tests/ rm dist/tests/.gitignore - name: Build package - if: "(${{ matrix.os }}=='ubuntu-20.04')&&(${{ matrix.python-version }}=='3.8')" + if: "matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8'" run: | source .venv/bin/activate python3 -m build - name: Build documentation - if: "(${{ matrix.os }}=='ubuntu-20.04')&&(${{ matrix.python-version }}=='3.8')" + if: "matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8'" run: | source .venv/bin/activate pip install -e .[doc] @@ -102,7 +102,7 @@ jobs: cp README.md CHANGELOG.md dist/ - name: Upload packages, tests results and documentation - if: "(${{ matrix.os }}=='ubuntu-20.04')&&(${{ matrix.python-version }}=='3.8')" + if: "matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8'" uses: actions/upload-artifact@v3 with: name: dist-py3 @@ -189,6 +189,7 @@ jobs: cat artifact/CHANGELOG.md >>docs/versions/${{ github.ref_name }}/CHANGELOG.md sed "s#__version__#${{ github.ref_name }}#" templates/README.template.md >docs/versions/${{ github.ref_name }}/README.md cat artifact/README.md >>docs/versions/${{ github.ref_name }}/README.md + sed -i "s#x.y.z#${{ github.ref_name }}#g" docs/versions/${{ github.ref_name }}/README.md sed "s#__version__#${{ github.ref_name }}#" templates/index.template.md >docs/index.md echo "# Versions" >docs/versions.md echo "" >>docs/versions.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a7e242b..8dd332d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,28 @@ ## Summary -Fonction de lecture d'une tuile vecteur décodée. +Lecture par système de fichier virtuel avec GDAL ## Changelog ### [Added] -* Pyramid - * Décodage d'une tuile vecteur avec `get_tile_data_vector` (le pendant vecteur de `get_tile_data_raster`) : le résultat est un "dictionnaire GeoJSON", et les coordonnées sont en relatif à la tuile (souvent entre 0 et 4096) -* Utils - * Ajout d'un cache pour la création de spatial reference (via la fonction `srs_to_spatialreference`) +* Storage + * Fonction `get_osgeo_path` permettant de configurer le bon sytème de fichier virtuel en fonction du chemin fourni, et retourne celui à utiliser dans le Open de gdal ou ogr ### [Changed] * Storage - * La lecture d'un fichier ou objet qui n'existe pas émet toujours une exception `FileNotFoundError` -* Pyramid - * Si la tuile que l'on veut lire est dans une dalle qui n'existe pas, on retourne `None` + * la récupération d'un client S3 (`__get_s3_client`) permet de récupérer le client, l'hôte, les clés d'accès et secrète, ainsi que le nom du bucket sans l'éventuel hôte du cluster + +### [Fixed] + +* Storage + * Lecture binaire S3 : mauvaise configuration du nom du bucket et de l'objet et mauvaise lecture partielle + +### [Removed] +* Exceptions + * `NotImplementedError` est une exceptions native +* Librairie d'abstraction du stockage (S3, CEPH ou FILE) + * récupération du contenu sous forme de string + * écriture d'un contenu string + * création d'un lien symbolique + * copie fichier/objet <-> fichier/objet +* Librairie de chargement d'un Tile Matrix Set +* Librairie de gestion d'un descripteur de pyramide + * chargement depuis un descripteur ou par clone (avec changement de stockage) + * écriture du descripteur +* Tests unitaires couvrant ces librairies \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index caa2d20..568d889 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,6 @@ # Directives de contribution -Tout d'abord, merci d'envisager de contribuer à ce projet ! - -Il s'agit principalement de lignes directrices, et non de règles. Faites preuve de discernement, et n'hésitez pas à proposer des modifications à ce document dans une pull request. +Merci d'envisager de contribuer à ce projet ! ## Git hooks @@ -10,11 +8,13 @@ Nous utilisons les git hooks via [pre-commit](https://pre-commit.com/) pour appl Voir le fichier de configuration correspondant : `.pre-commit-config.yaml`. -## CHANGELOG +## Changelog Pour éviter les conflits d'édition du changelog en cas de multiples contributions parallèles, n'éditez pas directement le fichier `CHANGELOG.md` sur votre branche de travail. + A la place, indiquez en description ou commentaire de votre pull request, bien en évidence, le changelog qui concerne spécifiquement cette PR, avec le même formalisme que le fichier `CHANGELOG.md`. -Le mainteneur qui validera la PR éditera alors ce fichier dans le même temps, avec deux objectifs : + +Lors de la release de la prochaine version, le mainteneur ajoutera vos notes de modifications dans le fichier `CHANGELOG.md` ce fichier dans le même temps, avec deux objectifs : * la date de modification sera celle de la fusion de branches. * le contenu pourra tenir compte de toutes les modifications depuis la dernière release. @@ -22,6 +22,8 @@ Le mainteneur qui validera la PR éditera alors ce fichier dans le même temps, Le formalisme du changelog est le suivant, en markdown : ```md +Résumé des objectifs des modifications apportées + ### [Added] Liste de nouvelles fonctionnalités. From 757d4e5fd04bbc06f855dc929b9c2f5d55255a03 Mon Sep 17 00:00:00 2001 From: Theo Satabin Date: Mon, 10 Jul 2023 14:27:57 +0200 Subject: [PATCH 54/89] =?UTF-8?q?Pr=C3=A9paration=20=C3=A0=20la=20publicat?= =?UTF-8?q?ion=20de=20la=201.7.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Complétion du Changelog * Adaptation aux changements du site de documentation --- .github/workflows/build-and-release.yaml | 25 ++++++++-------- CHANGELOG.md | 38 ++++++++++++++++++++++++ CONTRIBUTING.md | 13 ++------ README.md | 6 +--- pyproject.toml | 2 +- 5 files changed, 55 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 835b3fb..5f3767f 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -182,17 +182,18 @@ jobs: - name: Add pages from templates run: | - sed "s#__version__#${{ github.ref_name }}#" templates/TESTS.template.md >docs/versions/${{ github.ref_name }}/TESTS.md - sed "s#__version__#${{ github.ref_name }}#" templates/DOCUMENTATION.template.md >docs/versions/${{ github.ref_name }}/DOCUMENTATION.md - sed "s#__version__#${{ github.ref_name }}#" templates/CHANGELOG.template.md >docs/versions/${{ github.ref_name }}/CHANGELOG.md - cat artifact/CHANGELOG.md >>docs/versions/${{ github.ref_name }}/CHANGELOG.md - sed "s#__version__#${{ github.ref_name }}#" templates/README.template.md >docs/versions/${{ github.ref_name }}/README.md - cat artifact/README.md >>docs/versions/${{ github.ref_name }}/README.md - sed -i "s#x.y.z#${{ github.ref_name }}#g" docs/versions/${{ github.ref_name }}/README.md - sed "s#__version__#${{ github.ref_name }}#" templates/index.template.md >docs/index.md - echo "# Versions" >docs/versions.md - echo "" >>docs/versions.md - for v in `ls -t docs/versions | grep -v latest`; do sed "s#__version__#$v#" templates/versions.template.md >>docs/versions.md; done + sed "s#__version__#${{ github.ref_name }}#" templates/mkdocs.template.yml >mkdocs.yml + + sed "s#__version__#${{ github.ref_name }}#" templates/tests.template.md >docs/versions/${{ github.ref_name }}/tests.md + sed "s#__version__#${{ github.ref_name }}#" templates/documentation.template.md >docs/versions/${{ github.ref_name }}/documentation.md + + sed "s#__version__#${{ github.ref_name }}#" templates/index-version.template.md >docs/versions/${{ github.ref_name }}/index.md + cat artifact/README.md >>docs/versions/${{ github.ref_name }}/index.md + sed -i "s#x.y.z#${{ github.ref_name }}#g" docs/versions/${{ github.ref_name }}/index.md + + cp templates/index-versions.template.md docs/versions/index.md + sed "s/^## \(.*\)$/## \1 \n\n[➔ Lien vers la documentation](\1\/index.md) /" artifact/CHANGELOG.md >>docs/versions/index.md + sed "s#__version__#${{ github.ref_name }}#" templates/latest.template.html >docs/versions/latest/index.html rm -r artifact @@ -200,7 +201,7 @@ jobs: run: | git config user.name github-actions git config user.email github-actions@github.com - git add -v docs/versions/${{ github.ref_name }}/ docs/versions/latest/ docs/index.md docs/versions.md + git add -v docs/versions/${{ github.ref_name }}/ docs/versions/latest/ docs/versions/index.md mkdocs.yml git commit -m "Add documentation for version ${{ github.ref_name }}" git push diff --git a/CHANGELOG.md b/CHANGELOG.md index 0474158..be082c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,41 @@ +## 1.7.1 + +### [Added] + +* Raster + * Classe RasterSet, réprésentant une collection d'objets de la classe Raster, avec des informations supplémentaires + * Méthodes d'import et export des informations extraites par une instance RasterSet, au travers d'un descripteur (fichier ou objet json, voire sortie standard) + * Documentation interne + * Tests unitaires pour la classe RasterSet + * Classe Raster : constructeur à partir des paramètres + +* Pyramid + * Fonction de calcul de la taille d'une pyramide + * Générateur de lecture de la liste du contenu + +* Storage + * Fonction de calcul de la taille des fichiers d'un chemin selon le stockage + * Ajout de la copie de HTTP vers FILE/S3/CEPH + * Ajout de la fonction de lecture d'un fichier HTTP, de l'existence d'un fichier HTTP et du calcul de taille d'un fichier HTTP + +### [Changed] + +* Raster + * Homogénéisation du code + * Mise en conformité PEP-8 +* test_Raster + * Homogénéisation du code + * Mise en conformité PEP-8 +* Utils + * Mise en conformité PEP-8 des fonctions `compute_bbox` et `compute_format` + + +### [Fixed] + +* Utils + * Correction d'un nom de variable dans la fonction `compute_format`, qui écrasait une fonction du noyau python. + + ## 1.6.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 568d889..ff54d46 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,22 +8,13 @@ Nous utilisons les git hooks via [pre-commit](https://pre-commit.com/) pour appl Voir le fichier de configuration correspondant : `.pre-commit-config.yaml`. -## Changelog +## Pull request -Pour éviter les conflits d'édition du changelog en cas de multiples contributions parallèles, n'éditez pas directement le fichier `CHANGELOG.md` sur votre branche de travail. - -A la place, indiquez en description ou commentaire de votre pull request, bien en évidence, le changelog qui concerne spécifiquement cette PR, avec le même formalisme que le fichier `CHANGELOG.md`. - -Lors de la release de la prochaine version, le mainteneur ajoutera vos notes de modifications dans le fichier `CHANGELOG.md` ce fichier dans le même temps, avec deux objectifs : - -* la date de modification sera celle de la fusion de branches. -* le contenu pourra tenir compte de toutes les modifications depuis la dernière release. +Le titre de la PR est utilisé pour constituer automatiquement les notes de release. Vous pouvez préciser en commentaire de votre PR des détails qui seront ajoutés dans le fichier `CHANGELOG.md` par les mainteneurs du projet. Le formalisme du changelog est le suivant, en markdown : ```md -Résumé des objectifs des modifications apportées - ### [Added] Liste de nouvelles fonctionnalités. diff --git a/README.md b/README.md index b4399ba..29bf082 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,6 @@ Ces librairies facilitent la manipulation d'entités du projet ROK4 comme les Tile Matrix Sets, les pyramides ou encore les couches, ainsi que la manipulation des stockages associés. -- [Installer la librairie](#installer-la-librairie) -- [Utiliser la librairie](#utiliser-la-librairie) -- [Compiler la librairie](#compiler-la-librairie) -- [Publier la librairie sur Pypi](#publier-la-librairie-sur-pypi) - ## Installer la librairie Installations système requises : @@ -73,6 +68,7 @@ python3 -m build ``` Remarque : + Lors de l'installation du paquet apt `python3-gdal`, une dépendance, peut demander des interactions de configuration. Pour installer dans un environnement non-interactif, définir la variable shell `DEBIAN_FRONTEND=noninteractive` permet d'adopter une configuration par défaut. ## Publier la librairie sur Pypi diff --git a/pyproject.toml b/pyproject.toml index 9f58bdd..4e1599b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ test = [ [project.urls] "Homepage" = "https://rok4.github.io/core-python" "Bug Reports" = "https://github.com/rok4/core-python/issues" -"Changelog" = "https://github.com/rok4/core-python/releases" +"Changelog" = "https://rok4.github.io/core-python/versions/" "Source" = "https://github.com/rok4/core-python" [build-system] From ecd969ebbfe759a88586bab75019a902e0d652f9 Mon Sep 17 00:00:00 2001 From: Theo Satabin Date: Tue, 11 Jul 2023 11:10:56 +0200 Subject: [PATCH 55/89] Correction de la page des tests unitaires --- .github/workflows/build-and-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 5f3767f..8fb0b1e 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -184,7 +184,7 @@ jobs: run: | sed "s#__version__#${{ github.ref_name }}#" templates/mkdocs.template.yml >mkdocs.yml - sed "s#__version__#${{ github.ref_name }}#" templates/tests.template.md >docs/versions/${{ github.ref_name }}/tests.md + sed "s#__version__#${{ github.ref_name }}#" templates/unit-tests.template.md >docs/versions/${{ github.ref_name }}/unit-tests.md sed "s#__version__#${{ github.ref_name }}#" templates/documentation.template.md >docs/versions/${{ github.ref_name }}/documentation.md sed "s#__version__#${{ github.ref_name }}#" templates/index-version.template.md >docs/versions/${{ github.ref_name }}/index.md From f5f8a98dcce311bfb9527405dae8cd8c5fc7f67e Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Tue, 18 Jul 2023 12:34:29 +0200 Subject: [PATCH 56/89] Add isort as git hook --- .pre-commit-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 31a1cee..033c0cd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,3 +25,9 @@ repos: rev: 23.3.0 hooks: - id: black + + - repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + args: ["--profile", "black", "--filter-files"] From 17d1d39682e22fe0169f67873d2489c6aa836ade Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Tue, 18 Jul 2023 12:37:48 +0200 Subject: [PATCH 57/89] Add isort to dev dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 4e1599b..f172c36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ doc = [ dev = [ "black", + "isort >= 5.12.0", "pre-commit >3,<4" ] From 91b0d3d05a3827dd5c2d2e436a66e1675d48e1af Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Tue, 18 Jul 2023 12:41:43 +0200 Subject: [PATCH 58/89] Apply isort --- CHANGELOG.md | 10 +-- src/rok4/Layer.py | 6 +- src/rok4/Pyramid.py | 31 ++++----- src/rok4/Raster.py | 4 +- src/rok4/Storage.py | 98 +++++++++++++++------------- src/rok4/TileMatrixSet.py | 10 +-- src/rok4/Utils.py | 6 +- src/rok4/Vector.py | 8 ++- tests/test_Layer.py | 12 ++-- tests/test_Pyramid.py | 15 +++-- tests/test_Raster.py | 8 +-- tests/test_Storage.py | 124 ++++++++++++++++++++++++------------ tests/test_TileMatrixSet.py | 11 ++-- tests/test_Utils.py | 14 ++-- tests/test_Vector.py | 13 ++-- 15 files changed, 211 insertions(+), 159 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be082c3..24eaed0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ * Documentation interne * Tests unitaires pour la classe RasterSet * Classe Raster : constructeur à partir des paramètres - + * Pyramid * Fonction de calcul de la taille d'une pyramide * Générateur de lecture de la liste du contenu @@ -75,7 +75,7 @@ Lecture par système de fichier virtuel avec GDAL * Utils * Meilleure gestion de reprojection par `reproject_bbox` : on détecte des systèmes identiques en entrée ou quand seul l'ordre des axes changent, pour éviter le calcul * Ajout de la fonction de reprojection d'un point `reproject_point` : on détecte des systèmes identiques en entrée ou quand seul l'ordre des axes changent, pour éviter le calcul - + ### [Changed] * Utils : @@ -98,8 +98,8 @@ Ajout de fonctionnalités de lecture de donnée d'une pyramide et suivi des reco * Storage : * Fonction de lecture binaire, complète ou partielle, d'un fichier ou objet S3 ou CEPH * Exceptions : NotImplementedError permet de préciser qu'une fonctionnalité n'a pas été implémentée pour tous les cas. Ici, on ne gère pas la décompression des données raster pour les compressions packbit et LZW - -* Ajout de la publication PyPI dans la CI GitHub + +* Ajout de la publication PyPI dans la CI GitHub ### [Changed] @@ -181,4 +181,4 @@ Initialisation des librairies Python utilisées par les outils python à venir d * Librairie de gestion d'un descripteur de pyramide * chargement depuis un descripteur ou par clone (avec changement de stockage) * écriture du descripteur -* Tests unitaires couvrant ces librairies \ No newline at end of file +* Tests unitaires couvrant ces librairies diff --git a/src/rok4/Layer.py b/src/rok4/Layer.py index 316a9ca..74387ae 100644 --- a/src/rok4/Layer.py +++ b/src/rok4/Layer.py @@ -5,16 +5,16 @@ - `Layer` - Descriptor to broadcast pyramids' data """ -from typing import Dict, List, Tuple, Union import json -from json.decoder import JSONDecodeError import os import re +from json.decoder import JSONDecodeError +from typing import Dict, List, Tuple, Union from rok4.Exceptions import * from rok4.Pyramid import Pyramid, PyramidType -from rok4.TileMatrixSet import TileMatrixSet from rok4.Storage import * +from rok4.TileMatrixSet import TileMatrixSet from rok4.Utils import * diff --git a/src/rok4/Pyramid.py b/src/rok4/Pyramid.py index 7667244..a16465a 100644 --- a/src/rok4/Pyramid.py +++ b/src/rok4/Pyramid.py @@ -6,20 +6,21 @@ - `Level` - Level of a pyramid """ -from typing import Dict, List, Tuple, Union, Iterator +import io import json -from json.decoder import JSONDecodeError import os import re -import numpy import zlib -import io +from json.decoder import JSONDecodeError +from typing import Dict, Iterator, List, Tuple, Union + import mapbox_vector_tile +import numpy from PIL import Image from rok4.Exceptions import * -from rok4.TileMatrixSet import TileMatrixSet, TileMatrix from rok4.Storage import * +from rok4.TileMatrixSet import TileMatrix, TileMatrixSet from rok4.Utils import * @@ -549,11 +550,8 @@ def serializable(self) -> Dict: Returns: Dict: descriptor structured object description """ - - serialization = { - "tile_matrix_set": self.__tms.name, - "format": self.__format - } + + serialization = {"tile_matrix_set": self.__tms.name, "format": self.__format} serialization["levels"] = [] sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution, reverse=True) @@ -620,7 +618,7 @@ def storage_root(self) -> str: Returns: str: Pyramid's storage root """ - + return self.__storage["root"].split("@", 1)[ 0 ] # Suppression de l'éventuel hôte de spécification du cluster S3 @@ -670,7 +668,6 @@ def format(self) -> str: @property def tile_extension(self) -> str: - if self.__format in [ "TIFF_RAW_UINT8", "TIFF_LZW_UINT8", @@ -835,7 +832,7 @@ def get_level(self, level_id: str) -> "Level": Returns: The corresponding pyramid's level, None if not present """ - + return self.__levels.get(level_id, None) def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: @@ -1019,7 +1016,6 @@ def get_slab_path_from_infos( else: return slab_path - def get_tile_data_binary(self, level: str, column: int, row: int) -> str: """Get a pyramid's tile as binary string @@ -1182,7 +1178,6 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr level_object = self.get_level(level) if self.__format == "TIFF_JPG_UINT8" or self.__format == "TIFF_JPG90_UINT8": - try: img = Image.open(io.BytesIO(binary_tile)) except Exception as e: @@ -1379,6 +1374,8 @@ def size(self) -> int: Returns: int: size of the pyramid """ - if not hasattr(self,"_Pyramid__size") : - self.__size = size_path(get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name)) + if not hasattr(self, "_Pyramid__size"): + self.__size = size_path( + get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name) + ) return self.__size diff --git a/src/rok4/Raster.py b/src/rok4/Raster.py index 640f4bf..271fc6a 100644 --- a/src/rok4/Raster.py +++ b/src/rok4/Raster.py @@ -10,9 +10,9 @@ import json import re from enum import Enum -from typing import Tuple, Dict +from typing import Dict, Tuple -from osgeo import ogr, gdal +from osgeo import gdal, ogr from rok4.Storage import exists, get_osgeo_path, put_data_str from rok4.Utils import ColorFormat, compute_bbox, compute_format diff --git a/src/rok4/Storage.py b/src/rok4/Storage.py index bcbee52..66e3d6d 100644 --- a/src/rok4/Storage.py +++ b/src/rok4/Storage.py @@ -30,17 +30,18 @@ To precise the cluster to use, bucket name should be bucket_name@s3.storage.fr or bucket_name@s4.storage.fr. If no host is defined (no @) in the bucket name, first S3 cluster is used """ +import hashlib +import os +import re +import tempfile +from enum import Enum +from shutil import copyfile +from typing import Dict, List, Tuple, Union + import boto3 import botocore.exceptions -import tempfile -import re -import os import rados -import hashlib import requests -from typing import Dict, List, Tuple, Union -from enum import Enum -from shutil import copyfile from osgeo import gdal gdal.UseExceptions() @@ -75,7 +76,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s Returns: Tuple[Dict[str, Union['boto3.client',str]], str, str]: the S3 informations (client, host, key, secret) and the simple bucket name """ - + global __S3_CLIENTS, __S3_DEFAULT_CLIENT if not __S3_CLIENTS: @@ -134,7 +135,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s def disconnect_s3_clients() -> None: """Clean S3 clients""" - + global __S3_CLIENTS, __S3_DEFAULT_CLIENT __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None @@ -363,16 +364,15 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: raise StorageError("FILE", e) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - - if range is None : + if range is None: try: reponse = requests.get(f"{storage_type.value}{path}", stream=True) data = reponse.content - if reponse.status_code == 404 : + if reponse.status_code == 404: raise FileNotFoundError(f"{storage_type.value}{path}") except Exception as e: raise StorageError(storage_type.name, e) - else : + else: raise NotImplementedError else: @@ -471,7 +471,6 @@ def get_size(path: str) -> int: raise StorageError("FILE", e) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - try: # Le stream=True permet de ne télécharger que le header initialement reponse = requests.get(storage_type.value + path, stream=True).headers["content-length"] @@ -526,12 +525,11 @@ def exists(path: str) -> bool: return os.path.exists(path) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - try: response = requests.get(storage_type.value + path, stream=True) - if response.status_code == 200 : + if response.status_code == 200: return True - else : + else: return False except Exception as e: raise StorageError(storage_type.name, e) @@ -839,43 +837,52 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: f"CEPH and S3", f"Cannot copy CEPH object {from_path} to S3 object {to_path} : {e}" ) - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.FILE : - + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.FILE: try: - response = requests.get(from_type.value + from_path, stream = True) + response = requests.get(from_type.value + from_path, stream=True) with open(to_path, "wb") as f: - for chunk in response.iter_content(chunk_size=65536) : + for chunk in response.iter_content(chunk_size=65536): if chunk: f.write(chunk) except Exception as e: - raise StorageError(f"HTTP(S) and FILE", f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}") - - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.CEPH : + raise StorageError( + f"HTTP(S) and FILE", + f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}", + ) + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.CEPH: to_ioctx = __get_ceph_ioctx(to_tray) try: - response = requests.get(from_type.value + from_path, stream = True) + response = requests.get(from_type.value + from_path, stream=True) offset = 0 - for chunk in response.iter_content(chunk_size=65536) : + for chunk in response.iter_content(chunk_size=65536): if chunk: size = len(chunk) to_ioctx.write(to_base_name, chunk, offset) offset += size except Exception as e: - raise StorageError(f"HTTP(S) and CEPH", f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}") - - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.S3 : + raise StorageError( + f"HTTP(S) and CEPH", + f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}", + ) + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.S3: to_s3_client, to_bucket = __get_s3_client(to_tray) try: - response = requests.get(from_type.value + from_path, stream = True) - with tempfile.NamedTemporaryFile("w+b",delete=False) as f: + response = requests.get(from_type.value + from_path, stream=True) + with tempfile.NamedTemporaryFile("w+b", delete=False) as f: name_fich = f.name - for chunk in response.iter_content(chunk_size=65536) : + for chunk in response.iter_content(chunk_size=65536): if chunk: f.write(chunk) @@ -884,7 +891,10 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: os.remove(name_fich) except Exception as e: - raise StorageError(f"HTTP(S) and S3", f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}") + raise StorageError( + f"HTTP(S) and S3", + f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}", + ) else: raise StorageError( @@ -998,7 +1008,8 @@ def get_osgeo_path(path: str) -> str: else: raise NotImplementedError(f"Cannot get a GDAL/OGR compliant path from {path}") -def size_path(path: str) -> int : + +def size_path(path: str) -> int: """Return the size of the path given (or, for the CEPH, the sum of the size of each object of the .list) Args: @@ -1011,10 +1022,10 @@ def size_path(path: str) -> int : Returns: int: size of the path """ - storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) + storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) if storage_type == StorageType.FILE: - try : + try: total = 0 with os.scandir(unprefixed_path) as it: for entry in it: @@ -1029,24 +1040,23 @@ def size_path(path: str) -> int : elif storage_type == StorageType.S3: s3_client, bucket_name = __get_s3_client(tray_name) - try : - paginator = s3_client["client"].get_paginator('list_objects_v2') + try: + paginator = s3_client["client"].get_paginator("list_objects_v2") pages = paginator.paginate( Bucket=bucket_name, - Prefix=base_name+"/", + Prefix=base_name + "/", PaginationConfig={ - 'PageSize': 10000, - } + "PageSize": 10000, + }, ) total = 0 for page in pages: - for key in page['Contents']: - total += key['Size'] + for key in page["Contents"]: + total += key["Size"] except Exception as e: raise StorageError("S3", e) - elif storage_type == StorageType.CEPH: raise NotImplementedError else: diff --git a/src/rok4/TileMatrixSet.py b/src/rok4/TileMatrixSet.py index 472810c..3000492 100644 --- a/src/rok4/TileMatrixSet.py +++ b/src/rok4/TileMatrixSet.py @@ -9,15 +9,15 @@ - ROK4_TMS_DIRECTORY """ +import json +import os +from json.decoder import JSONDecodeError +from typing import Dict, List, Tuple + from rok4.Exceptions import * from rok4.Storage import get_data_str from rok4.Utils import * -from typing import Dict, List, Tuple -from json.decoder import JSONDecodeError -import json -import os - class TileMatrix: """A tile matrix is a tile matrix set's level. diff --git a/src/rok4/Utils.py b/src/rok4/Utils.py index da15aef..125e391 100644 --- a/src/rok4/Utils.py +++ b/src/rok4/Utils.py @@ -3,10 +3,10 @@ import os import re - -from typing import Dict, List, Tuple, Union -from osgeo import ogr, osr, gdal from enum import Enum +from typing import Dict, List, Tuple, Union + +from osgeo import gdal, ogr, osr ogr.UseExceptions() osr.UseExceptions() diff --git a/src/rok4/Vector.py b/src/rok4/Vector.py index 94e6def..46656c2 100644 --- a/src/rok4/Vector.py +++ b/src/rok4/Vector.py @@ -6,12 +6,14 @@ """ -from osgeo import ogr -from rok4.Storage import get_osgeo_path, copy -from rok4.Exceptions import * import os import tempfile +from osgeo import ogr + +from rok4.Exceptions import * +from rok4.Storage import copy, get_osgeo_path + # Enable GDAL/OGR exceptions ogr.UseExceptions() diff --git a/tests/test_Layer.py b/tests/test_Layer.py index 0a60d5c..a35f649 100644 --- a/tests/test_Layer.py +++ b/tests/test_Layer.py @@ -1,12 +1,12 @@ -from rok4.Layer import Layer -from rok4.Pyramid import PyramidType -from rok4.Exceptions import * +import os +from unittest import mock +from unittest.mock import * import pytest -import os -from unittest.mock import * -from unittest import mock +from rok4.Exceptions import * +from rok4.Layer import Layer +from rok4.Pyramid import PyramidType @mock.patch.dict(os.environ, {}, clear=True) diff --git a/tests/test_Pyramid.py b/tests/test_Pyramid.py index 6eebfd7..f5f1d60 100644 --- a/tests/test_Pyramid.py +++ b/tests/test_Pyramid.py @@ -1,13 +1,14 @@ +import os +from unittest import mock +from unittest.mock import * + +import pytest + +from rok4.Exceptions import * from rok4.Pyramid import * -from rok4.TileMatrixSet import TileMatrixSet from rok4.Storage import StorageType +from rok4.TileMatrixSet import TileMatrixSet from rok4.Utils import * -from rok4.Exceptions import * - -import pytest -import os -from unittest.mock import * -from unittest import mock @mock.patch("rok4.Pyramid.get_data_str", side_effect=StorageError("FILE", "Not found")) diff --git a/tests/test_Raster.py b/tests/test_Raster.py index de35de1..cc79846 100644 --- a/tests/test_Raster.py +++ b/tests/test_Raster.py @@ -6,17 +6,17 @@ """ import copy -import math import json +import math import random +from unittest import TestCase, mock +from unittest.mock import MagicMock, Mock, call, mock_open, patch + import pytest -from unittest import mock, TestCase -from unittest.mock import call, MagicMock, Mock, mock_open, patch from rok4.Raster import Raster, RasterSet from rok4.Utils import ColorFormat - # rok4.Raster.Raster class tests diff --git a/tests/test_Storage.py b/tests/test_Storage.py index 9779954..f7b665d 100644 --- a/tests/test_Storage.py +++ b/tests/test_Storage.py @@ -1,14 +1,13 @@ -from rok4.Storage import * -from rok4.Exceptions import * - -import pytest import os +from unittest import mock +from unittest.mock import * import botocore.exceptions +import pytest from rados import ObjectNotFound -from unittest import mock -from unittest.mock import * +from rok4.Exceptions import * +from rok4.Storage import * @mock.patch.dict(os.environ, {}, clear=True) @@ -21,6 +20,7 @@ def test_hash_file_ok(mock_file): except Exception as exc: assert False, f"FILE md5 sum raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) def test_get_infos_from_path(): assert (StorageType.S3, "toto/titi", "toto", "titi") == get_infos_from_path("s3://toto/titi") @@ -103,6 +103,7 @@ def test_file_read_ok(mock_file): except Exception as exc: assert False, f"FILE read raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -117,6 +118,7 @@ def test_s3_read_nok(mocked_s3_client): with pytest.raises(StorageError): data = get_data_str("s3://bucket/path/to/object") + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -159,8 +161,9 @@ def test_ceph_read_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH read raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch("requests.get", side_effect={"status_code":404}) +@mock.patch("requests.get", side_effect={"status_code": 404}) def test_http_read_error(mock_http): with pytest.raises(StorageError): requests_instance = MagicMock() @@ -171,28 +174,31 @@ def test_http_read_error(mock_http): mock_http.assert_called_with("http://path/to/file.ext", stream=True) + @mock.patch.dict(os.environ, {}, clear=True) def test_http_read_range_error(): with pytest.raises(NotImplementedError): - data = get_data_binary("http://path/to/file.ext", (0,100)) + data = get_data_binary("http://path/to/file.ext", (0, 100)) + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("requests.get") def test_http_read_ok(mock_http): - try : + try: requests_instance = MagicMock() - requests_instance.content = b'data' + requests_instance.content = b"data" mock_http.return_value = requests_instance data = get_data_str("http://path/to/file.ext") mock_http.assert_called_with("http://path/to/file.ext", stream=True) - assert data == 'data' + assert data == "data" except Exception as exc: assert False, f"HTTP read raises an exception: {exc}" ############ put_data_str + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -225,6 +231,7 @@ def test_s3_write_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 write raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -302,6 +309,7 @@ def test_copy_s3_file_nok(mock_hash_file, mock_makedirs, mocked_s3_client): copy("s3://bucket/source.ext", "file:///path/to/destination.ext", "toto") mock_makedirs.assert_called_once_with("/path/to", exist_ok=True) + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -320,6 +328,7 @@ def test_copy_file_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"FILE -> S3 copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -338,6 +347,7 @@ def test_copy_s3_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 -> S3 copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -373,6 +383,7 @@ def test_copy_s3_s3_intercluster_nok(mocked_s3_client): with pytest.raises(StorageError): copy("s3://bucket@a/source.ext", "s3://bucket@c/destination.ext", "toto") + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -399,6 +410,7 @@ def test_copy_ceph_file_ok(mock_file, mock_makedirs, mocked_rados_client): except Exception as exc: assert False, f"CEPH -> FILE copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -423,6 +435,7 @@ def test_copy_file_ceph_ok(mock_file, mocked_rados_client): except Exception as exc: assert False, f"FILE -> CEPH copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -487,14 +500,14 @@ def test_copy_ceph_s3_ok(mock_file, mocked_s3_client, mocked_rados_client): except Exception as exc: assert False, f"CEPH -> S3 copy raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') -@patch('builtins.open', new_callable=mock_open) +@mock.patch("requests.get") +@patch("builtins.open", new_callable=mock_open) def test_copy_http_file_ok(mock_open, mock_requests): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance copy("http://path/to/source.ext", "file:///path/to/destination.ext") @@ -503,17 +516,20 @@ def test_copy_http_file_ok(mock_open, mock_requests): except Exception as exc: assert False, f"HTTP -> FILE copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -@mock.patch('rok4.Storage.rados.Rados') -@mock.patch('requests.get') + +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +@mock.patch("rok4.Storage.rados.Rados") +@mock.patch("requests.get") def test_copy_http_ceph_ok(mock_requests, mocked_rados_client): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance - disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.write.return_value = None @@ -526,16 +542,20 @@ def test_copy_http_ceph_ok(mock_requests, mocked_rados_client): except Exception as exc: assert False, f"HTTP -> CEPH copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') -@mock.patch('requests.get') -@patch('tempfile.NamedTemporaryFile', new_callable=mock_open) -@mock.patch('os.remove') + +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") +@mock.patch("requests.get") +@patch("tempfile.NamedTemporaryFile", new_callable=mock_open) +@mock.patch("os.remove") def test_copy_http_s3_ok(mock_remove, mock_tempfile, mock_requests, mocked_s3_client): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance disconnect_s3_clients() @@ -546,7 +566,7 @@ def test_copy_http_s3_ok(mock_remove, mock_tempfile, mock_requests, mocked_s3_cl copy("http://path/to/source.ext", "s3://bucket/destination.ext") mock_requests.assert_called_once_with("http://path/to/source.ext", stream=True) - mock_tempfile.assert_called_once_with("w+b",delete=False) + mock_tempfile.assert_called_once_with("w+b", delete=False) except Exception as exc: assert False, f"HTTP -> CEPH copy raises an exception: {exc}" @@ -563,6 +583,7 @@ def test_link_hard_nok(): with pytest.raises(StorageError): link("ceph://pool1/source.ext", "ceph://pool2/destination.ext", True) + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.symlink", return_value=None) def test_link_file_ok(mock_link): @@ -582,6 +603,7 @@ def test_hlink_file_ok(mock_link): except Exception as exc: assert False, f"FILE hard link raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -619,6 +641,7 @@ def test_link_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 link raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -637,6 +660,7 @@ def test_link_s3_nok(mocked_s3_client): ############ get_size + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.stat") def test_size_file_ok(mock_stat): @@ -647,6 +671,7 @@ def test_size_file_ok(mock_stat): except Exception as exc: assert False, f"FILE size raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -667,6 +692,7 @@ def test_size_ceph_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH size raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -685,12 +711,12 @@ def test_size_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 size raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') +@mock.patch("requests.get") def test_size_http_ok(mock_requests): - http_instance = MagicMock() - http_instance.headers = {"content-length":12} + http_instance.headers = {"content-length": 12} mock_requests.return_value = http_instance try: @@ -717,6 +743,7 @@ def test_exists_file_ok(mock_exists): except Exception as exc: assert False, f"FILE not exists raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -768,10 +795,10 @@ def test_exists_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"CEPH not exists raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') +@mock.patch("requests.get") def test_exists_http_ok(mock_requests): - http_instance = MagicMock() http_instance.status_code = 200 mock_requests.return_value = http_instance @@ -792,6 +819,7 @@ def test_exists_http_ok(mock_requests): ############ remove + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.remove") def test_remove_file_ok(mock_remove): @@ -807,6 +835,7 @@ def test_remove_file_ok(mock_remove): except Exception as exc: assert False, f"FILE deletion (not found) raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -832,6 +861,7 @@ def test_remove_ceph_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH deletion (not found) raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -861,6 +891,7 @@ def test_get_osgeo_path_file_ok(): except Exception as exc: assert False, f"FILE osgeo path raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -881,6 +912,7 @@ def test_get_osgeo_path_nok(): with pytest.raises(NotImplementedError): get_osgeo_path("ceph://pool/data.ext") + ############ size_path def test_size_path_file_ok(): try: @@ -889,22 +921,31 @@ def test_size_path_file_ok(): except Exception as exc: assert False, f"FILE size of the path raises an exception: {exc}" + def test_size_file_nok(): - with pytest.raises(StorageError) : + with pytest.raises(StorageError): size = size_path("file://tests/fixtures/TIFF_PBF_M") -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -def test_size_path_ceph_nok(): +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +def test_size_path_ceph_nok(): with pytest.raises(NotImplementedError): size = size_path("ceph://pool/path") -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') -def test_size_path_s3_ok(mocked_s3_client): +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") +def test_size_path_s3_ok(mocked_s3_client): disconnect_s3_clients() - pages = [{"Contents" : [{"Size" : 10},{"Size" : 20}]}, {"Contents" : [{"Size" : 50}]}] + pages = [{"Contents": [{"Size": 10}, {"Size": 20}]}, {"Contents": [{"Size": 50}]}] paginator = MagicMock() paginator.paginate.return_value = pages client = MagicMock() @@ -916,4 +957,3 @@ def test_size_path_s3_ok(mocked_s3_client): assert size == 80 except Exception as exc: assert False, f"S3 size of the path raises an exception: {exc}" - diff --git a/tests/test_TileMatrixSet.py b/tests/test_TileMatrixSet.py index 4750f50..5cf6062 100644 --- a/tests/test_TileMatrixSet.py +++ b/tests/test_TileMatrixSet.py @@ -1,10 +1,11 @@ -from rok4.TileMatrixSet import TileMatrixSet -from rok4.Exceptions import * - -import pytest import os -from unittest.mock import * from unittest import mock +from unittest.mock import * + +import pytest + +from rok4.Exceptions import * +from rok4.TileMatrixSet import TileMatrixSet @mock.patch.dict(os.environ, {}, clear=True) diff --git a/tests/test_Utils.py b/tests/test_Utils.py index cdba571..0f5d67d 100644 --- a/tests/test_Utils.py +++ b/tests/test_Utils.py @@ -1,14 +1,14 @@ -from rok4.Utils import * -from rok4.Exceptions import * +import math +import os +import random +from unittest import mock +from unittest.mock import * import pytest -import os from osgeo import gdal, osr -import math -import random -from unittest.mock import * -from unittest import mock +from rok4.Exceptions import * +from rok4.Utils import * def test_srs_to_spatialreference_ignf_ok(): diff --git a/tests/test_Vector.py b/tests/test_Vector.py index 221f5e4..3fe0a1c 100644 --- a/tests/test_Vector.py +++ b/tests/test_Vector.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -from rok4.Vector import * -from rok4.Exceptions import * -from rok4.Storage import disconnect_ceph_clients - -import pytest import os -from unittest.mock import * from unittest import mock +from unittest.mock import * + +import pytest + +from rok4.Exceptions import * +from rok4.Storage import disconnect_ceph_clients +from rok4.Vector import * @mock.patch.dict(os.environ, {}, clear=True) From 3b2e580270f1d283ba6f350786596737f19d41fb Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Tue, 18 Jul 2023 12:43:44 +0200 Subject: [PATCH 59/89] Specify target Python version to black --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 31a1cee..2470e26 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,3 +25,4 @@ repos: rev: 23.3.0 hooks: - id: black + args: ["--target-version=py38"] From 3fd1c93ac2ba14385ada5e760d5e2bf0870d3964 Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Tue, 18 Jul 2023 12:50:35 +0200 Subject: [PATCH 60/89] Rename files to comply with Python convention PEP8 --- src/rok4/{Exceptions.py => exceptions.py} | 0 src/rok4/{Layer.py => layer.py} | 10 +- src/rok4/{Pyramid.py => pyramid.py} | 28 ++--- src/rok4/{Raster.py => raster.py} | 4 +- src/rok4/{Storage.py => storage.py} | 85 +++++++------ .../{TileMatrixSet.py => tile_matrix_set.py} | 6 +- src/rok4/{Utils.py => utils.py} | 0 src/rok4/{Vector.py => vector.py} | 4 +- tests/{test_Layer.py => test_layer.py} | 6 +- tests/{test_Pyramid.py => test_pyramid.py} | 10 +- tests/{test_Raster.py => test_raster.py} | 4 +- tests/{test_Storage.py => test_storage.py} | 117 ++++++++++++------ ...leMatrixSet.py => test_tile_matrix_set.py} | 4 +- tests/{test_Utils.py => test_utils.py} | 4 +- tests/{test_Vector.py => test_vector.py} | 6 +- 15 files changed, 167 insertions(+), 121 deletions(-) rename src/rok4/{Exceptions.py => exceptions.py} (100%) rename src/rok4/{Layer.py => layer.py} (98%) rename src/rok4/{Pyramid.py => pyramid.py} (99%) rename src/rok4/{Raster.py => raster.py} (98%) rename src/rok4/{Storage.py => storage.py} (95%) rename src/rok4/{TileMatrixSet.py => tile_matrix_set.py} (98%) rename src/rok4/{Utils.py => utils.py} (100%) rename src/rok4/{Vector.py => vector.py} (99%) rename tests/{test_Layer.py => test_layer.py} (98%) rename tests/{test_Pyramid.py => test_pyramid.py} (98%) rename tests/{test_Raster.py => test_raster.py} (99%) rename tests/{test_Storage.py => test_storage.py} (94%) rename tests/{test_TileMatrixSet.py => test_tile_matrix_set.py} (99%) rename tests/{test_Utils.py => test_utils.py} (99%) rename tests/{test_Vector.py => test_vector.py} (97%) diff --git a/src/rok4/Exceptions.py b/src/rok4/exceptions.py similarity index 100% rename from src/rok4/Exceptions.py rename to src/rok4/exceptions.py diff --git a/src/rok4/Layer.py b/src/rok4/layer.py similarity index 98% rename from src/rok4/Layer.py rename to src/rok4/layer.py index 316a9ca..f7a891b 100644 --- a/src/rok4/Layer.py +++ b/src/rok4/layer.py @@ -11,11 +11,11 @@ import os import re -from rok4.Exceptions import * -from rok4.Pyramid import Pyramid, PyramidType -from rok4.TileMatrixSet import TileMatrixSet -from rok4.Storage import * -from rok4.Utils import * +from rok4.exceptions import * +from rok4.pyramid import Pyramid, PyramidType +from rok4.tile_matrix_set import TileMatrixSet +from rok4.storage import * +from rok4.utils import * class Layer: diff --git a/src/rok4/Pyramid.py b/src/rok4/pyramid.py similarity index 99% rename from src/rok4/Pyramid.py rename to src/rok4/pyramid.py index 7667244..8ca8afc 100644 --- a/src/rok4/Pyramid.py +++ b/src/rok4/pyramid.py @@ -17,10 +17,10 @@ import mapbox_vector_tile from PIL import Image -from rok4.Exceptions import * -from rok4.TileMatrixSet import TileMatrixSet, TileMatrix -from rok4.Storage import * -from rok4.Utils import * +from rok4.exceptions import * +from rok4.tile_matrix_set import TileMatrixSet, TileMatrix +from rok4.storage import * +from rok4.utils import * class PyramidType(Enum): @@ -549,11 +549,8 @@ def serializable(self) -> Dict: Returns: Dict: descriptor structured object description """ - - serialization = { - "tile_matrix_set": self.__tms.name, - "format": self.__format - } + + serialization = {"tile_matrix_set": self.__tms.name, "format": self.__format} serialization["levels"] = [] sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution, reverse=True) @@ -620,7 +617,7 @@ def storage_root(self) -> str: Returns: str: Pyramid's storage root """ - + return self.__storage["root"].split("@", 1)[ 0 ] # Suppression de l'éventuel hôte de spécification du cluster S3 @@ -670,7 +667,6 @@ def format(self) -> str: @property def tile_extension(self) -> str: - if self.__format in [ "TIFF_RAW_UINT8", "TIFF_LZW_UINT8", @@ -835,7 +831,7 @@ def get_level(self, level_id: str) -> "Level": Returns: The corresponding pyramid's level, None if not present """ - + return self.__levels.get(level_id, None) def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: @@ -1019,7 +1015,6 @@ def get_slab_path_from_infos( else: return slab_path - def get_tile_data_binary(self, level: str, column: int, row: int) -> str: """Get a pyramid's tile as binary string @@ -1182,7 +1177,6 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr level_object = self.get_level(level) if self.__format == "TIFF_JPG_UINT8" or self.__format == "TIFF_JPG90_UINT8": - try: img = Image.open(io.BytesIO(binary_tile)) except Exception as e: @@ -1379,6 +1373,8 @@ def size(self) -> int: Returns: int: size of the pyramid """ - if not hasattr(self,"_Pyramid__size") : - self.__size = size_path(get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name)) + if not hasattr(self, "_Pyramid__size"): + self.__size = size_path( + get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name) + ) return self.__size diff --git a/src/rok4/Raster.py b/src/rok4/raster.py similarity index 98% rename from src/rok4/Raster.py rename to src/rok4/raster.py index 640f4bf..c86901b 100644 --- a/src/rok4/Raster.py +++ b/src/rok4/raster.py @@ -14,8 +14,8 @@ from osgeo import ogr, gdal -from rok4.Storage import exists, get_osgeo_path, put_data_str -from rok4.Utils import ColorFormat, compute_bbox, compute_format +from rok4.storage import exists, get_osgeo_path, put_data_str +from rok4.utils import ColorFormat, compute_bbox, compute_format # Enable GDAL/OGR exceptions ogr.UseExceptions() diff --git a/src/rok4/Storage.py b/src/rok4/storage.py similarity index 95% rename from src/rok4/Storage.py rename to src/rok4/storage.py index bcbee52..3015e5e 100644 --- a/src/rok4/Storage.py +++ b/src/rok4/storage.py @@ -45,7 +45,7 @@ gdal.UseExceptions() -from rok4.Exceptions import * +from rok4.exceptions import * class StorageType(Enum): @@ -75,7 +75,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s Returns: Tuple[Dict[str, Union['boto3.client',str]], str, str]: the S3 informations (client, host, key, secret) and the simple bucket name """ - + global __S3_CLIENTS, __S3_DEFAULT_CLIENT if not __S3_CLIENTS: @@ -134,7 +134,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s def disconnect_s3_clients() -> None: """Clean S3 clients""" - + global __S3_CLIENTS, __S3_DEFAULT_CLIENT __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None @@ -363,16 +363,15 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: raise StorageError("FILE", e) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - - if range is None : + if range is None: try: reponse = requests.get(f"{storage_type.value}{path}", stream=True) data = reponse.content - if reponse.status_code == 404 : + if reponse.status_code == 404: raise FileNotFoundError(f"{storage_type.value}{path}") except Exception as e: raise StorageError(storage_type.name, e) - else : + else: raise NotImplementedError else: @@ -471,7 +470,6 @@ def get_size(path: str) -> int: raise StorageError("FILE", e) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - try: # Le stream=True permet de ne télécharger que le header initialement reponse = requests.get(storage_type.value + path, stream=True).headers["content-length"] @@ -526,12 +524,11 @@ def exists(path: str) -> bool: return os.path.exists(path) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - try: response = requests.get(storage_type.value + path, stream=True) - if response.status_code == 200 : + if response.status_code == 200: return True - else : + else: return False except Exception as e: raise StorageError(storage_type.name, e) @@ -839,43 +836,52 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: f"CEPH and S3", f"Cannot copy CEPH object {from_path} to S3 object {to_path} : {e}" ) - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.FILE : - + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.FILE: try: - response = requests.get(from_type.value + from_path, stream = True) + response = requests.get(from_type.value + from_path, stream=True) with open(to_path, "wb") as f: - for chunk in response.iter_content(chunk_size=65536) : + for chunk in response.iter_content(chunk_size=65536): if chunk: f.write(chunk) except Exception as e: - raise StorageError(f"HTTP(S) and FILE", f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}") - - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.CEPH : + raise StorageError( + f"HTTP(S) and FILE", + f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}", + ) + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.CEPH: to_ioctx = __get_ceph_ioctx(to_tray) try: - response = requests.get(from_type.value + from_path, stream = True) + response = requests.get(from_type.value + from_path, stream=True) offset = 0 - for chunk in response.iter_content(chunk_size=65536) : + for chunk in response.iter_content(chunk_size=65536): if chunk: size = len(chunk) to_ioctx.write(to_base_name, chunk, offset) offset += size except Exception as e: - raise StorageError(f"HTTP(S) and CEPH", f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}") - - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.S3 : + raise StorageError( + f"HTTP(S) and CEPH", + f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}", + ) + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.S3: to_s3_client, to_bucket = __get_s3_client(to_tray) try: - response = requests.get(from_type.value + from_path, stream = True) - with tempfile.NamedTemporaryFile("w+b",delete=False) as f: + response = requests.get(from_type.value + from_path, stream=True) + with tempfile.NamedTemporaryFile("w+b", delete=False) as f: name_fich = f.name - for chunk in response.iter_content(chunk_size=65536) : + for chunk in response.iter_content(chunk_size=65536): if chunk: f.write(chunk) @@ -884,7 +890,10 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: os.remove(name_fich) except Exception as e: - raise StorageError(f"HTTP(S) and S3", f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}") + raise StorageError( + f"HTTP(S) and S3", + f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}", + ) else: raise StorageError( @@ -998,7 +1007,8 @@ def get_osgeo_path(path: str) -> str: else: raise NotImplementedError(f"Cannot get a GDAL/OGR compliant path from {path}") -def size_path(path: str) -> int : + +def size_path(path: str) -> int: """Return the size of the path given (or, for the CEPH, the sum of the size of each object of the .list) Args: @@ -1011,10 +1021,10 @@ def size_path(path: str) -> int : Returns: int: size of the path """ - storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) + storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) if storage_type == StorageType.FILE: - try : + try: total = 0 with os.scandir(unprefixed_path) as it: for entry in it: @@ -1029,24 +1039,23 @@ def size_path(path: str) -> int : elif storage_type == StorageType.S3: s3_client, bucket_name = __get_s3_client(tray_name) - try : - paginator = s3_client["client"].get_paginator('list_objects_v2') + try: + paginator = s3_client["client"].get_paginator("list_objects_v2") pages = paginator.paginate( Bucket=bucket_name, - Prefix=base_name+"/", + Prefix=base_name + "/", PaginationConfig={ - 'PageSize': 10000, - } + "PageSize": 10000, + }, ) total = 0 for page in pages: - for key in page['Contents']: - total += key['Size'] + for key in page["Contents"]: + total += key["Size"] except Exception as e: raise StorageError("S3", e) - elif storage_type == StorageType.CEPH: raise NotImplementedError else: diff --git a/src/rok4/TileMatrixSet.py b/src/rok4/tile_matrix_set.py similarity index 98% rename from src/rok4/TileMatrixSet.py rename to src/rok4/tile_matrix_set.py index 472810c..037e6b6 100644 --- a/src/rok4/TileMatrixSet.py +++ b/src/rok4/tile_matrix_set.py @@ -9,9 +9,9 @@ - ROK4_TMS_DIRECTORY """ -from rok4.Exceptions import * -from rok4.Storage import get_data_str -from rok4.Utils import * +from rok4.exceptions import * +from rok4.storage import get_data_str +from rok4.utils import * from typing import Dict, List, Tuple from json.decoder import JSONDecodeError diff --git a/src/rok4/Utils.py b/src/rok4/utils.py similarity index 100% rename from src/rok4/Utils.py rename to src/rok4/utils.py diff --git a/src/rok4/Vector.py b/src/rok4/vector.py similarity index 99% rename from src/rok4/Vector.py rename to src/rok4/vector.py index 94e6def..b12261a 100644 --- a/src/rok4/Vector.py +++ b/src/rok4/vector.py @@ -7,8 +7,8 @@ """ from osgeo import ogr -from rok4.Storage import get_osgeo_path, copy -from rok4.Exceptions import * +from rok4.storage import get_osgeo_path, copy +from rok4.exceptions import * import os import tempfile diff --git a/tests/test_Layer.py b/tests/test_layer.py similarity index 98% rename from tests/test_Layer.py rename to tests/test_layer.py index 0a60d5c..ecc140e 100644 --- a/tests/test_Layer.py +++ b/tests/test_layer.py @@ -1,6 +1,6 @@ -from rok4.Layer import Layer -from rok4.Pyramid import PyramidType -from rok4.Exceptions import * +from rok4.layer import Layer +from rok4.pyramid import PyramidType +from rok4.exceptions import * import pytest import os diff --git a/tests/test_Pyramid.py b/tests/test_pyramid.py similarity index 98% rename from tests/test_Pyramid.py rename to tests/test_pyramid.py index 6eebfd7..14cda2d 100644 --- a/tests/test_Pyramid.py +++ b/tests/test_pyramid.py @@ -1,8 +1,8 @@ -from rok4.Pyramid import * -from rok4.TileMatrixSet import TileMatrixSet -from rok4.Storage import StorageType -from rok4.Utils import * -from rok4.Exceptions import * +from rok4.pyramid import * +from rok4.tile_matrix_set import TileMatrixSet +from rok4.storage import StorageType +from rok4.utils import * +from rok4.exceptions import * import pytest import os diff --git a/tests/test_Raster.py b/tests/test_raster.py similarity index 99% rename from tests/test_Raster.py rename to tests/test_raster.py index de35de1..ae466a5 100644 --- a/tests/test_Raster.py +++ b/tests/test_raster.py @@ -13,8 +13,8 @@ from unittest import mock, TestCase from unittest.mock import call, MagicMock, Mock, mock_open, patch -from rok4.Raster import Raster, RasterSet -from rok4.Utils import ColorFormat +from rok4.raster import Raster, RasterSet +from rok4.utils import ColorFormat # rok4.Raster.Raster class tests diff --git a/tests/test_Storage.py b/tests/test_storage.py similarity index 94% rename from tests/test_Storage.py rename to tests/test_storage.py index 9779954..8e0f90a 100644 --- a/tests/test_Storage.py +++ b/tests/test_storage.py @@ -1,5 +1,5 @@ -from rok4.Storage import * -from rok4.Exceptions import * +from rok4.storage import * +from rok4.exceptions import * import pytest import os @@ -21,6 +21,7 @@ def test_hash_file_ok(mock_file): except Exception as exc: assert False, f"FILE md5 sum raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) def test_get_infos_from_path(): assert (StorageType.S3, "toto/titi", "toto", "titi") == get_infos_from_path("s3://toto/titi") @@ -103,6 +104,7 @@ def test_file_read_ok(mock_file): except Exception as exc: assert False, f"FILE read raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -117,6 +119,7 @@ def test_s3_read_nok(mocked_s3_client): with pytest.raises(StorageError): data = get_data_str("s3://bucket/path/to/object") + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -159,8 +162,9 @@ def test_ceph_read_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH read raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch("requests.get", side_effect={"status_code":404}) +@mock.patch("requests.get", side_effect={"status_code": 404}) def test_http_read_error(mock_http): with pytest.raises(StorageError): requests_instance = MagicMock() @@ -171,28 +175,31 @@ def test_http_read_error(mock_http): mock_http.assert_called_with("http://path/to/file.ext", stream=True) + @mock.patch.dict(os.environ, {}, clear=True) def test_http_read_range_error(): with pytest.raises(NotImplementedError): - data = get_data_binary("http://path/to/file.ext", (0,100)) + data = get_data_binary("http://path/to/file.ext", (0, 100)) + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("requests.get") def test_http_read_ok(mock_http): - try : + try: requests_instance = MagicMock() - requests_instance.content = b'data' + requests_instance.content = b"data" mock_http.return_value = requests_instance data = get_data_str("http://path/to/file.ext") mock_http.assert_called_with("http://path/to/file.ext", stream=True) - assert data == 'data' + assert data == "data" except Exception as exc: assert False, f"HTTP read raises an exception: {exc}" ############ put_data_str + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -225,6 +232,7 @@ def test_s3_write_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 write raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -302,6 +310,7 @@ def test_copy_s3_file_nok(mock_hash_file, mock_makedirs, mocked_s3_client): copy("s3://bucket/source.ext", "file:///path/to/destination.ext", "toto") mock_makedirs.assert_called_once_with("/path/to", exist_ok=True) + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -320,6 +329,7 @@ def test_copy_file_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"FILE -> S3 copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -338,6 +348,7 @@ def test_copy_s3_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 -> S3 copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -373,6 +384,7 @@ def test_copy_s3_s3_intercluster_nok(mocked_s3_client): with pytest.raises(StorageError): copy("s3://bucket@a/source.ext", "s3://bucket@c/destination.ext", "toto") + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -399,6 +411,7 @@ def test_copy_ceph_file_ok(mock_file, mock_makedirs, mocked_rados_client): except Exception as exc: assert False, f"CEPH -> FILE copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -423,6 +436,7 @@ def test_copy_file_ceph_ok(mock_file, mocked_rados_client): except Exception as exc: assert False, f"FILE -> CEPH copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -487,14 +501,14 @@ def test_copy_ceph_s3_ok(mock_file, mocked_s3_client, mocked_rados_client): except Exception as exc: assert False, f"CEPH -> S3 copy raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') -@patch('builtins.open', new_callable=mock_open) +@mock.patch("requests.get") +@patch("builtins.open", new_callable=mock_open) def test_copy_http_file_ok(mock_open, mock_requests): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance copy("http://path/to/source.ext", "file:///path/to/destination.ext") @@ -503,17 +517,20 @@ def test_copy_http_file_ok(mock_open, mock_requests): except Exception as exc: assert False, f"HTTP -> FILE copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -@mock.patch('rok4.Storage.rados.Rados') -@mock.patch('requests.get') + +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +@mock.patch("rok4.Storage.rados.Rados") +@mock.patch("requests.get") def test_copy_http_ceph_ok(mock_requests, mocked_rados_client): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance - disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.write.return_value = None @@ -526,16 +543,20 @@ def test_copy_http_ceph_ok(mock_requests, mocked_rados_client): except Exception as exc: assert False, f"HTTP -> CEPH copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') -@mock.patch('requests.get') -@patch('tempfile.NamedTemporaryFile', new_callable=mock_open) -@mock.patch('os.remove') + +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") +@mock.patch("requests.get") +@patch("tempfile.NamedTemporaryFile", new_callable=mock_open) +@mock.patch("os.remove") def test_copy_http_s3_ok(mock_remove, mock_tempfile, mock_requests, mocked_s3_client): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance disconnect_s3_clients() @@ -546,7 +567,7 @@ def test_copy_http_s3_ok(mock_remove, mock_tempfile, mock_requests, mocked_s3_cl copy("http://path/to/source.ext", "s3://bucket/destination.ext") mock_requests.assert_called_once_with("http://path/to/source.ext", stream=True) - mock_tempfile.assert_called_once_with("w+b",delete=False) + mock_tempfile.assert_called_once_with("w+b", delete=False) except Exception as exc: assert False, f"HTTP -> CEPH copy raises an exception: {exc}" @@ -563,6 +584,7 @@ def test_link_hard_nok(): with pytest.raises(StorageError): link("ceph://pool1/source.ext", "ceph://pool2/destination.ext", True) + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.symlink", return_value=None) def test_link_file_ok(mock_link): @@ -582,6 +604,7 @@ def test_hlink_file_ok(mock_link): except Exception as exc: assert False, f"FILE hard link raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -619,6 +642,7 @@ def test_link_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 link raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -637,6 +661,7 @@ def test_link_s3_nok(mocked_s3_client): ############ get_size + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.stat") def test_size_file_ok(mock_stat): @@ -647,6 +672,7 @@ def test_size_file_ok(mock_stat): except Exception as exc: assert False, f"FILE size raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -667,6 +693,7 @@ def test_size_ceph_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH size raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -685,12 +712,12 @@ def test_size_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 size raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') +@mock.patch("requests.get") def test_size_http_ok(mock_requests): - http_instance = MagicMock() - http_instance.headers = {"content-length":12} + http_instance.headers = {"content-length": 12} mock_requests.return_value = http_instance try: @@ -717,6 +744,7 @@ def test_exists_file_ok(mock_exists): except Exception as exc: assert False, f"FILE not exists raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -768,10 +796,10 @@ def test_exists_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"CEPH not exists raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') +@mock.patch("requests.get") def test_exists_http_ok(mock_requests): - http_instance = MagicMock() http_instance.status_code = 200 mock_requests.return_value = http_instance @@ -792,6 +820,7 @@ def test_exists_http_ok(mock_requests): ############ remove + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.remove") def test_remove_file_ok(mock_remove): @@ -807,6 +836,7 @@ def test_remove_file_ok(mock_remove): except Exception as exc: assert False, f"FILE deletion (not found) raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -832,6 +862,7 @@ def test_remove_ceph_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH deletion (not found) raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -861,6 +892,7 @@ def test_get_osgeo_path_file_ok(): except Exception as exc: assert False, f"FILE osgeo path raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -881,6 +913,7 @@ def test_get_osgeo_path_nok(): with pytest.raises(NotImplementedError): get_osgeo_path("ceph://pool/data.ext") + ############ size_path def test_size_path_file_ok(): try: @@ -889,22 +922,31 @@ def test_size_path_file_ok(): except Exception as exc: assert False, f"FILE size of the path raises an exception: {exc}" + def test_size_file_nok(): - with pytest.raises(StorageError) : + with pytest.raises(StorageError): size = size_path("file://tests/fixtures/TIFF_PBF_M") -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -def test_size_path_ceph_nok(): +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +def test_size_path_ceph_nok(): with pytest.raises(NotImplementedError): size = size_path("ceph://pool/path") -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') -def test_size_path_s3_ok(mocked_s3_client): +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") +def test_size_path_s3_ok(mocked_s3_client): disconnect_s3_clients() - pages = [{"Contents" : [{"Size" : 10},{"Size" : 20}]}, {"Contents" : [{"Size" : 50}]}] + pages = [{"Contents": [{"Size": 10}, {"Size": 20}]}, {"Contents": [{"Size": 50}]}] paginator = MagicMock() paginator.paginate.return_value = pages client = MagicMock() @@ -916,4 +958,3 @@ def test_size_path_s3_ok(mocked_s3_client): assert size == 80 except Exception as exc: assert False, f"S3 size of the path raises an exception: {exc}" - diff --git a/tests/test_TileMatrixSet.py b/tests/test_tile_matrix_set.py similarity index 99% rename from tests/test_TileMatrixSet.py rename to tests/test_tile_matrix_set.py index 4750f50..350669b 100644 --- a/tests/test_TileMatrixSet.py +++ b/tests/test_tile_matrix_set.py @@ -1,5 +1,5 @@ -from rok4.TileMatrixSet import TileMatrixSet -from rok4.Exceptions import * +from rok4.tile_matrix_set import TileMatrixSet +from rok4.exceptions import * import pytest import os diff --git a/tests/test_Utils.py b/tests/test_utils.py similarity index 99% rename from tests/test_Utils.py rename to tests/test_utils.py index cdba571..a5d93b3 100644 --- a/tests/test_Utils.py +++ b/tests/test_utils.py @@ -1,5 +1,5 @@ -from rok4.Utils import * -from rok4.Exceptions import * +from rok4.utils import * +from rok4.exceptions import * import pytest import os diff --git a/tests/test_Vector.py b/tests/test_vector.py similarity index 97% rename from tests/test_Vector.py rename to tests/test_vector.py index 221f5e4..dbecde2 100644 --- a/tests/test_Vector.py +++ b/tests/test_vector.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -from rok4.Vector import * -from rok4.Exceptions import * -from rok4.Storage import disconnect_ceph_clients +from rok4.vector import * +from rok4.exceptions import * +from rok4.storage import disconnect_ceph_clients import pytest import os From c8a51a0158bfc6bb737d67af77b178c10168f9cd Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Tue, 18 Jul 2023 14:50:00 +0200 Subject: [PATCH 61/89] Move StorageType to enums module --- src/rok4/Pyramid.py | 21 ++++---- src/rok4/Storage.py | 92 +++++++++++++++++----------------- src/rok4/enums.py | 14 ++++++ tests/test_Pyramid.py | 2 +- tests/test_Storage.py | 114 +++++++++++++++++++++++++++++------------- 5 files changed, 149 insertions(+), 94 deletions(-) create mode 100644 src/rok4/enums.py diff --git a/src/rok4/Pyramid.py b/src/rok4/Pyramid.py index 7667244..fd01f1d 100644 --- a/src/rok4/Pyramid.py +++ b/src/rok4/Pyramid.py @@ -21,6 +21,7 @@ from rok4.TileMatrixSet import TileMatrixSet, TileMatrix from rok4.Storage import * from rok4.Utils import * +from rok4.enums import StorageType class PyramidType(Enum): @@ -549,11 +550,8 @@ def serializable(self) -> Dict: Returns: Dict: descriptor structured object description """ - - serialization = { - "tile_matrix_set": self.__tms.name, - "format": self.__format - } + + serialization = {"tile_matrix_set": self.__tms.name, "format": self.__format} serialization["levels"] = [] sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution, reverse=True) @@ -620,7 +618,7 @@ def storage_root(self) -> str: Returns: str: Pyramid's storage root """ - + return self.__storage["root"].split("@", 1)[ 0 ] # Suppression de l'éventuel hôte de spécification du cluster S3 @@ -670,7 +668,6 @@ def format(self) -> str: @property def tile_extension(self) -> str: - if self.__format in [ "TIFF_RAW_UINT8", "TIFF_LZW_UINT8", @@ -835,7 +832,7 @@ def get_level(self, level_id: str) -> "Level": Returns: The corresponding pyramid's level, None if not present """ - + return self.__levels.get(level_id, None) def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: @@ -1019,7 +1016,6 @@ def get_slab_path_from_infos( else: return slab_path - def get_tile_data_binary(self, level: str, column: int, row: int) -> str: """Get a pyramid's tile as binary string @@ -1182,7 +1178,6 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr level_object = self.get_level(level) if self.__format == "TIFF_JPG_UINT8" or self.__format == "TIFF_JPG90_UINT8": - try: img = Image.open(io.BytesIO(binary_tile)) except Exception as e: @@ -1379,6 +1374,8 @@ def size(self) -> int: Returns: int: size of the pyramid """ - if not hasattr(self,"_Pyramid__size") : - self.__size = size_path(get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name)) + if not hasattr(self, "_Pyramid__size"): + self.__size = size_path( + get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name) + ) return self.__size diff --git a/src/rok4/Storage.py b/src/rok4/Storage.py index bcbee52..406e431 100644 --- a/src/rok4/Storage.py +++ b/src/rok4/Storage.py @@ -45,17 +45,10 @@ gdal.UseExceptions() +from rok4.enums import StorageType from rok4.Exceptions import * -class StorageType(Enum): - FILE = "file://" - S3 = "s3://" - CEPH = "ceph://" - HTTP = "http://" - HTTPS = "https://" - - __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None @@ -75,7 +68,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s Returns: Tuple[Dict[str, Union['boto3.client',str]], str, str]: the S3 informations (client, host, key, secret) and the simple bucket name """ - + global __S3_CLIENTS, __S3_DEFAULT_CLIENT if not __S3_CLIENTS: @@ -134,7 +127,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s def disconnect_s3_clients() -> None: """Clean S3 clients""" - + global __S3_CLIENTS, __S3_DEFAULT_CLIENT __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None @@ -363,16 +356,15 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: raise StorageError("FILE", e) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - - if range is None : + if range is None: try: reponse = requests.get(f"{storage_type.value}{path}", stream=True) data = reponse.content - if reponse.status_code == 404 : + if reponse.status_code == 404: raise FileNotFoundError(f"{storage_type.value}{path}") except Exception as e: raise StorageError(storage_type.name, e) - else : + else: raise NotImplementedError else: @@ -471,7 +463,6 @@ def get_size(path: str) -> int: raise StorageError("FILE", e) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - try: # Le stream=True permet de ne télécharger que le header initialement reponse = requests.get(storage_type.value + path, stream=True).headers["content-length"] @@ -526,12 +517,11 @@ def exists(path: str) -> bool: return os.path.exists(path) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - try: response = requests.get(storage_type.value + path, stream=True) - if response.status_code == 200 : + if response.status_code == 200: return True - else : + else: return False except Exception as e: raise StorageError(storage_type.name, e) @@ -839,43 +829,52 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: f"CEPH and S3", f"Cannot copy CEPH object {from_path} to S3 object {to_path} : {e}" ) - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.FILE : - + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.FILE: try: - response = requests.get(from_type.value + from_path, stream = True) + response = requests.get(from_type.value + from_path, stream=True) with open(to_path, "wb") as f: - for chunk in response.iter_content(chunk_size=65536) : + for chunk in response.iter_content(chunk_size=65536): if chunk: f.write(chunk) except Exception as e: - raise StorageError(f"HTTP(S) and FILE", f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}") - - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.CEPH : + raise StorageError( + f"HTTP(S) and FILE", + f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}", + ) + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.CEPH: to_ioctx = __get_ceph_ioctx(to_tray) try: - response = requests.get(from_type.value + from_path, stream = True) + response = requests.get(from_type.value + from_path, stream=True) offset = 0 - for chunk in response.iter_content(chunk_size=65536) : + for chunk in response.iter_content(chunk_size=65536): if chunk: size = len(chunk) to_ioctx.write(to_base_name, chunk, offset) offset += size except Exception as e: - raise StorageError(f"HTTP(S) and CEPH", f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}") - - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.S3 : + raise StorageError( + f"HTTP(S) and CEPH", + f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}", + ) + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.S3: to_s3_client, to_bucket = __get_s3_client(to_tray) try: - response = requests.get(from_type.value + from_path, stream = True) - with tempfile.NamedTemporaryFile("w+b",delete=False) as f: + response = requests.get(from_type.value + from_path, stream=True) + with tempfile.NamedTemporaryFile("w+b", delete=False) as f: name_fich = f.name - for chunk in response.iter_content(chunk_size=65536) : + for chunk in response.iter_content(chunk_size=65536): if chunk: f.write(chunk) @@ -884,7 +883,10 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: os.remove(name_fich) except Exception as e: - raise StorageError(f"HTTP(S) and S3", f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}") + raise StorageError( + f"HTTP(S) and S3", + f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}", + ) else: raise StorageError( @@ -998,7 +1000,8 @@ def get_osgeo_path(path: str) -> str: else: raise NotImplementedError(f"Cannot get a GDAL/OGR compliant path from {path}") -def size_path(path: str) -> int : + +def size_path(path: str) -> int: """Return the size of the path given (or, for the CEPH, the sum of the size of each object of the .list) Args: @@ -1011,10 +1014,10 @@ def size_path(path: str) -> int : Returns: int: size of the path """ - storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) + storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) if storage_type == StorageType.FILE: - try : + try: total = 0 with os.scandir(unprefixed_path) as it: for entry in it: @@ -1029,24 +1032,23 @@ def size_path(path: str) -> int : elif storage_type == StorageType.S3: s3_client, bucket_name = __get_s3_client(tray_name) - try : - paginator = s3_client["client"].get_paginator('list_objects_v2') + try: + paginator = s3_client["client"].get_paginator("list_objects_v2") pages = paginator.paginate( Bucket=bucket_name, - Prefix=base_name+"/", + Prefix=base_name + "/", PaginationConfig={ - 'PageSize': 10000, - } + "PageSize": 10000, + }, ) total = 0 for page in pages: - for key in page['Contents']: - total += key['Size'] + for key in page["Contents"]: + total += key["Size"] except Exception as e: raise StorageError("S3", e) - elif storage_type == StorageType.CEPH: raise NotImplementedError else: diff --git a/src/rok4/enums.py b/src/rok4/enums.py new file mode 100644 index 0000000..2fca210 --- /dev/null +++ b/src/rok4/enums.py @@ -0,0 +1,14 @@ +#! python3 # noqa: E265 + +# standard lib +from enum import Enum + + +class StorageType(Enum): + """Matrice de correspondance entre type de stockage et protocole.""" + + CEPH = "ceph://" + FILE = "file://" + HTTP = "http://" + HTTPS = "https://" + S3 = "s3://" diff --git a/tests/test_Pyramid.py b/tests/test_Pyramid.py index 6eebfd7..d72ce2f 100644 --- a/tests/test_Pyramid.py +++ b/tests/test_Pyramid.py @@ -1,6 +1,6 @@ from rok4.Pyramid import * from rok4.TileMatrixSet import TileMatrixSet -from rok4.Storage import StorageType +from rok4.enums import StorageType from rok4.Utils import * from rok4.Exceptions import * diff --git a/tests/test_Storage.py b/tests/test_Storage.py index 9779954..540632c 100644 --- a/tests/test_Storage.py +++ b/tests/test_Storage.py @@ -1,5 +1,6 @@ from rok4.Storage import * from rok4.Exceptions import * +from rok4.enums import StorageType import pytest import os @@ -21,6 +22,7 @@ def test_hash_file_ok(mock_file): except Exception as exc: assert False, f"FILE md5 sum raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) def test_get_infos_from_path(): assert (StorageType.S3, "toto/titi", "toto", "titi") == get_infos_from_path("s3://toto/titi") @@ -103,6 +105,7 @@ def test_file_read_ok(mock_file): except Exception as exc: assert False, f"FILE read raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -117,6 +120,7 @@ def test_s3_read_nok(mocked_s3_client): with pytest.raises(StorageError): data = get_data_str("s3://bucket/path/to/object") + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -159,8 +163,9 @@ def test_ceph_read_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH read raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch("requests.get", side_effect={"status_code":404}) +@mock.patch("requests.get", side_effect={"status_code": 404}) def test_http_read_error(mock_http): with pytest.raises(StorageError): requests_instance = MagicMock() @@ -171,28 +176,31 @@ def test_http_read_error(mock_http): mock_http.assert_called_with("http://path/to/file.ext", stream=True) + @mock.patch.dict(os.environ, {}, clear=True) def test_http_read_range_error(): with pytest.raises(NotImplementedError): - data = get_data_binary("http://path/to/file.ext", (0,100)) + data = get_data_binary("http://path/to/file.ext", (0, 100)) + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("requests.get") def test_http_read_ok(mock_http): - try : + try: requests_instance = MagicMock() - requests_instance.content = b'data' + requests_instance.content = b"data" mock_http.return_value = requests_instance data = get_data_str("http://path/to/file.ext") mock_http.assert_called_with("http://path/to/file.ext", stream=True) - assert data == 'data' + assert data == "data" except Exception as exc: assert False, f"HTTP read raises an exception: {exc}" ############ put_data_str + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -225,6 +233,7 @@ def test_s3_write_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 write raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -302,6 +311,7 @@ def test_copy_s3_file_nok(mock_hash_file, mock_makedirs, mocked_s3_client): copy("s3://bucket/source.ext", "file:///path/to/destination.ext", "toto") mock_makedirs.assert_called_once_with("/path/to", exist_ok=True) + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -320,6 +330,7 @@ def test_copy_file_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"FILE -> S3 copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -338,6 +349,7 @@ def test_copy_s3_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 -> S3 copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -373,6 +385,7 @@ def test_copy_s3_s3_intercluster_nok(mocked_s3_client): with pytest.raises(StorageError): copy("s3://bucket@a/source.ext", "s3://bucket@c/destination.ext", "toto") + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -399,6 +412,7 @@ def test_copy_ceph_file_ok(mock_file, mock_makedirs, mocked_rados_client): except Exception as exc: assert False, f"CEPH -> FILE copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -423,6 +437,7 @@ def test_copy_file_ceph_ok(mock_file, mocked_rados_client): except Exception as exc: assert False, f"FILE -> CEPH copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -487,14 +502,14 @@ def test_copy_ceph_s3_ok(mock_file, mocked_s3_client, mocked_rados_client): except Exception as exc: assert False, f"CEPH -> S3 copy raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') -@patch('builtins.open', new_callable=mock_open) +@mock.patch("requests.get") +@patch("builtins.open", new_callable=mock_open) def test_copy_http_file_ok(mock_open, mock_requests): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance copy("http://path/to/source.ext", "file:///path/to/destination.ext") @@ -503,17 +518,20 @@ def test_copy_http_file_ok(mock_open, mock_requests): except Exception as exc: assert False, f"HTTP -> FILE copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -@mock.patch('rok4.Storage.rados.Rados') -@mock.patch('requests.get') + +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +@mock.patch("rok4.Storage.rados.Rados") +@mock.patch("requests.get") def test_copy_http_ceph_ok(mock_requests, mocked_rados_client): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance - disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.write.return_value = None @@ -526,16 +544,20 @@ def test_copy_http_ceph_ok(mock_requests, mocked_rados_client): except Exception as exc: assert False, f"HTTP -> CEPH copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') -@mock.patch('requests.get') -@patch('tempfile.NamedTemporaryFile', new_callable=mock_open) -@mock.patch('os.remove') + +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") +@mock.patch("requests.get") +@patch("tempfile.NamedTemporaryFile", new_callable=mock_open) +@mock.patch("os.remove") def test_copy_http_s3_ok(mock_remove, mock_tempfile, mock_requests, mocked_s3_client): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance disconnect_s3_clients() @@ -546,7 +568,7 @@ def test_copy_http_s3_ok(mock_remove, mock_tempfile, mock_requests, mocked_s3_cl copy("http://path/to/source.ext", "s3://bucket/destination.ext") mock_requests.assert_called_once_with("http://path/to/source.ext", stream=True) - mock_tempfile.assert_called_once_with("w+b",delete=False) + mock_tempfile.assert_called_once_with("w+b", delete=False) except Exception as exc: assert False, f"HTTP -> CEPH copy raises an exception: {exc}" @@ -563,6 +585,7 @@ def test_link_hard_nok(): with pytest.raises(StorageError): link("ceph://pool1/source.ext", "ceph://pool2/destination.ext", True) + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.symlink", return_value=None) def test_link_file_ok(mock_link): @@ -582,6 +605,7 @@ def test_hlink_file_ok(mock_link): except Exception as exc: assert False, f"FILE hard link raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -619,6 +643,7 @@ def test_link_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 link raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -637,6 +662,7 @@ def test_link_s3_nok(mocked_s3_client): ############ get_size + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.stat") def test_size_file_ok(mock_stat): @@ -647,6 +673,7 @@ def test_size_file_ok(mock_stat): except Exception as exc: assert False, f"FILE size raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -667,6 +694,7 @@ def test_size_ceph_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH size raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -685,12 +713,12 @@ def test_size_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 size raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') +@mock.patch("requests.get") def test_size_http_ok(mock_requests): - http_instance = MagicMock() - http_instance.headers = {"content-length":12} + http_instance.headers = {"content-length": 12} mock_requests.return_value = http_instance try: @@ -717,6 +745,7 @@ def test_exists_file_ok(mock_exists): except Exception as exc: assert False, f"FILE not exists raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -768,10 +797,10 @@ def test_exists_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"CEPH not exists raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') +@mock.patch("requests.get") def test_exists_http_ok(mock_requests): - http_instance = MagicMock() http_instance.status_code = 200 mock_requests.return_value = http_instance @@ -792,6 +821,7 @@ def test_exists_http_ok(mock_requests): ############ remove + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.remove") def test_remove_file_ok(mock_remove): @@ -807,6 +837,7 @@ def test_remove_file_ok(mock_remove): except Exception as exc: assert False, f"FILE deletion (not found) raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -832,6 +863,7 @@ def test_remove_ceph_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH deletion (not found) raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -861,6 +893,7 @@ def test_get_osgeo_path_file_ok(): except Exception as exc: assert False, f"FILE osgeo path raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -881,6 +914,7 @@ def test_get_osgeo_path_nok(): with pytest.raises(NotImplementedError): get_osgeo_path("ceph://pool/data.ext") + ############ size_path def test_size_path_file_ok(): try: @@ -889,22 +923,31 @@ def test_size_path_file_ok(): except Exception as exc: assert False, f"FILE size of the path raises an exception: {exc}" + def test_size_file_nok(): - with pytest.raises(StorageError) : + with pytest.raises(StorageError): size = size_path("file://tests/fixtures/TIFF_PBF_M") -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -def test_size_path_ceph_nok(): +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +def test_size_path_ceph_nok(): with pytest.raises(NotImplementedError): size = size_path("ceph://pool/path") -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') -def test_size_path_s3_ok(mocked_s3_client): +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") +def test_size_path_s3_ok(mocked_s3_client): disconnect_s3_clients() - pages = [{"Contents" : [{"Size" : 10},{"Size" : 20}]}, {"Contents" : [{"Size" : 50}]}] + pages = [{"Contents": [{"Size": 10}, {"Size": 20}]}, {"Contents": [{"Size": 50}]}] paginator = MagicMock() paginator.paginate.return_value = pages client = MagicMock() @@ -916,4 +959,3 @@ def test_size_path_s3_ok(mocked_s3_client): assert size == 80 except Exception as exc: assert False, f"S3 size of the path raises an exception: {exc}" - From 175152bf6cf1f17fc71b0a8ba14d2d7895829c31 Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Tue, 18 Jul 2023 14:55:22 +0200 Subject: [PATCH 62/89] Move Pyramidtype to enums module --- src/rok4/Layer.py | 3 ++- src/rok4/Pyramid.py | 9 +-------- src/rok4/enums.py | 7 +++++++ tests/test_Layer.py | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/rok4/Layer.py b/src/rok4/Layer.py index 316a9ca..e57e3ab 100644 --- a/src/rok4/Layer.py +++ b/src/rok4/Layer.py @@ -12,10 +12,11 @@ import re from rok4.Exceptions import * -from rok4.Pyramid import Pyramid, PyramidType +from rok4.Pyramid import Pyramid from rok4.TileMatrixSet import TileMatrixSet from rok4.Storage import * from rok4.Utils import * +from rok4.enums import PyramidType class Layer: diff --git a/src/rok4/Pyramid.py b/src/rok4/Pyramid.py index fd01f1d..0d176d4 100644 --- a/src/rok4/Pyramid.py +++ b/src/rok4/Pyramid.py @@ -21,14 +21,7 @@ from rok4.TileMatrixSet import TileMatrixSet, TileMatrix from rok4.Storage import * from rok4.Utils import * -from rok4.enums import StorageType - - -class PyramidType(Enum): - """Pyramid's data type""" - - RASTER = "RASTER" - VECTOR = "VECTOR" +from rok4.enums import PyramidType, StorageType class SlabType(Enum): diff --git a/src/rok4/enums.py b/src/rok4/enums.py index 2fca210..9a0a486 100644 --- a/src/rok4/enums.py +++ b/src/rok4/enums.py @@ -4,6 +4,13 @@ from enum import Enum +class PyramidType(Enum): + """Pyramid's data type""" + + RASTER = "RASTER" + VECTOR = "VECTOR" + + class StorageType(Enum): """Matrice de correspondance entre type de stockage et protocole.""" diff --git a/tests/test_Layer.py b/tests/test_Layer.py index 0a60d5c..7a3788e 100644 --- a/tests/test_Layer.py +++ b/tests/test_Layer.py @@ -1,5 +1,5 @@ from rok4.Layer import Layer -from rok4.Pyramid import PyramidType +from rok4.enums import PyramidType from rok4.Exceptions import * import pytest From 5672c878a517804a33eff3ad7abc9d924581219f Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Tue, 18 Jul 2023 15:02:21 +0200 Subject: [PATCH 63/89] Move SlabType to enums module --- src/rok4/Pyramid.py | 9 +-------- src/rok4/enums.py | 7 +++++++ tests/test_Pyramid.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/rok4/Pyramid.py b/src/rok4/Pyramid.py index 0d176d4..0d7e382 100644 --- a/src/rok4/Pyramid.py +++ b/src/rok4/Pyramid.py @@ -21,14 +21,7 @@ from rok4.TileMatrixSet import TileMatrixSet, TileMatrix from rok4.Storage import * from rok4.Utils import * -from rok4.enums import PyramidType, StorageType - - -class SlabType(Enum): - """Slab's type""" - - DATA = "DATA" # Slab of data, raster or vector - MASK = "MASK" # Slab of mask, only for raster pyramid, image with one band : 0 is nodata, other values are data +from rok4.enums import PyramidType, SlabType, StorageType ROK4_IMAGE_HEADER_SIZE = 2048 diff --git a/src/rok4/enums.py b/src/rok4/enums.py index 9a0a486..76300ce 100644 --- a/src/rok4/enums.py +++ b/src/rok4/enums.py @@ -11,6 +11,13 @@ class PyramidType(Enum): VECTOR = "VECTOR" +class SlabType(Enum): + """Slab's type""" + + DATA = "DATA" # Slab of data, raster or vector + MASK = "MASK" # Slab of mask, only for raster pyramid, image with one band : 0 is nodata, other values are data + + class StorageType(Enum): """Matrice de correspondance entre type de stockage et protocole.""" diff --git a/tests/test_Pyramid.py b/tests/test_Pyramid.py index d72ce2f..60576f3 100644 --- a/tests/test_Pyramid.py +++ b/tests/test_Pyramid.py @@ -1,6 +1,6 @@ from rok4.Pyramid import * from rok4.TileMatrixSet import TileMatrixSet -from rok4.enums import StorageType +from rok4.enums import SlabType, StorageType from rok4.Utils import * from rok4.Exceptions import * From 726a18dfb4bacbfb919e9c7dbbc182e13b66afff Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Tue, 18 Jul 2023 15:12:11 +0200 Subject: [PATCH 64/89] Rm unused ColorFormat --- src/rok4/Utils.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/rok4/Utils.py b/src/rok4/Utils.py index da15aef..627686c 100644 --- a/src/rok4/Utils.py +++ b/src/rok4/Utils.py @@ -6,25 +6,12 @@ from typing import Dict, List, Tuple, Union from osgeo import ogr, osr, gdal -from enum import Enum ogr.UseExceptions() osr.UseExceptions() gdal.UseExceptions() -class ColorFormat(Enum): - """A color format enumeration. - Except from "BIT", the member's name matches - a common variable format name. The member's value is - the allocated bit size associated to this format. - """ - - BIT = 1 - UINT8 = 8 - FLOAT32 = 32 - - __SR_BOOK = {} From 48fb66a586f56e6a0edb5127cfd503797d53965e Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Tue, 18 Jul 2023 12:50:35 +0200 Subject: [PATCH 65/89] Rename files to comply with Python convention PEP8 --- README.md | 4 +- README.pypi.md | 2 +- src/rok4/Pyramid.py | 79 +- src/rok4/enums.py | 13 +- src/rok4/{Exceptions.py => exceptions.py} | 0 src/rok4/{Layer.py => layer.py} | 22 +- src/rok4/pyramid.py | 1373 +++++++++++++++++ src/rok4/{Raster.py => raster.py} | 13 +- src/rok4/{Storage.py => storage.py} | 112 +- .../{TileMatrixSet.py => tile_matrix_set.py} | 10 +- src/rok4/{Utils.py => utils.py} | 1 + src/rok4/{Vector.py => vector.py} | 11 +- tests/{test_Layer.py => test_layer.py} | 28 +- tests/{test_Pyramid.py => test_pyramid.py} | 53 +- tests/{test_Raster.py => test_raster.py} | 78 +- tests/{test_Storage.py => test_storage.py} | 187 +-- ...leMatrixSet.py => test_tile_matrix_set.py} | 36 +- tests/{test_Utils.py => test_utils.py} | 59 +- tests/{test_Vector.py => test_vector.py} | 21 +- 19 files changed, 1724 insertions(+), 378 deletions(-) rename src/rok4/{Exceptions.py => exceptions.py} (100%) rename src/rok4/{Layer.py => layer.py} (96%) create mode 100644 src/rok4/pyramid.py rename src/rok4/{Raster.py => raster.py} (97%) rename src/rok4/{Storage.py => storage.py} (94%) rename src/rok4/{TileMatrixSet.py => tile_matrix_set.py} (98%) rename src/rok4/{Utils.py => utils.py} (99%) rename src/rok4/{Vector.py => vector.py} (98%) rename tests/{test_Layer.py => test_layer.py} (91%) rename tests/{test_Pyramid.py => test_pyramid.py} (92%) rename tests/{test_Raster.py => test_raster.py} (91%) rename tests/{test_Storage.py => test_storage.py} (90%) rename tests/{test_TileMatrixSet.py => test_tile_matrix_set.py} (93%) rename tests/{test_Utils.py => test_utils.py} (90%) rename tests/{test_Vector.py => test_vector.py} (92%) diff --git a/README.md b/README.md index 29bf082..2117b06 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ L'environnement d'exécution doit avoir accès aux librairies système. Dans le ## Utiliser la librairie ```python -from rok4.TileMatrixSet import TileMatrixSet +from rok4.tile_matrix_set import TileMatrixSet try: tms = TileMatrixSet("file:///path/to/tms.json") @@ -34,7 +34,7 @@ Plus d'exemple dans la documentation développeur. * Installer les dépendances de développement : ```sh - python3 -m pip install -e[dev] + python3 -m pip install -e .[dev] ``` * Consulter les [directives de contribution](./CONTRIBUTING.md) diff --git a/README.pypi.md b/README.pypi.md index fc3417b..d5e9d1a 100644 --- a/README.pypi.md +++ b/README.pypi.md @@ -16,7 +16,7 @@ The `rok4` package is available on : ## Usage ```python -from rok4.TileMatrixSet import TileMatrixSet +from rok4.tile_matrix_set import TileMatrixSet try: tms = TileMatrixSet("file:///path/to/tms.json") diff --git a/src/rok4/Pyramid.py b/src/rok4/Pyramid.py index f6c670f..19fa9c7 100644 --- a/src/rok4/Pyramid.py +++ b/src/rok4/Pyramid.py @@ -6,23 +6,36 @@ - `Level` - Level of a pyramid """ -import io +from typing import Dict, List, Tuple, Union, Iterator import json +from json.decoder import JSONDecodeError import os import re +import numpy import zlib -from json.decoder import JSONDecodeError -from typing import Dict, Iterator, List, Tuple, Union - +import io import mapbox_vector_tile -import numpy from PIL import Image from rok4.Exceptions import * -from rok4.Storage import * -from rok4.TileMatrixSet import TileMatrix, TileMatrixSet -from rok4.Utils import * -from rok4.enums import PyramidType, SlabType, StorageType +from rok4.tile_matrix_set import TileMatrixSet, TileMatrix +from rok4.storage import * +from rok4.utils import * +from rok4.enums import StorageType + + +class PyramidType(Enum): + """Pyramid's data type""" + + RASTER = "RASTER" + VECTOR = "VECTOR" + + +class SlabType(Enum): + """Slab's type""" + + DATA = "DATA" # Slab of data, raster or vector + MASK = "MASK" # Slab of mask, only for raster pyramid, image with one band : 0 is nodata, other values are data ROK4_IMAGE_HEADER_SIZE = 2048 @@ -360,10 +373,10 @@ class Pyramid: __name (str): pyramid's name __descriptor (str): pyramid's descriptor path __list (str): pyramid's list path - __tms (rok4.TileMatrixSet.TileMatrixSet): Used grid + __tms (rok4.tile_matrix_set.TileMatrixSet): Used grid __levels (Dict[str, Level]): Pyramid's levels __format (str): Data format - __storage (Dict[str, Union[rok4.Storage.StorageType,str,int]]): Pyramid's storage informations (type, root and depth if FILE storage) + __storage (Dict[str, Union[rok4.storage.StorageType,str,int]]): Pyramid's storage informations (type, root and depth if FILE storage) __raster_specifications (Dict): If raster pyramid, raster specifications __content (Dict): Loading status (loaded) and list content (cache). @@ -400,7 +413,7 @@ def from_descriptor(cls, descriptor: str) -> "Pyramid": S3 stored descriptor - from rok4.Pyramid import Pyramid + from rok4.pyramid import Pyramid try: pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") @@ -537,8 +550,11 @@ def serializable(self) -> Dict: Returns: Dict: descriptor structured object description """ - - serialization = {"tile_matrix_set": self.__tms.name, "format": self.__format} + + serialization = { + "tile_matrix_set": self.__tms.name, + "format": self.__format + } serialization["levels"] = [] sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution, reverse=True) @@ -605,7 +621,7 @@ def storage_root(self) -> str: Returns: str: Pyramid's storage root """ - + return self.__storage["root"].split("@", 1)[ 0 ] # Suppression de l'éventuel hôte de spécification du cluster S3 @@ -655,6 +671,7 @@ def format(self) -> str: @property def tile_extension(self) -> str: + if self.__format in [ "TIFF_RAW_UINT8", "TIFF_LZW_UINT8", @@ -729,7 +746,7 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict] S3 stored descriptor - from rok4.Pyramid import Pyramid + from rok4.pyramid import Pyramid try: pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") @@ -769,7 +786,7 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict] roots = {} s3_cluster = self.storage_s3_cluster - with open(list_file) as listin: + with open(list_file, "r") as listin: # Lecture des racines for line in listin: line = line.rstrip() @@ -819,7 +836,7 @@ def get_level(self, level_id: str) -> "Level": Returns: The corresponding pyramid's level, None if not present """ - + return self.__levels.get(level_id, None) def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: @@ -836,7 +853,7 @@ def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: All levels - from rok4.Pyramid import Pyramid + from rok4.pyramid import Pyramid try: pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") @@ -847,7 +864,7 @@ def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: From pyramid's bottom to provided top (level 5) - from rok4.Pyramid import Pyramid + from rok4.pyramid import Pyramid try: pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") @@ -918,7 +935,7 @@ def get_infos_from_slab_path(self, path: str) -> Tuple[SlabType, str, int, int]: FILE stored pyramid - from rok4.Pyramid import Pyramid + from rok4.pyramid import Pyramid try: pyramid = Pyramid.from_descriptor("/path/to/descriptor.json") @@ -929,7 +946,7 @@ def get_infos_from_slab_path(self, path: str) -> Tuple[SlabType, str, int, int]: S3 stored pyramid - from rok4.Pyramid import Pyramid + from rok4.pyramid import Pyramid try: pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/pyramid.json") @@ -1003,6 +1020,7 @@ def get_slab_path_from_infos( else: return slab_path + def get_tile_data_binary(self, level: str, column: int, row: int) -> str: """Get a pyramid's tile as binary string @@ -1023,7 +1041,7 @@ def get_tile_data_binary(self, level: str, column: int, row: int) -> str: FILE stored raster pyramid, to extract a tile containing a point and save it as independent image - from rok4.Pyramid import Pyramid + from rok4.pyramid import Pyramid try: pyramid = Pyramid.from_descriptor("/data/pyramids/SCAN1000.json") @@ -1135,7 +1153,7 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr FILE stored DTM (raster) pyramid, to get the altitude value at a point in the best level - from rok4.Pyramid import Pyramid + from rok4.pyramid import Pyramid try: pyramid = Pyramid.from_descriptor("/data/pyramids/RGEALTI.json") @@ -1165,6 +1183,7 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr level_object = self.get_level(level) if self.__format == "TIFF_JPG_UINT8" or self.__format == "TIFF_JPG90_UINT8": + try: img = Image.open(io.BytesIO(binary_tile)) except Exception as e: @@ -1246,7 +1265,7 @@ def get_tile_data_vector(self, level: str, column: int, row: int) -> Dict: S3 stored vector pyramid, to print a tile as GeoJSON - from rok4.Pyramid import Pyramid + from rok4.pyramid import Pyramid import json try: @@ -1307,7 +1326,7 @@ def get_tile_indices( FILE stored DTM (raster) pyramid, to get the altitude value at a point in the best level - from rok4.Pyramid import Pyramid + from rok4.pyramid import Pyramid try: pyramid = Pyramid.from_descriptor("/data/pyramids/RGEALTI.json") @@ -1349,7 +1368,7 @@ def size(self) -> int: Examples: - from rok4.Pyramid import Pyramid + from rok4.pyramid import Pyramid try: pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") @@ -1361,8 +1380,6 @@ def size(self) -> int: Returns: int: size of the pyramid """ - if not hasattr(self, "_Pyramid__size"): - self.__size = size_path( - get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name) - ) + if not hasattr(self,"_Pyramid__size") : + self.__size = size_path(get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name)) return self.__size diff --git a/src/rok4/enums.py b/src/rok4/enums.py index 76300ce..861464f 100644 --- a/src/rok4/enums.py +++ b/src/rok4/enums.py @@ -19,10 +19,21 @@ class SlabType(Enum): class StorageType(Enum): - """Matrice de correspondance entre type de stockage et protocole.""" + """Storage type and path's protocol""" CEPH = "ceph://" FILE = "file://" HTTP = "http://" HTTPS = "https://" S3 = "s3://" + +class ColorFormat(Enum): + """A color format enumeration. + Except from "BIT", the member's name matches + a common variable format name. The member's value is + the allocated bit size associated to this format. + """ + + BIT = 1 + UINT8 = 8 + FLOAT32 = 32 \ No newline at end of file diff --git a/src/rok4/Exceptions.py b/src/rok4/exceptions.py similarity index 100% rename from src/rok4/Exceptions.py rename to src/rok4/exceptions.py diff --git a/src/rok4/Layer.py b/src/rok4/layer.py similarity index 96% rename from src/rok4/Layer.py rename to src/rok4/layer.py index b13e55a..ba2f211 100644 --- a/src/rok4/Layer.py +++ b/src/rok4/layer.py @@ -5,17 +5,17 @@ - `Layer` - Descriptor to broadcast pyramids' data """ +from typing import Dict, List, Tuple, Union import json +from json.decoder import JSONDecodeError import os import re -from json.decoder import JSONDecodeError -from typing import Dict, List, Tuple, Union -from rok4.Exceptions import * -from rok4.Pyramid import Pyramid -from rok4.TileMatrixSet import TileMatrixSet -from rok4.Storage import * -from rok4.Utils import * +from rok4.exceptions import * +from rok4.pyramid import Pyramid, PyramidType +from rok4.tile_matrix_set import TileMatrixSet +from rok4.storage import * +from rok4.utils import * from rok4.enums import PyramidType @@ -24,12 +24,12 @@ class Layer: Attributes: __name (str): layer's technical name - __pyramids (Dict[str, Union[rok4.Pyramid.Pyramid,str,str]]): used pyramids + __pyramids (Dict[str, Union[rok4.pyramid.Pyramid,str,str]]): used pyramids __format (str): pyramid's list path - __tms (rok4.TileMatrixSet.TileMatrixSet): Used grid + __tms (rok4.tile_matrix_set.TileMatrixSet): Used grid __keywords (List[str]): Keywords - __levels (Dict[str, rok4.Pyramid.Level]): Used pyramids' levels - __best_level (rok4.Pyramid.Level): Used pyramids best level + __levels (Dict[str, rok4.pyramid.Level]): Used pyramids' levels + __best_level (rok4.pyramid.Level): Used pyramids best level __resampling (str): Interpolation to use fot resampling __bbox (Tuple[float, float, float, float]): data bounding box, TMS coordinates system __geobbox (Tuple[float, float, float, float]): data bounding box, EPSG:4326 diff --git a/src/rok4/pyramid.py b/src/rok4/pyramid.py new file mode 100644 index 0000000..6c83f0e --- /dev/null +++ b/src/rok4/pyramid.py @@ -0,0 +1,1373 @@ +"""Provide classes to use pyramid's data. + +The module contains the following classes: + +- `Pyramid` - Data container +- `Level` - Level of a pyramid +""" + +from typing import Dict, List, Tuple, Union, Iterator +import json +from json.decoder import JSONDecodeError +import os +import re +import numpy +import zlib +import io +import mapbox_vector_tile +from PIL import Image + +from rok4.exceptions import * +from rok4.tile_matrix_set import TileMatrixSet, TileMatrix +from rok4.storage import * +from rok4.utils import * + + +from rok4.enums import PyramidType, SlabType, StorageType + + +ROK4_IMAGE_HEADER_SIZE = 2048 +"""Slab's header size, 2048 bytes""" + + +def b36_number_encode(number: int) -> str: + """Convert base-10 number to base-36 + + Used alphabet is '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + + Args: + number (int): base-10 number + + Returns: + str: base-36 number + """ + + alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + + base36 = "" + + if 0 <= number < len(alphabet): + return alphabet[number] + + while number != 0: + number, i = divmod(number, len(alphabet)) + base36 = alphabet[i] + base36 + + return base36 + + +def b36_number_decode(number: str) -> int: + """Convert base-36 number to base-10 + + Args: + number (str): base-36 number + + Returns: + int: base-10 number + """ + return int(number, 36) + + +def b36_path_decode(path: str) -> Tuple[int, int]: + """Get slab's column and row from a base-36 based path + + Args: + path (str): slab's path + + Returns: + Tuple[int, int]: slab's column and row + """ + + path = path.replace("/", "") + path = re.sub(r"(\.TIFF?)", "", path.upper()) + + b36_column = "" + b36_row = "" + + while len(path) > 0: + b36_column += path[0] + b36_row += path[1] + path = path[2:] + + return b36_number_decode(b36_column), b36_number_decode(b36_row) + + +def b36_path_encode(column: int, row: int, slashs: int) -> str: + """Convert slab indices to base-36 based path, with .tif extension + + Args: + column (int): slab's column + row (int): slab's row + slashs (int): slashs' number (to split path) + + Returns: + str: base-36 based path + """ + + b36_column = b36_number_encode(column) + b36_row = b36_number_encode(row) + + max_len = max(slashs + 1, len(b36_column), len(b36_row)) + + b36_column = b36_column.rjust(max_len, "0") + b36_row = b36_row.rjust(max_len, "0") + + b36_path = "" + + while len(b36_column) > 0: + b36_path = b36_row[-1] + b36_path + b36_path = b36_column[-1] + b36_path + + b36_column = b36_column[:-1] + b36_row = b36_row[:-1] + + if slashs > 0: + b36_path = "/" + b36_path + slashs -= 1 + + return f"{b36_path}.tif" + + +class Level: + """A pyramid's level, raster or vector + + Attributes: + __id (str): level's identifier. have to exist in the pyramid's used TMS + __tile_limits (Dict[str, int]): minimum and maximum tiles' columns and rows of pyramid's content + __slab_size (Tuple[int, int]): number of tile in a slab, widthwise and heightwise + __tables (List[Dict]): for a VECTOR pyramid, description of vector content, tables and attributes + """ + + @classmethod + def from_descriptor(cls, data: Dict, pyramid: "Pyramid") -> "Level": + """Create a pyramid's level from the pyramid's descriptor levels element + + Args: + data (Dict): level's information from the pyramid's descriptor + pyramid (Pyramid): pyramid containing the level to create + + Raises: + Exception: different storage or masks presence between the level and the pyramid + MissingAttributeError: Attribute is missing in the content + + Returns: + Pyramid: a Level instance + """ + level = cls() + + level.__pyramid = pyramid + + # Attributs communs + try: + level.__id = data["id"] + level.__tile_limits = data["tile_limits"] + level.__slab_size = ( + data["tiles_per_width"], + data["tiles_per_height"], + ) + + # Informations sur le stockage : on les valide et stocke dans la pyramide + if pyramid.storage_type.name != data["storage"]["type"]: + raise Exception( + f"Pyramid {pyramid.descriptor} owns levels using different storage types ({ data['storage']['type'] }) than its one ({pyramid.storage_type.name})" + ) + + if pyramid.storage_type == StorageType.FILE: + pyramid.storage_depth = data["storage"]["path_depth"] + + if "mask_directory" in data["storage"] or "mask_prefix" in data["storage"]: + if not pyramid.own_masks: + raise Exception( + f"Pyramid {pyramid.__descriptor} does not define a mask format but level {level.__id} define mask storage informations" + ) + else: + if pyramid.own_masks: + raise Exception( + f"Pyramid {pyramid.__descriptor} define a mask format but level {level.__id} does not define mask storage informations" + ) + + except KeyError as e: + raise MissingAttributeError(pyramid.descriptor, f"levels[].{e}") + + # Attributs dans le cas d'un niveau vecteur + if level.__pyramid.type == PyramidType.VECTOR: + try: + level.__tables = data["tables"] + + except KeyError as e: + raise MissingAttributeError(pyramid.descriptor, f"levels[].{e}") + + return level + + @classmethod + def from_other(cls, other: "Level", pyramid: "Pyramid") -> "Level": + """Create a pyramid's level from another one + + Args: + other (Level): level to clone + pyramid (Pyramid): new pyramid containing the new level + + Raises: + Exception: different storage or masks presence between the level and the pyramid + MissingAttributeError: Attribute is missing in the content + + Returns: + Pyramid: a Level instance + """ + + level = cls() + + # Attributs communs + level.__id = other.__id + level.__pyramid = pyramid + level.__tile_limits = other.__tile_limits + level.__slab_size = other.__slab_size + + # Attributs dans le cas d'un niveau vecteur + if level.__pyramid.type == PyramidType.VECTOR: + level.__tables = other.__tables + + return level + + def __str__(self) -> str: + return f"{self.__pyramid.type.name} pyramid's level '{self.__id}' ({self.__pyramid.storage_type.name} storage)" + + @property + def serializable(self) -> Dict: + """Get the dict version of the pyramid object, pyramid's descriptor compliant + + Returns: + Dict: pyramid's descriptor structured object description + """ + serialization = { + "id": self.__id, + "tiles_per_width": self.__slab_size[0], + "tiles_per_height": self.__slab_size[1], + "tile_limits": self.__tile_limits, + } + + if self.__pyramid.type == PyramidType.VECTOR: + serialization["tables"] = self.__tables + + if self.__pyramid.storage_type == StorageType.FILE: + serialization["storage"] = { + "type": "FILE", + "image_directory": f"{self.__pyramid.name}/DATA/{self.__id}", + "path_depth": self.__pyramid.storage_depth, + } + if self.__pyramid.own_masks: + serialization["storage"][ + "mask_directory" + ] = f"{self.__pyramid.name}/MASK/{self.__id}" + + elif self.__pyramid.storage_type == StorageType.CEPH: + serialization["storage"] = { + "type": "CEPH", + "image_prefix": f"{self.__pyramid.name}/DATA_{self.__id}", + "pool_name": self.__pyramid.storage_root, + } + if self.__pyramid.own_masks: + serialization["storage"]["mask_prefix"] = f"{self.__pyramid.name}/MASK_{self.__id}" + + elif self.__pyramid.storage_type == StorageType.S3: + serialization["storage"] = { + "type": "S3", + "image_prefix": f"{self.__pyramid.name}/DATA_{self.__id}", + "bucket_name": self.__pyramid.storage_root, + } + if self.__pyramid.own_masks: + serialization["storage"]["mask_prefix"] = f"{self.__pyramid.name}/MASK_{self.__id}" + + return serialization + + @property + def id(self) -> str: + return self.__id + + @property + def bbox(self) -> Tuple[float, float, float, float]: + """Return level extent, based on tile limits + + Returns: + Tuple[float, float, float, float]: level terrain extent (xmin, ymin, xmax, ymax) + """ + + min_bbox = self.__pyramid.tms.get_level(self.__id).tile_to_bbox( + self.__tile_limits["min_col"], self.__tile_limits["max_row"] + ) + max_bbox = self.__pyramid.tms.get_level(self.__id).tile_to_bbox( + self.__tile_limits["max_col"], self.__tile_limits["min_row"] + ) + + return (min_bbox[0], min_bbox[1], max_bbox[2], max_bbox[3]) + + @property + def resolution(self) -> str: + return self.__pyramid.tms.get_level(self.__id).resolution + + @property + def tile_matrix(self) -> TileMatrix: + return self.__pyramid.tms.get_level(self.__id) + + @property + def slab_width(self) -> int: + return self.__slab_size[0] + + @property + def slab_height(self) -> int: + return self.__slab_size[1] + + def is_in_limits(self, column: int, row: int) -> bool: + """Is the tile indices in limits ? + + Args: + column (int): tile's column + row (int): tile's row + + Returns: + bool: True if tiles' limits contain the provided tile's indices + """ + return ( + self.__tile_limits["min_row"] <= row + and self.__tile_limits["max_row"] >= row + and self.__tile_limits["min_col"] <= column + and self.__tile_limits["max_col"] >= column + ) + + def set_limits_from_bbox(self, bbox: Tuple[float, float, float, float]) -> None: + """Set tile limits, based on provided bounding box + + Args: + bbox (Tuple[float, float, float, float]): terrain extent (xmin, ymin, xmax, ymax), in TMS coordinates system + + """ + + col_min, row_min, col_max, row_max = self.__pyramid.tms.get_level(self.__id).bbox_to_tiles( + bbox + ) + self.__tile_limits = { + "min_row": row_min, + "max_col": col_max, + "max_row": row_max, + "min_col": col_min, + } + + +class Pyramid: + + """A data pyramid, raster or vector + + Attributes: + __name (str): pyramid's name + __descriptor (str): pyramid's descriptor path + __list (str): pyramid's list path + __tms (rok4.tile_matrix_set.TileMatrixSet): Used grid + __levels (Dict[str, Level]): Pyramid's levels + __format (str): Data format + __storage (Dict[str, Union[rok4.storage.StorageType,str,int]]): Pyramid's storage informations (type, root and depth if FILE storage) + __raster_specifications (Dict): If raster pyramid, raster specifications + __content (Dict): Loading status (loaded) and list content (cache). + + Example (S3 storage): + + { + 'cache': { + (, '18', 5424, 7526): { + 'link': False, + 'md5': None, + 'root': 'pyramids@localhost:9000/LIMADM', + 'slab': 'DATA_18_5424_7526' + } + }, + 'loaded': True + } + """ + + @classmethod + def from_descriptor(cls, descriptor: str) -> "Pyramid": + """Create a pyramid from its descriptor + + Args: + descriptor (str): pyramid's descriptor path + + Raises: + FormatError: Provided path or the TMS is not a well formed JSON + Exception: Level issue : no one in the pyramid or the used TMS, or level ID not defined in the TMS + MissingAttributeError: Attribute is missing in the content + StorageError: Storage read issue (pyramid descriptor or TMS) + MissingEnvironmentError: Missing object storage informations or TMS root directory + + Examples: + + S3 stored descriptor + + from rok4.pyramid import Pyramid + + try: + pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") + except Exception as e: + print("Cannot load the pyramid from its descriptor") + + Returns: + Pyramid: a Pyramid instance + """ + try: + data = json.loads(get_data_str(descriptor)) + + except JSONDecodeError as e: + raise FormatError("JSON", descriptor, e) + + pyramid = cls() + + pyramid.__storage["type"], path, pyramid.__storage["root"], base_name = get_infos_from_path( + descriptor + ) + pyramid.__name = base_name[:-5] # on supprime l'extension.json + pyramid.__descriptor = descriptor + pyramid.__list = get_path_from_infos( + pyramid.__storage["type"], pyramid.__storage["root"], f"{pyramid.__name}.list" + ) + + try: + # Attributs communs + pyramid.__tms = TileMatrixSet(data["tile_matrix_set"]) + pyramid.__format = data["format"] + + # Attributs d'une pyramide raster + if pyramid.type == PyramidType.RASTER: + pyramid.__raster_specifications = data["raster_specifications"] + + if "mask_format" in data: + pyramid.__masks = True + else: + pyramid.__masks = False + + # Niveaux + for l in data["levels"]: + lev = Level.from_descriptor(l, pyramid) + pyramid.__levels[lev.id] = lev + + if pyramid.__tms.get_level(lev.id) is None: + raise Exception( + f"Pyramid {descriptor} owns a level with the ID '{lev.id}', not defined in the TMS '{pyramid.tms.name}'" + ) + + except KeyError as e: + raise MissingAttributeError(descriptor, e) + + if len(pyramid.__levels.keys()) == 0: + raise Exception(f"Pyramid '{descriptor}' has no level") + + return pyramid + + @classmethod + def from_other(cls, other: "Pyramid", name: str, storage: Dict) -> "Pyramid": + """Create a pyramid from another one + + Args: + other (Pyramid): pyramid to clone + name (str): new pyramid's name + storage (Dict[str, Union[str, int]]): new pyramid's storage informations + + Raises: + FormatError: Provided path or the TMS is not a well formed JSON + Exception: Level issue : no one in the pyramid or the used TMS, or level ID not defined in the TMS + MissingAttributeError: Attribute is missing in the content + + Returns: + Pyramid: a Pyramid instance + """ + try: + # On convertit le type de stockage selon l'énumération + storage["type"] = StorageType[storage["type"]] + + if storage["type"] == StorageType.FILE and name.find("/") != -1: + raise Exception(f"A FILE stored pyramid's name cannot contain '/' : '{name}'") + + if storage["type"] == StorageType.FILE and "depth" not in storage: + storage["depth"] = 2 + + pyramid = cls() + + # Attributs communs + pyramid.__name = name + pyramid.__storage = storage + pyramid.__masks = other.__masks + + pyramid.__descriptor = get_path_from_infos( + pyramid.__storage["type"], pyramid.__storage["root"], f"{pyramid.__name}.json" + ) + pyramid.__list = get_path_from_infos( + pyramid.__storage["type"], pyramid.__storage["root"], f"{pyramid.__name}.list" + ) + pyramid.__tms = other.__tms + pyramid.__format = other.__format + + # Attributs d'une pyramide raster + if pyramid.type == PyramidType.RASTER: + if other.own_masks: + pyramid.__masks = True + else: + pyramid.__masks = False + pyramid.__raster_specifications = other.__raster_specifications + + # Niveaux + for l in other.__levels.values(): + lev = Level.from_other(l, pyramid) + pyramid.__levels[lev.id] = lev + + except KeyError as e: + raise MissingAttributeError(descriptor, e) + + return pyramid + + def __init__(self) -> None: + self.__storage = {} + self.__levels = {} + self.__masks = None + + self.__content = {"loaded": False, "cache": {}} + + def __str__(self) -> str: + return f"{self.type.name} pyramid '{self.__name}' ({self.__storage['type'].name} storage)" + + @property + def serializable(self) -> Dict: + """Get the dict version of the pyramid object, descriptor compliant + + Returns: + Dict: descriptor structured object description + """ + + serialization = { + "tile_matrix_set": self.__tms.name, + "format": self.__format + } + + serialization["levels"] = [] + sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution, reverse=True) + + for l in sorted_levels: + serialization["levels"].append(l.serializable) + + if self.type == PyramidType.RASTER: + serialization["raster_specifications"] = self.__raster_specifications + + if self.__masks: + serialization["mask_format"] = "TIFF_ZIP_UINT8" + + return serialization + + @property + def list(self) -> str: + return self.__list + + @property + def descriptor(self) -> str: + return self.__descriptor + + @property + def name(self) -> str: + return self.__name + + @property + def tms(self) -> TileMatrixSet: + return self.__tms + + @property + def raster_specifications(self) -> Dict: + """Get raster specifications for a RASTER pyramid + + Example: + { + "channels": 3, + "nodata": "255,0,0", + "photometric": "rgb", + "interpolation": "bicubic" + } + + Returns: + Dict: Raster specifications, None if VECTOR pyramid + """ + return self.__raster_specifications + + @property + def storage_type(self) -> StorageType: + """Get the storage type + + Returns: + StorageType: FILE, S3 or CEPH + """ + return self.__storage["type"] + + @property + def storage_root(self) -> str: + """Get the pyramid's storage root. + + If storage is S3, the used cluster is removed. + + Returns: + str: Pyramid's storage root + """ + + return self.__storage["root"].split("@", 1)[ + 0 + ] # Suppression de l'éventuel hôte de spécification du cluster S3 + + @property + def storage_depth(self) -> int: + return self.__storage.get("depth", None) + + @property + def storage_s3_cluster(self) -> str: + """Get the pyramid's storage S3 cluster (host name) + + Returns: + str: the host if known, None if the default one have to be used or if storage is not S3 + """ + if self.__storage["type"] == StorageType.S3: + try: + return self.__storage["root"].split("@")[1] + except IndexError: + return None + else: + return None + + @storage_depth.setter + def storage_depth(self, d: int) -> None: + """Set the tree depth for a FILE storage + + Args: + d (int): file storage depth + + Raises: + Exception: the depth is not equal to the already known depth + """ + if "depth" in self.__storage and self.__storage["depth"] != d: + raise Exception( + f"Pyramid {pyramid.__descriptor} owns levels with different path depths" + ) + self.__storage["depth"] = d + + @property + def own_masks(self) -> bool: + return self.__masks + + @property + def format(self) -> str: + return self.__format + + @property + def tile_extension(self) -> str: + + if self.__format in [ + "TIFF_RAW_UINT8", + "TIFF_LZW_UINT8", + "TIFF_ZIP_UINT8", + "TIFF_PKB_UINT8", + "TIFF_RAW_FLOAT32", + "TIFF_LZW_FLOAT32", + "TIFF_ZIP_FLOAT32", + "TIFF_PKB_FLOAT32", + ]: + return "tif" + elif self.__format in ["TIFF_JPG_UINT8", "TIFF_JPG90_UINT8"]: + return "jpg" + elif self.__format == "TIFF_PNG_UINT8": + return "png" + elif self.__format == "TIFF_PBF_MVT": + return "pbf" + else: + raise Exception( + f"Unknown pyramid's format ({self.__format}), cannot return the tile extension" + ) + + @property + def bottom_level(self) -> "Level": + """Get the best resolution level in the pyramid + + Returns: + Level: the bottom level + """ + return sorted(self.__levels.values(), key=lambda l: l.resolution)[0] + + @property + def top_level(self) -> "Level": + """Get the low resolution level in the pyramid + + Returns: + Level: the top level + """ + return sorted(self.__levels.values(), key=lambda l: l.resolution)[-1] + + @property + def type(self) -> PyramidType: + """Get the pyramid's type (RASTER or VECTOR) from its format + + Returns: + PyramidType: RASTER or VECTOR + """ + if self.__format == "TIFF_PBF_MVT": + return PyramidType.VECTOR + else: + return PyramidType.RASTER + + def load_list(self) -> None: + """Load list content and cache it + + If list is already loaded, nothing done + """ + if self.__content["loaded"]: + return + + for slab, infos in self.list_generator(): + self.__content["cache"][slab] = infos + + self.__content["loaded"] = True + + def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict]]: + """Get list content + + List is copied as temporary file, roots are read and informations about each slab is returned. If list is already loaded, we yield the cached content + + Examples: + + S3 stored descriptor + + from rok4.pyramid import Pyramid + + try: + pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") + + for (slab_type, level, column, row), infos in pyramid.list_generator(): + print(infos) + + except Exception as e: + print("Cannot load the pyramid from its descriptor and read the list") + + Yields: + Iterator[Tuple[Tuple[SlabType,str,int,int], Dict]]: Slab indices and storage informations + + Value example: + + ( + (, '18', 5424, 7526), + { + 'link': False, + 'md5': None, + 'root': 'pyramids@localhost:9000/LIMADM', + 'slab': 'DATA_18_5424_7526' + } + ) + + """ + if self.__content["loaded"]: + for slab, infos in self.__content["cache"].items(): + yield slab, infos + else: + # Copie de la liste dans un fichier temporaire (cette liste peut être un objet) + list_obj = tempfile.NamedTemporaryFile(mode="r", delete=False) + list_file = list_obj.name + copy(self.__list, f"file://{list_file}") + list_obj.close() + + roots = {} + s3_cluster = self.storage_s3_cluster + + with open(list_file, "r") as listin: + # Lecture des racines + for line in listin: + line = line.rstrip() + + if line == "#": + break + + root_id, root_path = line.split("=", 1) + + if s3_cluster is None: + roots[root_id] = root_path + else: + # On a un nom de cluster S3, on l'ajoute au nom du bucket dans les racines + root_bucket, root_path = root_path.split("/", 1) + roots[root_id] = f"{root_bucket}@{s3_cluster}/{root_path}" + + # Lecture des dalles + for line in listin: + line = line.rstrip() + + parts = line.split(" ", 1) + slab_path = parts[0] + slab_md5 = None + if len(parts) == 2: + slab_md5 = parts[1] + + root_id, slab_path = slab_path.split("/", 1) + + slab_type, level, column, row = self.get_infos_from_slab_path(slab_path) + infos = { + "root": roots[root_id], + "link": root_id != "0", + "slab": slab_path, + "md5": slab_md5, + } + + yield ((slab_type, level, column, row), infos) + + remove(f"file://{list_file}") + + def get_level(self, level_id: str) -> "Level": + """Get one level according to its identifier + + Args: + level_id: Level identifier + + Returns: + The corresponding pyramid's level, None if not present + """ + + return self.__levels.get(level_id, None) + + def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: + """Get sorted levels in the provided range from bottom to top + + Args: + bottom_id (str, optionnal): specific bottom level id. Defaults to None. + top_id (str, optionnal): specific top level id. Defaults to None. + + Raises: + Exception: Provided levels are not consistent (bottom > top or not in the pyramid) + + Examples: + + All levels + + from rok4.pyramid import Pyramid + + try: + pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") + levels = pyramid.get_levels() + + except Exception as e: + print("Cannot load the pyramid from its descriptor and get levels") + + From pyramid's bottom to provided top (level 5) + + from rok4.pyramid import Pyramid + + try: + pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") + levels = pyramid.get_levels(None, "5") + + except Exception as e: + print("Cannot load the pyramid from its descriptor and get levels") + + Returns: + List[Level]: asked sorted levels + """ + + sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution) + + levels = [] + + begin = False + if bottom_id is None: + # Pas de niveau du bas fourni, on commence tout en bas + begin = True + else: + if self.get_level(bottom_id) is None: + raise Exception( + f"Pyramid {self.name} does not contain the provided bottom level {bottom_id}" + ) + + if top_id is not None and self.get_level(top_id) is None: + raise Exception(f"Pyramid {self.name} does not contain the provided top level {top_id}") + + end = False + + for l in sorted_levels: + if not begin and l.id == bottom_id: + begin = True + + if begin: + levels.append(l) + if top_id is not None and l.id == top_id: + end = True + break + else: + continue + + if top_id is None: + # Pas de niveau du haut fourni, on a été jusqu'en haut et c'est normal + end = True + + if not begin or not end: + raise Exception( + f"Provided levels ids are not consistent to extract levels from the pyramid {self.name}" + ) + + return levels + + def write_descriptor(self) -> None: + """Write the pyramid's descriptor to the final location (in the pyramid's storage root)""" + + content = json.dumps(self.serializable) + put_data_str(content, self.__descriptor) + + def get_infos_from_slab_path(self, path: str) -> Tuple[SlabType, str, int, int]: + """Get the slab's indices from its storage path + + Args: + path (str): Slab's storage path + + Examples: + + FILE stored pyramid + + from rok4.pyramid import Pyramid + + try: + pyramid = Pyramid.from_descriptor("/path/to/descriptor.json") + slab_type, level, column, row = self.get_infos_from_slab_path("DATA/12/00/4A/F7.tif") + # (SlabType.DATA, "12", 159, 367) + except Exception as e: + print("Cannot load the pyramid from its descriptor and convert a slab path") + + S3 stored pyramid + + from rok4.pyramid import Pyramid + + try: + pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/pyramid.json") + slab_type, level, column, row = self.get_infos_from_slab_path("s3://bucket_name/path/to/pyramid/MASK_15_9164_5846") + # (SlabType.MASK, "15", 9164, 5846) + except Exception as e: + print("Cannot load the pyramid from its descriptor and convert a slab path") + + Returns: + Tuple[SlabType, str, int, int]: Slab's type (DATA or MASK), level identifier, slab's column and slab's row + """ + if self.__storage["type"] == StorageType.FILE: + parts = path.split("/") + + # Le partie du chemin qui contient la colonne et ligne de la dalle est à la fin, en fonction de la profondeur choisie + # depth = 2 -> on doit utiliser les 3 dernières parties pour la conversion + column, row = b36_path_decode("/".join(parts[-(self.__storage["depth"] + 1) :])) + level = parts[-(self.__storage["depth"] + 2)] + raw_slab_type = parts[-(self.__storage["depth"] + 3)] + + # Pour être retro compatible avec l'ancien nommage + if raw_slab_type == "IMAGE": + raw_slab_type = "DATA" + + slab_type = SlabType[raw_slab_type] + + return slab_type, level, column, row + else: + parts = re.split(r"[/_]", path) + column = parts[-2] + row = parts[-1] + level = parts[-3] + raw_slab_type = parts[-4] + + # Pour être retro compatible avec l'ancien nommage + if raw_slab_type == "IMG": + raw_slab_type = "DATA" + elif raw_slab_type == "MSK": + raw_slab_type = "MASK" + + slab_type = SlabType[raw_slab_type] + + return slab_type, level, int(column), int(row) + + def get_slab_path_from_infos( + self, slab_type: SlabType, level: str, column: int, row: int, full: bool = True + ) -> str: + """Get slab's storage path from the indices + + Args: + slab_type (SlabType): DATA or MASK + level (str): Level identifier + column (int): Slab's column + row (int): Slab's row + full (bool, optional): Full path or just relative path from pyramid storage root. Defaults to True. + + Returns: + str: Absolute or relative slab's storage path + """ + if self.__storage["type"] == StorageType.FILE: + slab_path = os.path.join( + slab_type.value, level, b36_path_encode(column, row, self.__storage["depth"]) + ) + else: + slab_path = f"{slab_type.value}_{level}_{column}_{row}" + + if full: + return get_path_from_infos( + self.__storage["type"], self.__storage["root"], self.__name, slab_path + ) + else: + return slab_path + + + def get_tile_data_binary(self, level: str, column: int, row: int) -> str: + """Get a pyramid's tile as binary string + + To get a tile, 3 steps : + * calculate slab path from tile index + * read slab index to get offsets and sizes of slab's tiles + * read the tile into the slab + + Args: + level (str): Tile's level + column (int): Tile's column + row (int): Tile's row + + Limitations: + Pyramids with one-tile slab are not handled + + Examples: + + FILE stored raster pyramid, to extract a tile containing a point and save it as independent image + + from rok4.pyramid import Pyramid + + try: + pyramid = Pyramid.from_descriptor("/data/pyramids/SCAN1000.json") + level, col, row, pcol, prow = pyramid.get_tile_indices(992904.46, 6733643.15, "9", srs = "IGNF:LAMB93") + data = pyramid.get_tile_data_binary(level, col, row) + + if data is None: + print("No data") + else: + tile_name = f"tile_{level}_{col}_{row}.{pyramid.tile_extension}" + with open(tile_name, "wb") as image: + image.write(data) + print (f"Tile written in {tile_name}") + + except Exception as e: + print("Cannot save a pyramid's tile : {e}") + + Raises: + Exception: Level not found in the pyramid + NotImplementedError: Pyramid owns one-tile slabs + MissingEnvironmentError: Missing object storage informations + StorageError: Storage read issue + + Returns: + str: data, as binary string, None if no data + """ + + level_object = self.get_level(level) + + if level_object is None: + raise Exception(f"No level {level} in the pyramid") + + if level_object.slab_width == 1 and level_object.slab_height == 1: + raise NotImplementedError(f"One-tile slab pyramid is not handled") + + if not level_object.is_in_limits(column, row): + return None + + # Indices de la dalle + slab_column = column // level_object.slab_width + slab_row = row // level_object.slab_height + + # Indices de la tuile dans la dalle + relative_tile_column = column % level_object.slab_width + relative_tile_row = row % level_object.slab_height + + # Numéro de la tuile dans le header + tile_index = relative_tile_row * level_object.slab_width + relative_tile_column + + # Calcul du chemin de la dalle contenant la tuile voulue + slab_path = self.get_slab_path_from_infos(SlabType.DATA, level, slab_column, slab_row) + + # Récupération des offset et tailles des tuiles dans la dalle + # Une dalle ROK4 a une en-tête fixe de 2048 octets, + # puis sont stockés les offsets (chacun sur 4 octets) + # puis les tailles (chacune sur 4 octets) + try: + binary_index = get_data_binary( + slab_path, + ( + ROK4_IMAGE_HEADER_SIZE, + 2 * 4 * level_object.slab_width * level_object.slab_height, + ), + ) + except FileNotFoundError as e: + # L'absence de la dalle est gérée comme simplement une absence de données + return None + + offsets = numpy.frombuffer( + binary_index, + dtype=numpy.dtype("uint32"), + count=level_object.slab_width * level_object.slab_height, + ) + sizes = numpy.frombuffer( + binary_index, + dtype=numpy.dtype("uint32"), + offset=4 * level_object.slab_width * level_object.slab_height, + count=level_object.slab_width * level_object.slab_height, + ) + + if sizes[tile_index] == 0: + return None + + return get_data_binary(slab_path, (offsets[tile_index], sizes[tile_index])) + + def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarray: + """Get a raster pyramid's tile as 3-dimension numpy ndarray + + First dimension is the row, second one is column, third one is band. + + Args: + level (str): Tile's level + column (int): Tile's column + row (int): Tile's row + + Limitations: + Packbits (pyramid formats TIFF_PKB_FLOAT32 and TIFF_PKB_UINT8) and LZW (pyramid formats TIFF_LZW_FLOAT32 and TIFF_LZW_UINT8) compressions are not handled. + + Raises: + Exception: Cannot get raster data for a vector pyramid + Exception: Level not found in the pyramid + NotImplementedError: Pyramid owns one-tile slabs + NotImplementedError: Raster pyramid format not handled + MissingEnvironmentError: Missing object storage informations + StorageError: Storage read issue + FormatError: Cannot decode tile + + Examples: + + FILE stored DTM (raster) pyramid, to get the altitude value at a point in the best level + + from rok4.pyramid import Pyramid + + try: + pyramid = Pyramid.from_descriptor("/data/pyramids/RGEALTI.json") + level, col, row, pcol, prow = pyramid.get_tile_indices(44, 5, srs = "EPSG:4326") + data = pyramid.get_tile_data_raster(level, col, row) + + if data is None: + print("No data") + else: + print(data[prow][pcol]) + + except Exception as e: + print("Cannot get a pyramid's pixel value : {e}") + + Returns: + str: data, as numpy array, None if no data + """ + + if self.type == PyramidType.VECTOR: + raise Exception("Cannot get tile as raster data : it's a vector pyramid") + + binary_tile = self.get_tile_data_binary(level, column, row) + + if binary_tile is None: + return None + + level_object = self.get_level(level) + + if self.__format == "TIFF_JPG_UINT8" or self.__format == "TIFF_JPG90_UINT8": + + try: + img = Image.open(io.BytesIO(binary_tile)) + except Exception as e: + raise FormatError("JPEG", "binary tile", e) + + data = numpy.asarray(img) + + elif self.__format == "TIFF_RAW_UINT8": + data = numpy.frombuffer(binary_tile, dtype=numpy.dtype("uint8")) + data.shape = ( + level_object.tile_matrix.tile_size[0], + level_object.tile_matrix.tile_size[1], + self.__raster_specifications["channels"], + ) + + elif self.__format == "TIFF_PNG_UINT8": + try: + img = Image.open(io.BytesIO(binary_tile)) + except Exception as e: + raise FormatError("PNG", "binary tile", e) + + data = numpy.asarray(img) + + elif self.__format == "TIFF_ZIP_UINT8": + try: + data = numpy.frombuffer(zlib.decompress(binary_tile), dtype=numpy.dtype("uint8")) + except Exception as e: + raise FormatError("ZIP", "binary tile", e) + + data.shape = ( + level_object.tile_matrix.tile_size[0], + level_object.tile_matrix.tile_size[1], + self.__raster_specifications["channels"], + ) + + elif self.__format == "TIFF_ZIP_FLOAT32": + try: + data = numpy.frombuffer(zlib.decompress(binary_tile), dtype=numpy.dtype("float32")) + except Exception as e: + raise FormatError("ZIP", "binary tile", e) + + data.shape = ( + level_object.tile_matrix.tile_size[0], + level_object.tile_matrix.tile_size[1], + self.__raster_specifications["channels"], + ) + + elif self.__format == "TIFF_RAW_FLOAT32": + data = numpy.frombuffer(binary_tile, dtype=numpy.dtype("float32")) + data.shape = ( + level_object.tile_matrix.tile_size[0], + level_object.tile_matrix.tile_size[1], + self.__raster_specifications["channels"], + ) + + else: + raise NotImplementedError(f"Cannot get tile as raster data for format {self.__format}") + + return data + + def get_tile_data_vector(self, level: str, column: int, row: int) -> Dict: + """Get a vector pyramid's tile as GeoJSON dictionnary + + Args: + level (str): Tile's level + column (int): Tile's column + row (int): Tile's row + + Raises: + Exception: Cannot get vector data for a raster pyramid + Exception: Level not found in the pyramid + NotImplementedError: Pyramid owns one-tile slabs + NotImplementedError: Vector pyramid format not handled + MissingEnvironmentError: Missing object storage informations + StorageError: Storage read issue + FormatError: Cannot decode tile + + Examples: + + S3 stored vector pyramid, to print a tile as GeoJSON + + from rok4.pyramid import Pyramid + import json + + try: + pyramid = Pyramid.from_descriptor("s3://pyramids/vectors/BDTOPO.json") + level, col, row, pcol, prow = pyramid.get_tile_indices(40.325, 3.123, srs = "EPSG:4326") + data = pyramid.get_tile_data_vector(level, col, row) + + if data is None: + print("No data") + else: + print(json.dumps(data)) + + except Exception as e: + print("Cannot print a vector pyramid's tile as GeoJSON : {e}") + + Returns: + str: data, as GeoJSON dictionnary. None if no data + """ + + if self.type == PyramidType.RASTER: + raise Exception("Cannot get tile as vector data : it's a raster pyramid") + + binary_tile = self.get_tile_data_binary(level, column, row) + + if binary_tile is None: + return None + + level_object = self.get_level(level) + + if self.__format == "TIFF_PBF_MVT": + try: + data = mapbox_vector_tile.decode(binary_tile) + except Exception as e: + raise FormatError("PBF (MVT)", "binary tile", e) + else: + raise NotImplementedError(f"Cannot get tile as vector data for format {self.__format}") + + return data + + def get_tile_indices( + self, x: float, y: float, level: str = None, **kwargs + ) -> Tuple[str, int, int, int, int]: + """Get pyramid's tile and pixel indices from point's coordinates + + Used coordinates system have to be the pyramid one. If EPSG:4326, x is latitude and y longitude. + + Args: + x (float): point's x + y (float): point's y + level (str, optional): Pyramid's level to take into account, the bottom one if None . Defaults to None. + **srs (string): spatial reference system of provided coordinates, with authority and code (same as the pyramid's one if not provided) + + Raises: + Exception: Cannot find level to calculate indices + RuntimeError: Provided SRS is invalid for OSR + + Examples: + + FILE stored DTM (raster) pyramid, to get the altitude value at a point in the best level + + from rok4.pyramid import Pyramid + + try: + pyramid = Pyramid.from_descriptor("/data/pyramids/RGEALTI.json") + level, col, row, pcol, prow = pyramid.get_tile_indices(44, 5, srs = "EPSG:4326") + data = pyramid.get_tile_data_raster(level, col, row) + + if data is None: + print("No data") + else: + print(data[prow][pcol]) + + except Exception as e: + print("Cannot get a pyramid's pixel value : {e}") + + Returns: + Tuple[str, int, int, int, int]: Level identifier, tile's column, tile's row, pixel's (in the tile) column, pixel's row + """ + + level_object = self.bottom_level + if level is not None: + level_object = self.get_level(level) + + if level_object is None: + raise Exception(f"Cannot found the level to calculate indices") + + if ( + "srs" in kwargs + and kwargs["srs"] is not None + and kwargs["srs"].upper() != self.__tms.srs.upper() + ): + sr = srs_to_spatialreference(kwargs["srs"]) + x, y = reproject_point((x, y), sr, self.__tms.sr) + + return (level_object.id,) + level_object.tile_matrix.point_to_indices(x, y) + + @property + def size(self) -> int: + """Get the size of the pyramid + + Examples: + + from rok4.pyramid import Pyramid + + try: + pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") + size = pyramid.size() + + except Exception as e: + print("Cannot load the pyramid from its descriptor and get his size") + + Returns: + int: size of the pyramid + """ + if not hasattr(self,"_Pyramid__size") : + self.__size = size_path(get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name)) + return self.__size diff --git a/src/rok4/Raster.py b/src/rok4/raster.py similarity index 97% rename from src/rok4/Raster.py rename to src/rok4/raster.py index 271fc6a..1c1ac36 100644 --- a/src/rok4/Raster.py +++ b/src/rok4/raster.py @@ -14,8 +14,9 @@ from osgeo import gdal, ogr -from rok4.Storage import exists, get_osgeo_path, put_data_str -from rok4.Utils import ColorFormat, compute_bbox, compute_format +from rok4.storage import exists, get_osgeo_path, put_data_str +from rok4.utils import compute_bbox, compute_format +from rok4.enums import ColorFormat # Enable GDAL/OGR exceptions ogr.UseExceptions() @@ -59,7 +60,7 @@ def from_file(cls, path: str) -> "Raster": Loading informations from a file stored raster TIFF image - from rok4.Raster import Raster + from rok4.raster import Raster try: raster = Raster.from_file( @@ -141,7 +142,7 @@ def from_parameters( Loading informations from parameters, related to a TIFF main image coupled to a TIFF mask image - from rok4.Raster import Raster + from rok4.raster import Raster try: raster = Raster.from_parameters( @@ -213,7 +214,7 @@ def from_list(cls, path: str, srs: str) -> "RasterSet": Loading informations from a file stored list - from rok4.Raster import RasterSet + from rok4.raster import RasterSet try: raster_set = RasterSet.from_list( @@ -276,7 +277,7 @@ def from_descriptor(cls, path: str) -> "RasterSet": Loading informations from a file stored descriptor - from rok4.Raster import RasterSet + from rok4.raster import RasterSet try: raster_set = RasterSet.from_descriptor( diff --git a/src/rok4/Storage.py b/src/rok4/storage.py similarity index 94% rename from src/rok4/Storage.py rename to src/rok4/storage.py index efc3696..54e2837 100644 --- a/src/rok4/Storage.py +++ b/src/rok4/storage.py @@ -30,24 +30,23 @@ To precise the cluster to use, bucket name should be bucket_name@s3.storage.fr or bucket_name@s4.storage.fr. If no host is defined (no @) in the bucket name, first S3 cluster is used """ -import hashlib -import os -import re -import tempfile -from enum import Enum -from shutil import copyfile -from typing import Dict, List, Tuple, Union - import boto3 import botocore.exceptions +import tempfile +import re +import os import rados +import hashlib import requests +from typing import Dict, List, Tuple, Union +from enum import Enum +from shutil import copyfile from osgeo import gdal gdal.UseExceptions() +from rok4.exceptions import * from rok4.enums import StorageType -from rok4.Exceptions import * __S3_CLIENTS = {} @@ -69,7 +68,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s Returns: Tuple[Dict[str, Union['boto3.client',str]], str, str]: the S3 informations (client, host, key, secret) and the simple bucket name """ - + global __S3_CLIENTS, __S3_DEFAULT_CLIENT if not __S3_CLIENTS: @@ -128,7 +127,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s def disconnect_s3_clients() -> None: """Clean S3 clients""" - + global __S3_CLIENTS, __S3_DEFAULT_CLIENT __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None @@ -357,15 +356,16 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: raise StorageError("FILE", e) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - if range is None: + + if range is None : try: reponse = requests.get(f"{storage_type.value}{path}", stream=True) data = reponse.content - if reponse.status_code == 404: + if reponse.status_code == 404 : raise FileNotFoundError(f"{storage_type.value}{path}") except Exception as e: raise StorageError(storage_type.name, e) - else: + else : raise NotImplementedError else: @@ -464,6 +464,7 @@ def get_size(path: str) -> int: raise StorageError("FILE", e) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: + try: # Le stream=True permet de ne télécharger que le header initialement reponse = requests.get(storage_type.value + path, stream=True).headers["content-length"] @@ -518,11 +519,12 @@ def exists(path: str) -> bool: return os.path.exists(path) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: + try: response = requests.get(storage_type.value + path, stream=True) - if response.status_code == 200: + if response.status_code == 200 : return True - else: + else : return False except Exception as e: raise StorageError(storage_type.name, e) @@ -830,52 +832,43 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: f"CEPH and S3", f"Cannot copy CEPH object {from_path} to S3 object {to_path} : {e}" ) - elif ( - from_type == StorageType.HTTP or from_type == StorageType.HTTPS - ) and to_type == StorageType.FILE: + elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.FILE : + try: - response = requests.get(from_type.value + from_path, stream=True) + response = requests.get(from_type.value + from_path, stream = True) with open(to_path, "wb") as f: - for chunk in response.iter_content(chunk_size=65536): + for chunk in response.iter_content(chunk_size=65536) : if chunk: f.write(chunk) except Exception as e: - raise StorageError( - f"HTTP(S) and FILE", - f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}", - ) + raise StorageError(f"HTTP(S) and FILE", f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}") + + elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.CEPH : - elif ( - from_type == StorageType.HTTP or from_type == StorageType.HTTPS - ) and to_type == StorageType.CEPH: to_ioctx = __get_ceph_ioctx(to_tray) try: - response = requests.get(from_type.value + from_path, stream=True) + response = requests.get(from_type.value + from_path, stream = True) offset = 0 - for chunk in response.iter_content(chunk_size=65536): + for chunk in response.iter_content(chunk_size=65536) : if chunk: size = len(chunk) to_ioctx.write(to_base_name, chunk, offset) offset += size except Exception as e: - raise StorageError( - f"HTTP(S) and CEPH", - f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}", - ) + raise StorageError(f"HTTP(S) and CEPH", f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}") + + elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.S3 : - elif ( - from_type == StorageType.HTTP or from_type == StorageType.HTTPS - ) and to_type == StorageType.S3: to_s3_client, to_bucket = __get_s3_client(to_tray) try: - response = requests.get(from_type.value + from_path, stream=True) - with tempfile.NamedTemporaryFile("w+b", delete=False) as f: + response = requests.get(from_type.value + from_path, stream = True) + with tempfile.NamedTemporaryFile("w+b",delete=False) as f: name_fich = f.name - for chunk in response.iter_content(chunk_size=65536): + for chunk in response.iter_content(chunk_size=65536) : if chunk: f.write(chunk) @@ -884,10 +877,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: os.remove(name_fich) except Exception as e: - raise StorageError( - f"HTTP(S) and S3", - f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}", - ) + raise StorageError(f"HTTP(S) and S3", f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}") else: raise StorageError( @@ -934,9 +924,11 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: try: target_s3_client["client"].put_object( - Body = f"{__OBJECT_SYMLINK_SIGNATURE}{target_bucket}/{target_base_name}".encode(), - Bucket = link_bucket, - Key = link_base_name + Body=f"{__OBJECT_SYMLINK_SIGNATURE}{target_bucket}/{target_base_name}".encode( + "utf-8" + ), + Bucket=link_bucket, + Key=link_base_name, ) except Exception as e: raise StorageError("S3", e) @@ -945,7 +937,9 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: ioctx = __get_ceph_ioctx(link_tray) try: - ioctx.write_full(link_base_name, f"{__OBJECT_SYMLINK_SIGNATURE}{target_path}".encode()) + ioctx.write_full( + link_base_name, f"{__OBJECT_SYMLINK_SIGNATURE}{target_path}".encode("utf-8") + ) except Exception as e: raise StorageError("CEPH", e) @@ -997,8 +991,7 @@ def get_osgeo_path(path: str) -> str: else: raise NotImplementedError(f"Cannot get a GDAL/OGR compliant path from {path}") - -def size_path(path: str) -> int: +def size_path(path: str) -> int : """Return the size of the path given (or, for the CEPH, the sum of the size of each object of the .list) Args: @@ -1011,10 +1004,10 @@ def size_path(path: str) -> int: Returns: int: size of the path """ - storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) + storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) if storage_type == StorageType.FILE: - try: + try : total = 0 with os.scandir(unprefixed_path) as it: for entry in it: @@ -1029,23 +1022,24 @@ def size_path(path: str) -> int: elif storage_type == StorageType.S3: s3_client, bucket_name = __get_s3_client(tray_name) - try: - paginator = s3_client["client"].get_paginator("list_objects_v2") + try : + paginator = s3_client["client"].get_paginator('list_objects_v2') pages = paginator.paginate( Bucket=bucket_name, - Prefix=base_name + "/", + Prefix=base_name+"/", PaginationConfig={ - "PageSize": 10000, - }, + 'PageSize': 10000, + } ) total = 0 for page in pages: - for key in page["Contents"]: - total += key["Size"] + for key in page['Contents']: + total += key['Size'] except Exception as e: raise StorageError("S3", e) + elif storage_type == StorageType.CEPH: raise NotImplementedError else: diff --git a/src/rok4/TileMatrixSet.py b/src/rok4/tile_matrix_set.py similarity index 98% rename from src/rok4/TileMatrixSet.py rename to src/rok4/tile_matrix_set.py index 3000492..06d4466 100644 --- a/src/rok4/TileMatrixSet.py +++ b/src/rok4/tile_matrix_set.py @@ -9,14 +9,14 @@ - ROK4_TMS_DIRECTORY """ +from typing import Dict, List, Tuple +from json.decoder import JSONDecodeError import json import os -from json.decoder import JSONDecodeError -from typing import Dict, List, Tuple -from rok4.Exceptions import * -from rok4.Storage import get_data_str -from rok4.Utils import * +from rok4.exceptions import * +from rok4.storage import get_data_str +from rok4.utils import * class TileMatrix: diff --git a/src/rok4/Utils.py b/src/rok4/utils.py similarity index 99% rename from src/rok4/Utils.py rename to src/rok4/utils.py index 53450d7..d7d3ef7 100644 --- a/src/rok4/Utils.py +++ b/src/rok4/utils.py @@ -13,6 +13,7 @@ osr.UseExceptions() gdal.UseExceptions() +from rok4.enums import ColorFormat __SR_BOOK = {} diff --git a/src/rok4/Vector.py b/src/rok4/vector.py similarity index 98% rename from src/rok4/Vector.py rename to src/rok4/vector.py index 46656c2..5478056 100644 --- a/src/rok4/Vector.py +++ b/src/rok4/vector.py @@ -6,17 +6,16 @@ """ +from osgeo import ogr import os import tempfile -from osgeo import ogr - -from rok4.Exceptions import * -from rok4.Storage import copy, get_osgeo_path - # Enable GDAL/OGR exceptions ogr.UseExceptions() +from rok4.storage import get_osgeo_path, copy +from rok4.exceptions import * + class Vector: """A data vector @@ -41,7 +40,7 @@ def from_file(cls, path: str, **kwargs) -> "Vector": Examples: - from rok4.Vector import Vector + from rok4.vector import Vector try: vector = Vector.from_file("file://tests/fixtures/ARRONDISSEMENT.shp") diff --git a/tests/test_Layer.py b/tests/test_layer.py similarity index 91% rename from tests/test_Layer.py rename to tests/test_layer.py index bbf71eb..8d515da 100644 --- a/tests/test_Layer.py +++ b/tests/test_layer.py @@ -1,21 +1,21 @@ +import pytest import os -from unittest import mock -from unittest.mock import * -import pytest +from unittest.mock import * +from unittest import mock -from rok4.Exceptions import * -from rok4.Layer import Layer +from rok4.layer import Layer from rok4.enums import PyramidType +from rok4.exceptions import * @mock.patch.dict(os.environ, {}, clear=True) @mock.patch( - "rok4.Layer.get_data_str", + "rok4.layer.get_data_str", return_value='{"pyramids" : [{"bottom_level" : "10","top_level" : "10","path" : "s3://pyramids/SCAN1000.json"}],"title" : "SCAN1000","bbox":{"east": 11.250000000000997,"west": -5.624999999999043,"north": 52.48278022207774,"south": 40.9798980696195},"styles" : ["normal","hypso"],"abstract" : "Diffusion de la donnée BDORTHO","resampling" : "linear","keywords" : ["PM","TIFF_JPG_UINT8"]}', ) -@mock.patch("rok4.Layer.Pyramid.from_descriptor") -@mock.patch("rok4.Layer.put_data_str", return_value=None) +@mock.patch("rok4.layer.Pyramid.from_descriptor") +@mock.patch("rok4.layer.put_data_str", return_value=None) def test_descriptor_ok(mocked_put_data_str, mocked_pyramid_class, mocked_get_data_str): tms_instance = MagicMock() tms_instance.srs = "EPSG:3857" @@ -51,9 +51,9 @@ def test_descriptor_ok(mocked_put_data_str, mocked_pyramid_class, mocked_get_dat assert False, f"Layer creation from descriptor raises an exception: {exc}" -@mock.patch("rok4.Layer.Pyramid.from_descriptor") -@mock.patch("rok4.Layer.reproject_bbox", return_value=(0, 0, 100, 100)) -@mock.patch("rok4.Layer.put_data_str", return_value=None) +@mock.patch("rok4.layer.Pyramid.from_descriptor") +@mock.patch("rok4.layer.reproject_bbox", return_value=(0, 0, 100, 100)) +@mock.patch("rok4.layer.put_data_str", return_value=None) def test_parameters_vector_ok( mocked_put_data_str, mocked_utils_reproject_bbox, mocked_pyramid_class ): @@ -97,9 +97,9 @@ def test_parameters_vector_ok( assert False, f"Layer creation from parameters raises an exception: {exc}" -@mock.patch("rok4.Layer.Pyramid.from_descriptor") -@mock.patch("rok4.Layer.reproject_bbox", return_value=(0, 0, 100, 100)) -@mock.patch("rok4.Layer.put_data_str", return_value=None) +@mock.patch("rok4.layer.Pyramid.from_descriptor") +@mock.patch("rok4.layer.reproject_bbox", return_value=(0, 0, 100, 100)) +@mock.patch("rok4.layer.put_data_str", return_value=None) def test_parameters_raster_ok( mocked_put_data_str, mocked_utils_reproject_bbox, mocked_pyramid_class ): diff --git a/tests/test_Pyramid.py b/tests/test_pyramid.py similarity index 92% rename from tests/test_Pyramid.py rename to tests/test_pyramid.py index 3b75be5..fa04a01 100644 --- a/tests/test_Pyramid.py +++ b/tests/test_pyramid.py @@ -1,25 +1,22 @@ +import pytest import os -from unittest import mock from unittest.mock import * +from unittest import mock -import pytest - -from rok4.Exceptions import * -from rok4.Pyramid import * -from rok4.TileMatrixSet import TileMatrixSet +from rok4.pyramid import * +from rok4.tile_matrix_set import TileMatrixSet from rok4.enums import SlabType, StorageType +from rok4.utils import * +from rok4.exceptions import * -from rok4.Utils import * - - -@mock.patch("rok4.Pyramid.get_data_str", side_effect=StorageError("FILE", "Not found")) +@mock.patch("rok4.pyramid.get_data_str", side_effect=StorageError("FILE", "Not found")) def test_wrong_file(mocked_get_data_str): with pytest.raises(StorageError): pyramid = Pyramid.from_descriptor("file:///pyramid.json") @mock.patch( - "rok4.Pyramid.get_data_str", + "rok4.pyramid.get_data_str", return_value='{"format": "TIFF_PBF_MVT","levels":[{"id": "100","tables":', ) def test_bad_json(mocked_get_data_str): @@ -33,7 +30,7 @@ def test_bad_json(mocked_get_data_str): mocked_get_data_str.assert_called_once_with("file:///pyramid.json") -@mock.patch("rok4.Pyramid.get_data_str", return_value='{"format": "TIFF_PBF_MVT","levels":[]}') +@mock.patch("rok4.pyramid.get_data_str", return_value='{"format": "TIFF_PBF_MVT","levels":[]}') def test_missing_tms(mocked_get_data_str): with pytest.raises(MissingAttributeError) as exc: pyramid = Pyramid.from_descriptor("file:///pyramid.json") @@ -44,10 +41,10 @@ def test_missing_tms(mocked_get_data_str): @mock.patch.dict(os.environ, {}, clear=True) @mock.patch( - "rok4.Pyramid.get_data_str", + "rok4.pyramid.get_data_str", return_value='{"format": "TIFF_PBF_MVT","levels":[{}], "tile_matrix_set": "PM"}', ) -@mock.patch("rok4.Pyramid.TileMatrixSet", side_effect=StorageError("FILE", "TMS not found")) +@mock.patch("rok4.pyramid.TileMatrixSet", side_effect=StorageError("FILE", "TMS not found")) def test_wrong_tms(mocked_tms_constructor, mocked_get_data_str): with pytest.raises(StorageError) as exc: pyramid = Pyramid.from_descriptor("file:///pyramid.json") @@ -59,10 +56,10 @@ def test_wrong_tms(mocked_tms_constructor, mocked_get_data_str): @mock.patch.dict(os.environ, {}, clear=True) @mock.patch( - "rok4.Pyramid.get_data_str", + "rok4.pyramid.get_data_str", return_value='{"format": "TIFF_JPG_UINT8","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_directory":"SCAN1000/DATA/0","path_depth":2,"type":"FILE"},"tiles_per_width":16,"id":"0"}], "tile_matrix_set": "PM"}', ) -@mock.patch("rok4.Pyramid.TileMatrixSet") +@mock.patch("rok4.pyramid.TileMatrixSet") def test_raster_missing_raster_specifications(mocked_tms_class, mocked_get_data_str): with pytest.raises(MissingAttributeError) as exc: pyramid = Pyramid.from_descriptor("file:///pyramid.json") @@ -73,10 +70,10 @@ def test_raster_missing_raster_specifications(mocked_tms_class, mocked_get_data_ @mock.patch.dict(os.environ, {}, clear=True) @mock.patch( - "rok4.Pyramid.get_data_str", + "rok4.pyramid.get_data_str", return_value='{"raster_specifications":{"channels":3,"nodata":"255,0,0","photometric":"rgb","interpolation":"bicubic"}, "format": "TIFF_JPG_UINT8","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_directory":"SCAN1000/DATA/0","path_depth":2,"type":"FILE"},"tiles_per_width":16,"id":"unknown"}], "tile_matrix_set": "PM"}', ) -@mock.patch("rok4.Pyramid.TileMatrixSet") +@mock.patch("rok4.pyramid.TileMatrixSet") def test_wrong_level(mocked_tms_class, mocked_get_data_str): tms_instance = MagicMock() tms_instance.get_level.return_value = None @@ -97,10 +94,10 @@ def test_wrong_level(mocked_tms_class, mocked_get_data_str): @mock.patch.dict(os.environ, {}, clear=True) @mock.patch( - "rok4.Pyramid.get_data_str", + "rok4.pyramid.get_data_str", return_value='{"format": "TIFF_PBF_MVT","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_directory":"SCAN1000/DATA/0","path_depth":2,"type":"FILE"},"tiles_per_width":16,"id":"0"}], "tile_matrix_set": "PM"}', ) -@mock.patch("rok4.Pyramid.TileMatrixSet", autospec=True) +@mock.patch("rok4.pyramid.TileMatrixSet", autospec=True) def test_vector_missing_tables(mocked_tms_class, mocked_get_data_str): with pytest.raises(MissingAttributeError) as exc: pyramid = Pyramid.from_descriptor("file:///pyramid.json") @@ -111,11 +108,11 @@ def test_vector_missing_tables(mocked_tms_class, mocked_get_data_str): @mock.patch.dict(os.environ, {}, clear=True) @mock.patch( - "rok4.Pyramid.get_data_str", + "rok4.pyramid.get_data_str", return_value='{"raster_specifications":{"channels":3,"nodata":"255,0,0","photometric":"rgb","interpolation":"bicubic"}, "format": "TIFF_JPG_UINT8","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_prefix":"SCAN1000/DATA_0","pool_name":"pool1","type":"CEPH"},"tiles_per_width":16,"id":"0"}], "tile_matrix_set": "PM"}', ) -@mock.patch("rok4.Pyramid.TileMatrixSet") -@mock.patch("rok4.Pyramid.put_data_str", return_value=None) +@mock.patch("rok4.pyramid.TileMatrixSet") +@mock.patch("rok4.pyramid.put_data_str", return_value=None) def test_raster_ok(mocked_put_data_str, mocked_tms_class, mocked_get_data_str): tms_instance = MagicMock() tms_instance.name = "PM" @@ -169,10 +166,10 @@ def test_raster_ok(mocked_put_data_str, mocked_tms_class, mocked_get_data_str): @mock.patch.dict(os.environ, {}, clear=True) @mock.patch( - "rok4.Pyramid.get_data_str", + "rok4.pyramid.get_data_str", return_value='{"format": "TIFF_PBF_MVT","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_directory":"SCAN1000/DATA/0","path_depth":2,"type":"FILE"},"tiles_per_width":16,"id":"0","tables":[{"name":"table","geometry":"POINT","attributes":[{"type":"bigint","name":"fid","count":1531}]}]}], "tile_matrix_set": "PM"}', ) -@mock.patch("rok4.Pyramid.TileMatrixSet") +@mock.patch("rok4.pyramid.TileMatrixSet") def test_vector_ok(mocked_tms_class, mocked_get_data_str): try: pyramid = Pyramid.from_descriptor("file:///pyramid.json") @@ -197,7 +194,7 @@ def test_vector_ok(mocked_tms_class, mocked_get_data_str): @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch("rok4.Pyramid.TileMatrixSet") +@mock.patch("rok4.pyramid.TileMatrixSet") def test_tile_read_raster(mocked_tms_class): tms_instance = MagicMock() tms_instance.name = "UTM20W84MART_1M_MNT" @@ -222,7 +219,7 @@ def test_tile_read_raster(mocked_tms_class): @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch("rok4.Pyramid.TileMatrixSet") +@mock.patch("rok4.pyramid.TileMatrixSet") def test_tile_read_vector(mocked_tms_class): tms_instance = MagicMock() tms_instance.name = "PM" @@ -250,7 +247,7 @@ def test_tile_read_vector(mocked_tms_class): @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch("rok4.Pyramid.TileMatrixSet") +@mock.patch("rok4.pyramid.TileMatrixSet") def test_list_read(mocked_tms_class): tms_instance = MagicMock() tms_instance.name = "PM" diff --git a/tests/test_Raster.py b/tests/test_raster.py similarity index 91% rename from tests/test_Raster.py rename to tests/test_raster.py index cc79846..c91f527 100644 --- a/tests/test_Raster.py +++ b/tests/test_raster.py @@ -1,4 +1,4 @@ -"""Describes unit tests for the rok4.Raster module. +"""Describes unit tests for the rok4.raster module. There is one test class for each tested functionnality. See internal docstrings for more information. @@ -14,14 +14,14 @@ import pytest -from rok4.Raster import Raster, RasterSet -from rok4.Utils import ColorFormat +from rok4.raster import Raster, RasterSet +from rok4.enums import ColorFormat -# rok4.Raster.Raster class tests +# rok4.raster.Raster class tests class TestRasterInit(TestCase): - """rok4.Raster.Raster default constructor.""" + """rok4.raster.Raster default constructor.""" def test_default(self): """Default property values.""" @@ -44,7 +44,7 @@ def test_default(self): class TestRasterFromFile(TestCase): - """rok4.Raster.Raster.from_file(path) class constructor.""" + """rok4.raster.Raster.from_file(path) class constructor.""" def setUp(self): self.source_image_path = "file:///home/user/image.tif" @@ -60,18 +60,18 @@ def test_empty(self): with pytest.raises(TypeError): Raster.from_file() - @mock.patch("rok4.Raster.exists", return_value=False) + @mock.patch("rok4.raster.exists", return_value=False) def test_image_not_found(self, m_exists): """Constructor called on a path matching no file or object.""" with pytest.raises(Exception): Raster.from_file(self.source_image_path) m_exists.assert_called_once_with(self.source_image_path) - @mock.patch("rok4.Raster.get_osgeo_path") - @mock.patch("rok4.Raster.compute_format", return_value=ColorFormat.UINT8) - @mock.patch("rok4.Raster.gdal.Open") - @mock.patch("rok4.Raster.compute_bbox") - @mock.patch("rok4.Raster.exists", side_effect=[True, False]) + @mock.patch("rok4.raster.get_osgeo_path") + @mock.patch("rok4.raster.compute_format", return_value=ColorFormat.UINT8) + @mock.patch("rok4.raster.gdal.Open") + @mock.patch("rok4.raster.compute_bbox") + @mock.patch("rok4.raster.exists", side_effect=[True, False]) def test_image(self, m_exists, m_compute_bbox, m_gdal_open, m_compute_format, m_get_osgeo_path): """Constructor called nominally on an image without mask.""" m_compute_bbox.return_value = self.bbox @@ -104,12 +104,12 @@ def test_image(self, m_exists, m_compute_bbox, m_gdal_open, m_compute_format, m_ assert raster.format == ColorFormat.UINT8 assert raster.dimensions == self.image_size - @mock.patch("rok4.Raster.get_osgeo_path") - @mock.patch("rok4.Raster.compute_format", return_value=ColorFormat.UINT8) - @mock.patch("rok4.Raster.gdal.IdentifyDriver") - @mock.patch("rok4.Raster.gdal.Open") - @mock.patch("rok4.Raster.compute_bbox") - @mock.patch("rok4.Raster.exists", side_effect=[True, True]) + @mock.patch("rok4.raster.get_osgeo_path") + @mock.patch("rok4.raster.compute_format", return_value=ColorFormat.UINT8) + @mock.patch("rok4.raster.gdal.IdentifyDriver") + @mock.patch("rok4.raster.gdal.Open") + @mock.patch("rok4.raster.compute_bbox") + @mock.patch("rok4.raster.exists", side_effect=[True, True]) def test_image_and_mask( self, m_exists, @@ -154,9 +154,9 @@ def test_image_and_mask( assert raster.format == ColorFormat.UINT8 assert raster.dimensions == self.image_size - @mock.patch("rok4.Raster.get_osgeo_path") - @mock.patch("rok4.Raster.gdal.Open", side_effect=RuntimeError) - @mock.patch("rok4.Raster.exists", side_effect=[True, False]) + @mock.patch("rok4.raster.get_osgeo_path") + @mock.patch("rok4.raster.gdal.Open", side_effect=RuntimeError) + @mock.patch("rok4.raster.exists", side_effect=[True, False]) def test_unsupported_image_format(self, m_exists, m_gdal_open, m_get_osgeo_path): """Test case : Constructor called on an unsupported image file or object.""" m_get_osgeo_path.return_value = self.osgeo_image_path @@ -168,10 +168,10 @@ def test_unsupported_image_format(self, m_exists, m_gdal_open, m_get_osgeo_path) m_get_osgeo_path.assert_called_once_with(self.source_image_path) m_gdal_open.assert_called_once_with(self.osgeo_image_path) - @mock.patch("rok4.Raster.get_osgeo_path") - @mock.patch("rok4.Raster.gdal.IdentifyDriver") - @mock.patch("rok4.Raster.gdal.Open", side_effect=None) - @mock.patch("rok4.Raster.exists", side_effect=[True, True]) + @mock.patch("rok4.raster.get_osgeo_path") + @mock.patch("rok4.raster.gdal.IdentifyDriver") + @mock.patch("rok4.raster.gdal.Open", side_effect=None) + @mock.patch("rok4.raster.exists", side_effect=[True, True]) def test_unsupported_mask_format( self, m_exists, m_gdal_open, m_identifydriver, m_get_osgeo_path ): @@ -191,7 +191,7 @@ def test_unsupported_mask_format( class TestRasterFromParameters(TestCase): - """rok4.Raster.Raster.from_parameters(**kwargs) class constructor.""" + """rok4.raster.Raster.from_parameters(**kwargs) class constructor.""" def test_image(self): """Parameters describing an image without mask""" @@ -247,11 +247,11 @@ def test_image_and_mask(self): assert raster.mask == parameters["mask"] -# rok4.Raster.RasterSet class tests +# rok4.raster.RasterSet class tests class TestRasterSetInit(TestCase): - """rok4.Raster.RasterSet default constructor.""" + """rok4.raster.RasterSet default constructor.""" def test_default(self): """Default property values.""" @@ -268,10 +268,10 @@ def test_default(self): class TestRasterSetFromList(TestCase): - """rok4.Raster.RasterSet.from_list(path, srs) class constructor.""" + """rok4.raster.RasterSet.from_list(path, srs) class constructor.""" - @mock.patch("rok4.Raster.get_osgeo_path") - @mock.patch("rok4.Raster.Raster.from_file") + @mock.patch("rok4.raster.get_osgeo_path") + @mock.patch("rok4.raster.Raster.from_file") def test_ok_at_least_3_files(self, m_from_file, m_get_osgeo_path): """List of 3 or more valid image files""" file_number = random.randint(3, 100) @@ -326,7 +326,7 @@ def test_ok_at_least_3_files(self, m_from_file, m_get_osgeo_path): bbox = (-0.75, -1.33, 0.25 + math.floor((file_number - 1) / 3), 1.67) serial_in["bbox"] = list(bbox) - with mock.patch("rok4.Raster.open", m_open): + with mock.patch("rok4.raster.open", m_open): rasterset = RasterSet.from_list(list_path, srs) serial_out = rasterset.serializable @@ -347,10 +347,10 @@ def test_ok_at_least_3_files(self, m_from_file, m_get_osgeo_path): class TestRasterSetFromDescriptor(TestCase): - """rok4.Raster.RasterSet.from_descriptor(path) class constructor.""" + """rok4.raster.RasterSet.from_descriptor(path) class constructor.""" - @mock.patch("rok4.Raster.get_osgeo_path") - @mock.patch("rok4.Raster.Raster.from_parameters") + @mock.patch("rok4.raster.get_osgeo_path") + @mock.patch("rok4.raster.Raster.from_parameters") def test_simple_ok(self, m_from_parameters, m_get_osgeo_path): serial_in = { "bbox": [550000.000, 6210000.000, 570000.000, 6230000.000], @@ -401,7 +401,7 @@ def test_simple_ok(self, m_from_parameters, m_get_osgeo_path): m_get_osgeo_path.return_value = local_path m_open = mock_open(read_data=desc_content) - with mock.patch("rok4.Raster.open", m_open): + with mock.patch("rok4.raster.open", m_open): rasterset = RasterSet.from_descriptor(desc_path) m_get_osgeo_path.assert_called_once_with(desc_path) @@ -429,9 +429,9 @@ def test_simple_ok(self, m_from_parameters, m_get_osgeo_path): class TestRasterSetWriteDescriptor(TestCase): - """rok4.Raster.RasterSet.write_descriptor(path) class method.""" + """rok4.raster.RasterSet.write_descriptor(path) class method.""" - @mock.patch("rok4.Raster.put_data_str") + @mock.patch("rok4.raster.put_data_str") def test_ok_with_output_path(self, m_put_data_str): serial_in = { "bbox": [550000.000, 6210000.000, 570000.000, 6230000.000], @@ -489,7 +489,7 @@ def test_ok_with_output_path(self, m_put_data_str): m_put_data_str.assert_called_once_with(content, path) - @mock.patch("rok4.Raster.print") + @mock.patch("rok4.raster.print") def test_ok_no_output_path(self, m_print): serial_in = { "bbox": [550000.000, 6210000.000, 570000.000, 6230000.000], diff --git a/tests/test_Storage.py b/tests/test_storage.py similarity index 90% rename from tests/test_Storage.py rename to tests/test_storage.py index 8426ccf..a3f8799 100644 --- a/tests/test_Storage.py +++ b/tests/test_storage.py @@ -1,15 +1,15 @@ +import pytest import os -from unittest import mock -from unittest.mock import * import botocore.exceptions -import pytest from rados import ObjectNotFound -from rok4.Exceptions import * -from rok4.Storage import * -from rok4.enums import StorageType +from unittest import mock +from unittest.mock import * +from rok4.storage import * +from rok4.exceptions import * +from rok4.enums import StorageType @mock.patch.dict(os.environ, {}, clear=True) @patch("builtins.open", new_callable=mock_open, read_data=b"data") @@ -21,7 +21,6 @@ def test_hash_file_ok(mock_file): except Exception as exc: assert False, f"FILE md5 sum raises an exception: {exc}" - @mock.patch.dict(os.environ, {}, clear=True) def test_get_infos_from_path(): assert (StorageType.S3, "toto/titi", "toto", "titi") == get_infos_from_path("s3://toto/titi") @@ -77,7 +76,7 @@ def test_s3_invalid_envs(): @mock.patch.dict( os.environ, {"ROK4_S3_URL": "a", "ROK4_S3_SECRETKEY": "b", "ROK4_S3_KEY": "c"}, clear=True ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") def test_s3_invalid_endpoint(mocked_s3_client): s3_instance = MagicMock() mocked_s3_client.side_effect = Exception("Invalid URL") @@ -104,13 +103,12 @@ def test_file_read_ok(mock_file): except Exception as exc: assert False, f"FILE read raises an exception: {exc}" - @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True, ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") def test_s3_read_nok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() @@ -119,13 +117,12 @@ def test_s3_read_nok(mocked_s3_client): with pytest.raises(StorageError): data = get_data_str("s3://bucket/path/to/object") - @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True, ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") def test_s3_read_ok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() @@ -146,7 +143,7 @@ def test_s3_read_ok(mocked_s3_client): {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True, ) -@mock.patch("rok4.Storage.rados.Rados") +@mock.patch("rok4.storage.rados.Rados") def test_ceph_read_ok(mocked_rados_client): disconnect_ceph_clients() ioctx_instance = MagicMock() @@ -162,9 +159,8 @@ def test_ceph_read_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH read raises an exception: {exc}" - @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch("requests.get", side_effect={"status_code": 404}) +@mock.patch("requests.get", side_effect={"status_code":404}) def test_http_read_error(mock_http): with pytest.raises(StorageError): requests_instance = MagicMock() @@ -175,37 +171,34 @@ def test_http_read_error(mock_http): mock_http.assert_called_with("http://path/to/file.ext", stream=True) - @mock.patch.dict(os.environ, {}, clear=True) def test_http_read_range_error(): with pytest.raises(NotImplementedError): - data = get_data_binary("http://path/to/file.ext", (0, 100)) - + data = get_data_binary("http://path/to/file.ext", (0,100)) @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("requests.get") def test_http_read_ok(mock_http): - try: + try : requests_instance = MagicMock() - requests_instance.content = b"data" + requests_instance.content = b'data' mock_http.return_value = requests_instance data = get_data_str("http://path/to/file.ext") mock_http.assert_called_with("http://path/to/file.ext", stream=True) - assert data == "data" + assert data == 'data' except Exception as exc: assert False, f"HTTP read raises an exception: {exc}" ############ put_data_str - @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True, ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") def test_s3_write_nok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() @@ -221,7 +214,7 @@ def test_s3_write_nok(mocked_s3_client): {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True, ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") def test_s3_write_ok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() @@ -232,13 +225,12 @@ def test_s3_write_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 write raises an exception: {exc}" - @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True, ) -@mock.patch("rok4.Storage.rados.Rados") +@mock.patch("rok4.storage.rados.Rados") def test_ceph_write_ok(mocked_rados_client): disconnect_ceph_clients() ioctx_instance = MagicMock() @@ -258,8 +250,8 @@ def test_ceph_write_ok(mocked_rados_client): @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.makedirs", return_value=None) -@mock.patch("rok4.Storage.copyfile", return_value=None) -@mock.patch("rok4.Storage.hash_file", return_value="toto") +@mock.patch("rok4.storage.copyfile", return_value=None) +@mock.patch("rok4.storage.hash_file", return_value="toto") def test_copy_file_file_ok(mock_hash_file, mock_copyfile, mock_makedirs): try: copy("file:///path/to/source.ext", "file:///path/to/destination.ext", "toto") @@ -275,9 +267,9 @@ def test_copy_file_file_ok(mock_hash_file, mock_copyfile, mock_makedirs): {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True, ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") @mock.patch("os.makedirs", return_value=None) -@mock.patch("rok4.Storage.hash_file", return_value="toto") +@mock.patch("rok4.storage.hash_file", return_value="toto") def test_copy_s3_file_ok(mock_hash_file, mock_makedirs, mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() @@ -297,9 +289,9 @@ def test_copy_s3_file_ok(mock_hash_file, mock_makedirs, mocked_s3_client): {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True, ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") @mock.patch("os.makedirs", return_value=None) -@mock.patch("rok4.Storage.hash_file", return_value="toto") +@mock.patch("rok4.storage.hash_file", return_value="toto") def test_copy_s3_file_nok(mock_hash_file, mock_makedirs, mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() @@ -310,13 +302,12 @@ def test_copy_s3_file_nok(mock_hash_file, mock_makedirs, mocked_s3_client): copy("s3://bucket/source.ext", "file:///path/to/destination.ext", "toto") mock_makedirs.assert_called_once_with("/path/to", exist_ok=True) - @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True, ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") def test_copy_file_s3_ok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() @@ -329,13 +320,12 @@ def test_copy_file_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"FILE -> S3 copy raises an exception: {exc}" - @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True, ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") def test_copy_s3_s3_ok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() @@ -348,13 +338,12 @@ def test_copy_s3_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 -> S3 copy raises an exception: {exc}" - @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True, ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") def test_copy_s3_s3_intercluster_ok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() @@ -373,7 +362,7 @@ def test_copy_s3_s3_intercluster_ok(mocked_s3_client): {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True, ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") def test_copy_s3_s3_intercluster_nok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() @@ -384,13 +373,12 @@ def test_copy_s3_s3_intercluster_nok(mocked_s3_client): with pytest.raises(StorageError): copy("s3://bucket@a/source.ext", "s3://bucket@c/destination.ext", "toto") - @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True, ) -@mock.patch("rok4.Storage.rados.Rados") +@mock.patch("rok4.storage.rados.Rados") @mock.patch("os.makedirs", return_value=None) @patch("builtins.open", new_callable=mock_open) def test_copy_ceph_file_ok(mock_file, mock_makedirs, mocked_rados_client): @@ -411,13 +399,12 @@ def test_copy_ceph_file_ok(mock_file, mock_makedirs, mocked_rados_client): except Exception as exc: assert False, f"CEPH -> FILE copy raises an exception: {exc}" - @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True, ) -@mock.patch("rok4.Storage.rados.Rados") +@mock.patch("rok4.storage.rados.Rados") @patch("builtins.open", new_callable=mock_open, read_data=b"data") def test_copy_file_ceph_ok(mock_file, mocked_rados_client): disconnect_ceph_clients() @@ -436,13 +423,12 @@ def test_copy_file_ceph_ok(mock_file, mocked_rados_client): except Exception as exc: assert False, f"FILE -> CEPH copy raises an exception: {exc}" - @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True, ) -@mock.patch("rok4.Storage.rados.Rados") +@mock.patch("rok4.storage.rados.Rados") @patch("builtins.open", new_callable=mock_open, read_data=b"data") def test_copy_ceph_ceph_ok(mock_file, mocked_rados_client): disconnect_ceph_clients() @@ -475,8 +461,8 @@ def test_copy_ceph_ceph_ok(mock_file, mocked_rados_client): }, clear=True, ) -@mock.patch("rok4.Storage.rados.Rados") -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.rados.Rados") +@mock.patch("rok4.storage.boto3.client") @patch("builtins.open", new_callable=mock_open, read_data=b"data") def test_copy_ceph_s3_ok(mock_file, mocked_s3_client, mocked_rados_client): disconnect_ceph_clients() @@ -501,14 +487,14 @@ def test_copy_ceph_s3_ok(mock_file, mocked_s3_client, mocked_rados_client): except Exception as exc: assert False, f"CEPH -> S3 copy raises an exception: {exc}" - @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch("requests.get") -@patch("builtins.open", new_callable=mock_open) +@mock.patch('requests.get') +@patch('builtins.open', new_callable=mock_open) def test_copy_http_file_ok(mock_open, mock_requests): try: + http_instance = MagicMock() - http_instance.iter_content.return_value = ["data", "data2"] + http_instance.iter_content.return_value = ["data","data2"] mock_requests.return_value = http_instance copy("http://path/to/source.ext", "file:///path/to/destination.ext") @@ -517,20 +503,17 @@ def test_copy_http_file_ok(mock_open, mock_requests): except Exception as exc: assert False, f"HTTP -> FILE copy raises an exception: {exc}" - -@mock.patch.dict( - os.environ, - {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, - clear=True, -) -@mock.patch("rok4.Storage.rados.Rados") -@mock.patch("requests.get") +@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) +@mock.patch('rok4.storage.rados.Rados') +@mock.patch('requests.get') def test_copy_http_ceph_ok(mock_requests, mocked_rados_client): try: + http_instance = MagicMock() - http_instance.iter_content.return_value = ["data", "data2"] + http_instance.iter_content.return_value = ["data","data2"] mock_requests.return_value = http_instance + disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.write.return_value = None @@ -543,20 +526,16 @@ def test_copy_http_ceph_ok(mock_requests, mocked_rados_client): except Exception as exc: assert False, f"HTTP -> CEPH copy raises an exception: {exc}" - -@mock.patch.dict( - os.environ, - {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, - clear=True, -) -@mock.patch("rok4.Storage.boto3.client") -@mock.patch("requests.get") -@patch("tempfile.NamedTemporaryFile", new_callable=mock_open) -@mock.patch("os.remove") +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch('rok4.storage.boto3.client') +@mock.patch('requests.get') +@patch('tempfile.NamedTemporaryFile', new_callable=mock_open) +@mock.patch('os.remove') def test_copy_http_s3_ok(mock_remove, mock_tempfile, mock_requests, mocked_s3_client): try: + http_instance = MagicMock() - http_instance.iter_content.return_value = ["data", "data2"] + http_instance.iter_content.return_value = ["data","data2"] mock_requests.return_value = http_instance disconnect_s3_clients() @@ -567,7 +546,7 @@ def test_copy_http_s3_ok(mock_remove, mock_tempfile, mock_requests, mocked_s3_cl copy("http://path/to/source.ext", "s3://bucket/destination.ext") mock_requests.assert_called_once_with("http://path/to/source.ext", stream=True) - mock_tempfile.assert_called_once_with("w+b", delete=False) + mock_tempfile.assert_called_once_with("w+b",delete=False) except Exception as exc: assert False, f"HTTP -> CEPH copy raises an exception: {exc}" @@ -584,7 +563,6 @@ def test_link_hard_nok(): with pytest.raises(StorageError): link("ceph://pool1/source.ext", "ceph://pool2/destination.ext", True) - @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.symlink", return_value=None) def test_link_file_ok(mock_link): @@ -604,13 +582,12 @@ def test_hlink_file_ok(mock_link): except Exception as exc: assert False, f"FILE hard link raises an exception: {exc}" - @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True, ) -@mock.patch("rok4.Storage.rados.Rados") +@mock.patch("rok4.storage.rados.Rados") def test_link_ceph_ok(mocked_rados_client): disconnect_ceph_clients() ioctx_instance = MagicMock() @@ -630,7 +607,7 @@ def test_link_ceph_ok(mocked_rados_client): {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True, ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") def test_link_s3_ok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() @@ -642,13 +619,12 @@ def test_link_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 link raises an exception: {exc}" - @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True, ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") def test_link_s3_nok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() @@ -661,7 +637,6 @@ def test_link_s3_nok(mocked_s3_client): ############ get_size - @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.stat") def test_size_file_ok(mock_stat): @@ -672,13 +647,12 @@ def test_size_file_ok(mock_stat): except Exception as exc: assert False, f"FILE size raises an exception: {exc}" - @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True, ) -@mock.patch("rok4.Storage.rados.Rados") +@mock.patch("rok4.storage.rados.Rados") def test_size_ceph_ok(mocked_rados_client): disconnect_ceph_clients() ioctx_instance = MagicMock() @@ -693,13 +667,12 @@ def test_size_ceph_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH size raises an exception: {exc}" - @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True, ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") def test_size_s3_ok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() @@ -712,12 +685,12 @@ def test_size_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 size raises an exception: {exc}" - @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch("requests.get") +@mock.patch('requests.get') def test_size_http_ok(mock_requests): + http_instance = MagicMock() - http_instance.headers = {"content-length": 12} + http_instance.headers = {"content-length":12} mock_requests.return_value = http_instance try: @@ -744,13 +717,12 @@ def test_exists_file_ok(mock_exists): except Exception as exc: assert False, f"FILE not exists raises an exception: {exc}" - @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True, ) -@mock.patch("rok4.Storage.rados.Rados") +@mock.patch("rok4.storage.rados.Rados") def test_exists_ceph_ok(mocked_rados_client): disconnect_ceph_clients() ioctx_instance = MagicMock() @@ -776,7 +748,7 @@ def test_exists_ceph_ok(mocked_rados_client): {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True, ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") def test_exists_s3_ok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() @@ -796,10 +768,10 @@ def test_exists_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"CEPH not exists raises an exception: {exc}" - @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch("requests.get") +@mock.patch('requests.get') def test_exists_http_ok(mock_requests): + http_instance = MagicMock() http_instance.status_code = 200 mock_requests.return_value = http_instance @@ -820,7 +792,6 @@ def test_exists_http_ok(mock_requests): ############ remove - @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.remove") def test_remove_file_ok(mock_remove): @@ -836,13 +807,12 @@ def test_remove_file_ok(mock_remove): except Exception as exc: assert False, f"FILE deletion (not found) raises an exception: {exc}" - @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True, ) -@mock.patch("rok4.Storage.rados.Rados") +@mock.patch("rok4.storage.rados.Rados") def test_remove_ceph_ok(mocked_rados_client): disconnect_ceph_clients() ioctx_instance = MagicMock() @@ -862,13 +832,12 @@ def test_remove_ceph_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH deletion (not found) raises an exception: {exc}" - @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True, ) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch("rok4.storage.boto3.client") def test_remove_s3_ok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() @@ -892,7 +861,6 @@ def test_get_osgeo_path_file_ok(): except Exception as exc: assert False, f"FILE osgeo path raises an exception: {exc}" - @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -913,7 +881,6 @@ def test_get_osgeo_path_nok(): with pytest.raises(NotImplementedError): get_osgeo_path("ceph://pool/data.ext") - ############ size_path def test_size_path_file_ok(): try: @@ -922,31 +889,22 @@ def test_size_path_file_ok(): except Exception as exc: assert False, f"FILE size of the path raises an exception: {exc}" - def test_size_file_nok(): - with pytest.raises(StorageError): + with pytest.raises(StorageError) : size = size_path("file://tests/fixtures/TIFF_PBF_M") - -@mock.patch.dict( - os.environ, - {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, - clear=True, -) +@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) def test_size_path_ceph_nok(): + with pytest.raises(NotImplementedError): size = size_path("ceph://pool/path") - -@mock.patch.dict( - os.environ, - {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, - clear=True, -) -@mock.patch("rok4.Storage.boto3.client") +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch('rok4.storage.boto3.client') def test_size_path_s3_ok(mocked_s3_client): + disconnect_s3_clients() - pages = [{"Contents": [{"Size": 10}, {"Size": 20}]}, {"Contents": [{"Size": 50}]}] + pages = [{"Contents" : [{"Size" : 10},{"Size" : 20}]}, {"Contents" : [{"Size" : 50}]}] paginator = MagicMock() paginator.paginate.return_value = pages client = MagicMock() @@ -958,3 +916,4 @@ def test_size_path_s3_ok(mocked_s3_client): assert size == 80 except Exception as exc: assert False, f"S3 size of the path raises an exception: {exc}" + diff --git a/tests/test_TileMatrixSet.py b/tests/test_tile_matrix_set.py similarity index 93% rename from tests/test_TileMatrixSet.py rename to tests/test_tile_matrix_set.py index 5cf6062..b4e58a9 100644 --- a/tests/test_TileMatrixSet.py +++ b/tests/test_tile_matrix_set.py @@ -1,12 +1,10 @@ +import pytest import os -from unittest import mock from unittest.mock import * +from unittest import mock -import pytest - -from rok4.Exceptions import * -from rok4.TileMatrixSet import TileMatrixSet - +from rok4.tile_matrix_set import TileMatrixSet +from rok4.exceptions import * @mock.patch.dict(os.environ, {}, clear=True) def test_missing_env(): @@ -15,7 +13,7 @@ def test_missing_env(): @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) -@mock.patch("rok4.TileMatrixSet.get_data_str", side_effect=StorageError("FILE", "Not found")) +@mock.patch("rok4.tile_matrix_set.get_data_str", side_effect=StorageError("FILE", "Not found")) def test_wrong_file(mocked_get_data_str): with pytest.raises(StorageError): tms = TileMatrixSet("tms") @@ -23,7 +21,7 @@ def test_wrong_file(mocked_get_data_str): @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) @mock.patch( - "rok4.TileMatrixSet.get_data_str", + "rok4.tile_matrix_set.get_data_str", return_value='"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}', ) def test_bad_json(mocked_get_data_str): @@ -34,7 +32,7 @@ def test_bad_json(mocked_get_data_str): @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) @mock.patch( - "rok4.TileMatrixSet.get_data_str", + "rok4.tile_matrix_set.get_data_str", return_value='{"tileMatrices":[{"id":"0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"crs":"EPSG:3857","orderedAxes":["X","Y"]}', ) def test_missing_id(mocked_get_data_str): @@ -46,7 +44,7 @@ def test_missing_id(mocked_get_data_str): @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) @mock.patch( - "rok4.TileMatrixSet.get_data_str", + "rok4.tile_matrix_set.get_data_str", return_value='{"tileMatrices":[{"id":"0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"orderedAxes":["X","Y"],"id":"PM"}', ) def test_missing_crs(mocked_get_data_str): @@ -58,7 +56,7 @@ def test_missing_crs(mocked_get_data_str): @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) @mock.patch( - "rok4.TileMatrixSet.get_data_str", + "rok4.tile_matrix_set.get_data_str", return_value='{"crs":"epsg:123456","orderedAxes":["X","Y"],"tileMatrices":[{"id":"0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"orderedAxes":["X","Y"],"id":"PM"}', ) def test_wrong_crs(mocked_get_data_str): @@ -73,7 +71,7 @@ def test_wrong_crs(mocked_get_data_str): @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) @mock.patch( - "rok4.TileMatrixSet.get_data_str", + "rok4.tile_matrix_set.get_data_str", return_value='{"crs":"epsg:4326","orderedAxes":["Lat","Lon"],"tileMatrices":[{"id":"0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"id":"PM"}', ) def test_wrong_axes_order(mocked_get_data_str): @@ -88,7 +86,7 @@ def test_wrong_axes_order(mocked_get_data_str): @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) @mock.patch( - "rok4.TileMatrixSet.get_data_str", + "rok4.tile_matrix_set.get_data_str", return_value='{"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}', ) def test_missing_levels(mocked_get_data_str): @@ -100,7 +98,7 @@ def test_missing_levels(mocked_get_data_str): @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) @mock.patch( - "rok4.TileMatrixSet.get_data_str", + "rok4.tile_matrix_set.get_data_str", return_value='{"tileMatrices":[],"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}', ) def test_no_levels(mocked_get_data_str): @@ -112,7 +110,7 @@ def test_no_levels(mocked_get_data_str): @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) @mock.patch( - "rok4.TileMatrixSet.get_data_str", + "rok4.tile_matrix_set.get_data_str", return_value='{"tileMatrices":[{"tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"orderedAxes":["X","Y"],"id":"PM","crs":"EPSG:3857"}', ) def test_wrong_level(mocked_get_data_str): @@ -124,7 +122,7 @@ def test_wrong_level(mocked_get_data_str): @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) @mock.patch( - "rok4.TileMatrixSet.get_data_str", + "rok4.tile_matrix_set.get_data_str", return_value='{"tileMatrices":[{"id":"level_0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}', ) def test_wrong_level_id(mocked_get_data_str): @@ -140,7 +138,7 @@ def test_wrong_level_id(mocked_get_data_str): @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) @mock.patch( - "rok4.TileMatrixSet.get_data_str", + "rok4.tile_matrix_set.get_data_str", return_value='{"tileMatrices":[{"id":"0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}', ) def test_ok(mocked_get_data_str): @@ -155,7 +153,7 @@ def test_ok(mocked_get_data_str): @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) @mock.patch( - "rok4.TileMatrixSet.get_data_str", + "rok4.tile_matrix_set.get_data_str", return_value='{"tileMatrices":[{"id":"17","cellSize":1.19432856695588,"matrixHeight":131072,"pointOfOrigin":[-20037508.3427892,20037508.3427892],"tileHeight":256,"tileWidth":256,"scaleDenominator":4265.45916769957,"matrixWidth":131072}],"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}', ) def test_pm_conversions(mocked_get_data_str): @@ -180,7 +178,7 @@ def test_pm_conversions(mocked_get_data_str): @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) @mock.patch( - "rok4.TileMatrixSet.get_data_str", + "rok4.tile_matrix_set.get_data_str", return_value='{"crs":"EPSG:4326","tileMatrices":[{"tileWidth":256,"scaleDenominator":1066.36480348451,"matrixWidth":524288,"cellSize":2.68220901489258e-06,"matrixHeight":262144,"pointOfOrigin":[-180,90],"tileHeight":256,"id":"18"}],"orderedAxes":["Lon","Lat"],"id":"4326"}', ) def test_4326_conversions(mocked_get_data_str): diff --git a/tests/test_Utils.py b/tests/test_utils.py similarity index 90% rename from tests/test_Utils.py rename to tests/test_utils.py index 0f5d67d..37941aa 100644 --- a/tests/test_Utils.py +++ b/tests/test_utils.py @@ -1,15 +1,14 @@ -import math -import os -import random -from unittest import mock -from unittest.mock import * - import pytest +import os from osgeo import gdal, osr +import math +import random -from rok4.Exceptions import * -from rok4.Utils import * +from unittest.mock import * +from unittest import mock +from rok4.utils import * +from rok4.exceptions import * def test_srs_to_spatialreference_ignf_ok(): try: @@ -74,7 +73,7 @@ def test_reproject_point_ok(): assert False, f"Bbox reprojection raises an exception: {exc}" -# Tests for the rok4.Utils.compute_bbox function. +# Tests for the rok4.utils.compute_bbox function. def test_compute_bbox_epsg_3857_ok(): @@ -215,13 +214,13 @@ def test_compute_bbox_no_srs_ok(): assert False, f"Bbox computation raises an exception: {exc}" -# Tests for the rok4.Utils.compute_format function. +# Tests for the rok4.utils.compute_format function. -@mock.patch("rok4.Utils.gdal.Info") -@mock.patch("rok4.Utils.gdal.GetColorInterpretationName", return_value="Palette") -@mock.patch("rok4.Utils.gdal.GetDataTypeSize", return_value=8) -@mock.patch("rok4.Utils.gdal.GetDataTypeName", return_value="Byte") +@mock.patch("rok4.utils.gdal.Info") +@mock.patch("rok4.utils.gdal.GetColorInterpretationName", return_value="Palette") +@mock.patch("rok4.utils.gdal.GetDataTypeSize", return_value=8) +@mock.patch("rok4.utils.gdal.GetDataTypeName", return_value="Byte") def test_compute_format_bit_ok( mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info ): @@ -247,10 +246,10 @@ def test_compute_format_bit_ok( assert False, f"Color format computation raises an exception: {exc}" -@mock.patch("rok4.Utils.gdal.Info") -@mock.patch("rok4.Utils.gdal.GetColorInterpretationName") -@mock.patch("rok4.Utils.gdal.GetDataTypeSize", return_value=8) -@mock.patch("rok4.Utils.gdal.GetDataTypeName", return_value="Byte") +@mock.patch("rok4.utils.gdal.Info") +@mock.patch("rok4.utils.gdal.GetColorInterpretationName") +@mock.patch("rok4.utils.gdal.GetDataTypeSize", return_value=8) +@mock.patch("rok4.utils.gdal.GetDataTypeName", return_value="Byte") def test_compute_format_uint8_ok( mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info ): @@ -284,10 +283,10 @@ def test_compute_format_uint8_ok( assert False, f"Color format computation raises an exception: {exc}" -@mock.patch("rok4.Utils.gdal.Info") -@mock.patch("rok4.Utils.gdal.GetColorInterpretationName") -@mock.patch("rok4.Utils.gdal.GetDataTypeSize", return_value=32) -@mock.patch("rok4.Utils.gdal.GetDataTypeName", return_value="Float32") +@mock.patch("rok4.utils.gdal.Info") +@mock.patch("rok4.utils.gdal.GetColorInterpretationName") +@mock.patch("rok4.utils.gdal.GetDataTypeSize", return_value=32) +@mock.patch("rok4.utils.gdal.GetDataTypeName", return_value="Float32") def test_compute_format_float32_ok( mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info ): @@ -321,10 +320,10 @@ def test_compute_format_float32_ok( assert False, f"Color format computation raises an exception: {exc}" -@mock.patch("rok4.Utils.gdal.Info") -@mock.patch("rok4.Utils.gdal.GetColorInterpretationName") -@mock.patch("rok4.Utils.gdal.GetDataTypeSize", return_value=16) -@mock.patch("rok4.Utils.gdal.GetDataTypeName", return_value="UInt16") +@mock.patch("rok4.utils.gdal.Info") +@mock.patch("rok4.utils.gdal.GetColorInterpretationName") +@mock.patch("rok4.utils.gdal.GetDataTypeSize", return_value=16) +@mock.patch("rok4.utils.gdal.GetDataTypeName", return_value="UInt16") def test_compute_format_unsupported_nok( mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info ): @@ -358,10 +357,10 @@ def test_compute_format_unsupported_nok( assert False, f"Color format computation raises an exception: {exc}" -@mock.patch("rok4.Utils.gdal.Info") -@mock.patch("rok4.Utils.gdal.GetColorInterpretationName") -@mock.patch("rok4.Utils.gdal.GetDataTypeSize", return_value=16) -@mock.patch("rok4.Utils.gdal.GetDataTypeName", return_value="UInt16") +@mock.patch("rok4.utils.gdal.Info") +@mock.patch("rok4.utils.gdal.GetColorInterpretationName") +@mock.patch("rok4.utils.gdal.GetDataTypeSize", return_value=16) +@mock.patch("rok4.utils.gdal.GetDataTypeName", return_value="UInt16") def test_compute_format_no_band_nok( mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info ): diff --git a/tests/test_Vector.py b/tests/test_vector.py similarity index 92% rename from tests/test_Vector.py rename to tests/test_vector.py index 3fe0a1c..513c559 100644 --- a/tests/test_Vector.py +++ b/tests/test_vector.py @@ -1,14 +1,11 @@ -#!/usr/bin/env python3 +import pytest import os -from unittest import mock from unittest.mock import * +from unittest import mock -import pytest - -from rok4.Exceptions import * -from rok4.Storage import disconnect_ceph_clients -from rok4.Vector import * - +from rok4.vector import * +from rok4.exceptions import * +from rok4.storage import disconnect_ceph_clients @mock.patch.dict(os.environ, {}, clear=True) def test_missing_env(): @@ -17,7 +14,7 @@ def test_missing_env(): vector = Vector.from_file("ceph:///ign_std/vector.shp") -@mock.patch("rok4.Vector.copy", side_effect=StorageError("CEPH", "Not found")) +@mock.patch("rok4.vector.copy", side_effect=StorageError("CEPH", "Not found")) def test_wrong_file(mocked_copy): with pytest.raises(StorageError): vector = Vector.from_file("ceph:///vector.geojson") @@ -29,15 +26,15 @@ def test_wrong_format(): assert str(exc.value) == "This format of file cannot be loaded" -@mock.patch("rok4.Vector.ogr.Open", return_value="not a shape") +@mock.patch("rok4.vector.ogr.Open", return_value="not a shape") def test_wrong_content(mocked_copy): with pytest.raises(Exception) as exc: vector = Vector.from_file("file:///vector.shp") assert str(exc.value) == "The content of file:///vector.shp cannot be read" -@mock.patch("rok4.Vector.copy") -@mock.patch("rok4.Vector.ogr.Open", return_value="not a shape") +@mock.patch("rok4.vector.copy") +@mock.patch("rok4.vector.ogr.Open", return_value="not a shape") def test_wrong_content_ceph(mocked_open, mocked_copy): with pytest.raises(Exception) as exc: vector = Vector.from_file("file:///vector.shp") From bc67460466fa4d5263990927ee7675d091fdc2e2 Mon Sep 17 00:00:00 2001 From: Theo Satabin Date: Tue, 18 Jul 2023 23:24:26 +0200 Subject: [PATCH 66/89] Suppression de l'ancien fichier des classes Pyramid et Level --- src/rok4/Pyramid.py | 1385 ------------------------------------------- src/rok4/pyramid.py | 4 +- 2 files changed, 3 insertions(+), 1386 deletions(-) delete mode 100644 src/rok4/Pyramid.py diff --git a/src/rok4/Pyramid.py b/src/rok4/Pyramid.py deleted file mode 100644 index 19fa9c7..0000000 --- a/src/rok4/Pyramid.py +++ /dev/null @@ -1,1385 +0,0 @@ -"""Provide classes to use pyramid's data. - -The module contains the following classes: - -- `Pyramid` - Data container -- `Level` - Level of a pyramid -""" - -from typing import Dict, List, Tuple, Union, Iterator -import json -from json.decoder import JSONDecodeError -import os -import re -import numpy -import zlib -import io -import mapbox_vector_tile -from PIL import Image - -from rok4.Exceptions import * -from rok4.tile_matrix_set import TileMatrixSet, TileMatrix -from rok4.storage import * -from rok4.utils import * -from rok4.enums import StorageType - - -class PyramidType(Enum): - """Pyramid's data type""" - - RASTER = "RASTER" - VECTOR = "VECTOR" - - -class SlabType(Enum): - """Slab's type""" - - DATA = "DATA" # Slab of data, raster or vector - MASK = "MASK" # Slab of mask, only for raster pyramid, image with one band : 0 is nodata, other values are data - - -ROK4_IMAGE_HEADER_SIZE = 2048 -"""Slab's header size, 2048 bytes""" - - -def b36_number_encode(number: int) -> str: - """Convert base-10 number to base-36 - - Used alphabet is '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' - - Args: - number (int): base-10 number - - Returns: - str: base-36 number - """ - - alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" - - base36 = "" - - if 0 <= number < len(alphabet): - return alphabet[number] - - while number != 0: - number, i = divmod(number, len(alphabet)) - base36 = alphabet[i] + base36 - - return base36 - - -def b36_number_decode(number: str) -> int: - """Convert base-36 number to base-10 - - Args: - number (str): base-36 number - - Returns: - int: base-10 number - """ - return int(number, 36) - - -def b36_path_decode(path: str) -> Tuple[int, int]: - """Get slab's column and row from a base-36 based path - - Args: - path (str): slab's path - - Returns: - Tuple[int, int]: slab's column and row - """ - - path = path.replace("/", "") - path = re.sub(r"(\.TIFF?)", "", path.upper()) - - b36_column = "" - b36_row = "" - - while len(path) > 0: - b36_column += path[0] - b36_row += path[1] - path = path[2:] - - return b36_number_decode(b36_column), b36_number_decode(b36_row) - - -def b36_path_encode(column: int, row: int, slashs: int) -> str: - """Convert slab indices to base-36 based path, with .tif extension - - Args: - column (int): slab's column - row (int): slab's row - slashs (int): slashs' number (to split path) - - Returns: - str: base-36 based path - """ - - b36_column = b36_number_encode(column) - b36_row = b36_number_encode(row) - - max_len = max(slashs + 1, len(b36_column), len(b36_row)) - - b36_column = b36_column.rjust(max_len, "0") - b36_row = b36_row.rjust(max_len, "0") - - b36_path = "" - - while len(b36_column) > 0: - b36_path = b36_row[-1] + b36_path - b36_path = b36_column[-1] + b36_path - - b36_column = b36_column[:-1] - b36_row = b36_row[:-1] - - if slashs > 0: - b36_path = "/" + b36_path - slashs -= 1 - - return f"{b36_path}.tif" - - -class Level: - """A pyramid's level, raster or vector - - Attributes: - __id (str): level's identifier. have to exist in the pyramid's used TMS - __tile_limits (Dict[str, int]): minimum and maximum tiles' columns and rows of pyramid's content - __slab_size (Tuple[int, int]): number of tile in a slab, widthwise and heightwise - __tables (List[Dict]): for a VECTOR pyramid, description of vector content, tables and attributes - """ - - @classmethod - def from_descriptor(cls, data: Dict, pyramid: "Pyramid") -> "Level": - """Create a pyramid's level from the pyramid's descriptor levels element - - Args: - data (Dict): level's information from the pyramid's descriptor - pyramid (Pyramid): pyramid containing the level to create - - Raises: - Exception: different storage or masks presence between the level and the pyramid - MissingAttributeError: Attribute is missing in the content - - Returns: - Pyramid: a Level instance - """ - level = cls() - - level.__pyramid = pyramid - - # Attributs communs - try: - level.__id = data["id"] - level.__tile_limits = data["tile_limits"] - level.__slab_size = ( - data["tiles_per_width"], - data["tiles_per_height"], - ) - - # Informations sur le stockage : on les valide et stocke dans la pyramide - if pyramid.storage_type.name != data["storage"]["type"]: - raise Exception( - f"Pyramid {pyramid.descriptor} owns levels using different storage types ({ data['storage']['type'] }) than its one ({pyramid.storage_type.name})" - ) - - if pyramid.storage_type == StorageType.FILE: - pyramid.storage_depth = data["storage"]["path_depth"] - - if "mask_directory" in data["storage"] or "mask_prefix" in data["storage"]: - if not pyramid.own_masks: - raise Exception( - f"Pyramid {pyramid.__descriptor} does not define a mask format but level {level.__id} define mask storage informations" - ) - else: - if pyramid.own_masks: - raise Exception( - f"Pyramid {pyramid.__descriptor} define a mask format but level {level.__id} does not define mask storage informations" - ) - - except KeyError as e: - raise MissingAttributeError(pyramid.descriptor, f"levels[].{e}") - - # Attributs dans le cas d'un niveau vecteur - if level.__pyramid.type == PyramidType.VECTOR: - try: - level.__tables = data["tables"] - - except KeyError as e: - raise MissingAttributeError(pyramid.descriptor, f"levels[].{e}") - - return level - - @classmethod - def from_other(cls, other: "Level", pyramid: "Pyramid") -> "Level": - """Create a pyramid's level from another one - - Args: - other (Level): level to clone - pyramid (Pyramid): new pyramid containing the new level - - Raises: - Exception: different storage or masks presence between the level and the pyramid - MissingAttributeError: Attribute is missing in the content - - Returns: - Pyramid: a Level instance - """ - - level = cls() - - # Attributs communs - level.__id = other.__id - level.__pyramid = pyramid - level.__tile_limits = other.__tile_limits - level.__slab_size = other.__slab_size - - # Attributs dans le cas d'un niveau vecteur - if level.__pyramid.type == PyramidType.VECTOR: - level.__tables = other.__tables - - return level - - def __str__(self) -> str: - return f"{self.__pyramid.type.name} pyramid's level '{self.__id}' ({self.__pyramid.storage_type.name} storage)" - - @property - def serializable(self) -> Dict: - """Get the dict version of the pyramid object, pyramid's descriptor compliant - - Returns: - Dict: pyramid's descriptor structured object description - """ - serialization = { - "id": self.__id, - "tiles_per_width": self.__slab_size[0], - "tiles_per_height": self.__slab_size[1], - "tile_limits": self.__tile_limits, - } - - if self.__pyramid.type == PyramidType.VECTOR: - serialization["tables"] = self.__tables - - if self.__pyramid.storage_type == StorageType.FILE: - serialization["storage"] = { - "type": "FILE", - "image_directory": f"{self.__pyramid.name}/DATA/{self.__id}", - "path_depth": self.__pyramid.storage_depth, - } - if self.__pyramid.own_masks: - serialization["storage"][ - "mask_directory" - ] = f"{self.__pyramid.name}/MASK/{self.__id}" - - elif self.__pyramid.storage_type == StorageType.CEPH: - serialization["storage"] = { - "type": "CEPH", - "image_prefix": f"{self.__pyramid.name}/DATA_{self.__id}", - "pool_name": self.__pyramid.storage_root, - } - if self.__pyramid.own_masks: - serialization["storage"]["mask_prefix"] = f"{self.__pyramid.name}/MASK_{self.__id}" - - elif self.__pyramid.storage_type == StorageType.S3: - serialization["storage"] = { - "type": "S3", - "image_prefix": f"{self.__pyramid.name}/DATA_{self.__id}", - "bucket_name": self.__pyramid.storage_root, - } - if self.__pyramid.own_masks: - serialization["storage"]["mask_prefix"] = f"{self.__pyramid.name}/MASK_{self.__id}" - - return serialization - - @property - def id(self) -> str: - return self.__id - - @property - def bbox(self) -> Tuple[float, float, float, float]: - """Return level extent, based on tile limits - - Returns: - Tuple[float, float, float, float]: level terrain extent (xmin, ymin, xmax, ymax) - """ - - min_bbox = self.__pyramid.tms.get_level(self.__id).tile_to_bbox( - self.__tile_limits["min_col"], self.__tile_limits["max_row"] - ) - max_bbox = self.__pyramid.tms.get_level(self.__id).tile_to_bbox( - self.__tile_limits["max_col"], self.__tile_limits["min_row"] - ) - - return (min_bbox[0], min_bbox[1], max_bbox[2], max_bbox[3]) - - @property - def resolution(self) -> str: - return self.__pyramid.tms.get_level(self.__id).resolution - - @property - def tile_matrix(self) -> TileMatrix: - return self.__pyramid.tms.get_level(self.__id) - - @property - def slab_width(self) -> int: - return self.__slab_size[0] - - @property - def slab_height(self) -> int: - return self.__slab_size[1] - - def is_in_limits(self, column: int, row: int) -> bool: - """Is the tile indices in limits ? - - Args: - column (int): tile's column - row (int): tile's row - - Returns: - bool: True if tiles' limits contain the provided tile's indices - """ - return ( - self.__tile_limits["min_row"] <= row - and self.__tile_limits["max_row"] >= row - and self.__tile_limits["min_col"] <= column - and self.__tile_limits["max_col"] >= column - ) - - def set_limits_from_bbox(self, bbox: Tuple[float, float, float, float]) -> None: - """Set tile limits, based on provided bounding box - - Args: - bbox (Tuple[float, float, float, float]): terrain extent (xmin, ymin, xmax, ymax), in TMS coordinates system - - """ - - col_min, row_min, col_max, row_max = self.__pyramid.tms.get_level(self.__id).bbox_to_tiles( - bbox - ) - self.__tile_limits = { - "min_row": row_min, - "max_col": col_max, - "max_row": row_max, - "min_col": col_min, - } - - -class Pyramid: - - """A data pyramid, raster or vector - - Attributes: - __name (str): pyramid's name - __descriptor (str): pyramid's descriptor path - __list (str): pyramid's list path - __tms (rok4.tile_matrix_set.TileMatrixSet): Used grid - __levels (Dict[str, Level]): Pyramid's levels - __format (str): Data format - __storage (Dict[str, Union[rok4.storage.StorageType,str,int]]): Pyramid's storage informations (type, root and depth if FILE storage) - __raster_specifications (Dict): If raster pyramid, raster specifications - __content (Dict): Loading status (loaded) and list content (cache). - - Example (S3 storage): - - { - 'cache': { - (, '18', 5424, 7526): { - 'link': False, - 'md5': None, - 'root': 'pyramids@localhost:9000/LIMADM', - 'slab': 'DATA_18_5424_7526' - } - }, - 'loaded': True - } - """ - - @classmethod - def from_descriptor(cls, descriptor: str) -> "Pyramid": - """Create a pyramid from its descriptor - - Args: - descriptor (str): pyramid's descriptor path - - Raises: - FormatError: Provided path or the TMS is not a well formed JSON - Exception: Level issue : no one in the pyramid or the used TMS, or level ID not defined in the TMS - MissingAttributeError: Attribute is missing in the content - StorageError: Storage read issue (pyramid descriptor or TMS) - MissingEnvironmentError: Missing object storage informations or TMS root directory - - Examples: - - S3 stored descriptor - - from rok4.pyramid import Pyramid - - try: - pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") - except Exception as e: - print("Cannot load the pyramid from its descriptor") - - Returns: - Pyramid: a Pyramid instance - """ - try: - data = json.loads(get_data_str(descriptor)) - - except JSONDecodeError as e: - raise FormatError("JSON", descriptor, e) - - pyramid = cls() - - pyramid.__storage["type"], path, pyramid.__storage["root"], base_name = get_infos_from_path( - descriptor - ) - pyramid.__name = base_name[:-5] # on supprime l'extension.json - pyramid.__descriptor = descriptor - pyramid.__list = get_path_from_infos( - pyramid.__storage["type"], pyramid.__storage["root"], f"{pyramid.__name}.list" - ) - - try: - # Attributs communs - pyramid.__tms = TileMatrixSet(data["tile_matrix_set"]) - pyramid.__format = data["format"] - - # Attributs d'une pyramide raster - if pyramid.type == PyramidType.RASTER: - pyramid.__raster_specifications = data["raster_specifications"] - - if "mask_format" in data: - pyramid.__masks = True - else: - pyramid.__masks = False - - # Niveaux - for l in data["levels"]: - lev = Level.from_descriptor(l, pyramid) - pyramid.__levels[lev.id] = lev - - if pyramid.__tms.get_level(lev.id) is None: - raise Exception( - f"Pyramid {descriptor} owns a level with the ID '{lev.id}', not defined in the TMS '{pyramid.tms.name}'" - ) - - except KeyError as e: - raise MissingAttributeError(descriptor, e) - - if len(pyramid.__levels.keys()) == 0: - raise Exception(f"Pyramid '{descriptor}' has no level") - - return pyramid - - @classmethod - def from_other(cls, other: "Pyramid", name: str, storage: Dict) -> "Pyramid": - """Create a pyramid from another one - - Args: - other (Pyramid): pyramid to clone - name (str): new pyramid's name - storage (Dict[str, Union[str, int]]): new pyramid's storage informations - - Raises: - FormatError: Provided path or the TMS is not a well formed JSON - Exception: Level issue : no one in the pyramid or the used TMS, or level ID not defined in the TMS - MissingAttributeError: Attribute is missing in the content - - Returns: - Pyramid: a Pyramid instance - """ - try: - # On convertit le type de stockage selon l'énumération - storage["type"] = StorageType[storage["type"]] - - if storage["type"] == StorageType.FILE and name.find("/") != -1: - raise Exception(f"A FILE stored pyramid's name cannot contain '/' : '{name}'") - - if storage["type"] == StorageType.FILE and "depth" not in storage: - storage["depth"] = 2 - - pyramid = cls() - - # Attributs communs - pyramid.__name = name - pyramid.__storage = storage - pyramid.__masks = other.__masks - - pyramid.__descriptor = get_path_from_infos( - pyramid.__storage["type"], pyramid.__storage["root"], f"{pyramid.__name}.json" - ) - pyramid.__list = get_path_from_infos( - pyramid.__storage["type"], pyramid.__storage["root"], f"{pyramid.__name}.list" - ) - pyramid.__tms = other.__tms - pyramid.__format = other.__format - - # Attributs d'une pyramide raster - if pyramid.type == PyramidType.RASTER: - if other.own_masks: - pyramid.__masks = True - else: - pyramid.__masks = False - pyramid.__raster_specifications = other.__raster_specifications - - # Niveaux - for l in other.__levels.values(): - lev = Level.from_other(l, pyramid) - pyramid.__levels[lev.id] = lev - - except KeyError as e: - raise MissingAttributeError(descriptor, e) - - return pyramid - - def __init__(self) -> None: - self.__storage = {} - self.__levels = {} - self.__masks = None - - self.__content = {"loaded": False, "cache": {}} - - def __str__(self) -> str: - return f"{self.type.name} pyramid '{self.__name}' ({self.__storage['type'].name} storage)" - - @property - def serializable(self) -> Dict: - """Get the dict version of the pyramid object, descriptor compliant - - Returns: - Dict: descriptor structured object description - """ - - serialization = { - "tile_matrix_set": self.__tms.name, - "format": self.__format - } - - serialization["levels"] = [] - sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution, reverse=True) - - for l in sorted_levels: - serialization["levels"].append(l.serializable) - - if self.type == PyramidType.RASTER: - serialization["raster_specifications"] = self.__raster_specifications - - if self.__masks: - serialization["mask_format"] = "TIFF_ZIP_UINT8" - - return serialization - - @property - def list(self) -> str: - return self.__list - - @property - def descriptor(self) -> str: - return self.__descriptor - - @property - def name(self) -> str: - return self.__name - - @property - def tms(self) -> TileMatrixSet: - return self.__tms - - @property - def raster_specifications(self) -> Dict: - """Get raster specifications for a RASTER pyramid - - Example: - { - "channels": 3, - "nodata": "255,0,0", - "photometric": "rgb", - "interpolation": "bicubic" - } - - Returns: - Dict: Raster specifications, None if VECTOR pyramid - """ - return self.__raster_specifications - - @property - def storage_type(self) -> StorageType: - """Get the storage type - - Returns: - StorageType: FILE, S3 or CEPH - """ - return self.__storage["type"] - - @property - def storage_root(self) -> str: - """Get the pyramid's storage root. - - If storage is S3, the used cluster is removed. - - Returns: - str: Pyramid's storage root - """ - - return self.__storage["root"].split("@", 1)[ - 0 - ] # Suppression de l'éventuel hôte de spécification du cluster S3 - - @property - def storage_depth(self) -> int: - return self.__storage.get("depth", None) - - @property - def storage_s3_cluster(self) -> str: - """Get the pyramid's storage S3 cluster (host name) - - Returns: - str: the host if known, None if the default one have to be used or if storage is not S3 - """ - if self.__storage["type"] == StorageType.S3: - try: - return self.__storage["root"].split("@")[1] - except IndexError: - return None - else: - return None - - @storage_depth.setter - def storage_depth(self, d: int) -> None: - """Set the tree depth for a FILE storage - - Args: - d (int): file storage depth - - Raises: - Exception: the depth is not equal to the already known depth - """ - if "depth" in self.__storage and self.__storage["depth"] != d: - raise Exception( - f"Pyramid {pyramid.__descriptor} owns levels with different path depths" - ) - self.__storage["depth"] = d - - @property - def own_masks(self) -> bool: - return self.__masks - - @property - def format(self) -> str: - return self.__format - - @property - def tile_extension(self) -> str: - - if self.__format in [ - "TIFF_RAW_UINT8", - "TIFF_LZW_UINT8", - "TIFF_ZIP_UINT8", - "TIFF_PKB_UINT8", - "TIFF_RAW_FLOAT32", - "TIFF_LZW_FLOAT32", - "TIFF_ZIP_FLOAT32", - "TIFF_PKB_FLOAT32", - ]: - return "tif" - elif self.__format in ["TIFF_JPG_UINT8", "TIFF_JPG90_UINT8"]: - return "jpg" - elif self.__format == "TIFF_PNG_UINT8": - return "png" - elif self.__format == "TIFF_PBF_MVT": - return "pbf" - else: - raise Exception( - f"Unknown pyramid's format ({self.__format}), cannot return the tile extension" - ) - - @property - def bottom_level(self) -> "Level": - """Get the best resolution level in the pyramid - - Returns: - Level: the bottom level - """ - return sorted(self.__levels.values(), key=lambda l: l.resolution)[0] - - @property - def top_level(self) -> "Level": - """Get the low resolution level in the pyramid - - Returns: - Level: the top level - """ - return sorted(self.__levels.values(), key=lambda l: l.resolution)[-1] - - @property - def type(self) -> PyramidType: - """Get the pyramid's type (RASTER or VECTOR) from its format - - Returns: - PyramidType: RASTER or VECTOR - """ - if self.__format == "TIFF_PBF_MVT": - return PyramidType.VECTOR - else: - return PyramidType.RASTER - - def load_list(self) -> None: - """Load list content and cache it - - If list is already loaded, nothing done - """ - if self.__content["loaded"]: - return - - for slab, infos in self.list_generator(): - self.__content["cache"][slab] = infos - - self.__content["loaded"] = True - - def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict]]: - """Get list content - - List is copied as temporary file, roots are read and informations about each slab is returned. If list is already loaded, we yield the cached content - - Examples: - - S3 stored descriptor - - from rok4.pyramid import Pyramid - - try: - pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") - - for (slab_type, level, column, row), infos in pyramid.list_generator(): - print(infos) - - except Exception as e: - print("Cannot load the pyramid from its descriptor and read the list") - - Yields: - Iterator[Tuple[Tuple[SlabType,str,int,int], Dict]]: Slab indices and storage informations - - Value example: - - ( - (, '18', 5424, 7526), - { - 'link': False, - 'md5': None, - 'root': 'pyramids@localhost:9000/LIMADM', - 'slab': 'DATA_18_5424_7526' - } - ) - - """ - if self.__content["loaded"]: - for slab, infos in self.__content["cache"].items(): - yield slab, infos - else: - # Copie de la liste dans un fichier temporaire (cette liste peut être un objet) - list_obj = tempfile.NamedTemporaryFile(mode="r", delete=False) - list_file = list_obj.name - copy(self.__list, f"file://{list_file}") - list_obj.close() - - roots = {} - s3_cluster = self.storage_s3_cluster - - with open(list_file, "r") as listin: - # Lecture des racines - for line in listin: - line = line.rstrip() - - if line == "#": - break - - root_id, root_path = line.split("=", 1) - - if s3_cluster is None: - roots[root_id] = root_path - else: - # On a un nom de cluster S3, on l'ajoute au nom du bucket dans les racines - root_bucket, root_path = root_path.split("/", 1) - roots[root_id] = f"{root_bucket}@{s3_cluster}/{root_path}" - - # Lecture des dalles - for line in listin: - line = line.rstrip() - - parts = line.split(" ", 1) - slab_path = parts[0] - slab_md5 = None - if len(parts) == 2: - slab_md5 = parts[1] - - root_id, slab_path = slab_path.split("/", 1) - - slab_type, level, column, row = self.get_infos_from_slab_path(slab_path) - infos = { - "root": roots[root_id], - "link": root_id != "0", - "slab": slab_path, - "md5": slab_md5, - } - - yield ((slab_type, level, column, row), infos) - - remove(f"file://{list_file}") - - def get_level(self, level_id: str) -> "Level": - """Get one level according to its identifier - - Args: - level_id: Level identifier - - Returns: - The corresponding pyramid's level, None if not present - """ - - return self.__levels.get(level_id, None) - - def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: - """Get sorted levels in the provided range from bottom to top - - Args: - bottom_id (str, optionnal): specific bottom level id. Defaults to None. - top_id (str, optionnal): specific top level id. Defaults to None. - - Raises: - Exception: Provided levels are not consistent (bottom > top or not in the pyramid) - - Examples: - - All levels - - from rok4.pyramid import Pyramid - - try: - pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") - levels = pyramid.get_levels() - - except Exception as e: - print("Cannot load the pyramid from its descriptor and get levels") - - From pyramid's bottom to provided top (level 5) - - from rok4.pyramid import Pyramid - - try: - pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") - levels = pyramid.get_levels(None, "5") - - except Exception as e: - print("Cannot load the pyramid from its descriptor and get levels") - - Returns: - List[Level]: asked sorted levels - """ - - sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution) - - levels = [] - - begin = False - if bottom_id is None: - # Pas de niveau du bas fourni, on commence tout en bas - begin = True - else: - if self.get_level(bottom_id) is None: - raise Exception( - f"Pyramid {self.name} does not contain the provided bottom level {bottom_id}" - ) - - if top_id is not None and self.get_level(top_id) is None: - raise Exception(f"Pyramid {self.name} does not contain the provided top level {top_id}") - - end = False - - for l in sorted_levels: - if not begin and l.id == bottom_id: - begin = True - - if begin: - levels.append(l) - if top_id is not None and l.id == top_id: - end = True - break - else: - continue - - if top_id is None: - # Pas de niveau du haut fourni, on a été jusqu'en haut et c'est normal - end = True - - if not begin or not end: - raise Exception( - f"Provided levels ids are not consistent to extract levels from the pyramid {self.name}" - ) - - return levels - - def write_descriptor(self) -> None: - """Write the pyramid's descriptor to the final location (in the pyramid's storage root)""" - - content = json.dumps(self.serializable) - put_data_str(content, self.__descriptor) - - def get_infos_from_slab_path(self, path: str) -> Tuple[SlabType, str, int, int]: - """Get the slab's indices from its storage path - - Args: - path (str): Slab's storage path - - Examples: - - FILE stored pyramid - - from rok4.pyramid import Pyramid - - try: - pyramid = Pyramid.from_descriptor("/path/to/descriptor.json") - slab_type, level, column, row = self.get_infos_from_slab_path("DATA/12/00/4A/F7.tif") - # (SlabType.DATA, "12", 159, 367) - except Exception as e: - print("Cannot load the pyramid from its descriptor and convert a slab path") - - S3 stored pyramid - - from rok4.pyramid import Pyramid - - try: - pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/pyramid.json") - slab_type, level, column, row = self.get_infos_from_slab_path("s3://bucket_name/path/to/pyramid/MASK_15_9164_5846") - # (SlabType.MASK, "15", 9164, 5846) - except Exception as e: - print("Cannot load the pyramid from its descriptor and convert a slab path") - - Returns: - Tuple[SlabType, str, int, int]: Slab's type (DATA or MASK), level identifier, slab's column and slab's row - """ - if self.__storage["type"] == StorageType.FILE: - parts = path.split("/") - - # Le partie du chemin qui contient la colonne et ligne de la dalle est à la fin, en fonction de la profondeur choisie - # depth = 2 -> on doit utiliser les 3 dernières parties pour la conversion - column, row = b36_path_decode("/".join(parts[-(self.__storage["depth"] + 1) :])) - level = parts[-(self.__storage["depth"] + 2)] - raw_slab_type = parts[-(self.__storage["depth"] + 3)] - - # Pour être retro compatible avec l'ancien nommage - if raw_slab_type == "IMAGE": - raw_slab_type = "DATA" - - slab_type = SlabType[raw_slab_type] - - return slab_type, level, column, row - else: - parts = re.split(r"[/_]", path) - column = parts[-2] - row = parts[-1] - level = parts[-3] - raw_slab_type = parts[-4] - - # Pour être retro compatible avec l'ancien nommage - if raw_slab_type == "IMG": - raw_slab_type = "DATA" - elif raw_slab_type == "MSK": - raw_slab_type = "MASK" - - slab_type = SlabType[raw_slab_type] - - return slab_type, level, int(column), int(row) - - def get_slab_path_from_infos( - self, slab_type: SlabType, level: str, column: int, row: int, full: bool = True - ) -> str: - """Get slab's storage path from the indices - - Args: - slab_type (SlabType): DATA or MASK - level (str): Level identifier - column (int): Slab's column - row (int): Slab's row - full (bool, optional): Full path or just relative path from pyramid storage root. Defaults to True. - - Returns: - str: Absolute or relative slab's storage path - """ - if self.__storage["type"] == StorageType.FILE: - slab_path = os.path.join( - slab_type.value, level, b36_path_encode(column, row, self.__storage["depth"]) - ) - else: - slab_path = f"{slab_type.value}_{level}_{column}_{row}" - - if full: - return get_path_from_infos( - self.__storage["type"], self.__storage["root"], self.__name, slab_path - ) - else: - return slab_path - - - def get_tile_data_binary(self, level: str, column: int, row: int) -> str: - """Get a pyramid's tile as binary string - - To get a tile, 3 steps : - * calculate slab path from tile index - * read slab index to get offsets and sizes of slab's tiles - * read the tile into the slab - - Args: - level (str): Tile's level - column (int): Tile's column - row (int): Tile's row - - Limitations: - Pyramids with one-tile slab are not handled - - Examples: - - FILE stored raster pyramid, to extract a tile containing a point and save it as independent image - - from rok4.pyramid import Pyramid - - try: - pyramid = Pyramid.from_descriptor("/data/pyramids/SCAN1000.json") - level, col, row, pcol, prow = pyramid.get_tile_indices(992904.46, 6733643.15, "9", srs = "IGNF:LAMB93") - data = pyramid.get_tile_data_binary(level, col, row) - - if data is None: - print("No data") - else: - tile_name = f"tile_{level}_{col}_{row}.{pyramid.tile_extension}" - with open(tile_name, "wb") as image: - image.write(data) - print (f"Tile written in {tile_name}") - - except Exception as e: - print("Cannot save a pyramid's tile : {e}") - - Raises: - Exception: Level not found in the pyramid - NotImplementedError: Pyramid owns one-tile slabs - MissingEnvironmentError: Missing object storage informations - StorageError: Storage read issue - - Returns: - str: data, as binary string, None if no data - """ - - level_object = self.get_level(level) - - if level_object is None: - raise Exception(f"No level {level} in the pyramid") - - if level_object.slab_width == 1 and level_object.slab_height == 1: - raise NotImplementedError(f"One-tile slab pyramid is not handled") - - if not level_object.is_in_limits(column, row): - return None - - # Indices de la dalle - slab_column = column // level_object.slab_width - slab_row = row // level_object.slab_height - - # Indices de la tuile dans la dalle - relative_tile_column = column % level_object.slab_width - relative_tile_row = row % level_object.slab_height - - # Numéro de la tuile dans le header - tile_index = relative_tile_row * level_object.slab_width + relative_tile_column - - # Calcul du chemin de la dalle contenant la tuile voulue - slab_path = self.get_slab_path_from_infos(SlabType.DATA, level, slab_column, slab_row) - - # Récupération des offset et tailles des tuiles dans la dalle - # Une dalle ROK4 a une en-tête fixe de 2048 octets, - # puis sont stockés les offsets (chacun sur 4 octets) - # puis les tailles (chacune sur 4 octets) - try: - binary_index = get_data_binary( - slab_path, - ( - ROK4_IMAGE_HEADER_SIZE, - 2 * 4 * level_object.slab_width * level_object.slab_height, - ), - ) - except FileNotFoundError as e: - # L'absence de la dalle est gérée comme simplement une absence de données - return None - - offsets = numpy.frombuffer( - binary_index, - dtype=numpy.dtype("uint32"), - count=level_object.slab_width * level_object.slab_height, - ) - sizes = numpy.frombuffer( - binary_index, - dtype=numpy.dtype("uint32"), - offset=4 * level_object.slab_width * level_object.slab_height, - count=level_object.slab_width * level_object.slab_height, - ) - - if sizes[tile_index] == 0: - return None - - return get_data_binary(slab_path, (offsets[tile_index], sizes[tile_index])) - - def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarray: - """Get a raster pyramid's tile as 3-dimension numpy ndarray - - First dimension is the row, second one is column, third one is band. - - Args: - level (str): Tile's level - column (int): Tile's column - row (int): Tile's row - - Limitations: - Packbits (pyramid formats TIFF_PKB_FLOAT32 and TIFF_PKB_UINT8) and LZW (pyramid formats TIFF_LZW_FLOAT32 and TIFF_LZW_UINT8) compressions are not handled. - - Raises: - Exception: Cannot get raster data for a vector pyramid - Exception: Level not found in the pyramid - NotImplementedError: Pyramid owns one-tile slabs - NotImplementedError: Raster pyramid format not handled - MissingEnvironmentError: Missing object storage informations - StorageError: Storage read issue - FormatError: Cannot decode tile - - Examples: - - FILE stored DTM (raster) pyramid, to get the altitude value at a point in the best level - - from rok4.pyramid import Pyramid - - try: - pyramid = Pyramid.from_descriptor("/data/pyramids/RGEALTI.json") - level, col, row, pcol, prow = pyramid.get_tile_indices(44, 5, srs = "EPSG:4326") - data = pyramid.get_tile_data_raster(level, col, row) - - if data is None: - print("No data") - else: - print(data[prow][pcol]) - - except Exception as e: - print("Cannot get a pyramid's pixel value : {e}") - - Returns: - str: data, as numpy array, None if no data - """ - - if self.type == PyramidType.VECTOR: - raise Exception("Cannot get tile as raster data : it's a vector pyramid") - - binary_tile = self.get_tile_data_binary(level, column, row) - - if binary_tile is None: - return None - - level_object = self.get_level(level) - - if self.__format == "TIFF_JPG_UINT8" or self.__format == "TIFF_JPG90_UINT8": - - try: - img = Image.open(io.BytesIO(binary_tile)) - except Exception as e: - raise FormatError("JPEG", "binary tile", e) - - data = numpy.asarray(img) - - elif self.__format == "TIFF_RAW_UINT8": - data = numpy.frombuffer(binary_tile, dtype=numpy.dtype("uint8")) - data.shape = ( - level_object.tile_matrix.tile_size[0], - level_object.tile_matrix.tile_size[1], - self.__raster_specifications["channels"], - ) - - elif self.__format == "TIFF_PNG_UINT8": - try: - img = Image.open(io.BytesIO(binary_tile)) - except Exception as e: - raise FormatError("PNG", "binary tile", e) - - data = numpy.asarray(img) - - elif self.__format == "TIFF_ZIP_UINT8": - try: - data = numpy.frombuffer(zlib.decompress(binary_tile), dtype=numpy.dtype("uint8")) - except Exception as e: - raise FormatError("ZIP", "binary tile", e) - - data.shape = ( - level_object.tile_matrix.tile_size[0], - level_object.tile_matrix.tile_size[1], - self.__raster_specifications["channels"], - ) - - elif self.__format == "TIFF_ZIP_FLOAT32": - try: - data = numpy.frombuffer(zlib.decompress(binary_tile), dtype=numpy.dtype("float32")) - except Exception as e: - raise FormatError("ZIP", "binary tile", e) - - data.shape = ( - level_object.tile_matrix.tile_size[0], - level_object.tile_matrix.tile_size[1], - self.__raster_specifications["channels"], - ) - - elif self.__format == "TIFF_RAW_FLOAT32": - data = numpy.frombuffer(binary_tile, dtype=numpy.dtype("float32")) - data.shape = ( - level_object.tile_matrix.tile_size[0], - level_object.tile_matrix.tile_size[1], - self.__raster_specifications["channels"], - ) - - else: - raise NotImplementedError(f"Cannot get tile as raster data for format {self.__format}") - - return data - - def get_tile_data_vector(self, level: str, column: int, row: int) -> Dict: - """Get a vector pyramid's tile as GeoJSON dictionnary - - Args: - level (str): Tile's level - column (int): Tile's column - row (int): Tile's row - - Raises: - Exception: Cannot get vector data for a raster pyramid - Exception: Level not found in the pyramid - NotImplementedError: Pyramid owns one-tile slabs - NotImplementedError: Vector pyramid format not handled - MissingEnvironmentError: Missing object storage informations - StorageError: Storage read issue - FormatError: Cannot decode tile - - Examples: - - S3 stored vector pyramid, to print a tile as GeoJSON - - from rok4.pyramid import Pyramid - import json - - try: - pyramid = Pyramid.from_descriptor("s3://pyramids/vectors/BDTOPO.json") - level, col, row, pcol, prow = pyramid.get_tile_indices(40.325, 3.123, srs = "EPSG:4326") - data = pyramid.get_tile_data_vector(level, col, row) - - if data is None: - print("No data") - else: - print(json.dumps(data)) - - except Exception as e: - print("Cannot print a vector pyramid's tile as GeoJSON : {e}") - - Returns: - str: data, as GeoJSON dictionnary. None if no data - """ - - if self.type == PyramidType.RASTER: - raise Exception("Cannot get tile as vector data : it's a raster pyramid") - - binary_tile = self.get_tile_data_binary(level, column, row) - - if binary_tile is None: - return None - - level_object = self.get_level(level) - - if self.__format == "TIFF_PBF_MVT": - try: - data = mapbox_vector_tile.decode(binary_tile) - except Exception as e: - raise FormatError("PBF (MVT)", "binary tile", e) - else: - raise NotImplementedError(f"Cannot get tile as vector data for format {self.__format}") - - return data - - def get_tile_indices( - self, x: float, y: float, level: str = None, **kwargs - ) -> Tuple[str, int, int, int, int]: - """Get pyramid's tile and pixel indices from point's coordinates - - Used coordinates system have to be the pyramid one. If EPSG:4326, x is latitude and y longitude. - - Args: - x (float): point's x - y (float): point's y - level (str, optional): Pyramid's level to take into account, the bottom one if None . Defaults to None. - **srs (string): spatial reference system of provided coordinates, with authority and code (same as the pyramid's one if not provided) - - Raises: - Exception: Cannot find level to calculate indices - RuntimeError: Provided SRS is invalid for OSR - - Examples: - - FILE stored DTM (raster) pyramid, to get the altitude value at a point in the best level - - from rok4.pyramid import Pyramid - - try: - pyramid = Pyramid.from_descriptor("/data/pyramids/RGEALTI.json") - level, col, row, pcol, prow = pyramid.get_tile_indices(44, 5, srs = "EPSG:4326") - data = pyramid.get_tile_data_raster(level, col, row) - - if data is None: - print("No data") - else: - print(data[prow][pcol]) - - except Exception as e: - print("Cannot get a pyramid's pixel value : {e}") - - Returns: - Tuple[str, int, int, int, int]: Level identifier, tile's column, tile's row, pixel's (in the tile) column, pixel's row - """ - - level_object = self.bottom_level - if level is not None: - level_object = self.get_level(level) - - if level_object is None: - raise Exception(f"Cannot found the level to calculate indices") - - if ( - "srs" in kwargs - and kwargs["srs"] is not None - and kwargs["srs"].upper() != self.__tms.srs.upper() - ): - sr = srs_to_spatialreference(kwargs["srs"]) - x, y = reproject_point((x, y), sr, self.__tms.sr) - - return (level_object.id,) + level_object.tile_matrix.point_to_indices(x, y) - - @property - def size(self) -> int: - """Get the size of the pyramid - - Examples: - - from rok4.pyramid import Pyramid - - try: - pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") - size = pyramid.size() - - except Exception as e: - print("Cannot load the pyramid from its descriptor and get his size") - - Returns: - int: size of the pyramid - """ - if not hasattr(self,"_Pyramid__size") : - self.__size = size_path(get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name)) - return self.__size diff --git a/src/rok4/pyramid.py b/src/rok4/pyramid.py index 6c83f0e..8aa51aa 100644 --- a/src/rok4/pyramid.py +++ b/src/rok4/pyramid.py @@ -759,7 +759,9 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict] 'slab': 'DATA_18_5424_7526' } ) - + Raises: + StorageError: Unhandled pyramid storage to copy list + MissingEnvironmentError: Missing object storage informations """ if self.__content["loaded"]: for slab, infos in self.__content["cache"].items(): From 965496af024ae28d73fb32792a6cf58cdb0589f3 Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Wed, 30 Aug 2023 10:58:32 +0200 Subject: [PATCH 67/89] Apply isort to existing codebase --- src/rok4/enums.py | 3 +- src/rok4/layer.py | 8 +-- src/rok4/pyramid.py | 32 +++++----- src/rok4/raster.py | 7 +-- src/rok4/storage.py | 108 ++++++++++++++++---------------- src/rok4/tile_matrix_set.py | 5 +- src/rok4/utils.py | 1 - src/rok4/vector.py | 6 +- tests/test_layer.py | 9 +-- tests/test_pyramid.py | 10 +-- tests/test_raster.py | 2 +- tests/test_storage.py | 113 ++++++++++++++++++++++------------ tests/test_tile_matrix_set.py | 9 +-- tests/test_utils.py | 15 +++-- tests/test_vector.py | 8 ++- 15 files changed, 188 insertions(+), 148 deletions(-) diff --git a/src/rok4/enums.py b/src/rok4/enums.py index 861464f..8ca58ba 100644 --- a/src/rok4/enums.py +++ b/src/rok4/enums.py @@ -27,6 +27,7 @@ class StorageType(Enum): HTTPS = "https://" S3 = "s3://" + class ColorFormat(Enum): """A color format enumeration. Except from "BIT", the member's name matches @@ -36,4 +37,4 @@ class ColorFormat(Enum): BIT = 1 UINT8 = 8 - FLOAT32 = 32 \ No newline at end of file + FLOAT32 = 32 diff --git a/src/rok4/layer.py b/src/rok4/layer.py index 631a870..6a72d0c 100644 --- a/src/rok4/layer.py +++ b/src/rok4/layer.py @@ -5,18 +5,18 @@ - `Layer` - Descriptor to broadcast pyramids' data """ -from typing import Dict, List, Tuple, Union import json -from json.decoder import JSONDecodeError import os import re +from json.decoder import JSONDecodeError +from typing import Dict, List, Tuple, Union +from rok4.enums import PyramidType from rok4.exceptions import * from rok4.pyramid import Pyramid -from rok4.tile_matrix_set import TileMatrixSet from rok4.storage import * +from rok4.tile_matrix_set import TileMatrixSet from rok4.utils import * -from rok4.enums import PyramidType class Layer: diff --git a/src/rok4/pyramid.py b/src/rok4/pyramid.py index ebb63b0..a2ae4a9 100644 --- a/src/rok4/pyramid.py +++ b/src/rok4/pyramid.py @@ -6,22 +6,23 @@ - `Level` - Level of a pyramid """ -from typing import Dict, List, Tuple, Union, Iterator +import io import json -from json.decoder import JSONDecodeError import os import re -import numpy import zlib -import io +from json.decoder import JSONDecodeError +from typing import Dict, Iterator, List, Tuple, Union + import mapbox_vector_tile +import numpy from PIL import Image +from rok4.enums import PyramidType, SlabType, StorageType from rok4.exceptions import * -from rok4.tile_matrix_set import TileMatrixSet, TileMatrix from rok4.storage import * +from rok4.tile_matrix_set import TileMatrix, TileMatrixSet from rok4.utils import * -from rok4.enums import PyramidType, SlabType, StorageType ROK4_IMAGE_HEADER_SIZE = 2048 """Slab's header size, 2048 bytes""" @@ -535,11 +536,8 @@ def serializable(self) -> Dict: Returns: Dict: descriptor structured object description """ - - serialization = { - "tile_matrix_set": self.__tms.name, - "format": self.__format - } + + serialization = {"tile_matrix_set": self.__tms.name, "format": self.__format} serialization["levels"] = [] sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution, reverse=True) @@ -656,7 +654,6 @@ def format(self) -> str: @property def tile_extension(self) -> str: - if self.__format in [ "TIFF_RAW_UINT8", "TIFF_LZW_UINT8", @@ -732,7 +729,7 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict] S3 stored descriptor from rok4.pyramid import Pyramid - + try: pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") @@ -756,7 +753,7 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict] 'slab': 'DATA_18_5424_7526' } ) - + Raises: StorageError: Unhandled pyramid storage to copy list MissingEnvironmentError: Missing object storage informations @@ -774,7 +771,7 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict] roots = {} s3_cluster = self.storage_s3_cluster - with open(list_file, "r") as listin: + with open(list_file) as listin: # Lecture des racines for line in listin: line = line.rstrip() @@ -1170,7 +1167,6 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr level_object = self.get_level(level) if self.__format == "TIFF_JPG_UINT8" or self.__format == "TIFF_JPG90_UINT8": - try: img = Image.open(io.BytesIO(binary_tile)) except Exception as e: @@ -1368,10 +1364,10 @@ def size(self) -> int: Returns: int: size of the pyramid """ - + if not hasattr(self, "_Pyramid__size"): self.__size = size_path( get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name) ) - + return self.__size diff --git a/src/rok4/raster.py b/src/rok4/raster.py index 95ccf68..3d2d813 100644 --- a/src/rok4/raster.py +++ b/src/rok4/raster.py @@ -9,14 +9,13 @@ import copy import json import re - from typing import Dict, Tuple from osgeo import gdal, ogr +from rok4.enums import ColorFormat from rok4.storage import exists, get_osgeo_path, put_data_str from rok4.utils import compute_bbox, compute_format -from rok4.enums import ColorFormat # Enable GDAL/OGR exceptions ogr.UseExceptions() @@ -239,7 +238,7 @@ def from_list(cls, path: str, srs: str) -> "RasterSet": local_list_path = get_osgeo_path(path) image_list = [] - with open(file=local_list_path, mode="r") as list_file: + with open(file=local_list_path) as list_file: for line in list_file: image_path = line.strip(" \t\n\r") image_list.append(image_path) @@ -298,7 +297,7 @@ def from_descriptor(cls, path: str) -> "RasterSet": """ self = cls() descriptor_path = get_osgeo_path(path) - with open(file=descriptor_path, mode="r") as file_handle: + with open(file=descriptor_path) as file_handle: raw_content = file_handle.read() serialization = json.loads(raw_content) self.srs = serialization["srs"] diff --git a/src/rok4/storage.py b/src/rok4/storage.py index 6e2eca8..f0a5ffa 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -30,23 +30,23 @@ To precise the cluster to use, bucket name should be bucket_name@s3.storage.fr or bucket_name@s4.storage.fr. If no host is defined (no @) in the bucket name, first S3 cluster is used """ +import hashlib +import os +import re +import tempfile +from shutil import copyfile +from typing import Dict, List, Tuple, Union + import boto3 import botocore.exceptions -import tempfile -import re -import os import rados -import hashlib import requests -from typing import Dict, List, Tuple, Union -from shutil import copyfile from osgeo import gdal gdal.UseExceptions() -from rok4.exceptions import * from rok4.enums import StorageType - +from rok4.exceptions import * __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None @@ -355,16 +355,15 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: raise StorageError("FILE", e) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - - if range is None : + if range is None: try: reponse = requests.get(f"{storage_type.value}{path}", stream=True) data = reponse.content - if reponse.status_code == 404 : + if reponse.status_code == 404: raise FileNotFoundError(f"{storage_type.value}{path}") except Exception as e: raise StorageError(storage_type.name, e) - else : + else: raise NotImplementedError else: @@ -463,7 +462,6 @@ def get_size(path: str) -> int: raise StorageError("FILE", e) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - try: # Le stream=True permet de ne télécharger que le header initialement reponse = requests.get(storage_type.value + path, stream=True).headers["content-length"] @@ -518,12 +516,11 @@ def exists(path: str) -> bool: return os.path.exists(path) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - try: response = requests.get(storage_type.value + path, stream=True) - if response.status_code == 200 : + if response.status_code == 200: return True - else : + else: return False except Exception as e: raise StorageError(storage_type.name, e) @@ -831,47 +828,52 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: f"CEPH and S3", f"Cannot copy CEPH object {from_path} to S3 object {to_path} : {e}" ) - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.FILE : - + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.FILE: try: - response = requests.get(from_type.value + from_path, stream = True) + response = requests.get(from_type.value + from_path, stream=True) with open(to_path, "wb") as f: - for chunk in response.iter_content(chunk_size=65536) : - + for chunk in response.iter_content(chunk_size=65536): if chunk: f.write(chunk) except Exception as e: + raise StorageError( + f"HTTP(S) and FILE", + f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}", + ) - raise StorageError(f"HTTP(S) and FILE", f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}") - - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.CEPH : - + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.CEPH: to_ioctx = __get_ceph_ioctx(to_tray) try: - response = requests.get(from_type.value + from_path, stream = True) + response = requests.get(from_type.value + from_path, stream=True) offset = 0 - for chunk in response.iter_content(chunk_size=65536) : + for chunk in response.iter_content(chunk_size=65536): if chunk: size = len(chunk) to_ioctx.write(to_base_name, chunk, offset) offset += size except Exception as e: + raise StorageError( + f"HTTP(S) and CEPH", + f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}", + ) - raise StorageError(f"HTTP(S) and CEPH", f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}") - - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.S3 : - + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.S3: to_s3_client, to_bucket = __get_s3_client(to_tray) try: - response = requests.get(from_type.value + from_path, stream = True) - with tempfile.NamedTemporaryFile("w+b",delete=False) as f: + response = requests.get(from_type.value + from_path, stream=True) + with tempfile.NamedTemporaryFile("w+b", delete=False) as f: name_fich = f.name - for chunk in response.iter_content(chunk_size=65536) : - + for chunk in response.iter_content(chunk_size=65536): if chunk: f.write(chunk) @@ -880,8 +882,10 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: os.remove(name_fich) except Exception as e: - raise StorageError(f"HTTP(S) and S3", f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}") - + raise StorageError( + f"HTTP(S) and S3", + f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}", + ) else: raise StorageError( @@ -928,9 +932,7 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: try: target_s3_client["client"].put_object( - Body=f"{__OBJECT_SYMLINK_SIGNATURE}{target_bucket}/{target_base_name}".encode( - "utf-8" - ), + Body=f"{__OBJECT_SYMLINK_SIGNATURE}{target_bucket}/{target_base_name}".encode(), Bucket=link_bucket, Key=link_base_name, ) @@ -941,9 +943,7 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: ioctx = __get_ceph_ioctx(link_tray) try: - ioctx.write_full( - link_base_name, f"{__OBJECT_SYMLINK_SIGNATURE}{target_path}".encode("utf-8") - ) + ioctx.write_full(link_base_name, f"{__OBJECT_SYMLINK_SIGNATURE}{target_path}".encode()) except Exception as e: raise StorageError("CEPH", e) @@ -995,7 +995,8 @@ def get_osgeo_path(path: str) -> str: else: raise NotImplementedError(f"Cannot get a GDAL/OGR compliant path from {path}") -def size_path(path: str) -> int : + +def size_path(path: str) -> int: """Return the size of the path given (or, for the CEPH, the sum of the size of each object of the .list) Args: @@ -1008,10 +1009,10 @@ def size_path(path: str) -> int : Returns: int: size of the path """ - storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) + storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) if storage_type == StorageType.FILE: - try : + try: total = 0 with os.scandir(unprefixed_path) as it: for entry in it: @@ -1026,20 +1027,19 @@ def size_path(path: str) -> int : elif storage_type == StorageType.S3: s3_client, bucket_name = __get_s3_client(tray_name) - - try : - paginator = s3_client["client"].get_paginator('list_objects_v2') + try: + paginator = s3_client["client"].get_paginator("list_objects_v2") pages = paginator.paginate( Bucket=bucket_name, - Prefix=base_name+"/", + Prefix=base_name + "/", PaginationConfig={ - 'PageSize': 10000, - } + "PageSize": 10000, + }, ) total = 0 for page in pages: - for key in page['Contents']: - total += key['Size'] + for key in page["Contents"]: + total += key["Size"] except Exception as e: raise StorageError("S3", e) diff --git a/src/rok4/tile_matrix_set.py b/src/rok4/tile_matrix_set.py index b6685bd..976a63e 100644 --- a/src/rok4/tile_matrix_set.py +++ b/src/rok4/tile_matrix_set.py @@ -9,15 +9,16 @@ - ROK4_TMS_DIRECTORY """ -from typing import Dict, List, Tuple -from json.decoder import JSONDecodeError import json import os +from json.decoder import JSONDecodeError +from typing import Dict, List, Tuple from rok4.exceptions import * from rok4.storage import get_data_str from rok4.utils import * + class TileMatrix: """A tile matrix is a tile matrix set's level. diff --git a/src/rok4/utils.py b/src/rok4/utils.py index 4db30d4..004852b 100644 --- a/src/rok4/utils.py +++ b/src/rok4/utils.py @@ -3,7 +3,6 @@ import os import re - from typing import Dict, List, Tuple, Union from osgeo import gdal, ogr, osr diff --git a/src/rok4/vector.py b/src/rok4/vector.py index c3c045b..2cdb0a2 100644 --- a/src/rok4/vector.py +++ b/src/rok4/vector.py @@ -6,15 +6,17 @@ """ -from osgeo import ogr import os import tempfile +from osgeo import ogr + # Enable GDAL/OGR exceptions ogr.UseExceptions() -from rok4.storage import get_osgeo_path, copy from rok4.exceptions import * +from rok4.storage import copy, get_osgeo_path + class Vector: """A data vector diff --git a/tests/test_layer.py b/tests/test_layer.py index a9a79a2..9cd314b 100644 --- a/tests/test_layer.py +++ b/tests/test_layer.py @@ -1,12 +1,13 @@ -import pytest import os - -from unittest.mock import * from unittest import mock +from unittest.mock import * + +import pytest -from rok4.layer import Layer from rok4.enums import PyramidType from rok4.exceptions import * +from rok4.layer import Layer + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch( diff --git a/tests/test_pyramid.py b/tests/test_pyramid.py index fa04a01..e71dd15 100644 --- a/tests/test_pyramid.py +++ b/tests/test_pyramid.py @@ -1,13 +1,15 @@ -import pytest import os -from unittest.mock import * from unittest import mock +from unittest.mock import * + +import pytest +from rok4.enums import SlabType, StorageType +from rok4.exceptions import * from rok4.pyramid import * from rok4.tile_matrix_set import TileMatrixSet -from rok4.enums import SlabType, StorageType from rok4.utils import * -from rok4.exceptions import * + @mock.patch("rok4.pyramid.get_data_str", side_effect=StorageError("FILE", "Not found")) def test_wrong_file(mocked_get_data_str): diff --git a/tests/test_raster.py b/tests/test_raster.py index c91f527..8bf80f0 100644 --- a/tests/test_raster.py +++ b/tests/test_raster.py @@ -14,8 +14,8 @@ import pytest -from rok4.raster import Raster, RasterSet from rok4.enums import ColorFormat +from rok4.raster import Raster, RasterSet # rok4.raster.Raster class tests diff --git a/tests/test_storage.py b/tests/test_storage.py index d60e8c0..f2649e5 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -1,15 +1,15 @@ -import pytest import os +from unittest import mock +from unittest.mock import * import botocore.exceptions +import pytest from rados import ObjectNotFound -from unittest import mock -from unittest.mock import * - -from rok4.storage import * -from rok4.exceptions import * from rok4.enums import StorageType +from rok4.exceptions import * +from rok4.storage import * + @mock.patch.dict(os.environ, {}, clear=True) @patch("builtins.open", new_callable=mock_open, read_data=b"data") @@ -21,6 +21,7 @@ def test_hash_file_ok(mock_file): except Exception as exc: assert False, f"FILE md5 sum raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) def test_get_infos_from_path(): assert (StorageType.S3, "toto/titi", "toto", "titi") == get_infos_from_path("s3://toto/titi") @@ -118,6 +119,7 @@ def test_s3_read_nok(mocked_s3_client): with pytest.raises(StorageError): data = get_data_str("s3://bucket/path/to/object") + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -160,8 +162,9 @@ def test_ceph_read_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH read raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch("requests.get", side_effect={"status_code":404}) +@mock.patch("requests.get", side_effect={"status_code": 404}) def test_http_read_error(mock_http): with pytest.raises(StorageError): requests_instance = MagicMock() @@ -172,18 +175,19 @@ def test_http_read_error(mock_http): mock_http.assert_called_with("http://path/to/file.ext", stream=True) + @mock.patch.dict(os.environ, {}, clear=True) def test_http_read_range_error(): with pytest.raises(NotImplementedError): - data = get_data_binary("http://path/to/file.ext", (0,100)) + data = get_data_binary("http://path/to/file.ext", (0, 100)) + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("requests.get") def test_http_read_ok(mock_http): - - try : + try: requests_instance = MagicMock() - requests_instance.content = b'data' + requests_instance.content = b"data" mock_http.return_value = requests_instance data = get_data_str("http://path/to/file.ext") @@ -195,6 +199,7 @@ def test_http_read_ok(mock_http): ############ put_data_str + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -227,6 +232,7 @@ def test_s3_write_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 write raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -304,6 +310,7 @@ def test_copy_s3_file_nok(mock_hash_file, mock_makedirs, mocked_s3_client): copy("s3://bucket/source.ext", "file:///path/to/destination.ext", "toto") mock_makedirs.assert_called_once_with("/path/to", exist_ok=True) + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -322,6 +329,7 @@ def test_copy_file_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"FILE -> S3 copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -340,6 +348,7 @@ def test_copy_s3_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 -> S3 copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -402,6 +411,7 @@ def test_copy_ceph_file_ok(mock_file, mock_makedirs, mocked_rados_client): except Exception as exc: assert False, f"CEPH -> FILE copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -491,14 +501,14 @@ def test_copy_ceph_s3_ok(mock_file, mocked_s3_client, mocked_rados_client): except Exception as exc: assert False, f"CEPH -> S3 copy raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') -@patch('builtins.open', new_callable=mock_open) +@mock.patch("requests.get") +@patch("builtins.open", new_callable=mock_open) def test_copy_http_file_ok(mock_open, mock_requests): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance copy("http://path/to/source.ext", "file:///path/to/destination.ext") @@ -507,14 +517,18 @@ def test_copy_http_file_ok(mock_open, mock_requests): except Exception as exc: assert False, f"HTTP -> FILE copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -@mock.patch('rok4.storage.rados.Rados') -@mock.patch('requests.get') + +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +@mock.patch("rok4.storage.rados.Rados") +@mock.patch("requests.get") def test_copy_http_ceph_ok(mock_requests, mocked_rados_client): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance disconnect_ceph_clients() @@ -529,16 +543,20 @@ def test_copy_http_ceph_ok(mock_requests, mocked_rados_client): except Exception as exc: assert False, f"HTTP -> CEPH copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.storage.boto3.client') -@mock.patch('requests.get') -@patch('tempfile.NamedTemporaryFile', new_callable=mock_open) -@mock.patch('os.remove') + +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.storage.boto3.client") +@mock.patch("requests.get") +@patch("tempfile.NamedTemporaryFile", new_callable=mock_open) +@mock.patch("os.remove") def test_copy_http_s3_ok(mock_remove, mock_tempfile, mock_requests, mocked_s3_client): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance disconnect_s3_clients() @@ -586,6 +604,7 @@ def test_hlink_file_ok(mock_link): except Exception as exc: assert False, f"FILE hard link raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -642,6 +661,7 @@ def test_link_s3_nok(mocked_s3_client): ############ get_size + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.stat") def test_size_file_ok(mock_stat): @@ -652,6 +672,7 @@ def test_size_file_ok(mock_stat): except Exception as exc: assert False, f"FILE size raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -672,6 +693,7 @@ def test_size_ceph_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH size raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -690,12 +712,12 @@ def test_size_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 size raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') +@mock.patch("requests.get") def test_size_http_ok(mock_requests): - http_instance = MagicMock() - http_instance.headers = {"content-length":12} + http_instance.headers = {"content-length": 12} mock_requests.return_value = http_instance try: @@ -722,6 +744,7 @@ def test_exists_file_ok(mock_exists): except Exception as exc: assert False, f"FILE not exists raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -773,10 +796,10 @@ def test_exists_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"CEPH not exists raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') +@mock.patch("requests.get") def test_exists_http_ok(mock_requests): - http_instance = MagicMock() http_instance.status_code = 200 mock_requests.return_value = http_instance @@ -839,6 +862,7 @@ def test_remove_ceph_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH deletion (not found) raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -868,6 +892,7 @@ def test_get_osgeo_path_file_ok(): except Exception as exc: assert False, f"FILE osgeo path raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -888,6 +913,7 @@ def test_get_osgeo_path_nok(): with pytest.raises(NotImplementedError): get_osgeo_path("ceph://pool/data.ext") + ############ size_path def test_size_path_file_ok(): try: @@ -898,21 +924,29 @@ def test_size_path_file_ok(): def test_size_file_nok(): - with pytest.raises(StorageError) : + with pytest.raises(StorageError): size = size_path("file://tests/fixtures/TIFF_PBF_M") -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -def test_size_path_ceph_nok(): +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +def test_size_path_ceph_nok(): with pytest.raises(NotImplementedError): size = size_path("ceph://pool/path") -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.storage.boto3.client') -def test_size_path_s3_ok(mocked_s3_client): +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.storage.boto3.client") +def test_size_path_s3_ok(mocked_s3_client): disconnect_s3_clients() - pages = [{"Contents" : [{"Size" : 10},{"Size" : 20}]}, {"Contents" : [{"Size" : 50}]}] + pages = [{"Contents": [{"Size": 10}, {"Size": 20}]}, {"Contents": [{"Size": 50}]}] paginator = MagicMock() paginator.paginate.return_value = pages client = MagicMock() @@ -924,4 +958,3 @@ def test_size_path_s3_ok(mocked_s3_client): assert size == 80 except Exception as exc: assert False, f"S3 size of the path raises an exception: {exc}" - diff --git a/tests/test_tile_matrix_set.py b/tests/test_tile_matrix_set.py index 98bccd6..faf03f1 100644 --- a/tests/test_tile_matrix_set.py +++ b/tests/test_tile_matrix_set.py @@ -1,11 +1,12 @@ - -import pytest import os -from unittest.mock import * from unittest import mock +from unittest.mock import * + +import pytest -from rok4.tile_matrix_set import TileMatrixSet from rok4.exceptions import * +from rok4.tile_matrix_set import TileMatrixSet + @mock.patch.dict(os.environ, {}, clear=True) def test_missing_env(): diff --git a/tests/test_utils.py b/tests/test_utils.py index bfcb636..0b73b9e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,14 +1,15 @@ -import pytest -import os -from osgeo import gdal, osr import math +import os import random - -from unittest.mock import * from unittest import mock +from unittest.mock import * + +import pytest +from osgeo import gdal, osr -from rok4.utils import * from rok4.exceptions import * +from rok4.utils import * + def test_srs_to_spatialreference_ignf_ok(): try: @@ -75,6 +76,7 @@ def test_reproject_point_ok(): # Tests for the rok4.utils.compute_bbox function. + def test_compute_bbox_epsg_3857_ok(): try: mocked_datasource = MagicMock(gdal.Dataset) @@ -215,6 +217,7 @@ def test_compute_bbox_no_srs_ok(): # Tests for the rok4.utils.compute_format function. + @mock.patch("rok4.utils.gdal.Info") @mock.patch("rok4.utils.gdal.GetColorInterpretationName", return_value="Palette") @mock.patch("rok4.utils.gdal.GetDataTypeSize", return_value=8) diff --git a/tests/test_vector.py b/tests/test_vector.py index 513c559..8f75618 100644 --- a/tests/test_vector.py +++ b/tests/test_vector.py @@ -1,11 +1,13 @@ -import pytest import os -from unittest.mock import * from unittest import mock +from unittest.mock import * + +import pytest -from rok4.vector import * from rok4.exceptions import * from rok4.storage import disconnect_ceph_clients +from rok4.vector import * + @mock.patch.dict(os.environ, {}, clear=True) def test_missing_env(): From 3a33229bb9811c11993640809ed53262a4b47a91 Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Wed, 30 Aug 2023 14:06:00 +0200 Subject: [PATCH 68/89] Add dependabot configuration --- .github/dependabot.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..9327b14 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,19 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: monthly + time: "17:00" + timezone: Europe/Paris + labels: + - dependencies + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: monthly + time: "22:00" + timezone: Europe/Paris + labels: + - ci-cd From 31e3fce7e50d4851c497cf1f1ac96771ed4c3483 Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Wed, 30 Aug 2023 14:37:32 +0200 Subject: [PATCH 69/89] Add requests as 3rd party dependency --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f172c36..29e103b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,9 +32,10 @@ classifiers = [ dependencies = [ "boto3 >= 1.26.54", - "pillow >= 9.4.0", + "mapbox-vector-tile >= 2.0.1", "numpy >= 1.24.2", - "mapbox-vector-tile >= 2.0.1" + "pillow >= 9.4.0", + "requests >= 2.31.0" ] [project.optional-dependencies] From f9d0b3897397492d77c147f246a6cc738ee334f4 Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Wed, 30 Aug 2023 14:39:12 +0200 Subject: [PATCH 70/89] Decrease requests min version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 29e103b..029bfdf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "mapbox-vector-tile >= 2.0.1", "numpy >= 1.24.2", "pillow >= 9.4.0", - "requests >= 2.31.0" + "requests >= 2.30.0" ] [project.optional-dependencies] From 66a706559989d13f623748ae5fa77b4046c5e8e3 Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Wed, 30 Aug 2023 17:30:45 +0200 Subject: [PATCH 71/89] Fix some of minor issues spotted by linters --- src/rok4/layer.py | 27 ++++++++------- src/rok4/pyramid.py | 41 +++++++++++++++-------- src/rok4/raster.py | 7 ++++ src/rok4/storage.py | 65 ++++++++++++++++++++----------------- src/rok4/tile_matrix_set.py | 16 ++++++--- src/rok4/utils.py | 13 +++++--- src/rok4/vector.py | 16 ++++++--- 7 files changed, 116 insertions(+), 69 deletions(-) diff --git a/src/rok4/layer.py b/src/rok4/layer.py index 6a72d0c..dfccac9 100644 --- a/src/rok4/layer.py +++ b/src/rok4/layer.py @@ -5,18 +5,21 @@ - `Layer` - Descriptor to broadcast pyramids' data """ +# -- IMPORTS -- + +# standard library import json import os import re from json.decoder import JSONDecodeError -from typing import Dict, List, Tuple, Union +from typing import Dict, List, Tuple +# package from rok4.enums import PyramidType -from rok4.exceptions import * +from rok4.exceptions import FormatError, MissingAttributeError from rok4.pyramid import Pyramid -from rok4.storage import * -from rok4.tile_matrix_set import TileMatrixSet -from rok4.utils import * +from rok4.storage import get_data_str, get_infos_from_path, put_data_str +from rok4.utils import reproject_bbox class Layer: @@ -92,8 +95,8 @@ def from_descriptor(cls, descriptor: str) -> "Layer": ) layer.__bbox = reproject_bbox(layer.__geobbox, "EPSG:4326", layer.__tms.srs, 5) # On force l'emprise de la couche, on recalcule donc les tuiles limites correspondantes pour chaque niveau - for l in layer.__levels.values(): - l.set_limits_from_bbox(layer.__bbox) + for level in layer.__levels.values(): + level.set_limits_from_bbox(layer.__bbox) else: layer.__bbox = layer.__best_level.bbox layer.__geobbox = reproject_bbox(layer.__bbox, layer.__tms.srs, "EPSG:4326", 5) @@ -184,7 +187,7 @@ def __load_pyramids(self, pyramids: List[Dict[str, str]]) -> None: Exception: Overlapping in usage pyramids' levels """ - ## Toutes les pyramides doivent avoir les même caractéristiques + # Toutes les pyramides doivent avoir les même caractéristiques channels = None for p in pyramids: pyramid = Pyramid.from_descriptor(p["path"]) @@ -221,10 +224,10 @@ def __load_pyramids(self, pyramids: List[Dict[str, str]]) -> None: self.__resampling = pyramid.raster_specifications["interpolation"] levels = pyramid.get_levels(bottom_level, top_level) - for l in levels: - if l.id in self.__levels: - raise Exception(f"Level {l.id} is present in two used pyramids") - self.__levels[l.id] = l + for level in levels: + if level.id in self.__levels: + raise Exception(f"Level {level.id} is present in two used pyramids") + self.__levels[level.id] = level self.__pyramids.append( {"pyramid": pyramid, "bottom_level": bottom_level, "top_level": top_level} diff --git a/src/rok4/pyramid.py b/src/rok4/pyramid.py index a2ae4a9..7566e93 100644 --- a/src/rok4/pyramid.py +++ b/src/rok4/pyramid.py @@ -6,24 +6,37 @@ - `Level` - Level of a pyramid """ +# -- IMPORTS -- + +# standard library import io import json import os import re +import tempfile import zlib from json.decoder import JSONDecodeError -from typing import Dict, Iterator, List, Tuple, Union +from typing import Dict, Iterator, List, Tuple +# 3rd party import mapbox_vector_tile import numpy from PIL import Image +# package from rok4.enums import PyramidType, SlabType, StorageType -from rok4.exceptions import * -from rok4.storage import * +from rok4.exceptions import FormatError, MissingAttributeError +from rok4.storage import ( + copy, + get_data_str, + get_infos_from_path, + get_path_from_infos, + put_data_str, + remove, +) from rok4.tile_matrix_set import TileMatrix, TileMatrixSet -from rok4.utils import * +# -- GLOBALS -- ROK4_IMAGE_HEADER_SIZE = 2048 """Slab's header size, 2048 bytes""" @@ -441,8 +454,8 @@ def from_descriptor(cls, descriptor: str) -> "Pyramid": pyramid.__masks = False # Niveaux - for l in data["levels"]: - lev = Level.from_descriptor(l, pyramid) + for level in data["levels"]: + lev = Level.from_descriptor(level, pyramid) pyramid.__levels[lev.id] = lev if pyramid.__tms.get_level(lev.id) is None: @@ -510,8 +523,8 @@ def from_other(cls, other: "Pyramid", name: str, storage: Dict) -> "Pyramid": pyramid.__raster_specifications = other.__raster_specifications # Niveaux - for l in other.__levels.values(): - lev = Level.from_other(l, pyramid) + for level in other.__levels.values(): + lev = Level.from_other(level, pyramid) pyramid.__levels[lev.id] = lev except KeyError as e: @@ -542,8 +555,8 @@ def serializable(self) -> Dict: serialization["levels"] = [] sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution, reverse=True) - for l in sorted_levels: - serialization["levels"].append(l.serializable) + for level in sorted_levels: + serialization["levels"].append(level.serializable) if self.type == PyramidType.RASTER: serialization["raster_specifications"] = self.__raster_specifications @@ -1059,7 +1072,7 @@ def get_tile_data_binary(self, level: str, column: int, row: int) -> str: raise Exception(f"No level {level} in the pyramid") if level_object.slab_width == 1 and level_object.slab_height == 1: - raise NotImplementedError(f"One-tile slab pyramid is not handled") + raise NotImplementedError("One-tile slab pyramid is not handled") if not level_object.is_in_limits(column, row): return None @@ -1090,7 +1103,7 @@ def get_tile_data_binary(self, level: str, column: int, row: int) -> str: 2 * 4 * level_object.slab_width * level_object.slab_height, ), ) - except FileNotFoundError as e: + except FileNotFoundError: # L'absence de la dalle est gérée comme simplement une absence de données return None @@ -1277,7 +1290,7 @@ def get_tile_data_vector(self, level: str, column: int, row: int) -> Dict: if binary_tile is None: return None - level_object = self.get_level(level) + self.get_level(level) if self.__format == "TIFF_PBF_MVT": try: @@ -1334,7 +1347,7 @@ def get_tile_indices( level_object = self.get_level(level) if level_object is None: - raise Exception(f"Cannot found the level to calculate indices") + raise Exception("Cannot found the level to calculate indices") if ( "srs" in kwargs diff --git a/src/rok4/raster.py b/src/rok4/raster.py index 3d2d813..d52617e 100644 --- a/src/rok4/raster.py +++ b/src/rok4/raster.py @@ -6,17 +6,24 @@ - RasterSet - Structure describing a set of raster data. """ +# -- IMPORTS -- + +# standard library import copy import json import re from typing import Dict, Tuple +# 3rd party from osgeo import gdal, ogr +# package from rok4.enums import ColorFormat from rok4.storage import exists, get_osgeo_path, put_data_str from rok4.utils import compute_bbox, compute_format +# -- GLOBALS -- + # Enable GDAL/OGR exceptions ogr.UseExceptions() gdal.UseExceptions() diff --git a/src/rok4/storage.py b/src/rok4/storage.py index f0a5ffa..3b17bea 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -30,24 +30,31 @@ To precise the cluster to use, bucket name should be bucket_name@s3.storage.fr or bucket_name@s4.storage.fr. If no host is defined (no @) in the bucket name, first S3 cluster is used """ +# -- IMPORTS -- + +# standard library import hashlib import os import re import tempfile from shutil import copyfile -from typing import Dict, List, Tuple, Union +from typing import Dict, Tuple, Union +# 3rd party import boto3 import botocore.exceptions import rados import requests from osgeo import gdal -gdal.UseExceptions() - +# package from rok4.enums import StorageType -from rok4.exceptions import * +from rok4.exceptions import MissingEnvironmentError, StorageError +# -- GLOBALS -- + +# Enable GDAL/OGR exceptions +gdal.UseExceptions() __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None @@ -331,7 +338,7 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: else: data = ioctx.read(base_name, range[1], range[0]) - except rados.ObjectNotFound as e: + except rados.ObjectNotFound: raise FileNotFoundError(f"{storage_type.value}{path}") except Exception as e: @@ -348,7 +355,7 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: f.close() - except FileNotFoundError as e: + except FileNotFoundError: raise FileNotFoundError(f"{storage_type.value}{path}") except Exception as e: @@ -507,7 +514,7 @@ def exists(path: str) -> bool: try: ioctx.stat(base_name) return True - except rados.ObjectNotFound as e: + except rados.ObjectNotFound: return False except Exception as e: raise StorageError("CEPH", e) @@ -554,7 +561,7 @@ def remove(path: str) -> None: try: ioctx.remove_object(base_name) - except rados.ObjectNotFound as e: + except rados.ObjectNotFound: pass except Exception as e: raise StorageError("CEPH", e) @@ -562,7 +569,7 @@ def remove(path: str) -> None: elif storage_type == StorageType.FILE: try: os.remove(path) - except FileNotFoundError as e: + except FileNotFoundError: pass except Exception as e: raise StorageError("FILE", e) @@ -599,12 +606,12 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: to_md5 = hash_file(to_path) if to_md5 != from_md5: raise StorageError( - f"FILE", + "FILE", f"Invalid MD5 sum control for copy file {from_path} to {to_path} : {from_md5} != {to_md5}", ) except Exception as e: - raise StorageError(f"FILE", f"Cannot copy file {from_path} to {to_path} : {e}") + raise StorageError("FILE", f"Cannot copy file {from_path} to {to_path} : {e}") elif from_type == StorageType.S3 and to_type == StorageType.FILE: s3_client, from_bucket = __get_s3_client(from_tray) @@ -625,7 +632,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: except Exception as e: raise StorageError( - f"S3 and FILE", f"Cannot copy S3 object {from_path} to file {to_path} : {e}" + "S3 and FILE", f"Cannot copy S3 object {from_path} to file {to_path} : {e}" ) elif from_type == StorageType.FILE and to_type == StorageType.S3: @@ -642,12 +649,12 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: ) if to_md5 != from_md5: raise StorageError( - f"FILE and S3", + "FILE and S3", f"Invalid MD5 sum control for copy file {from_path} to S3 object {to_path} : {from_md5} != {to_md5}", ) except Exception as e: raise StorageError( - f"FILE and S3", f"Cannot copy file {from_path} to S3 object {to_path} : {e}" + "FILE and S3", f"Cannot copy file {from_path} to S3 object {to_path} : {e}" ) elif from_type == StorageType.S3 and to_type == StorageType.S3: @@ -672,12 +679,12 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: ) if to_md5 != from_md5: raise StorageError( - f"S3", + "S3", f"Invalid MD5 sum control for copy S3 object {from_path} to {to_path} : {from_md5} != {to_md5}", ) except Exception as e: - raise StorageError(f"S3", f"Cannot copy S3 object {from_path} to {to_path} : {e}") + raise StorageError("S3", f"Cannot copy S3 object {from_path} to {to_path} : {e}") elif from_type == StorageType.CEPH and to_type == StorageType.FILE: ioctx = __get_ceph_ioctx(from_tray) @@ -709,13 +716,13 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: if from_md5 is not None and from_md5 != checker.hexdigest(): raise StorageError( - f"CEPH and FILE", + "CEPH and FILE", f"Invalid MD5 sum control for copy CEPH object {from_path} to file {to_path} : {from_md5} != {checker.hexdigest()}", ) except Exception as e: raise StorageError( - f"CEPH and FILE", f"Cannot copy CEPH object {from_path} to file {to_path} : {e}" + "CEPH and FILE", f"Cannot copy CEPH object {from_path} to file {to_path} : {e}" ) elif from_type == StorageType.FILE and to_type == StorageType.CEPH: @@ -746,13 +753,13 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: if from_md5 is not None and from_md5 != checker.hexdigest(): raise StorageError( - f"FILE and CEPH", + "FILE and CEPH", f"Invalid MD5 sum control for copy file {from_path} to CEPH object {to_path} : {from_md5} != {checker.hexdigest()}", ) except Exception as e: raise StorageError( - f"FILE and CEPH", f"Cannot copy file {from_path} to CEPH object {to_path} : {e}" + "FILE and CEPH", f"Cannot copy file {from_path} to CEPH object {to_path} : {e}" ) elif from_type == StorageType.CEPH and to_type == StorageType.CEPH: @@ -780,12 +787,12 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: if from_md5 is not None and from_md5 != checker.hexdigest(): raise StorageError( - f"FILE and CEPH", + "FILE and CEPH", f"Invalid MD5 sum control for copy CEPH object {from_path} to {to_path} : {from_md5} != {checker.hexdigest()}", ) except Exception as e: - raise StorageError(f"CEPH", f"Cannot copy CEPH object {from_path} to {to_path} : {e}") + raise StorageError("CEPH", f"Cannot copy CEPH object {from_path} to {to_path} : {e}") elif from_type == StorageType.CEPH and to_type == StorageType.S3: from_ioctx = __get_ceph_ioctx(from_tray) @@ -819,13 +826,13 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: if from_md5 is not None and from_md5 != checker.hexdigest(): raise StorageError( - f"CEPH and S3", + "CEPH and S3", f"Invalid MD5 sum control for copy CEPH object {from_path} to S3 object {to_path} : {from_md5} != {checker.hexdigest()}", ) except Exception as e: raise StorageError( - f"CEPH and S3", f"Cannot copy CEPH object {from_path} to S3 object {to_path} : {e}" + "CEPH and S3", f"Cannot copy CEPH object {from_path} to S3 object {to_path} : {e}" ) elif ( @@ -840,7 +847,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: except Exception as e: raise StorageError( - f"HTTP(S) and FILE", + "HTTP(S) and FILE", f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}", ) @@ -860,7 +867,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: except Exception as e: raise StorageError( - f"HTTP(S) and CEPH", + "HTTP(S) and CEPH", f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}", ) @@ -883,7 +890,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: except Exception as e: raise StorageError( - f"HTTP(S) and S3", + "HTTP(S) and S3", f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}", ) @@ -913,7 +920,7 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: if target_type != link_type: raise StorageError( f"{target_type.name} and {link_type.name}", - f"Cannot make link between two different storage types", + "Cannot make link between two different storage types", ) if hard and target_type != StorageType.FILE: @@ -926,7 +933,7 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: if target_s3_client["host"] != link_s3_client["host"]: raise StorageError( - f"S3", + "S3", f"Cannot make link {link_path} -> {target_path} : link works only on the same S3 cluster", ) diff --git a/src/rok4/tile_matrix_set.py b/src/rok4/tile_matrix_set.py index 976a63e..14bae9f 100644 --- a/src/rok4/tile_matrix_set.py +++ b/src/rok4/tile_matrix_set.py @@ -9,14 +9,20 @@ - ROK4_TMS_DIRECTORY """ +# -- IMPORTS -- + +# standard library import json import os from json.decoder import JSONDecodeError from typing import Dict, List, Tuple -from rok4.exceptions import * +# package +from rok4.exceptions import FormatError, MissingAttributeError, MissingEnvironmentError from rok4.storage import get_data_str -from rok4.utils import * +from rok4.utils import srs_to_spatialreference + +# -- GLOBALS -- class TileMatrix: @@ -212,8 +218,8 @@ def __init__(self, name: str) -> None: self.srs = data["crs"] self.sr = srs_to_spatialreference(self.srs) self.levels = {} - for l in data["tileMatrices"]: - lev = TileMatrix(l, self) + for level in data["tileMatrices"]: + lev = TileMatrix(level, self) self.levels[lev.id] = lev if len(self.levels.keys()) == 0: @@ -232,7 +238,7 @@ def __init__(self, name: str) -> None: except RuntimeError as e: raise Exception( - f"Wrong attribute 'crs' ('{self.srs}') in '{self.path}', not recognize by OSR" + f"Wrong attribute 'crs' ('{self.srs}') in '{self.path}', not recognize by OSR. Trace : {e}" ) def get_level(self, level_id: str) -> "TileMatrix": diff --git a/src/rok4/utils.py b/src/rok4/utils.py index 004852b..40c7884 100644 --- a/src/rok4/utils.py +++ b/src/rok4/utils.py @@ -1,18 +1,23 @@ """Provide functions to manipulate OGR / OSR entities """ +# -- IMPORTS -- +# standard library import os import re -from typing import Dict, List, Tuple, Union +from typing import Tuple +# 3rd party from osgeo import gdal, ogr, osr +# package +from rok4.enums import ColorFormat + +# -- GLOBALS -- ogr.UseExceptions() osr.UseExceptions() gdal.UseExceptions() -from rok4.enums import ColorFormat - __SR_BOOK = {} @@ -187,7 +192,7 @@ def compute_bbox(source_dataset: gdal.Dataset) -> Tuple: transform_vector = source_dataset.GetGeoTransform() if transform_vector is None: raise Exception( - f"No transform vector found in the dataset created from " + "No transform vector found in the dataset created from " + f"the following file : {source_dataset.GetFileList()[0]}" ) width = source_dataset.RasterXSize diff --git a/src/rok4/vector.py b/src/rok4/vector.py index 2cdb0a2..268d181 100644 --- a/src/rok4/vector.py +++ b/src/rok4/vector.py @@ -6,17 +6,23 @@ """ +# -- IMPORTS -- + +# standard library import os import tempfile +# 3rd party from osgeo import ogr +# package +from rok4.storage import copy, get_osgeo_path + +# -- GLOBALS -- + # Enable GDAL/OGR exceptions ogr.UseExceptions() -from rok4.exceptions import * -from rok4.storage import copy, get_osgeo_path - class Vector: """A data vector @@ -137,7 +143,7 @@ def from_file(cls, path: str, **kwargs) -> "Vector": vrt_file += "" + tmp_path + ".csv\n" vrt_file += "" + name_fich + "\n" vrt_file += "" + srs + "\n" - if column_wkt == None: + if column_wkt is None: vrt_file += ( ' Date: Wed, 30 Aug 2023 17:44:11 +0200 Subject: [PATCH 72/89] Fix linters errors and close #54 --- src/rok4/layer.py | 2 +- src/rok4/pyramid.py | 27 +++++++++++++++------------ src/rok4/tile_matrix_set.py | 2 +- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/rok4/layer.py b/src/rok4/layer.py index dfccac9..c6c21ed 100644 --- a/src/rok4/layer.py +++ b/src/rok4/layer.py @@ -233,7 +233,7 @@ def __load_pyramids(self, pyramids: List[Dict[str, str]]) -> None: {"pyramid": pyramid, "bottom_level": bottom_level, "top_level": top_level} ) - self.__best_level = sorted(self.__levels.values(), key=lambda l: l.resolution)[0] + self.__best_level = sorted(self.__levels.values(), key=lambda level: level.resolution)[0] def __str__(self) -> str: return f"{self.type.name} layer '{self.__name}'" diff --git a/src/rok4/pyramid.py b/src/rok4/pyramid.py index 7566e93..c87da7a 100644 --- a/src/rok4/pyramid.py +++ b/src/rok4/pyramid.py @@ -28,13 +28,16 @@ from rok4.exceptions import FormatError, MissingAttributeError from rok4.storage import ( copy, + get_data_binary, get_data_str, get_infos_from_path, get_path_from_infos, put_data_str, remove, + size_path, ) from rok4.tile_matrix_set import TileMatrix, TileMatrixSet +from rok4.utils import reproject_point, srs_to_spatialreference # -- GLOBALS -- ROK4_IMAGE_HEADER_SIZE = 2048 @@ -528,7 +531,7 @@ def from_other(cls, other: "Pyramid", name: str, storage: Dict) -> "Pyramid": pyramid.__levels[lev.id] = lev except KeyError as e: - raise MissingAttributeError(descriptor, e) + raise MissingAttributeError(pyramid.descriptor, e) return pyramid @@ -553,7 +556,9 @@ def serializable(self) -> Dict: serialization = {"tile_matrix_set": self.__tms.name, "format": self.__format} serialization["levels"] = [] - sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution, reverse=True) + sorted_levels = sorted( + self.__levels.values(), key=lambda level: level.resolution, reverse=True + ) for level in sorted_levels: serialization["levels"].append(level.serializable) @@ -652,9 +657,7 @@ def storage_depth(self, d: int) -> None: Exception: the depth is not equal to the already known depth """ if "depth" in self.__storage and self.__storage["depth"] != d: - raise Exception( - f"Pyramid {pyramid.__descriptor} owns levels with different path depths" - ) + raise Exception(f"Pyramid {self.__descriptor} owns levels with different path depths") self.__storage["depth"] = d @property @@ -696,7 +699,7 @@ def bottom_level(self) -> "Level": Returns: Level: the bottom level """ - return sorted(self.__levels.values(), key=lambda l: l.resolution)[0] + return sorted(self.__levels.values(), key=lambda level: level.resolution)[0] @property def top_level(self) -> "Level": @@ -705,7 +708,7 @@ def top_level(self) -> "Level": Returns: Level: the top level """ - return sorted(self.__levels.values(), key=lambda l: l.resolution)[-1] + return sorted(self.__levels.values(), key=lambda level: level.resolution)[-1] @property def type(self) -> PyramidType: @@ -875,7 +878,7 @@ def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: List[Level]: asked sorted levels """ - sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution) + sorted_levels = sorted(self.__levels.values(), key=lambda level: level.resolution) levels = [] @@ -894,13 +897,13 @@ def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: end = False - for l in sorted_levels: - if not begin and l.id == bottom_id: + for level in sorted_levels: + if not begin and level.id == bottom_id: begin = True if begin: - levels.append(l) - if top_id is not None and l.id == top_id: + levels.append(level) + if top_id is not None and level.id == top_id: end = True break else: diff --git a/src/rok4/tile_matrix_set.py b/src/rok4/tile_matrix_set.py index 14bae9f..84a3d22 100644 --- a/src/rok4/tile_matrix_set.py +++ b/src/rok4/tile_matrix_set.py @@ -255,4 +255,4 @@ def get_level(self, level_id: str) -> "TileMatrix": @property def sorted_levels(self) -> List[TileMatrix]: - return sorted(self.levels.values(), key=lambda l: l.resolution) + return sorted(self.levels.values(), key=lambda level: level.resolution) From 0346e08b37dd68bb41b518395ffa6fe92fed9a5b Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Wed, 30 Aug 2023 17:48:36 +0200 Subject: [PATCH 73/89] Add config section for pre-commit.ci --- .pre-commit-config.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3bbe193..4402755 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,6 @@ repos: - id: black args: ["--target-version=py38"] - - repo: https://github.com/pycqa/isort rev: 5.12.0 hooks: @@ -40,3 +39,9 @@ repos: - id: pyupgrade args: - "--py38-plus" + +ci: + autofix_commit_msg: "[pre-commit.ci] Corrections automatiques appliquées par les git hooks." + autofix_prs: true + autoupdate_commit_msg: "[pre-commit.ci] Mise à jour des git hooks." + autoupdate_schedule: quarterly From dcfd60975d4e8990c2756438d7dc5270a71bc387 Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Wed, 30 Aug 2023 17:58:59 +0200 Subject: [PATCH 74/89] Add ruff git hook --- .pre-commit-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3bbe193..9f470f7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,6 +21,12 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.0.281" + hooks: + - id: ruff + args: ["--fix-only", "--target-version=py38"] + - repo: https://github.com/psf/black rev: 23.3.0 hooks: From 86009615ed1d435a384ac19c9c67cecefdcfd5be Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Wed, 30 Aug 2023 17:59:19 +0200 Subject: [PATCH 75/89] Add ruff as dev dependency and configuration --- pyproject.toml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 029bfdf..ce31c97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,8 @@ doc = [ dev = [ "black", "isort >= 5.12.0", - "pre-commit >3,<4" + "pre-commit >3,<4", + "ruff >= 0.0.281" ] test = [ @@ -68,5 +69,14 @@ build-backend = "setuptools.build_meta" line-length = 100 target-version = ['py38'] +[tool.ruff] +line-length = 100 +src = ["src"] +target-version = "py38" + +# Allow autofix for all enabled rules (when `--fix`) is provided. +fixable = ["ALL"] +unfixable = [] + [tool.setuptools.packages.find] where = ["src"] From faf1a933b016b383b61d3530ea49fce51af83f54 Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Tue, 29 Aug 2023 17:07:46 +0200 Subject: [PATCH 76/89] Handle missing python rados --- src/rok4/storage.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/rok4/storage.py b/src/rok4/storage.py index 3b17bea..cc44ddf 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -30,24 +30,36 @@ To precise the cluster to use, bucket name should be bucket_name@s3.storage.fr or bucket_name@s4.storage.fr. If no host is defined (no @) in the bucket name, first S3 cluster is used """ -# -- IMPORTS -- - -# standard library import hashlib import os import re import tempfile from shutil import copyfile -from typing import Dict, Tuple, Union +from typing import Dict, List, Tuple, Union -# 3rd party import boto3 import botocore.exceptions +import tempfile +import re +import os import rados +import hashlib import requests +from typing import Dict, List, Tuple, Union +from shutil import copyfile from osgeo import gdal -# package +# conditional import +try: + import rados + + CEPH_RADOS_AVAILABLE: bool = True +except ImportError: + CEPH_RADOS_AVAILABLE: bool = False + rados = None + +gdal.UseExceptions() + from rok4.enums import StorageType from rok4.exceptions import MissingEnvironmentError, StorageError From 18fe8104af9f6138bcab4280d7586f47ce5d11aa Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Wed, 30 Aug 2023 14:48:23 +0200 Subject: [PATCH 77/89] Rm useless and failing rados import --- src/rok4/storage.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/rok4/storage.py b/src/rok4/storage.py index cc44ddf..2c02f09 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -39,11 +39,6 @@ import boto3 import botocore.exceptions -import tempfile -import re -import os -import rados -import hashlib import requests from typing import Dict, List, Tuple, Union from shutil import copyfile From 6376acef754635ee4862c0755ecd36ec6d234c5a Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Thu, 31 Aug 2023 14:47:44 +0200 Subject: [PATCH 78/89] Fix imports --- src/rok4/storage.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/rok4/storage.py b/src/rok4/storage.py index 2c02f09..baecc1f 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -35,13 +35,11 @@ import re import tempfile from shutil import copyfile -from typing import Dict, List, Tuple, Union +from typing import Dict, Tuple, Union import boto3 import botocore.exceptions import requests -from typing import Dict, List, Tuple, Union -from shutil import copyfile from osgeo import gdal # conditional import From 89936461878a1d14e3cd783c8765a1ca47c2a187 Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Thu, 31 Aug 2023 14:49:43 +0200 Subject: [PATCH 79/89] Clean globals --- src/rok4/storage.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/rok4/storage.py b/src/rok4/storage.py index baecc1f..6ea0faf 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -51,8 +51,7 @@ CEPH_RADOS_AVAILABLE: bool = False rados = None -gdal.UseExceptions() - +# package from rok4.enums import StorageType from rok4.exceptions import MissingEnvironmentError, StorageError @@ -60,6 +59,9 @@ # Enable GDAL/OGR exceptions gdal.UseExceptions() + +__CEPH_CLIENT = None +__CEPH_IOCTXS = {} __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None @@ -144,10 +146,6 @@ def disconnect_s3_clients() -> None: __S3_DEFAULT_CLIENT = None -__CEPH_CLIENT = None -__CEPH_IOCTXS = {} - - def __get_ceph_ioctx(pool: str) -> "rados.Ioctx": """Get the CEPH IO context From 4d805233521da5ce24a75cbbe41433ec8b559893 Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Thu, 31 Aug 2023 14:53:34 +0200 Subject: [PATCH 80/89] Move globals vars upper --- src/rok4/storage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rok4/storage.py b/src/rok4/storage.py index 6ea0faf..8b1d047 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -62,6 +62,7 @@ __CEPH_CLIENT = None __CEPH_IOCTXS = {} +__OBJECT_SYMLINK_SIGNATURE = "SYMLINK#" __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None From 4a84dc3268581bd115892057500427e8033d0314 Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Thu, 31 Aug 2023 14:56:07 +0200 Subject: [PATCH 81/89] Rm duplicated var --- src/rok4/storage.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/rok4/storage.py b/src/rok4/storage.py index 8b1d047..8fda078 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -195,9 +195,6 @@ def disconnect_ceph_clients() -> None: __CEPH_IOCTXS = {} -__OBJECT_SYMLINK_SIGNATURE = "SYMLINK#" - - def get_infos_from_path(path: str) -> Tuple[StorageType, str, str, str]: """Extract storage type, the unprefixed path, the container and the basename from path (Default: FILE storage) From 7e64927926fcd6a22f1de0ca6411e361cc00bedc Mon Sep 17 00:00:00 2001 From: Theo Satabin Date: Fri, 1 Sep 2023 10:32:28 +0200 Subject: [PATCH 82/89] =?UTF-8?q?Pas=20de=20documentation=20publi=C3=A9e?= =?UTF-8?q?=20pour=20les=20versions=20beta?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-and-release.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 8fb0b1e..8f8d8af 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -3,7 +3,8 @@ name: Test, build and publish artefacts and documentation on: push: tags: - - '**' + - '[0-9]+.[0-9]+.[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+-beta[0-9]+' jobs: @@ -158,7 +159,7 @@ jobs: commit_documentation: name: Add documentation and unit tests results into gh-pages branch needs: build_and_test - if: "always()&&(needs.create_release.outputs.job_status=='success')&&(needs.build_and_test.outputs.job_status=='success')" + if: "always()&&!contains(${{ github.ref_name }},'beta')&&(needs.create_release.outputs.job_status=='success')&&(needs.build_and_test.outputs.job_status=='success')" runs-on: ubuntu-latest steps: From 0040040b417074c16e6eff9b9ca7faf5eb101186 Mon Sep 17 00:00:00 2001 From: Theo Satabin Date: Fri, 1 Sep 2023 10:53:43 +0200 Subject: [PATCH 83/89] Correction des tests unitaires --- .github/workflows/build-and-release.yaml | 2 +- tests/test_raster.py | 4 ++-- tests/test_tile_matrix_set.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 8f8d8af..5dea00b 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -27,7 +27,7 @@ jobs: name: Release ${{ github.ref_name }} generateReleaseNotes: true draft: false - prerelease: false + prerelease: contains(${{ github.ref_name }},'beta') build_and_test: diff --git a/tests/test_raster.py b/tests/test_raster.py index 8bf80f0..85b71d4 100644 --- a/tests/test_raster.py +++ b/tests/test_raster.py @@ -332,7 +332,7 @@ def test_ok_at_least_3_files(self, m_from_file, m_get_osgeo_path): serial_out = rasterset.serializable assert rasterset.srs == srs m_get_osgeo_path.assert_called_once_with(list_path) - m_open.assert_called_once_with(file=list_local_path, mode="r") + m_open.assert_called_once_with(file=list_local_path) assert rasterset.raster_list == raster_list assert isinstance(serial_out["bbox"], list) for i in range(0, 4, 1): @@ -405,7 +405,7 @@ def test_simple_ok(self, m_from_parameters, m_get_osgeo_path): rasterset = RasterSet.from_descriptor(desc_path) m_get_osgeo_path.assert_called_once_with(desc_path) - m_open.assert_called_once_with(file=local_path, mode="r") + m_open.assert_called_once_with(file=local_path) assert rasterset.srs == serial_in["srs"] m_from_parameters.assert_called() assert m_from_parameters.call_count == 3 diff --git a/tests/test_tile_matrix_set.py b/tests/test_tile_matrix_set.py index faf03f1..0748128 100644 --- a/tests/test_tile_matrix_set.py +++ b/tests/test_tile_matrix_set.py @@ -66,7 +66,7 @@ def test_wrong_crs(mocked_get_data_str): tms = TileMatrixSet("tms") assert ( str(exc.value) - == "Wrong attribute 'crs' ('epsg:123456') in 'file:///path/to/tms.json', not recognize by OSR" + == "Wrong attribute 'crs' ('epsg:123456') in 'file:///path/to/tms.json', not recognize by OSR. Trace : PROJ: proj_create_from_database: crs not found" ) mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") From 3b8ef7405dc044f2eac6f05234dba2a86a837704 Mon Sep 17 00:00:00 2001 From: Theo Satabin Date: Fri, 1 Sep 2023 11:33:55 +0200 Subject: [PATCH 84/89] Correction de l'injection de version beta --- .bumpversion.cfg | 10 ++++++++++ .github/workflows/build-and-release.yaml | 5 ++--- 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 .bumpversion.cfg diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 0000000..25ed0e2 --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,10 @@ +[bumpversion] +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(-beta(?P\d+))? +serialize = + {major}.{minor}.{patch}-beta{beta} + {major}.{minor}.{patch} +current_version = 2.0.0-toto1 + +[bumpversion:file:pyproject.toml] + +[bumpversion:file:src/rok4/__init__.py] diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 5dea00b..dce24df 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -68,12 +68,11 @@ jobs: python3 -m venv .venv source .venv/bin/activate python3 -m pip install --upgrade build bump2version - bump2version --current-version 0.0.0 --new-version ${{ github.ref_name }} patch pyproject.toml src/rok4/__init__.py + bump2version --current-version 0.0.0 --new-version ${{ github.ref_name }} patch pip install -e . echo "/usr/lib/python3/dist-packages/" >.venv/lib/python${{ matrix.python-version }}/site-packages/system.pth - name: Run unit tests - if: "matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8'" run: | source .venv/bin/activate pip install -e .[test] @@ -159,7 +158,7 @@ jobs: commit_documentation: name: Add documentation and unit tests results into gh-pages branch needs: build_and_test - if: "always()&&!contains(${{ github.ref_name }},'beta')&&(needs.create_release.outputs.job_status=='success')&&(needs.build_and_test.outputs.job_status=='success')" + if: "!(contains(${{ github.ref_name }},'beta'))&&(needs.create_release.outputs.job_status=='success')&&(needs.build_and_test.outputs.job_status=='success')" runs-on: ubuntu-latest steps: From 1559d1af88f1d5d8b6379a1c5a43079dbe1a3114 Mon Sep 17 00:00:00 2001 From: Theo Satabin Date: Fri, 1 Sep 2023 12:40:19 +0200 Subject: [PATCH 85/89] Versionnement de beta compatible PEP440 --- .bumpversion.cfg | 5 ++--- .github/workflows/build-and-release.yaml | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 25ed0e2..33caec9 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,9 +1,8 @@ [bumpversion] -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(-beta(?P\d+))? +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(b(?P\d+))? serialize = - {major}.{minor}.{patch}-beta{beta} + {major}.{minor}.{patch}b{beta} {major}.{minor}.{patch} -current_version = 2.0.0-toto1 [bumpversion:file:pyproject.toml] diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index dce24df..4787ed7 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -4,7 +4,7 @@ on: push: tags: - '[0-9]+.[0-9]+.[0-9]+' - - '[0-9]+.[0-9]+.[0-9]+-beta[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+b[0-9]+' jobs: @@ -27,7 +27,7 @@ jobs: name: Release ${{ github.ref_name }} generateReleaseNotes: true draft: false - prerelease: contains(${{ github.ref_name }},'beta') + prerelease: contains(${{ github.ref_name }},'b') build_and_test: @@ -158,7 +158,7 @@ jobs: commit_documentation: name: Add documentation and unit tests results into gh-pages branch needs: build_and_test - if: "!(contains(${{ github.ref_name }},'beta'))&&(needs.create_release.outputs.job_status=='success')&&(needs.build_and_test.outputs.job_status=='success')" + if: "!(contains(${{ github.ref_name }},'b'))&&(needs.create_release.outputs.job_status=='success')&&(needs.build_and_test.outputs.job_status=='success')" runs-on: ubuntu-latest steps: From 206763f97c7cf3bbd32678fe0a70f2063eeb1eae Mon Sep 17 00:00:00 2001 From: Theo Satabin Date: Fri, 8 Sep 2023 13:24:00 +0200 Subject: [PATCH 86/89] Correction de la lecture d'une tuile d'unepyramide PNG 1 canal --- .github/workflows/build-and-release.yaml | 12 ++++++------ src/rok4/pyramid.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 4787ed7..eb7796c 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -27,7 +27,7 @@ jobs: name: Release ${{ github.ref_name }} generateReleaseNotes: true draft: false - prerelease: contains(${{ github.ref_name }},'b') + prerelease: ${{ contains(github.ref_name ,'b') }} build_and_test: @@ -80,7 +80,7 @@ jobs: coverage report -m - name: Build unit tests report - if: "matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8'" + if: "! contains(github.ref_name,'b') && matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8'" run: | source .venv/bin/activate coverage html -d dist/tests/ @@ -93,7 +93,7 @@ jobs: python3 -m build - name: Build documentation - if: "matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8'" + if: "! contains(github.ref_name,'b') && matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8'" run: | source .venv/bin/activate pip install -e .[doc] @@ -112,7 +112,7 @@ jobs: publish_artefacts: name: Add built artefacts to release and PyPI needs: [create_release, build_and_test] - if: "always()&&(needs.create_release.outputs.job_status=='success')&&(needs.build_and_test.outputs.job_status=='success')" + if: "always() && needs.create_release.outputs.job_status == 'success' && needs.build_and_test.outputs.job_status == 'success'" runs-on: ubuntu-latest steps: @@ -158,7 +158,7 @@ jobs: commit_documentation: name: Add documentation and unit tests results into gh-pages branch needs: build_and_test - if: "!(contains(${{ github.ref_name }},'b'))&&(needs.create_release.outputs.job_status=='success')&&(needs.build_and_test.outputs.job_status=='success')" + if: "always() && ! contains(github.ref_name,'b') && needs.create_release.outputs.job_status == 'success' && needs.build_and_test.outputs.job_status == 'success'" runs-on: ubuntu-latest steps: @@ -208,7 +208,7 @@ jobs: delete_version: name: Remove release and tag if error occured needs: build_and_test - if: "always()&&(needs.create_release.outputs.job_status=='success')&&(needs.build_and_test.outputs.job_status!='success')" + if: "always() && needs.create_release.outputs.job_status == 'success' && needs.build_and_test.outputs.job_status != 'success'" runs-on: ubuntu-latest steps: diff --git a/src/rok4/pyramid.py b/src/rok4/pyramid.py index c87da7a..f55433f 100644 --- a/src/rok4/pyramid.py +++ b/src/rok4/pyramid.py @@ -1189,6 +1189,11 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr raise FormatError("JPEG", "binary tile", e) data = numpy.asarray(img) + data.shape = ( + level_object.tile_matrix.tile_size[0], + level_object.tile_matrix.tile_size[1], + self.__raster_specifications["channels"], + ) elif self.__format == "TIFF_RAW_UINT8": data = numpy.frombuffer(binary_tile, dtype=numpy.dtype("uint8")) @@ -1205,6 +1210,11 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr raise FormatError("PNG", "binary tile", e) data = numpy.asarray(img) + data.shape = ( + level_object.tile_matrix.tile_size[0], + level_object.tile_matrix.tile_size[1], + self.__raster_specifications["channels"], + ) elif self.__format == "TIFF_ZIP_UINT8": try: From 2e2a202ecb611ae1c80e05ce2d0f1949222024c3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 8 Sep 2023 11:28:38 +0000 Subject: [PATCH 87/89] =?UTF-8?q?[pre-commit.ci]=20Corrections=20automatiq?= =?UTF-8?q?ues=20appliqu=C3=A9es=20par=20les=20git=20hooks.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- tests/test_layer.py | 2 -- tests/test_pyramid.py | 19 +++++++++---------- tests/test_raster.py | 2 +- tests/test_storage.py | 23 +++++++++++------------ tests/test_tile_matrix_set.py | 24 ++++++++++++------------ tests/test_utils.py | 13 ++++++------- tests/test_vector.py | 10 +++++----- 8 files changed, 45 insertions(+), 50 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 33caec9..2e04fb3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,6 +1,6 @@ [bumpversion] parse = (?P\d+)\.(?P\d+)\.(?P\d+)(b(?P\d+))? -serialize = +serialize = {major}.{minor}.{patch}b{beta} {major}.{minor}.{patch} diff --git a/tests/test_layer.py b/tests/test_layer.py index 9cd314b..a4df1b3 100644 --- a/tests/test_layer.py +++ b/tests/test_layer.py @@ -2,8 +2,6 @@ from unittest import mock from unittest.mock import * -import pytest - from rok4.enums import PyramidType from rok4.exceptions import * from rok4.layer import Layer diff --git a/tests/test_pyramid.py b/tests/test_pyramid.py index e71dd15..f44eed7 100644 --- a/tests/test_pyramid.py +++ b/tests/test_pyramid.py @@ -7,14 +7,13 @@ from rok4.enums import SlabType, StorageType from rok4.exceptions import * from rok4.pyramid import * -from rok4.tile_matrix_set import TileMatrixSet from rok4.utils import * @mock.patch("rok4.pyramid.get_data_str", side_effect=StorageError("FILE", "Not found")) def test_wrong_file(mocked_get_data_str): with pytest.raises(StorageError): - pyramid = Pyramid.from_descriptor("file:///pyramid.json") + Pyramid.from_descriptor("file:///pyramid.json") @mock.patch( @@ -23,7 +22,7 @@ def test_wrong_file(mocked_get_data_str): ) def test_bad_json(mocked_get_data_str): with pytest.raises(FormatError) as exc: - pyramid = Pyramid.from_descriptor("file:///pyramid.json") + Pyramid.from_descriptor("file:///pyramid.json") assert ( str(exc.value) @@ -35,7 +34,7 @@ def test_bad_json(mocked_get_data_str): @mock.patch("rok4.pyramid.get_data_str", return_value='{"format": "TIFF_PBF_MVT","levels":[]}') def test_missing_tms(mocked_get_data_str): with pytest.raises(MissingAttributeError) as exc: - pyramid = Pyramid.from_descriptor("file:///pyramid.json") + Pyramid.from_descriptor("file:///pyramid.json") assert str(exc.value) == "Missing attribute 'tile_matrix_set' in 'file:///pyramid.json'" mocked_get_data_str.assert_called_once_with("file:///pyramid.json") @@ -49,7 +48,7 @@ def test_missing_tms(mocked_get_data_str): @mock.patch("rok4.pyramid.TileMatrixSet", side_effect=StorageError("FILE", "TMS not found")) def test_wrong_tms(mocked_tms_constructor, mocked_get_data_str): with pytest.raises(StorageError) as exc: - pyramid = Pyramid.from_descriptor("file:///pyramid.json") + Pyramid.from_descriptor("file:///pyramid.json") assert str(exc.value) == "Issue occured using a FILE storage : TMS not found" mocked_tms_constructor.assert_called_once_with("PM") @@ -64,7 +63,7 @@ def test_wrong_tms(mocked_tms_constructor, mocked_get_data_str): @mock.patch("rok4.pyramid.TileMatrixSet") def test_raster_missing_raster_specifications(mocked_tms_class, mocked_get_data_str): with pytest.raises(MissingAttributeError) as exc: - pyramid = Pyramid.from_descriptor("file:///pyramid.json") + Pyramid.from_descriptor("file:///pyramid.json") assert str(exc.value) == "Missing attribute 'raster_specifications' in 'file:///pyramid.json'" mocked_get_data_str.assert_called_once_with("file:///pyramid.json") @@ -83,7 +82,7 @@ def test_wrong_level(mocked_tms_class, mocked_get_data_str): mocked_tms_class.return_value = tms_instance with pytest.raises(Exception) as exc: - pyramid = Pyramid.from_descriptor("file:///pyramid.json") + Pyramid.from_descriptor("file:///pyramid.json") mocked_tms_class.assert_called_once_with("PM") mocked_get_data_str.assert_called_once_with("file:///pyramid.json") @@ -102,7 +101,7 @@ def test_wrong_level(mocked_tms_class, mocked_get_data_str): @mock.patch("rok4.pyramid.TileMatrixSet", autospec=True) def test_vector_missing_tables(mocked_tms_class, mocked_get_data_str): with pytest.raises(MissingAttributeError) as exc: - pyramid = Pyramid.from_descriptor("file:///pyramid.json") + Pyramid.from_descriptor("file:///pyramid.json") assert str(exc.value) == "Missing attribute levels[].'tables' in 'file:///pyramid.json'" mocked_get_data_str.assert_called_once_with("file:///pyramid.json") @@ -119,7 +118,7 @@ def test_raster_ok(mocked_put_data_str, mocked_tms_class, mocked_get_data_str): tms_instance = MagicMock() tms_instance.name = "PM" tms_instance.srs = "EPSG:3857" - tms_instance.sr = sr_src = srs_to_spatialreference("EPSG:3857") + tms_instance.sr = srs_to_spatialreference("EPSG:3857") tm_instance = MagicMock() tm_instance.id = "0" @@ -191,7 +190,7 @@ def test_vector_ok(mocked_tms_class, mocked_get_data_str): except Exception as exc: assert False, f"Pyramid creation raises an exception: {exc}" - with pytest.raises(Exception) as exc: + with pytest.raises(Exception): pyramid.get_tile_data_raster("12", 5, 6) diff --git a/tests/test_raster.py b/tests/test_raster.py index 85b71d4..3222bc2 100644 --- a/tests/test_raster.py +++ b/tests/test_raster.py @@ -10,7 +10,7 @@ import math import random from unittest import TestCase, mock -from unittest.mock import MagicMock, Mock, call, mock_open, patch +from unittest.mock import MagicMock, call, mock_open import pytest diff --git a/tests/test_storage.py b/tests/test_storage.py index f2649e5..f5c4889 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -4,7 +4,6 @@ import botocore.exceptions import pytest -from rados import ObjectNotFound from rok4.enums import StorageType from rok4.exceptions import * @@ -55,13 +54,13 @@ def test_get_path_from_infos(): @mock.patch.dict(os.environ, {}, clear=True) def test_s3_missing_env(): with pytest.raises(MissingEnvironmentError): - data = get_data_str("s3://bucket/path/to/object") + get_data_str("s3://bucket/path/to/object") @mock.patch.dict(os.environ, {}, clear=True) def test_ceph_missing_env(): with pytest.raises(MissingEnvironmentError): - data = get_data_str("ceph://bucket/path/to/object") + get_data_str("ceph://bucket/path/to/object") @mock.patch.dict( @@ -71,7 +70,7 @@ def test_ceph_missing_env(): ) def test_s3_invalid_envs(): with pytest.raises(StorageError): - data = get_data_str("s3://bucket/path/to/object") + get_data_str("s3://bucket/path/to/object") @mock.patch.dict( @@ -79,17 +78,17 @@ def test_s3_invalid_envs(): ) @mock.patch("rok4.storage.boto3.client") def test_s3_invalid_endpoint(mocked_s3_client): - s3_instance = MagicMock() + MagicMock() mocked_s3_client.side_effect = Exception("Invalid URL") with pytest.raises(StorageError): - data = get_data_str("s3://bucket/path/to/object") + get_data_str("s3://bucket/path/to/object") @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("builtins.open", side_effect=FileNotFoundError("not_found")) def test_file_read_error(mock_file): with pytest.raises(FileNotFoundError): - data = get_data_str("file:///path/to/file.ext") + get_data_str("file:///path/to/file.ext") mock_file.assert_called_with("/path/to/file.ext", "rb") @@ -117,7 +116,7 @@ def test_s3_read_nok(mocked_s3_client): s3_instance.get_object.side_effect = Exception("Bucket or object not found") mocked_s3_client.return_value = s3_instance with pytest.raises(StorageError): - data = get_data_str("s3://bucket/path/to/object") + get_data_str("s3://bucket/path/to/object") @mock.patch.dict( @@ -171,7 +170,7 @@ def test_http_read_error(mock_http): requests_instance.content = "NULL" requests_instance.status_code = 404 mock_http.return_value = requests_instance - data = get_data_str("http://path/to/file.ext") + get_data_str("http://path/to/file.ext") mock_http.assert_called_with("http://path/to/file.ext", stream=True) @@ -179,7 +178,7 @@ def test_http_read_error(mock_http): @mock.patch.dict(os.environ, {}, clear=True) def test_http_read_range_error(): with pytest.raises(NotImplementedError): - data = get_data_binary("http://path/to/file.ext", (0, 100)) + get_data_binary("http://path/to/file.ext", (0, 100)) @mock.patch.dict(os.environ, {}, clear=True) @@ -925,7 +924,7 @@ def test_size_path_file_ok(): def test_size_file_nok(): with pytest.raises(StorageError): - size = size_path("file://tests/fixtures/TIFF_PBF_M") + size_path("file://tests/fixtures/TIFF_PBF_M") @mock.patch.dict( @@ -935,7 +934,7 @@ def test_size_file_nok(): ) def test_size_path_ceph_nok(): with pytest.raises(NotImplementedError): - size = size_path("ceph://pool/path") + size_path("ceph://pool/path") @mock.patch.dict( diff --git a/tests/test_tile_matrix_set.py b/tests/test_tile_matrix_set.py index 0748128..94f89f8 100644 --- a/tests/test_tile_matrix_set.py +++ b/tests/test_tile_matrix_set.py @@ -11,14 +11,14 @@ @mock.patch.dict(os.environ, {}, clear=True) def test_missing_env(): with pytest.raises(MissingEnvironmentError): - tms = TileMatrixSet("tms") + TileMatrixSet("tms") @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) @mock.patch("rok4.tile_matrix_set.get_data_str", side_effect=StorageError("FILE", "Not found")) def test_wrong_file(mocked_get_data_str): with pytest.raises(StorageError): - tms = TileMatrixSet("tms") + TileMatrixSet("tms") @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) @@ -27,8 +27,8 @@ def test_wrong_file(mocked_get_data_str): return_value='"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}', ) def test_bad_json(mocked_get_data_str): - with pytest.raises(FormatError) as exc: - tms = TileMatrixSet("tms") + with pytest.raises(FormatError): + TileMatrixSet("tms") mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") @@ -39,7 +39,7 @@ def test_bad_json(mocked_get_data_str): ) def test_missing_id(mocked_get_data_str): with pytest.raises(MissingAttributeError) as exc: - tms = TileMatrixSet("tms") + TileMatrixSet("tms") assert str(exc.value) == "Missing attribute 'id' in 'file:///path/to/tms.json'" mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") @@ -51,7 +51,7 @@ def test_missing_id(mocked_get_data_str): ) def test_missing_crs(mocked_get_data_str): with pytest.raises(MissingAttributeError) as exc: - tms = TileMatrixSet("tms") + TileMatrixSet("tms") assert str(exc.value) == "Missing attribute 'crs' in 'file:///path/to/tms.json'" mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") @@ -63,7 +63,7 @@ def test_missing_crs(mocked_get_data_str): ) def test_wrong_crs(mocked_get_data_str): with pytest.raises(Exception) as exc: - tms = TileMatrixSet("tms") + TileMatrixSet("tms") assert ( str(exc.value) == "Wrong attribute 'crs' ('epsg:123456') in 'file:///path/to/tms.json', not recognize by OSR. Trace : PROJ: proj_create_from_database: crs not found" @@ -78,7 +78,7 @@ def test_wrong_crs(mocked_get_data_str): ) def test_wrong_axes_order(mocked_get_data_str): with pytest.raises(Exception) as exc: - tms = TileMatrixSet("tms") + TileMatrixSet("tms") assert ( str(exc.value) == "TMS 'file:///path/to/tms.json' own invalid axes order : only X/Y or Lon/Lat are handled" @@ -93,7 +93,7 @@ def test_wrong_axes_order(mocked_get_data_str): ) def test_missing_levels(mocked_get_data_str): with pytest.raises(MissingAttributeError) as exc: - tms = TileMatrixSet("tms") + TileMatrixSet("tms") assert str(exc.value) == "Missing attribute 'tileMatrices' in 'file:///path/to/tms.json'" mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") @@ -105,7 +105,7 @@ def test_missing_levels(mocked_get_data_str): ) def test_no_levels(mocked_get_data_str): with pytest.raises(Exception) as exc: - tms = TileMatrixSet("tms") + TileMatrixSet("tms") assert str(exc.value) == "TMS 'file:///path/to/tms.json' has no level" mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") @@ -117,7 +117,7 @@ def test_no_levels(mocked_get_data_str): ) def test_wrong_level(mocked_get_data_str): with pytest.raises(MissingAttributeError) as exc: - tms = TileMatrixSet("tms") + TileMatrixSet("tms") assert str(exc.value) == "Missing attribute tileMatrices[].'id' in 'file:///path/to/tms.json'" mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") @@ -129,7 +129,7 @@ def test_wrong_level(mocked_get_data_str): ) def test_wrong_level_id(mocked_get_data_str): with pytest.raises(Exception) as exc: - tms = TileMatrixSet("tms") + TileMatrixSet("tms") assert ( str(exc.value) diff --git a/tests/test_utils.py b/tests/test_utils.py index 0b73b9e..584e34d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,4 @@ import math -import os import random from unittest import mock from unittest.mock import * @@ -13,28 +12,28 @@ def test_srs_to_spatialreference_ignf_ok(): try: - sr = srs_to_spatialreference("IGNF:LAMB93") - sr = srs_to_spatialreference("ignf:lamb93") + srs_to_spatialreference("IGNF:LAMB93") + srs_to_spatialreference("ignf:lamb93") except Exception as exc: assert False, f"SpatialReference creation raises an exception: {exc}" def test_srs_to_spatialreference_epsg_ok(): try: - sr = srs_to_spatialreference("EPSG:3857") - sr = srs_to_spatialreference("epsg:3857") + srs_to_spatialreference("EPSG:3857") + srs_to_spatialreference("epsg:3857") except Exception as exc: assert False, f"SpatialReference creation raises an exception: {exc}" def test_srs_to_spatialreference_ignf_nok(): with pytest.raises(Exception): - sr = srs_to_spatialreference("IGNF:TOTO") + srs_to_spatialreference("IGNF:TOTO") def test_srs_to_spatialreference_epsg_nok(): with pytest.raises(Exception): - sr = srs_to_spatialreference("EPSG:123456") + srs_to_spatialreference("EPSG:123456") def test_bbox_to_geometry_ok(): diff --git a/tests/test_vector.py b/tests/test_vector.py index 8f75618..72e9c50 100644 --- a/tests/test_vector.py +++ b/tests/test_vector.py @@ -13,25 +13,25 @@ def test_missing_env(): disconnect_ceph_clients() with pytest.raises(MissingEnvironmentError): - vector = Vector.from_file("ceph:///ign_std/vector.shp") + Vector.from_file("ceph:///ign_std/vector.shp") @mock.patch("rok4.vector.copy", side_effect=StorageError("CEPH", "Not found")) def test_wrong_file(mocked_copy): with pytest.raises(StorageError): - vector = Vector.from_file("ceph:///vector.geojson") + Vector.from_file("ceph:///vector.geojson") def test_wrong_format(): with pytest.raises(Exception) as exc: - vector = Vector.from_file("ceph:///vector.tif") + Vector.from_file("ceph:///vector.tif") assert str(exc.value) == "This format of file cannot be loaded" @mock.patch("rok4.vector.ogr.Open", return_value="not a shape") def test_wrong_content(mocked_copy): with pytest.raises(Exception) as exc: - vector = Vector.from_file("file:///vector.shp") + Vector.from_file("file:///vector.shp") assert str(exc.value) == "The content of file:///vector.shp cannot be read" @@ -39,7 +39,7 @@ def test_wrong_content(mocked_copy): @mock.patch("rok4.vector.ogr.Open", return_value="not a shape") def test_wrong_content_ceph(mocked_open, mocked_copy): with pytest.raises(Exception) as exc: - vector = Vector.from_file("file:///vector.shp") + Vector.from_file("file:///vector.shp") assert str(exc.value) == "The content of file:///vector.shp cannot be read" From a0a5f02b7161ef885a8b0158a8ba8c1364a780cf Mon Sep 17 00:00:00 2001 From: Theo Satabin Date: Wed, 13 Sep 2023 16:57:33 +0200 Subject: [PATCH 88/89] Client S3 avec connexions ouvertes et cache LRU de lecture --- CHANGELOG.md | 13 ++++++ src/rok4/storage.py | 103 +++++++++++++++++++++++++++++++------------- 2 files changed, 87 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24eaed0..59c51ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +## 2.0.0 + +### [Fixed] + +* Pyramid + * quand on lit une tuile dans une pyramide PNG 1 canal, on retourne bien aussi un numpy.array à 3 dimensions (la dernière dimension sera bien un array à un élément) + +### [Changed] + +* Storage + * Le client S3 garde ouverte des connexions + * La fonction get_data_binary a un système de cache de type LRU, avec un temps de validité de 5 minutes + ## 1.7.1 ### [Added] diff --git a/src/rok4/storage.py b/src/rok4/storage.py index 8fda078..14c689e 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -18,6 +18,7 @@ - ROK4_S3_KEY - ROK4_S3_SECRETKEY - ROK4_S3_URL +- ROK4_SSL_NO_VERIFY (optionnal) with a non empty value disables certificate check.. Define PYTHONWARNINGS to "ignore:Unverified HTTPS request" to disable warnings logs To use several S3 clusters, each environment variable have to contain a list (comma-separated), with the same number of elements @@ -34,12 +35,15 @@ import os import re import tempfile +from functools import lru_cache from shutil import copyfile from typing import Dict, Tuple, Union import boto3 import botocore.exceptions + import requests +import time from osgeo import gdal # conditional import @@ -67,6 +71,10 @@ __S3_DEFAULT_CLIENT = None +def __get_ttl_hash(): + """Return the same value withing 5 minutes time period""" + return round(time.time() / 300) + def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", str]], str, str]: """Get the S3 client @@ -85,7 +93,13 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s global __S3_CLIENTS, __S3_DEFAULT_CLIENT + if not __S3_CLIENTS: + + verify = True + if "ROK4_SSL_NO_VERIFY" in os.environ and os.environ["ROK4_SSL_NO_VERIFY"] != "": + verify = False + # C'est la première fois qu'on cherche à utiliser le stockage S3, chargeons les informations depuis les variables d'environnement try: keys = os.environ["ROK4_S3_KEY"].split(",") @@ -109,7 +123,12 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s "s3", aws_access_key_id=keys[i], aws_secret_access_key=secret_keys[i], + verify=verify, endpoint_url=urls[i], + config=botocore.config.Config( + tcp_keepalive = True, + max_pool_connections = 10 + ) ), "key": keys[i], "secret_key": secret_keys[i], @@ -271,6 +290,7 @@ def get_data_str(path: str) -> str: MissingEnvironmentError: Missing object storage informations StorageError: Storage read issue FileNotFoundError: File or object does not exist + NotImplementedError: Storage type not handled Returns: str: Data content @@ -279,17 +299,20 @@ def get_data_str(path: str) -> str: return get_data_binary(path).decode("utf-8") -def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: - """Load data into a binary string +@lru_cache(maxsize=50) +def __get_cached_data_binary(path: str, ttl_hash: int, range: Tuple[int, int] = None) -> str: + """Load data into a binary string, using a LRU cache Args: path (str): path to data + ttl_hash (int): time hash, to invalid cache range (Tuple[int, int], optional): offset and size, to make a partial read. Defaults to None. Raises: MissingEnvironmentError: Missing object storage informations StorageError: Storage read issue FileNotFoundError: File or object does not exist + NotImplementedError: Storage type not handled Returns: str: Data binary content @@ -329,7 +352,7 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: except Exception as e: raise StorageError("S3", e) - elif storage_type == StorageType.CEPH: + elif storage_type == StorageType.CEPH and CEPH_RADOS_AVAILABLE: ioctx = __get_ceph_ioctx(tray_name) try: @@ -372,13 +395,33 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: except Exception as e: raise StorageError(storage_type.name, e) else: - raise NotImplementedError + raise NotImplementedError(f"Cannot get partial data for storage type HTTP(S)") else: - raise StorageError("UNKNOWN", "Unhandled storage type to read binary data") + raise NotImplementedError(f"Cannot get data for storage type {storage_type.name}") return data +def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: + """Load data into a binary string + + This function uses a LRU cache, with a TTL of 5 minutes + + Args: + path (str): path to data + range (Tuple[int, int], optional): offset and size, to make a partial read. Defaults to None. + + Raises: + MissingEnvironmentError: Missing object storage informations + StorageError: Storage read issue + FileNotFoundError: File or object does not exist + NotImplementedError: Storage type not handled + + Returns: + str: Data binary content + """ + return __get_cached_data_binary(path, __get_ttl_hash(), range) + def put_data_str(data: str, path: str) -> None: """Store string data into a file or an object @@ -392,6 +435,7 @@ def put_data_str(data: str, path: str) -> None: Raises: MissingEnvironmentError: Missing object storage informations StorageError: Storage write issue + NotImplementedError: Storage type not handled """ storage_type, path, tray_name, base_name = get_infos_from_path(path) @@ -406,7 +450,7 @@ def put_data_str(data: str, path: str) -> None: except Exception as e: raise StorageError("S3", e) - elif storage_type == StorageType.CEPH: + elif storage_type == StorageType.CEPH and CEPH_RADOS_AVAILABLE: ioctx = __get_ceph_ioctx(tray_name) try: @@ -423,7 +467,7 @@ def put_data_str(data: str, path: str) -> None: raise StorageError("FILE", e) else: - raise StorageError("UNKNOWN", "Unhandled storage type to write string data") + raise NotImplementedError(f"Cannot write data for storage type {storage_type.name}") def get_size(path: str) -> int: @@ -435,6 +479,7 @@ def get_size(path: str) -> int: Raises: MissingEnvironmentError: Missing object storage informations StorageError: Storage read issue + NotImplementedError: Storage type not handled Returns: int: file/object size, in bytes @@ -453,7 +498,7 @@ def get_size(path: str) -> int: except Exception as e: raise StorageError("S3", e) - elif storage_type == StorageType.CEPH: + elif storage_type == StorageType.CEPH and CEPH_RADOS_AVAILABLE: ioctx = __get_ceph_ioctx(tray_name) try: @@ -478,7 +523,7 @@ def get_size(path: str) -> int: raise StorageError(storage_type.name, e) else: - raise StorageError("UNKNOWN", "Unhandled storage type to get size") + raise NotImplementedError(f"Cannot get size for storage type {storage_type.name}") def exists(path: str) -> bool: @@ -490,6 +535,7 @@ def exists(path: str) -> bool: Raises: MissingEnvironmentError: Missing object storage informations StorageError: Storage read issue + NotImplementedError: Storage type not handled Returns: bool: file/object existing status @@ -509,7 +555,7 @@ def exists(path: str) -> bool: else: raise StorageError("S3", e) - elif storage_type == StorageType.CEPH: + elif storage_type == StorageType.CEPH and CEPH_RADOS_AVAILABLE: ioctx = __get_ceph_ioctx(tray_name) try: @@ -534,7 +580,7 @@ def exists(path: str) -> bool: raise StorageError(storage_type.name, e) else: - raise StorageError("UNKNOWN", "Unhandled storage type to test if exists") + raise NotImplementedError(f"Cannot test existence for storage type {storage_type.name}") def remove(path: str) -> None: @@ -546,6 +592,7 @@ def remove(path: str) -> None: Raises: MissingEnvironmentError: Missing object storage informations StorageError: Storage removal issue + NotImplementedError: Storage type not handled """ storage_type, path, tray_name, base_name = get_infos_from_path(path) @@ -557,7 +604,7 @@ def remove(path: str) -> None: except Exception as e: raise StorageError("S3", e) - elif storage_type == StorageType.CEPH: + elif storage_type == StorageType.CEPH and CEPH_RADOS_AVAILABLE: ioctx = __get_ceph_ioctx(tray_name) try: @@ -576,7 +623,7 @@ def remove(path: str) -> None: raise StorageError("FILE", e) else: - raise StorageError("UNKNOWN", "Unhandled storage type to remove things") + raise NotImplementedError(f"Cannot remove data for storage type {storage_type.name}") def copy(from_path: str, to_path: str, from_md5: str = None) -> None: @@ -588,8 +635,9 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: from_md5 (str, optional): MD5 sum, re-processed after copy and controlled. Defaults to None. Raises: - StorageError: Unhandled copy or copy issue + StorageError: Copy issue MissingEnvironmentError: Missing object storage informations + NotImplementedError: Storage type not handled """ from_type, from_path, from_tray, from_base_name = get_infos_from_path(from_path) @@ -687,7 +735,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: except Exception as e: raise StorageError("S3", f"Cannot copy S3 object {from_path} to {to_path} : {e}") - elif from_type == StorageType.CEPH and to_type == StorageType.FILE: + elif from_type == StorageType.CEPH and CEPH_RADOS_AVAILABLE and to_type == StorageType.FILE: ioctx = __get_ceph_ioctx(from_tray) if from_md5 is not None: @@ -726,7 +774,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: "CEPH and FILE", f"Cannot copy CEPH object {from_path} to file {to_path} : {e}" ) - elif from_type == StorageType.FILE and to_type == StorageType.CEPH: + elif from_type == StorageType.FILE and to_type == StorageType.CEPH and CEPH_RADOS_AVAILABLE: ioctx = __get_ceph_ioctx(to_tray) if from_md5 is not None: @@ -763,7 +811,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: "FILE and CEPH", f"Cannot copy file {from_path} to CEPH object {to_path} : {e}" ) - elif from_type == StorageType.CEPH and to_type == StorageType.CEPH: + elif from_type == StorageType.CEPH and to_type == StorageType.CEPH and CEPH_RADOS_AVAILABLE: from_ioctx = __get_ceph_ioctx(from_tray) to_ioctx = __get_ceph_ioctx(to_tray) @@ -795,7 +843,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: except Exception as e: raise StorageError("CEPH", f"Cannot copy CEPH object {from_path} to {to_path} : {e}") - elif from_type == StorageType.CEPH and to_type == StorageType.S3: + elif from_type == StorageType.CEPH and CEPH_RADOS_AVAILABLE and to_type == StorageType.S3: from_ioctx = __get_ceph_ioctx(from_tray) s3_client, to_bucket = __get_s3_client(to_tray) @@ -854,7 +902,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: elif ( from_type == StorageType.HTTP or from_type == StorageType.HTTPS - ) and to_type == StorageType.CEPH: + ) and to_type == StorageType.CEPH and CEPH_RADOS_AVAILABLE: to_ioctx = __get_ceph_ioctx(to_tray) try: @@ -896,10 +944,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: ) else: - raise StorageError( - f"{from_type.name} and {to_type.name}", - f"Cannot copy from {from_type.name} to {to_type.name}", - ) + raise NotImplementedError(f"Cannot copy data from storage type {from_type.name} to storage type {to_type.name}") def link(target_path: str, link_path: str, hard: bool = False) -> None: @@ -911,8 +956,9 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: hard (bool, optional): hard link rather than symbolic. Only for FILE storage. Defaults to False. Raises: - StorageError: Unhandled link or link issue + StorageError: link issue MissingEnvironmentError: Missing object storage informations + NotImplementedError: Storage type not handled """ target_type, target_path, target_tray, target_base_name = get_infos_from_path(target_path) @@ -947,7 +993,7 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: except Exception as e: raise StorageError("S3", e) - elif target_type == StorageType.CEPH: + elif target_type == StorageType.CEPH and CEPH_RADOS_AVAILABLE: ioctx = __get_ceph_ioctx(link_tray) try: @@ -965,7 +1011,7 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: raise StorageError("FILE", e) else: - raise StorageError("UNKNOWN", "Unhandled storage type to make link") + raise NotImplementedError(f"Cannot make link for storage type {target_type.name}") def get_osgeo_path(path: str) -> str: @@ -1013,6 +1059,7 @@ def size_path(path: str) -> int: Raises: StorageError: Unhandled link or link issue MissingEnvironmentError: Missing object storage informations + NotImplementedError: Storage type not handled Returns: int: size of the path @@ -1052,9 +1099,7 @@ def size_path(path: str) -> int: except Exception as e: raise StorageError("S3", e) - elif storage_type == StorageType.CEPH: - raise NotImplementedError else: - raise StorageError("UNKNOWN", "Unhandled storage type to calculate size") + raise NotImplementedError(f"Cannot get prefix path size for storage type {storage_type.name}") return total From 73b49d79a45c71d557829b66be1377c4af35f405 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 15:00:40 +0000 Subject: [PATCH 89/89] =?UTF-8?q?[pre-commit.ci]=20Corrections=20automatiq?= =?UTF-8?q?ues=20appliqu=C3=A9es=20par=20les=20git=20hooks.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/rok4/storage.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/rok4/storage.py b/src/rok4/storage.py index 14c689e..fc7a775 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -35,15 +35,14 @@ import os import re import tempfile +import time from functools import lru_cache from shutil import copyfile from typing import Dict, Tuple, Union import boto3 import botocore.exceptions - import requests -import time from osgeo import gdal # conditional import @@ -75,6 +74,7 @@ def __get_ttl_hash(): """Return the same value withing 5 minutes time period""" return round(time.time() / 300) + def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", str]], str, str]: """Get the S3 client @@ -93,9 +93,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s global __S3_CLIENTS, __S3_DEFAULT_CLIENT - if not __S3_CLIENTS: - verify = True if "ROK4_SSL_NO_VERIFY" in os.environ and os.environ["ROK4_SSL_NO_VERIFY"] != "": verify = False @@ -125,10 +123,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s aws_secret_access_key=secret_keys[i], verify=verify, endpoint_url=urls[i], - config=botocore.config.Config( - tcp_keepalive = True, - max_pool_connections = 10 - ) + config=botocore.config.Config(tcp_keepalive=True, max_pool_connections=10), ), "key": keys[i], "secret_key": secret_keys[i], @@ -395,13 +390,14 @@ def __get_cached_data_binary(path: str, ttl_hash: int, range: Tuple[int, int] = except Exception as e: raise StorageError(storage_type.name, e) else: - raise NotImplementedError(f"Cannot get partial data for storage type HTTP(S)") + raise NotImplementedError("Cannot get partial data for storage type HTTP(S)") else: raise NotImplementedError(f"Cannot get data for storage type {storage_type.name}") return data + def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: """Load data into a binary string @@ -901,8 +897,10 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: ) elif ( - from_type == StorageType.HTTP or from_type == StorageType.HTTPS - ) and to_type == StorageType.CEPH and CEPH_RADOS_AVAILABLE: + (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) + and to_type == StorageType.CEPH + and CEPH_RADOS_AVAILABLE + ): to_ioctx = __get_ceph_ioctx(to_tray) try: @@ -944,7 +942,9 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: ) else: - raise NotImplementedError(f"Cannot copy data from storage type {from_type.name} to storage type {to_type.name}") + raise NotImplementedError( + f"Cannot copy data from storage type {from_type.name} to storage type {to_type.name}" + ) def link(target_path: str, link_path: str, hard: bool = False) -> None: @@ -1100,6 +1100,8 @@ def size_path(path: str) -> int: raise StorageError("S3", e) else: - raise NotImplementedError(f"Cannot get prefix path size for storage type {storage_type.name}") + raise NotImplementedError( + f"Cannot get prefix path size for storage type {storage_type.name}" + ) return total