From e2099d0c14875860ea74d07700613f22a3b62c51 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 25 May 2015 22:45:45 +0100 Subject: [PATCH 001/167] Initial files proving concepts WIP - nothing currently workable --- src/activefile_test.cdb | Bin 0 -> 2048 bytes src/eleveleddb.app.src | 17 + src/from_dict_test.cdb | Bin 0 -> 2108 bytes src/full.cdb | Bin 0 -> 582764 bytes src/hashtable1_test.cdb | Bin 0 -> 2336 bytes src/leveled_bst.beam | Bin 0 -> 4012 bytes src/leveled_bst.erl | 156 ++++++++ src/leveled_cdb.beam | Bin 0 -> 21212 bytes src/leveled_cdb.erl | 804 ++++++++++++++++++++++++++++++++++++++ src/leveled_internal.beam | Bin 0 -> 3068 bytes src/leveled_internal.erl | 118 ++++++ src/onstartfile.bst | Bin 0 -> 32 bytes src/rice.erl | 155 ++++++++ src/simple.cdb | Bin 0 -> 2124 bytes src/test.cdb | Bin 0 -> 32 bytes src/test_inconsole.cdb | 0 src/test_mem.cdb | Bin 0 -> 2115 bytes test/lookup_test.beam | Bin 0 -> 4096 bytes test/lookup_test.erl | 241 ++++++++++++ 19 files changed, 1491 insertions(+) create mode 100644 src/activefile_test.cdb create mode 100644 src/eleveleddb.app.src create mode 100644 src/from_dict_test.cdb create mode 100644 src/full.cdb create mode 100644 src/hashtable1_test.cdb create mode 100644 src/leveled_bst.beam create mode 100644 src/leveled_bst.erl create mode 100644 src/leveled_cdb.beam create mode 100644 src/leveled_cdb.erl create mode 100644 src/leveled_internal.beam create mode 100644 src/leveled_internal.erl create mode 100644 src/onstartfile.bst create mode 100644 src/rice.erl create mode 100644 src/simple.cdb create mode 100644 src/test.cdb create mode 100644 src/test_inconsole.cdb create mode 100644 src/test_mem.cdb create mode 100644 test/lookup_test.beam create mode 100644 test/lookup_test.erl diff --git a/src/activefile_test.cdb b/src/activefile_test.cdb new file mode 100644 index 0000000000000000000000000000000000000000..e9784eb4c8849062d374dd2058af8814b024f3bd GIT binary patch literal 2048 ccmZQz7zLvtFd71*Aut*OqaiRF0wXO100;m80RR91 literal 0 HcmV?d00001 diff --git a/src/eleveleddb.app.src b/src/eleveleddb.app.src new file mode 100644 index 0000000..1be8c00 --- /dev/null +++ b/src/eleveleddb.app.src @@ -0,0 +1,17 @@ +{application, eleveleddb, + [ + {description, ""}, + {vsn, "0.0.1"}, + {modules, []}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]}, + {mod, {eleveleddb_app, []}}, + {env, [ + %% Default max file size (in bytes) + {max_file_size, 32#80000000}, % 4GB default + + ]} + ]}. \ No newline at end of file diff --git a/src/from_dict_test.cdb b/src/from_dict_test.cdb new file mode 100644 index 0000000000000000000000000000000000000000..5cbf317443081e11d62055b2b9d52fdf90e452b2 GIT binary patch literal 2108 zcmb2)U;qP|QF=5SfFUrN4o33>Cwr2{4)LkC(vfM^ik2Ff3$A?akOh65we q`K&;kJmap{nG_f=v2fq}pGgp(F)$q2%EZ72jc<_HF(97-$Oiz@yfu>m literal 0 HcmV?d00001 diff --git a/src/full.cdb b/src/full.cdb new file mode 100644 index 0000000000000000000000000000000000000000..ffa584bafa2352c088b28bd9feec3aa6ab14563a GIT binary patch literal 582764 zcma%^b+}eV_xBH72L&ai6$vHw^bBDT2BCmTC?cJQE~NydK~OplpoG)`L6H!oloA9b zR8m?bq*LPEKW5GQ=d;%JT#vH{zTeq1vp@ID>@{=WB;8Dx&^93>>3N$=AOnS5F*t51`%rBA37|9zh)d^##e`h-{hue@A7y^tq;!fSE9 zR6d_36-b{@BhK$E`Ks+}G!@^1A%arwuEl zPk1~2`+G0=H2LN933cOqjVeC9_e%PN`f+~k>ptyMBYi@nIG^;UPaD?p<@b5Vr^)rx zCp3=#KKVUJ8~OVjZUXmhkv^em{P(3=`?Nwk->yFGeY&=zZ&&^zDpf|0fOb>Dod5KKF+Dw8BXL=c7jZ^zi4t9}*|{`j(mu{W{gBDapRy z56|#v;+N?Ys>SzjINPTwbNzi2=le8iVfutF@!#KD?9)-pe81mY?$e}I=@XLTzu&pW zr=`~Wbkru$ZBCz1C;t0U+kBdThi~VoUwnFRH~jq``1}3-x$OMIr+p5lPv{o=^B?u; z+T*^RiKl(q@T~vyd*^+cdDvFGUmy6iPeO)AeB@8#)3xa` zJbE6bGWs+*bB2UY|9Ah#e43m+!z2C@bNaMTo(zxpUz;z(qvw=ZD8r-vN`Bg>rHVm0 zB{DqX`rdOspH$kXhs$P2s1=vrko)wo^!4atpkC!NJZk^iiau@lvQO_-@oDmFP`?`f zekpJIKdrorAo8=gQaEv&I>prksWLorCsW^l9=XpI*4)xoCLZr#o-@c9y#9(^2<*dLbdB?}xNL zP08TX#4H&f@!97wpI*qG@e$7rbNF;;u8fcRJ29_MYvjxL==mfU@acs@{?AK2?bD>9 z86Q2*QN?|l@?6G8JRC0V(?pi>5%&$HPg6|BN1ula<$PM|g^Z8*PI<|vr7CB9^m*G^ z)u&0X`9Hr<&8JB<{hue+@@Yz)jF0%QQ8(kG&(+}uK23Vxr}rBBb~kM9)3vR`*ij9x!2vNeLnR4a-kRWQ(x$(k9}HUpl{cx!9G1a%-^TNDBr&+W1!#1 z`7~*wPj^o7X{qVHoaC84O`HYon&ZDpiXeH#Ai(|da|J{k`$?9ceZt$w zKF58UHy)cZ$N!Ib)Vy8Sq^?QLoBiK^I`)kJ+w$3>&#+nDliuyoB&mA~_Eap(^1r0f z$6KG6=e1G3SeW=f#nIYaWj+LTls}ec{onFv;p!+=&{GWT$hw3{!N3T2;8j-EE*%V< zU=U(ylQqF02nM-$-}OAfAPEL_aPLo}gFz7ty7IvZG8i<$VC0dxvx31840cY_RU#NH z!Ql3`7&bQ;91aHCk-gx*peH;w7JWM_)k{H70E4lg=Ca>|fpQ1_eoEt{U?70O#B+;k z-3L`{1%*Ab`Q$ z@wd(u4+a7l;@LtUZvs7`Q$;jje6#AlU4M`uAizv`HBFBVqa{(z9$$6U?}c3%xxVE1Td6%W9x(m!9V~* z$zNwDydMk%FqE1%C)yDVlwQ&=EN;>{7zkh}^ZLTte+L5r3}p}GtkEMF2w*6;h0EQMV&TXj}30k=YoL%h7o^43y_!&p%VMC;Q}j zvvx2Lz;G%{_K&v)0|5-D$L1;1Hy8+DICHvC!SlgD0K?htU!>_33tNQ2w-rNq2%J+!9W0m z+es6b!WxNj0vOyK`@BD_!WbujAu6PO1LEO~(5WKoUpLbYXn`PrA=s_ju30b;z!1Dc z{niJ9fdGaOc|WNCW-w6NAcvO!1?FZZ2w(_xxx#&z{h1(uA#@v+vLRH70ERFHd2-!g zAb=t4=uZd08i@%47{c8du?$vWOc213vl<_Qh(06qk&Jfdp2?vFgfJX}{)1_Bsb=crVFa4=B1)ikTx z`pIA*fT8WRyWIx{0|5-}2HmfkBNzx^nDT8#z74@3jgB z0vLv8mWz)B0|5*p`ikk*gMk2sktgeAJR1xIFpL^s%k~Qf0vJYTZP4bYU?6~DOs_@_ z>I4JjIoNZ@tNa)Y1Tc)7ex}kLrh!-clZ%d#YdSFalTt81C?I^FodAFe!6Uuhqvth88#i7>;FK_uD_g zKmfz>@hkhk6$}J0oH)63>E>V{fZ=4{eb2TG1_BsPW!{_faxf6UaC+RqDzGcT903ex zP95n6(Q4)hU^v^YX&cxnWsU%bb7@*MfE_gE@GvR=e0$fCzR4U7lk#YE;Tm^AjYGks zXhFqO9fO`?U`NaCyAupJMLgoKiounG0jG#ZB9h(P7#mXug4qi2nKw3;p|TFK1rY_ zJT?|BEw;Z*&=bI5Y-^FztAc@Y2i~Z}^3=gV0E3B=We1H91_BsNuB@Kj1OovKrZ&_l z0>@`KBY?qlgQ}hKg*yEA(5 zcnUn4mSxDrpeKOA*@eUvy@P=O2It+={k<_5C{+@vQ~yyf7zki+ncT9{^tlZf0|5-~PW%l+(L*y=5;3d!# zI#omiv#m@CdIA`N-N-$(Trd#85Im{i@BM>;0EQ6x@}9{R43sv=!F{`a7Yqb2gu2?R z@-x9e07K|@U7wf|3MIlShulb|Q` zk&J$+U8G&m6Tpz{`?}fh2Lk~N$)EY)rB{Q20EQHA*YEUOFc83y^5+Vty95KJ(dvit z`%eV}0Su{&@i$EHU`V(8(A}5WrAsPT^B? zgMrdZx?;)i3IziJ3}vd789Y1~2w*6C@bk%ef`I^ra*HR<9~}$?Fm$WR{{)Z-Ezr@! z*Z}1d!lYcYVU4@O%W#}Q4;gXIv$uH~URC1^!5~_8cmiIAxh8<2Du0vKA2 z$+{9=hPftyp>?jbXW&(pYf878CTV_$mtn36U}$?~?GAWV<(dG7c0<0$UnhbArzno9em%AtY=*}f^pS~3{W+l* zY=*}ff4MIy3a6;0nZ81Q*PMDDj|^u1ufhZn()Rj64S z^n}O8qGhMn=M8!S7>sQ>Ic7{SQ0~A#ICoS90|5*sN?iPRb}$gYU~u`-=Ab`Pnx2>Pv3mG56u2?1|KoF?z-^xOQA{xFt|y(_@j@4fdB@#5doz!1Fs;k7G+fdGaOPaPUnBp4`dkb~Ce zg}GS}PLm3uuB|Ewvwxhy!4SIbvJaPpDiOdCrof8VY%mbO5cbnd@4y;K5KfZ{;qE@x z0#;#h1_wiaR(JI*s1aehmC?>|)f)yq0Sw93jPLz(Fc83y{ORe>bqWRo7*f15^{F$# zKmbF^9S_D-4F*c1)!IAj_XYz245^>J{BN6JAb=rFom)pA1_J>MX*a$0V~1cMOeNBN z_vW?R!9V~*{at&w1U;e2D%wA+S6JE-zH~1Xn>D7|$Z!J!7>bu4l8*%g0SqN-40wKC zFc82{a?j~?X@h|PhEj7+j2;^dlwQ&m&i_>+7zkh}Q~koFb-_RYL)k-bT`e071Td6K zd3)39U?6~@`_=yw!m5JM0v$b=TkM zBl@j;{n=n3fMMjZy@!Vc0|5-9ChfbBEf@%382#9h9~K4!0Sse4I{0aYV4yq)`%jC{ z76$_X4C7`r{i1v@5Wuigj@=jzdP4a&YFsRBt)M5YF)YhnL~aQN0vMK$F43Y-Fc847 z;!fFj&IJPj3@ckz-|%`c5Wui1TaC+q1p@&LtA|(pvPCcuz_8{)rO&Sg0|5+cJKP`D zG#Drk&gH$k_IfZ7z;OTmy}1zR@i3`y(MJ_k_Mj(#;n-v4(hUs;0vL`@5^q-w1_BsP z9Fxr#1p@&LC;Qd9k}((vU^tbj-o_chKmfz(&l-JJG#ChAIP-UdFTM!|0vOKrJoCdd z!9W1RxrF1NeisaQm{fka@ON1BNc0s(Mt}A?4+Zm|O`IXzLCDqHVPi-}I7Kp&3y(B}Esr=uxPv-!FbOtQBq2;HBVA>0 zMF>g68NwZm*q60m=p~#Y8QIzGvO!oS&Jgb4_9YdDu!@ZE;YG5q`yPTpTM`}{i;U~t`S z1S~Kl;p54pnM3Y=04*SV>0WYncEHcugMk1B=beX5>lzG{Dv9)C7M%_T0vKFQKL5fi z!9W->xVm)Vv0sCM00!4RPJhrg7zki+lj=mrd%-{egWJ#F?*dylk`N}9+?{#15^UYX z8GI@cFL(VO7Lt1R0{Vn ze$wF%1Tf@Zxy!<0TN3(6M%xd+e*ns-EJ`F>cc}fF!9V~*^5T0c+z18&7*f>QpV%xI z2w+ILYuS_y!9Z!WTD@XH-C!VqA$8&Pw;?1UB>@a+>aE%aAqgo7U`V?)!+r>>NJ*GV zq+6PB8p0}462MUNvlW7uDoJRvihkXiG#UDc5GIw1&D+rMv0xy8q4*2ivn~$?0vJlX zzN_NX!9V~*$v+m(9}o-#FqHaY@#NgWKG8W!`gP=O#M3;C=br%|88ODU?7Yb?#r*<`V;hcm{hu`PnT!L2R#7{$Fg*PGIcNz zz;JwQpGvEOfdGaRr+arP6AT0}oa|lb?@xn)0ESbEdH%>33zT2v3W0Qjr+9oVKRTQOkxeC#< zID-z5Dq@GaZ-O{%oFN!^r9R`}ZCeq-q$(1s_iTvU#2LaJy7UmpwvFa&Q?dqcWl zAb=r6!Mc~n1p}oGvOjMDbF(6xCRIY+Qt!a*A7}8VAasWcY4e9F5x@{8Z+ST)7zkhp zJA6<{SR*Mym{bY(;FG+t3X3y17;;Nu!7xe^MpYSYD`@_O77)UuO0xBNGkh2f1TZ8o zmaXolU?6}YMeW=z-wp->7*g);dhJ#)P#UdPb^EbNFc83yx=7!#hk}6shBS41&8is; z1TduCvU&aCU?7Yb(kEK*rWL_J07I!SJ6(w8TLfdGcqPgcL&AQ&j!YMR&BusawC zU}$^&+L1=VKmbF#Pp)6w77PS1%m?Y-4C`k?g-mo(G%pS16Hb#V!^X?E*8~Fr48yZl zOqVJc2w)h|tDKq?3E{uc}cFs#b4aa8+YAb?@@r`wMF5ex({thu}U;>*E6 z0K?k0oi4+95=HpZy|S)A&ke7HI}pHdtA1!2-iPrpsd7=@rLWJ1@(EwMSB_;~`H>0+ z0vL{uTUTOCFc83S;?(Lqd4qufhLgRHeENMb5WsLM>%kvN1p@&Lr^oNTFfkYiU^sJf z-{FK{Ab{a)*LE8}4+g>~+PSnzS26?x9wyZj=d2v-077PS1m|R+E_4!~R zfWg$(s>AyR0|5-C8`b!8YcLSNV5VgCv$cbP00y(Guiw}Z3l^T@s!82yT4wGCFTB8A31F*7^+yhigKZR9jbw^@D(WoWa3x<33~WLYwh0sdmxF zH*ehydcsuVSjH<`lY)T&hU1^#p7?t(5WsNa%)_r<3I+lgPJTE|oD2p67*0K&oThs) z5WsMH;)FW8gMqNdaOU{frVWCD0EV-jOMeU-Xqxcbq}sXk&%N0w+yM`hn&Y#F!V9-S zDKgQudaKs^!U@xeiK4G>78?}wI7KF6e_qM;L@?kKnTWr9`_=iufKy~5p&#~qAsBFq zOeE(|I{};FaR&V?n@IhYyazVJ;|#%|Uzza9vQQTf_f*KK~TN&+`da{4e6Tpyc?ex95 zf`I^r9}fls7*aod=auh*fdGay z?>y-7OfV3@kaqLS?Y;>H!c-#NH?O=`G#ChAs81@^h(J$hvWoT$*t0R{31BETcgTr) z!9V~*@d{(Uy%-DxFqEi1?33QXKmbF@{TFim84Lt4l$v$E*c-t>=_OtMM9=HNKmbFT z8mC`v8Vm$5ls)|JyVrt&0ETjl-f!C?7zkkKr(}Ts0jv2x0+Vl za(o#K1TeI{)iYE0Koi2GM!WuN!L!EO-fF@!LwG3v7;RUl*-XYe(K(V6z;>=CL& z0K=Gmd!IcP43y_!|4p*6A!Z0+Qe)hdb{S!7HO}B`410f66g~`u5y?ia%BGBh77)Uu z#tG;&VP)e=ttSKnVZ^X1ch&l7f`I^r)uU@< zS{)1oFs!*#-IWan0vOh|xn2~`lNdsn)L2*WS`Ij|5@&EQ+=!70VOYh(q{c=4!^w{tG}spm1TdU>tWldv!9W1R z=}GnM?O-5);momG89M|60Ssq5UCQ||7zm$e=Q5mqwoNeLVN%;U|5W(!Sd=0gb=y6s zlP{bwTG%L>v#<$#aN-O)1lx%1U0fSJIB|wx;ML|Ng!&V{bZ;Y3adreB{y0OpgPgr} zE(~Cn5GJ*e+P7f>3}A7Fa0gv;`%f?kTf&#_ZDh*rx(=!a#s;Dcicr%5eiKd)#EADlRYH<|Ib)-(96@!+xAa55N8R$O=_9@sr^|vau;WCF!aQXR_txpF7KJQqr&fL7UJQxUIaQ*SauO15qeBQC# zWV}6baxf6U;CAw>BQpg9KJQrWF1@s7W-t)I5Er-H2;>nuRYYTk@?KCr;WVig>~{au z7lMHRhTu(xzg0IF2w(`2Yjm>>!9Zz)9Cq$zvtS^AA=JaTmx6%+hR`ifO?opJ2w(`4 z{p7p@!9V~**Z~c{e=QgYUzDQNNelQTg(9=^=RRukv1v=&Lqw}!Ab?@iv;k?Z1OovKqf-y7Q#}|6 zU>MVVOw%L5KzR=Kbo6nJU?6~D+*tGG!C)YOVSD8$72fp-<=ZH!>-l9+J|*yBS-x(E ziU$J$49f@h9Wgx^2w+%oqu2LYf`I^rmF)_aogWMYFyc8jZ=n~0fdGcpgR{LcC>RJ} zSaUUZzbAr$0EV?q=QsN#7$^_U<^1}soWVc^H=?eZ63C z)>%deJwETai0$cEK2I><6uF33Z`pQMFyIuqNK|O@t_%j8A{WV7>2^;F2Am=nsr{)R zrV0j}A{XfzS(dB`2Am=nnevGih< z<#d?+;|v;Baa{MSxYLCy5y0RkQ@MLz2Lk~NZl?@B4r?SwI8Exf`*+}PunLPaX#CuX z>qmYX2{j^gs)$BctzR?f31A3zr&8-f!9V~*@W$0$lVBi#Aw=#PnQjFGr44d;iDa0Y z9pN;o6Y4?P6qxEhA`QR{PJ%w5Wo<&|BPQ@jpPWYNu6-FrW}J+ zSe(JZkZo7jE&(+njH)u)n0D8QpeKMK*@`p|@&^L}49Ux8{`#w6Ab=r7{j4*iU?6}Y z<(4*?#{~nW(P~+Ta_NGB0EW~hTD4mr3T!Y@UGfdGaUZOh;PCKw1{XnFmE z*4cxB0ESkd)UQ7@7zki!{Z#Et6@!7&t)_ilwMQ0@Xl+&KmfzYi&Yxb z3I+lgMoq2UW=k*-z%V*ZHQOf`2w)h~t!Bn^!9aNq_SDs!y@P=OhH>L=JbN)12w>P| zj~5704?_7iYQJp25GbDzCUurQwPIPeU?6~D`6ug-Eer+%7*Ms|p(*{Z zgMk2sQ)xF;c`X`-T#{X^kY|&?U`92koJyP?hVp*2|C5_%b_p+}$kFd8L z9!2YFeOAPmjGrTo<802Uap0j5zcJ?0-jz1(8&8z-pV_!DI5n$}s8lT?;IiYm^desenkCHPS0VZCk zblLAva)u+o^2TI2gV8)e|KY|=bfY}9~m8=7D z90BHzPRjce$Z-Vt_ooW`L6zgZ&ZAK+ug>;m6Gj=%{%f*HfgDGG^QNuGjR85KG9qWk z*?B>ZBf#azEc$zp;|OqdFEM>7kmCq&-6CDRi6F-j;O2?ctr9?vBf#yzA9{TbavTBf zZhT)Y1ITd%#4jha`H7h0-6*07eSbXR%NF=3AlQjs*SdimM?mo2U1#kAIgWr3iQUH5 z2RWhca%{E?|A8DwK&aEXO?!~z2ngM$V9P&1jw2vUmb`Ue204y^u+zS({wv6F1cbZr zW$#x&jw2x78ami7-Zz z0!qAFM!^84a0Hb6soEC+yTTDr>f4&1g^nTgjDD`lM*pV86^?*1?^nL!-{iQ$5m5G* zdl%~ZXR2@nl>7d_!y7=3BcOj6ehR9N*IY+uGn;w7Y>gwJ+03jN7|3x1G*6$l?nsd1 z2x!qcP0Iox#}Ux-Qmbq0K#n7z)#S!MmH;`9fYzznj2#PdLPwe&9cHBkIgWs~XMS7n zV+CB}2x#~D?_+$-iEA7I^YiF^fFoWN6J5!k{*14=!4WWQNG{z86_@%~ z9|v+A0V_M7IvRl-N5HD|C;$BlWWw(a58y(K~C8M`D3R9z|61N+A|hTp zd79tbi?6Yl2N8*4(-!%iz4#iJ%T*6wcpNH&(jp?Y`}Si~K#tNPB3<{&2hV^UrA0)h z$jy%5fgGhpM0VNhU5bJnrA0(;%Nv!x0XaHg2v+XqOa2v4;17vKUk%?;&zFq{5(Q(w zjs9;V$O)zM*M=_iyLbW*Bnl>8>_64-;lhU+AGB^TE{eJSrKS7Qo!1SBv zIy412jsP?EVx#LI#}Qz5ex1rKK#n88++Xi>y#{g|0Y3X-cK_Ng@DCo3KAkgpye}IM zBnr;%&YqtdA}GE_5#Vyf_J)2g6?n8naCLuIlAo#L zYrJ&bd~3xop{Y0m+&sA zX#DyG`+dy`^M+t2S52u5avTA{doA1MPyYx4k3wW!al0dwPN=&ax2L_IONGEk0ijOq zf8Wp4Lf{Ap-S=?fF{liVfH0X4Re-rV{>O~sJ;KEQu}>(3o!Y#JU&F-LxZGWAmff$A zguoGy)smO{@EW0CWOV4ki%HkZ#t~Lsx(t zM?h7aHpque39PGV)9J_F@@30XdF-jL5?F}-0;Wubq6_)fUW%8ZNFVb zSZCO%XZ216ec5;*QCOBx97n+F871~V4{{smRs4jxVia+DSs@wfMWx*p^xEiw|% z?E9f4$WdBkB!5_1Vg$%hT4bbtUYRF9$WdBkq~BWi`d1)FX_1jBy85FC#+de=MU!w@H?kj)q=Ieq#Bo@uya%2yvaReCKxA9^XkP}MhHMgzx z8x;}{BuXaA?H=W~LE>w)UznV;D6bFpONJxB)ZUbmek(!ZClV#o)#iTaU-IK?yfUWZ ztk?WYzGOH8%q}kQjt@{sh9kh-!KYgI0EJ{Y0{p_dq5cRMe*8!BXxRDD+5I!c1BsHe zhjHm3#}VMX<>^ZmL5`0CTx37-*CLSP2yi)ONVSY0#}VM__JCe9K#q?BTsIl^d{L0& z2yl~YOulbGjw8VBkSM+1tdMvhQF3?1XunwzU!$Xdn6`SQKh}pI>5?M){POx*zUFu! zQ3`hE)|f3I#}N?x;|E9kfE-6ah>UmsJqL0e0U;+&@BKQ+aRh`qKDGK^Ajc69`onSY zC$C9T_ZCv%2nh4|_~)*H97jOdna>-)St0QSz7+0ksSJLzBECiukm-&*?H3vNF+C}x zqsvN8@--(c9VA<@B5x{?;|Roq%=*{XfE-6aidR>CSQ_Lw0#Y70d~r0$aRj7Va_C4N zkmCqQZT5`vFZmKbktn5kY5!XPk{@5A^Mq#KT5m0i&*2B9& zjw7Jd$_*D9fE-6a=`!1Y@GtoiKanVvX|U^4|B@eHqX;Owz4K>2Kq2uHiBh?BJ-+Y( ziuf8uK%d`R`48VxeEibUg`w4Rf*M~AYBsHZ?@vIEBcOTe;m!*SoM?mXzr~dT;e;NM{C9*{nj~QJ~9`ynL_!^hn6AgbX4wb=| zgW64a@7gku;|Q4KeO+JnYP>2Yx{=)LZBXL~7&dTPwM`($5imU8*nA&?97n*2qzTXe z3vwI*Bd=arR}7zW(@>6j(}CEJB-Z%avTAxr?vWV3CM8-thvzmnguzIfVF-8`gZ}y36IHT`Sa+D zAjc7Ky^|XG@R~$VBuW>xD^%~GFB=aeO2_i&X;lN{I0B9j&aQ8Q97n*3tGUuQ204y^ zlSy61{Rna#0jKhHpZyNVaRi(m*yrjckmCqAbEEgBk3fzi;B1SZ{yGbC90BK^`0-Ld zkfVV_H8;=v%8$efTQG{6>X}P+C-E zp6&gl-v)`V@zU*gKRq@Po)Sunirl6jUq}EsI$$W4l=8RoYWyLwXja;lKG?7D6N!qk z{b`2!t%Ud*Z7Ab4GXL&3Dij__R7{l5dd6>q#MgM~a&DWhKG?7DK%!!5Plw8WDlnGg$EK9GZh*??Z@f(8ZX_Z{Pm(=-zz+jsF*wS=i`1Y7+<5K0KYMJ#TbtO zI!r4b4KI1B7^v}4fU^f>_ALcDjsWMaiVw~VazbT9w!+^fgB(YI%h6Q|`?*x%ClVD` zcPeM|Gj)88J|wOiS9|$Ms0=;|aFe@c=fNPy5#V;{)g-@$QFtIxad-JfL%%|buTccV zz*$4``nurVD5B5!{#KyI5fJQeb@~d(aRdbKS7CW|kmCplk*WNkBOoW#T~7KSyPr!H ze)3KUb*z37KU2roC;~!%RJ&6hs0@yPFpt%J=_iom2najlpZEP5M&W@(CEP#%w)ZQf z_!>n(=HK$9->}BN?5Sk*SJ%p4_?qK^L?zk$Ze2CVaRlP+hrUmb0y&O=6tDDp;wg~h z2uS&7!J(@`j;{rz`a18*vLMG1h~oy?&P)I~j({{T=l(qn$Z-Uu{cZk^u^`9S0@8i; z^?zwWjw7JjZG5JbSK|#;(U#U-j)NK>1r%G>u~JWv;|M5TqUDpnfE-6ai4U4Q^B%}? z1eDyEZpCeo;|M6VBK43CASd*UE}P|#eIUmXP^Nz3-<3d)BcSZIZ}08}IgWsG>zDsr z734Sq`s%hWu-3$DuA_@pi#+Yi#=qLEG@Du}`*M)u2xy+B`b&?297jNlZZ$eh204y^ zmgh>GE)Q}X0j;K&-M<**I09OyEjB1O$O#>3x)xbJ0OU9V+Mb-TAREYW1hgAJWy%nc z;|Q3+yDP!_16~yqU2l}FnJ*jvYO^xzlLm!vfE-7_@TY2ZeiP(40!Fm2_woUd;|Lge zP3-FhavTAp2FX(wK#n6|bb*TB)de|@fH7^$4c-89LciO)gD2MpIgWsFpAMYA734Sq zHfN^~;C&V^+eTftB&PVX@e_&4va}m3lmj`AfaTM-weJsd904oN?S4NO$Z-U$?6zpz zcOb_RuqsW;?PoxaBVhH^xeKO%97n*Ki?gOY4ssj;Yx@=$lJpI08;RwQ_q?kmCqA{mHt?e}WuGz?ti-=f44R906yWr~l5Mb5Qt+MCII*sRsK~ z6!A6oql&sh_B(eTRYhYj4?2p53~KDP=u$8iuWn%PgPI^`yBcL|2Xd4a9r1eg8vF)w zlolO{!nNAG3UZVd9m#d_p5Mleudx?G9jWbNr{BhnuW`AqRc`t=s0>Prj!f~2iyMI) zrA0?}<-n!8L5_~=I&vEZ_iq4lbimN;)d9qSzALuGITn0cXC zav_l82r#>3%7D=z#}Q!ez>H;iK#n88j}ALs!>jRL=h4WH?|uww90AVmwQlO{er%Rr7JAaw8CyRv{BM?jdwY!9Y`97jOd zAY19C#o=+ZUYd2i_!@mFG)qoo|Is6WSH(m(^DOaUA&mzTwP6Da&G12| z_!@1f;dyfH_F*9%pH4&$c4{L!W`F2|PVqG^msh&C^4y$%267w$%V&<>djRA(0#=+Ie)3I_;|N&U<6@5sAjc7~D%H7HdVw5A z!0O2-b8i4Sj(|0nP8F{UavT9`KYp*A4-RWQkf^Q8*f6sXJ;&GRDBxayw}Ri})@UG6 zyQuZFPy7~9e2sqa94nOkt>1vs4oAT8Arnqa_Wj~;1e~}scF$uV#}RO{#j207>oc>*rN1cLC0w(^~R;JbY8Pq5(CStpjn>GYFN{fkj-3j6h$WdBM zB#Mkp(+T7#EhdudZ%*{vxbZbcx{1`bD_{9--1r)o>)N+(J%q}jw3x^gd$_d?$WdBM zWLLa&@HWWNIm1M5FshcH0 zjw8VA*YBNQ4RRa-=KgGWs4U2F1o+l*`?7j9-s?OXu{e4S)Hnj1-Cvk+6y!JpoHw6c zyE(`Sl@U+QX>t|hI09ULy5YK?OAUS^(QtKl>rZ~Bj;~P!xNf#+4) zfZ%;sHGUT4I08asUSI25kQ3@I#~<44=Td{8NHjv7Jbc5?)bTZnfY816^F!4VK9 z>z?roL5?FJ?DY1-{TjyLClZZt=i09JE2Q`uML?Fh+}ZlN;1i^b4&R!z3)DCQk}bMC zuRh3e1SIEoZk_-+j(`+ZA8hFcavT9E4^4gR50Dd@s#2ymdl}?70#d8-{6CQ62uM?D zT=nW6D4c6no#r;a1ace! zZI3mYP>2Yx^?g;SUeKmAr0$)g*OQX;+_$Xj>!PQM*Ld64##+WwiYQx}b z2>ovFq^j{EJS7|f<3^|NvjF5c0=D=o6`%BKylfkFs`Te3P~)S3Wf`iTeH-NXC}8=F z8ms>WIX((l@lW;PAA%eo1+45@Hvb`z^i+`J2so8j6i5U)j)2qs%hgx`avT9?ZdL44 z800tt&bAuZ#Gi98cp%X@mu+xue~Kc$#sZ1<(1Lvr9#zF+wAg4#x&{72if<*@DEfA5 zPXDopud$aZ8?m1@JnKIe@ii{z?{DwsKhhTeYO{^RbGxeek92&EmoAqt{4KN;rNu_- z$Hm9}a1>wTrR#U+EcHXF#lPBYBU5^Ie?OGQ*Ldmnhy2t1u`7%JMxu?}&-oU=g%*Tl}ldmI<-HyZ=bX z*Ldl2@v>unII?&k(K2;##jk!iim&m~b>;N~{7`D~uQpp|(UD2=c2((RlKDgM}% z#lPBYncJI?><@Lt*XSs~U&^u#=tYNV%cB8TdUp4C;lGh+IlFoD)srB{5#YSz!(0tO zPNk<)+Yt6TgETM}XTg z&pmt<`$Eo6Z zK#n6IM5>dyM}wSDcR6$Dz|tVc5fJKZ|8LfS97jOt&cjcp0y&O=FzH9{odj|m0bwVy zUFkuNBOu&y{opf@;|RzFt=q%wj`xd<{#f+uHD5MGJrFdi30DnLyzD5UADSzFP;~Z3m&{Q>VW6?e!#}SbFg>BupfE-6an%8$%tp#!% z0cj6(s=pEBI0Diw>Djs-$Z-VJ(T?xKiVbh5igqnM{*5mi|BXbe*y@$P6$LqtfZ~PM z^`8N9d=yZk-s+|PfR4rg)n=>Y_9M?Og3@sWlv;N%XGM?`dPW!DTV*K7aRij9wXa)t zkmCp_yP;j1{vgK@P;O;XgIplT5zw!;OoUZ9UUMCtp8ic|Up9Us(P}n!>cGE2jw7IX zmT`OE2RV*_7JbH_+yQcY6wva-FgeW&;y zkmCqwd;XPjl|happk4CInfHSnN5Di)D!`i+UKJDF8}doy%f;*ZFfE6cl@2U!N904o)bbap($Z-U$%F?Y}Cy?U^SUt9Hxt~FfBVf(xUYQ$$97n*~ z?wfP}0&>D*a;Y~Je-Gq10`89k*{gXq8c4J*YMO4zI#A;XIF>W@j1nNn5paA|mfd4P zjw9g2e~Ay%f*eP{$ri2K&I37)fKyL&e3yY7N5JWkEwhXSIgWrc_nMS10CF4wXOoT= z8wGM40q61^$n_M+(Lkb0S#ruxkSHxKT3K+ikFDSbQe6~%lkVdUkE#)0W9xMnvCXO9 ztP66K78mh`SqfeNIZBI*#Pf+|dVw6J#YOVF)@u%c9HqrYYE#FNZ-N}9#YOtPmIrTu z9HqrYrc{%2%|MRQ;v&2D=&h?DM`>}9+i_rPbCBZ*unl?V`*<0C$k?%HQNcF`__Fap zqGRlE-hR13jw8T$)of)KgB(YI37)%9d644>FqzVI#AJ};2rzZ1+xL%w97llZN`22S z2RV)aGpg62r$LS*!0g=3TfPN3jsSCeHr;#{K?tu?_z5Gj)884tQL* zskN#nR0cZ+ps-=`%N1PN(_P=P%? zLnsmkAz)BSceg=zcQ;CRNeWVuqM($tv~;(`fB)Wn&v%{A^ZwU*-sN(wv$^)np1IEK z*>lf1&wX6{-$-=Aow_?ZS|Nq6;~fHWX}hmhMQw?Hc9g-PCO>VZv*TY2NcKa^fuGXp z@ehIU*Mu4!yGWHYDA~UKLn&byqw>s)8l^-NVjBVqS|zN{6j$PXx}_nbawn5Rl(l2gZj|f z@ecvT)^u2xo=%T{2q>Pr^X2F(G)~1o1eB=U>1gx?9T)#bqEm8b&I*&MG4T%pr8Z{n zmXc2YAI#`H=?X5T)8iik%2djjBM+S({}51i^Ss2X>Gb%AfO4xAi`;a2{6j$3>{4uE zbawnL*TK13hd-dR;~xT=jjDR_4>~>mA)tB6N^6_a>G2N%ExJ@5@Ptl}e-_a4EYDVp zPLF>Ggx@n(@9d(};~xTArzzT{9i9F^h%_CGls`|W$3FzLz1+KEdpbS-A)wuqUbWBA z>G2N%lfPq{)X~}TH^l@`OI><_&W?Ww7&frvkr8xy{6oO-^kw?LO{d2{1dOO(e$CHx zdi+Dc$jA8$;FIVT8bo&3`-QJm2N73o= z4*}zPkFPX?PLF>G*tOfzPKnNrziu0JJbrH{ogM!Wuq@57Z8hoi_=kYyqxX$IN2kX> z1gtoFaAsRNJ^mqJWtU~_PdYvRAz)R?RS7?&)8iikR*%|P@hP1i{}8a|+?v|W==Au9 zfVJ&k=<=9O{~z8ampDQB=5%`eL%{9Xk}FkocDx&j&IR>$Y#l&n$3Fxd%dqKgPC7mQ zA>jCsU9)D<>G2N%Cm!z^Bk1(_hk%py=O-9Wr^i18oJzkaNJgi}KLnf}IJ3@rIz9d& z;LOt*mENM$;~xUfw#Zg)9i1Nk5OD6*tep$c=`Kl_DI#U~fBkm#+4K(0tN-WUS<>)d zQqjK#-FCbj1rRJq7XDjm|L?*2j0K}_pRpi#A^g`g|3`xPYlDPQRY6?FPHadp&NGJP zc&XKuqNasaa%4g*trPV!WCEF7bl`6EwUdxZWa{YttkPE}&hS@7fZgvC8Vn~ZIp#SVEEryQ@b zuy%Amvw)#7Cf-_5F`61ehQ^p&v3FbaU9FIzF{Xamc`y26R>;s8)73Z6?BLsi#+WIv zee_w+&=|8ne>WhSk;6LC7<0dNT^mjAAwy&Q)65km7&pnT?1*BmkfAYd$9{fvneRFpllm-Bjt#>AUByKNzaJrF^U(e?fhp)tYk zkNV7u7HCZHRudEXVJpy>5SgcNKM9C9*OPr8%{k_qg2sfp^=w>u&(N6AjUQ}(;29bd zCg;ujbv#33!Var&<)&w7Ot^<19c$2eTp)tk2-*tDNXJ{<^371W?q7STC zpwO5SpX?a(%$GxBO8zk;K@HE)m{N0Q2D?3r(@y>FqB?CnLu1N(JipREo}n>ikGxXu zYtPV_a*JQ?e9AL4roZSg$PahCnhu_v9oLUaX*8zUpmTH5d4|R`fBD+|1)iZXEgD?j zUd%HzrscmwKbz_q8q;dfK(ZFe*p)tz`w>@+ zCw)CbV^$AJoU^EBXv~@?Nea#P42@aad>t?1S)9JRtm_jk@(hi+ZNH8^PRPM3&IO%1 z4f@C;rzjYWjTu54K_D@{&XI5Dsl zu7MYHs;;RF#scAL?|A2?-+Klo!}U<<4n@V2Q2dj=+1AQxtw8hvua!h_g% zHGHxhOSd@sm?&gmk_Ea%=I4ohonVp$M&;Z)#xpR<0z2#beMvk6lPqxi=A0Sn8H_!~ z_N6RD%TR_rXTkE?&Ay>hacnZSt_iOo*nQEjWN4^-PzKfp)uxmuiwAlGc?A_ zChkg$dW>`!4a2d_DVP<@o z$K>?d$0#6+b0^4PZ>DLSu7Uq!5SorIYiT2j=42>yKA?fp7o}n=%|E$xrsb^?Rso4#y zKk+P1JN0|5QhneV8dK(@#zp?{42>y!_^&+gdxpl8Tk=Q7zdS=@x?IbjXtRLf)pYPQ z_tT|R3S$AL*}y!%=kW}UX`a5&7oG47jTzBJOeyOb8Z+{2mE;#aLt{pbu4ub@hQ^FeQ?1EX&(N4L z9cxzm)U!DKvzJeo`o%LeX55qu?Z5C0joAf<#-$>JrEE}d`XBFlgmlqre ze8|FOFMEc@ta!YjM{m#2n3eVSX2|Oq8nY_>&Nr5MhQ_QOxVgjsbpef8^K^UZDZU&U zv$n-|RsHQ{27p2hsGRBv*FK6o}n?vN3ZVw znP+IsiL*q|H@6z(3sPsj+UYTkU2Ew%()ZoDb!&OjXB%C zZWG#^U=EErm$*SS3d5O$Sb&e%)oxJK5+3^u<-yQgC{6o>Fq-S7~2O|7h??KPNBoE}`in+e_3{3Js9jTJ`lxJX)2fBF8_p5sbCV5~? zwKl(b1}1r6=UixB!!t0+1NX=2uXcL|V~?{(c{?{Jggs}$(mV$XdW6Op+mZYH8qeaC z<2CcGNaPtBV6Kbt*^ zQ>RFj=&vfCp)oEeH!N}2Gc?B4l{#(fdWK_+>$Z(wKkOMA<0fgV^q+W!#<(5x$BTb^ zhQ_!%^Vho{dxpkDru^B96T%*dpl90E3q3+(g5As1udrumOz`G84t4hojR}!8%Y}@d z#krpB)1}=?&(N4qw>y@6-7_>MbhCCZPw)(l36rh;YcF|*#)KWb`@N~2p)uk9{q3dX zo}n>0pnU(cgs_z|_`Oo@W*(t2$$qMw?x|;JO!C)j75~sPG$zI8Ra^h*85)!F*P`d! zdKRZG>gOT{&v}N%q<(|)b5Q z;hksT=R5IUZa&ocz{eaZ22;+NVrC0T0TtKk5(33qcN?9rCCiY60Xsh z))|vrpp^v=bMVD{4}fb^C-EU#k%R}x?5*{?X=TAR8q==tx_@a!!ZjK*X-ggPLlbK> z!G+FkY4yr67T|`B=v0DMz2O10#PH;8)6?pe8#HD_w+^qg5KF8U0euzi6q+4H`4%+syyas+XHM{j(S6Jf_tvH)zbbY2W`!t6pxoDI3%|*ODS6Zt)q58xLgE+;aO*PLLljws2e5*5 z51!oh(Edq6TOh?+S2@jCz_b0#I!q*97PXtC!%KEQoU{VBj z_8ZBXdItBM2;85q>nEPU*c0qj(eecdVb58x?A*pI9-%SDwx1m~%(FPe_?aCLb`W}1t_0~ljmx10a*4PPf3<0i?X@4oX4jd45r<@z*d3jAC^ zaCi2V&uOj=58$p8FYP5%k)3VvIc;&UzsY;wXWboUlvR`?G#w1%ey3?ORPqdsX`N~N;47ZR>4T}gd3_hp(3rM2yZ&(1Gc=}MzwbtL^bC!e6tyl=&=vj_ zWy}gynBbocnWs@Hj0J>Y!`J3Z;Tak;JjK%Qe)J5D8PRq1$8UOu#*93*|9D@|(3nx< z4qQ#^85%SC#S=fz_Y930^WD)QMLmntKYOXc@I{`XG2^Dz{l182Xv|JHb#DM6EMVWCXK2itXC+45@eGYw+w$q)x}L@9yUX%;{aw${n7jVZww#0@77#A@u9!;a5gK#s z#llJZd4|RuA16L9<{27u;*_kvz%w-FWY>zfl6!{6oJvt;^Hk5!nA5{+4u8WlH0I1d z)xKZu85(o;n+rd`?im_$F2U&`D?Ni)Kz=*_5Y3D-_8BUJ!yT{CK*7(?We_Y{I{rjd zTX+zAC(D2xT|Kv~XJAqWyyS*Q6r+X*SWd{byC}Am`1b-bkn>M`MKNP|faTPQqs=MS zlo$)hK$qHIjMiJ>0hTl3K$@<;onTT1c1E+bw7v@uu$(*4JQuC+B>ugCWJ|gnquWPf z&snfy!Mw?IHT;f{WbBuP6HryQ?7YbV9162CugCWbXHG($V@ZJb*FA zb^C!diAfv}Jebz^aV@$Uen&`hcCp8=yF5cJ-U_EjaHP8sl>Om3Kez z496H(H?F?;hi7Pv>#xt(YU&vp<0j#mR!=-bW899Y(U#)W@K5LAUEuCQ^%4}PhWRG6z+NG#g{{4^6yNA z=m8|Ll``0Myv7kK6=#l{MLk1fnq~Qf7DsX3 zA07OA`Mocx6vhHlv%%M1`rR`$rui%9YJTGx8q=cg*_Nk0Lt|P#>Gy95&(N4wgSziN z;29dz`sD!=DQ=Ybxq#HB!O-~>H--l=S802Xt#0D$L}S|Z)cu}&hQ`bXIlBH$2x~OK znOiTu>JhFk3>$O*-NBxrF~d_oZ>2p$V@7m(SnGSw(3p|uC;yt*Gc;z@$VrdKc!tJ| zPBn4?#f=g_7m&tu9y5vJ#_#})8GH4uzWIEeXw0}t1y=dh7=A7w?V%s)eL}^slnoj* zdHJA6Xw0&gTfSD(Gc;!TphoQ;d4|TUcv8D$E6>oFm35OG`pq*mX4NZ+FI4vojafZ7 z_dadWOcF9+R%`51yeh zXU=3?WjsS;&bFPmV76yyEc_^7@ua{rhy~P3i!RW}RI#fL6>P3^ltLh!^;HloY!H*oda@@Bqu%>GL|$AW|3$sK6auT!wDR@BoalRShpwV5qR? zELhp(m;Xa(jIk{(?|9MpcLs{_FB?s#7&SbARxJiv11?KG8>dxplCUAMgK1kYgXDdv7#+36+E&=@b&YMt)_ z_KpWrOAT*Kr7#vyoLwsU{WH(dH0Q0$+$!Z+oI1q|Vs%O1j%KG{)`lX_;xx48Ki{b-I7X=cBnc{HseCW1?!~ zL4H7B4@A(f)(`pVY8VSB!ERO^vC%U$CU}!do058l#)QaG`Q|9k;#^O5=M891QTVxl z66%5ag65*|0D32M%c4oL`8v^IhhQ^fmD0P)rJVRqj9%{U0m1k&7sUKS1&gEI0 zc52&j#zfE1m@*aWj7sep8dLV~lS8L?hQ^eedt^f@&(N68m#CYc8u4m6c%1Kx=2Qw} z0j1fHLJb~!hQ>6{kVh8x42@|~FL#ndo}n=4T|XZqr`R(3rM&@0_UV85+~B*WGJ7Jws!rR?-UoL4!4#;H;=$fJ))#0?M$_^7D0` zp)teL6ib@WGc;yI$HHoyXK2jGb2Y!0o}n?LMpYX=-7_?1bjpgGGkb={jOkM4)=tpGgIjop#;hK)^Tc1Cp)qS7@4fcEXK2jYrmb(% zUYx@32r28bf3xWWUk;7AH9yz&ixP+hlnc5n{dfkI!tV$v$5O8TPI-pL93QnI-!RY6 zm=ovLX3pXn8gsJai6K9EhQ^#qbM)tTJVRqnkKTWEtY>Jagj7tw?UqvF)k;yx}485G{)7{#z!}JhQ_$=P-jpQ&(Ih*i5jl+_ZH?6(%g?&(N4?SNbr8I`~atErLE-a<-;Y7z=2@Zs#a;!80@_c(Y7hzw!)? z36U-BC%<}z#)RzI{#X;w(3nv7+Fg0>85$G1dB>khdxpk@$=YS$e$Q}>2|H-ZxDuYB zG2xzUn{&W3G$ya*F8>c9Y^4l-tvX<}M-ZWF$$qZ2Hn(SJO!7A>U!3F_8k6F)N{7>U zhQ_4)U6o(#S)8`0pLiGR85)!N^&;=|^bCzj^Lf#189YN{(r)dQu#abGOu8R?v-F;! zF*Pwy7Yd8`%A95Po*#x(2C70)xWl9XiV{9<&PZo42>yKrp%>so}n=%4;1Wt z!!tCd)QtS)zVi%?DP1^M_Fp_hW6G4zTi^@N(3r9(#s^zHLu1O#pO)ZL&(N4oT<9GN z`f*QP>)>I_E3~o*58(Nv*{~+ZXhovqJTWxSSbHF?EOeZ?QHwf_exenLjx!5r`8;e3 zwG#hcKx@@MNlsdkga_Cboi5EMw6f4~`(PTT>`E&Vjh_o>ZSSwFM=Oi)0PA!;mVZVo z5{-W^piTRH*=P$TPK_ow-Qg&$UNy!7+OV;0FVm_wJbZCTIcW8&@$Ut+Q6q8|q*ZTtfaP@ZjNNGUs_}CHZA`ax6=>BP9$-0pWAUf7 zde!*%0@}Fo^Xk&7H#~s#m_0Wx;cxWDqGua4ytbVpB#p6vwk+M{`xIG)2hbAB`=6aN z&R2v@L8tgO>L(R9!7uCyv+KQ28(V^$9vT8|l ztUsX$Nn(y`>Xb~N`4jX6Hz;n+i- zp)n^eJfB_MGc@Mpx0A$K&(N4tFHKI|!80`G^w=?M853*L1b3>e*%%eUJJJNflKXjjdjuv;zz*Na__AkU z(gghdhab-M3{09p=;zlv6dft)+(4DHs02eISC1nTd}`)Ic&JV2&D7}IN+uMTF}Ig3spA6OD0`c;=t8 zJVRsLj>`HQ&6x&27cktN%l3rk+VBAGO3}3DrKD6P_CN%^k9GLaBQz%1o#Und^bCy& z-t=IGW}cxjA+qm#PKnrp)YFedL7?mU~S#JNKTyC|8fip}bKriy21O!1<_e!S)x z8dIWd|6ZLuLt{!FyqfW_XJ|~R=~wc6;#r(_>LO>px$7AkQ>NVc59@k{#*{r?z4{%` z(3o-yYBX)&85+~C%IpIP;nj5TXkm{;9-%SKhAvn(+A}n!d8X;73VMddw5UDnk2Rj5 zF)g2ME}hjgG^W*n?Hz`AhQ_o`yY~&_S)4wYMmsb7;29dz_Q5wP{9wUYz-ZUKwez3gUQnUaM4`3cMI>mukzV>yZF=M*!fAf@Qar$SkH@6gO8;k{vaTA&)r;Us7 z0Om3Kba3F`JRF%eXjEw7V7eN{0>-km`6j>W85*;EKu!rRTF!L#;hJ%F2!2U(3mxk%DO_Hp)qTl+!=8hFB=3i=hvb4G(3pCLu|nIFRDcEX?TE)mzkNsUkkq@WCKxbMnGdI zJiu~t#*SGuW-Z17Hc$sPjiE6c9$-0Le%DsIM=X9v$OfkHp1X99ga=s8&Tn&x_Kz)o zN5}^5#Meh?-#I)0V=VE`(HF=V_M8PjFRMuJsm0F)EMvc}s!i`{cmOS7{IiXW#*oGD z2w5iHSd);(P^ymBT(LG}EJ3^M3*AH%^dn7!- za`wljhiU)V;@=Bc=C-!DNc+y=0T|;GH*fO@?kvlLv6B-QA({BOfaUD;B(cadG{*V2 zBkN}JEKZ%`r7@rO@C=P{IpWri*F1x})pB*={(}{sp)szzJ)ixeXK=S#ZjwJ7JKi%i z#_jkI2c_@~?pDj)jpFO3d4|Tswe9!3XzYOqhV|nesT6)LUmx_B$>9or{&C z+TMG#PZ}P;T_WX<1+{1TI^(oOEnQfVdxplO&Oa+*f6vgEG&QHQ9G;;uY4>D*XP9Sb zOuDtNX3OFk8dFJTRip=r#g?ex$cZses1$xKU=>?@bXHT((3s+8|J^@4!!f2r@dH~w z@C=P9d2(r`^PZtGrRJ@!)6TOv?bN}BV3%iTOqmbYCaC2Z8dLVKWZCz4hQ^fpK4F2X zo}n>4Wnse7gz#!QxP5c`dmiDlQnNk}?$7fKjcK0k*_`yAp)oC*JsQ`?Gc=~N$}w^$d+^opn^bl%B=ugK0kcv&o*JF>Rl`{Ye_n(3p0E-sw8YGc;x{FJSu# zVT~rZa^*o+kIDnVOZT0p2g{(Js)%{=NTF^ZltMj)H5_@J7##tKkl)V4Vt&RvW!Z_ zS*ux=wf(WYo}n?zdv+N(#WOT!#l4O{rSc4oS=lT{p}C%+F{`p=$@#8lXw2$9X+P=h z85*f))qHu&4;yazQYwajKyn!QJWt zwy#x@%$|Ws7x1zTn@;x(Ou9f6ty5ik1}0q~rzhPz&NDFS0(CIa^Msy(Nf+pHsTQyE z3{1Mf6iGd`fM;OR1$M!Yz1DjMCSBl;ulVsT&(Iiqb^ok6gs|rn1s>?+UD7?;DvBAQde->8pW!-c;=|BswPb5VGJOm{8zoAY&|F>X>6ezL?f zG{)_OKBsBU41ZT1>vY$9{z-Ff_#^VM(UTK*Pi!4TRbmfBFtl{l@*bfv!5)=pe9SX6 zCV1_#u8wDDOo&Y7Qatc1&h_Mge3NNT34hWZdLY!ZLJMgw3V**HnQoNl>9@X4G$u^i z+`nJ<42=ofed-@HXNEt+j&-^R6Hd`w8~)}x7?VxcR?0_JVk>2^Imw=Z9-%SGRwaIx z%`-G6d7+d`W_yOlq^O!^THqNPlX82Ll%qV0(-yU?W#Ocrp)slRHEOoeGc+bmt=eDa z_Y93myZ7%8)_8`-q+4_N>w=!4F_nEooehMrB`P?a^Z9O%(3oONGViV785&burQ`5$tp)q9-ZT;qv zXJ|~hAAb3;m1k&7_g)<&A%s`c!5zLZxkqSBv)*d*RL{_u=Glw>{)T60OpB&Po-X$c zjcIweR^xP@p)sv`Rju03Gc>04tCdm|^DIsuOpD5HfoEt;+ozY_DC`*;({AAP3=2I& zW9H7%1S1JyjV8F9f6B)mp)tdz6kPPTXK2juBzd1S@C=O^(Jt3-cRWL5MqVpbt)gdW z%&3Van{4+CjTxP|jP2|h8Z)MS`Q(>9i_<@Q?)EF4JVRr~jk@>dHP6tPt@l!Pzfi+c zHfXV|M_($1v4FGe)m6*VdWOa<@3ryNe9zFB6?fPCQPeXuW@XdkrKfp@#;nSItV0UV z(3sV|_r39>XK2itI|nno=@}ZcwqDZ|D?E$ScbB1sd(AU6<}xHNFoqDs0^#o_x1Yas zhetTZ980{YdnM1%nBx;?{&~eSH0H#$8K=8=hQ^$1x1;?@&(N4tNj8=G$TKwN^pst% z-1ZEOIdgf>oAo_IW6pMLWAAx}#+*y@b@FuqO4i8y;aPwyVnbs|m&aqFMNp0~KrDinbNP*D=rV@COI- zR@)qHDul0r$hE~jw^Unf?xXO>2KE%LznDnuUqXjJHBd#U3`Ao8E+YJ)fm||aPa?73 zk_vxjVCkUD>50UCW)l9$z>dNB`VfhYtT=o9TAS#-4HNH{xEXC9a0YIe{60^E@2EC#!_;57zo55=fg7ej$(Q5{Dg$nq zafReBM8FNR^M>dDlnA(C?&P>ETZw=h{?oZ!(U|AZ8y*a9czZ@v0!JHX|JK=}h=3c; z>oy)Wj0n~yUTHNW3lVU`<)BphCnDg6t0$?Gy+Z`taNQtjm9a#?4L2_*YLtKoxZ$?v z&mBh)0XN*;`>D(eM8J*s{cPH3uHevS5sc~b%bBQzfO|@?Gac`=CjxE+@6>Mk9wOjI zh}7*zRwaVhkt5T-@GlW?Bh>j!rUemjBXs8+4gVqnZiGpdrSkhkz>TnzW|#eg2)GgM z>JObhAOddWXZ`v_b1a9J$>3m*7USMMWA!4;?PSjpX5f(W5GS6+5d)QBoy| z9&lWO8!1mz8WBAfxx^Nz`IUc&9;RG^8>vODo6#efOK>Aisj8cz2RE1CM%n}aoX8Xz zkl;qT8K(d0Tz>Q)na(zxylma)37s#KKZcqhol&D@% z(ePB@M#-&ZzNa@wfg7cMEI-^g4cn&QDz!O!*m4DKl&MkjR`iJG3fw6B`;)7cBLfQD zDEHI9$2SoHH~QxR=jbwU6&+kmY34*FG`P`hTACLa5pbh;L)$K*1lh=3a>?oaBtj|jMNvhnj9ZHRyyr*c00`yV3U#_7Jd zhSnegZk)Mwf5UDf;KtcHAJ3>x1l%~6@sm-%5s6(%i*lXPeEpw)7cqC52+H4T9X+Jr z`5h+a-#GGd#_0SY)|d#`&N07sClYHcOv+zr@}wy64qwOI>0wI#Jd+kgVR!f%h+Omh z-Iu7gSYu%Z{=E-hoIoVjm#h?3rtCd7zjfIK$mwnv!4I;6|!ZiHb zKPj=CNbG15tnmHf(PLUbKUpw)!0swh30N%)#{L}o?`9%chJVy=eiUL0SS<@C-s?Uw z3b4c1v2K|B;oM7~P;KCbsY7SqJ4^)JFkRtt%eq9s4Kwyy&AUXv4YPAUE!ltwxMA+^ zFWTK90&e(>=joz_ihy^T2Sa9#A03r|)w1C1@r=2Nh=3c;>&<@@Z3+rldkZcyEZP-q z35Ks@-EcW@*H_UT5a!qiqMq()9?ctJhJ7Hq{*GclPz%5fH|aN}F+{)(x4pWhjpnp4 zw?5S7?tYsqng_$o`my(vIPfrcv{Vz&W)Y0uIPYL|6};O7JG*8=Nh07z@Q%xNM%#Y^ zO$(7`)x%a)2CpMW?Q0Ru0bw@%*ftAw?qH2*-UxH)1JPZMr#?lsfg52`9xF;ShX6Oi zPOP6hn$yB8`cRv@Rxe#N4~9APW8KIylUGEGRcx6Ijy-$t<>>mbS{9NmeDqRpBH%`H zb+hKXL|_z3QQ|?%xkSK?l*dQ?n~Dgyk!r!{-BXBw8>#t}iFt{D8)-^UoWG0+xRLhY zo2zpX0XNc3f34ptBH%_9o76j6uH$>7f-UD?{46Q~tK~3J{>n4&ZX*J26o2PRt8PTV zjS^p8t#yeAxKVOj|6j`x0XIr5ANKe-5pbjQ+kF=_Bm!=f`KrgHn?%5kvcJ9Cw+<0- zque^T>H!gOqi2phm^K1%6&+mKSz|Z>cur|Hb#IGgM8Hk>_pRHDt|tO+v}nCK^;<;1 zjh5GEO&CA~+-Nmn`n;S(z>U@k7Cf9q1Y2#sS-4XW0XN$Ilm6gzBH%{5;Tg_LBH+gC zA9KEM1fVVx+}z*%PXge^uzm-=`j7~?F+9hKG*5|u8zY(=E!K<(xH0m;H+<=!3bRYt5Y{|DCMnNmiP&W9c zZ0j6R30N%)%Mz3;-k%7#v3x@5bessdvEq7(+%t)Q8!KDqS@8l9a1(x~EqCt`M8J*J zQ}Z2sn+UkE=2D^aKNA5r)_ynqVYEjiEV!}m#c?~MeI;SRjgtj-MqxXAJ{hL8Z&~sE zTG3TtwJaRVQl-rjn; zb4`hW8)t6H15b#68)s_|+7;~)2?uVR%QW~=w67$>q&4AhN2Cn@uivh|+?R8F^rV(C zPcIoHT6*Eke=Y&fs4!doPbaR9A`pA0g^BBbKYBbVkyvA4ruxtK57|g0)>xRQ{`CVt z=O+?tEX-2>^U{0+iNqQUbJYL3I&(H6vBts-^`C9{cs7w(V=^#rto<$^5^F5XPQM~q z{b;2WzK(gahq>v0N%(oRQVL(kx?vqwA8H>}2K{8gjO{1(5db%g9oT%W6cH@L%kNwt zMP*@r`q=9;QF!m*D3S}a(*uz+7i5W|r7$-=5Ve0{{wTrV|&NmM84$NgnjSGBjOe09MOk zGWzFX86x0@^M>bd6e9v|xJY;A?*&A_4VT0EmPt+o+;H`xCsZ_#*fw!0XI^7xaQjeM8J)dM~+_`N(9_UwfNYH%tXLVn96(K z;AjOCW|fcar!>V6u8&qQVNQ7<_K&7NL@T8*qdX8dr^Se9r4;6qk9DKk9(_5Q)M0O1 zs$j?b%!i_u!D=~7CBJl0zT!l{jpF%de)pINxKX0!jE|cW0XIr++Hrg@5pbi_>P=Ux z5dk+!7u@x8w1Nrq$H(?lnQD86L@Ssudpr=ktIhCer6gf*St_^T>+hqLQusR7jlQzK zJlR0M$0RghovS0ZnT|?^=XO-<##Eax^Ldt7Yj}w#<#n z5dk-j_erPk69G3)+|HP+HW6^+Wb?M8ejx&GoXXl^#ur4ujnh3l-@ZWv+&FWu)0Xdu zfE#BUZ2kKp5pd(&%fH;{N+fo*tY&4NTk=2uu3}~(6=YrT*No_&W5-OG&wX9azEQLk zzK*${!))%qWnL9USmA3R{EKw|L{V9o*Byv>E#qHNBo}6N2O`(EEfGaaVNQ1-YIldW zQG^v{bO)k8?fiNam4*4-ftWWtrH>-HFq=CNyK?J`W2vLD#=`9FTYh;r0g>3zq*(KX z|ELH+KUpw6$?7OtQn0tI7(1A_Uld`5uVb&zc)65^qNps);0{C-Npm5Jw`=lg-wTZO3R!$}Y zZnzv;DpxcIgn8Oy+sxIYl4+xPBh1ncMAt6!etN16+;Eere49Q*VBB!q?{@QOP7Cw1 zLv8Nny|1EqFwD*#>qhjP-Zx8B8MIjh!}-BR1i+18|ETk~h=3cxyB1whmI$~JB1Mtj zCy3y6M+Z_A~()H;OH5RpJ{W;70L$4b%Tl1l%Z5tIq3R z5&<_#Zce)DArWw+)T%^%TN1&x=|ZXgIzR;6C{s1{KP8BO8)bL?_;^1NaHHJD6~C4y z0&eu}oo#9QhO6k{TIt-cMJ3>y7-3rVi6zplAOdbQPh7V6i$uVU7VXQm9!~_^Xn8r` z`65KXjaE|%9b7~N+-RL7Pwz}bu+^qr?iD?VfE#VkPMw#Q2)NO1^n?k0iGUl^XKx95 zfI(d*xLY%Ay{H6y6C+HZ-m6-!dqlvE;jdO~Q-KJ$F``A4_m2<(H%8tO2Rae~H%9fA z=dKa~H%4bKwz4u2aAQocOu}%ipzUzWF!J^tZctv=Sm{r#;U{%AHGfm+*mzv z*1QQsz>PK6rcZc@2)MDfOZMK;9#NPZJ$5d(F6FB$l2aLQ{?U;R?A^h^z28|bRq(79PhpVtIb5fjT3hcG_OJg+&I}{>7%QO{uHIFb2)J>& z*M{+jiGUkt?yjBt2@!DPZ2e>_qdg)8d&|nX^a=Y!`%2;Kn7w6PG~J^||EWvIJiWp+ z=Y4zEj{Z4z9uEI~Z8iIp01~#RX7Xl4VvU8V&#P3a_9u~8V`1X+Tos#qNF>%+nCX0j zd=l+5gs)@X0XUU>}b;L!yYT{M*#ZCg6~VDSWf`lF!onz_ZAT>!#^q4csLPo!-OkaH5n0b z!{nSozeoF18dl4isXy{PjrOO)*RgJx{y6u-0aP2fVdmXDlXDUQH_R@c&|@eOaKqe@ zsmn4G0XO_)|MTS{0KMVCpjOqp5db%wJ!#zZ5)p92d4oDdJ|lv)iI*Fu+C~K2a5*^9 z)P_W0+;H`8(#1E4fE%vsrhZz}syjslTc8%zO7S%ja3i&>T=h31 z;6|E~l^Rzk0&b+;f4#$QBH%{4nU_k}AOdbw@vE(GL;$X@f~^IATSEZGjbcCMe^QVL zxKaGAT#H5%0XIt2$U7ww5pbjAFC~-zKm^<| z_Sbu*1`+``%Kd!1eKsQCMjtt!`-=#`RdjG8@wDjq9cCVnoe4D?pJYk&%n$R91K|nN zJd2*+Vb*aV;_H-qqGx`Xa~z1g*z(Kh`K>j$(P~9ZWfN3Io%!{LSUzlS& zwl2eZd{&; z%q$K>=j@a!TK9!{#etZ{or_GM?qGlH!(Z}5>%K6jIFxb2wq}ghieX0aSU2|F8mXgA z0JLd?Hr?Bz#Ib-ja3OR zf6$Q#xUqWt*-V>=fE#OWoXcC82)MDf+n0r-cp=Ol9_z-s8|^cMuVZ==W&+=NYj(8H5WWVYD?NPhoN9|T7N!Kx z^L$4WBC*E8gy5@+AALw9)>xPheDjBwTM~&KO@`H+*k^SFpr0)Gp?keS1i%erhx&b% zod}lU6^167Lj>Hg@YfPKBLZ%ioPBLhv_BPQ0FO1m)Ss8fMf+1>{%;`qqqEyLP;KCb znfK1!&qoB@FuUZ-D{F~>8|Dswb*vB(aKm?uI*=v;&>J2MToimn0Nil)bbf-9M8FN_ z^=DM7PXucd>1WotO$6LhX@P(Yz66`Uaxw?HSXBY6CaiWY{(9 z3K4L_ZSSwwL~~l0;~Q#ociQxi=D{$-ckDeSmX1C*DXI+GEP_$15~d&kZUj5Gj7=i~ zZUpbLruLgez>N?oH&*AV3^B0){QK9vyF`^gOO7Tlau zl?b?zoIkpMh6uQkqV%)v?TLUJDUVJ3>@OnN0<~~Tz4wWL8>!W3{x1=5BTb1>iCPc= zH`4BVC+8C);6}PxZx?Dx1l*`<58huK0l2;jwq2b)i~zV%Z26V3S%`oe#os=&V+Ij$ zqr_L|A1ET=M#(Kb>W(D>Zj@Tt_tOMKux` zLiPm_aHG}K1qq|Ys)5zA34f1gdd2Uk47S>|o>lt>5pbjJsaLynB?4}=8<)NOMIzwF ztbUZMX9S=w6FfM&l_qI?x*OL0#9f-AV6|)v&vIZEEeK$>Y>a5Re+)eiVYM7)+`hRw z0lhh}S`PDV_gfnHrop%|I>*{NG!ejR*%;GgLnXQm4ED!9N?7ha>L|D|ZfLU3^N4^O zoA-la=_3FoY|y&I;VlGU+=RcjQ~KiPL}1)lKDFH1>qKDOSaGTBfNzPwxUuq^LfMWH zfpKG1f_(3kCjxG)o{+oC10pbPtht`2d>ta-#@enEDn@%mVSeq{x!AfCQ)@^2N?~^G z*tytMd_PT}sIu6-<675ok8MA@7U$ZXyEHVS%Tej>-q6yRh=qC&2AAciy$HH9Mv2`(a;#h|0 z9SbvM1L39im5xS6m?s;E5C=O%BO}a`4MZ+lb}AZyVUBDd>gcLJq7fKo$OfWIZtM|_ z^e{g*5F^(ti$;2w9UF+9`NG0zmom(a4aDtFFge=U3^QZLjvHP))lPZ~pf^0|aqF87 zkxlsCa+nnR{{0Wn5&<`yw|btj8WF5bWPX@uFA;FVW#7p^HYWmZxVkl|*JC2!hU>;7 z_Z24sZn()g=FA}?;D*~_Z$1Bz2)Nod~!Q{OfahGZO(fLL@w!X($oAj-1x7X8|JMMyQM3m#-rNZiH?#;A}!7;3oXb zzM=ca5dk;Cj%9n25dk;Coz~BW69G4JUgH)ttw76U@YjMr?nEVEZ#hhK{r$pI4Tyl7 z@ay8UmPNa5VV3LIaVy0q(|bfabYYHbAo7px87@<8*a9_Y^BbLsfE%gb-PvI~5pW~T z$9qdxBm!=vJ<_`BW+LE5y2al#u0jOdsFST~(8LPYSHYg8r`tA2HYsM;pi*Hh+x}v-uT(7 zJ}6w02)NO1^7|VQA_dk=32)Hq# z!LWS;iGUj;|Gn~Iej?z;sKHmi*+>M&jnS{1&y$o0xG|>gnT(@|V1MlG8mSTy0XN3= zsa|9>5pZL_KbiY@1fYZsI^~%5oB%w-ElZtcNfRRA#_};~pY10CZmc+yX-{b);Ks_% z?Y_J~1l(Aas(rK8M8J*JBfAv-l?b@8=6uJLUl9Q})^^yI>31U7QSrsg98ZX6$+YVSxQ;Kqr6Q$J5a1l%~;pmEbVM8J(x zFSn}Bh=3cX2Q^GJhzPiG=1H9**@=J~XPckQGnfdtaW3nTjIR=jT`jwXi_b-Od#o`R ztj;k$T9Cn>dzfT;dD3p1{!>r*I_7y3rkLKAs6u5TvBts#(_f{^ag|7{u`s>#+o=n7 zBob>ZOfJ2$@wy{KVvU8VrMI*iRDnpWu`sdpmkp2JBNA&YOe_6Poy+xz#2O2eO0Pfp z;5Lz1V_{0^-A8uRCjxHRrYv)#B`JLB;8?IA$0t3a60ll!j2+L?H4_nV!+7bmg%%M3 zH%#zMIg1bhH%u;UH*h==aKqHG_CLKy1l%xPqRW*PM8FL*s^hWOh=3brXKma5BN1@J z+`cXM-y{NVc(t^TqD47;?&5gRU1Tp2m4MZ<&cM?Zefz`4T?0TVWV~Bto!N18` zBSeDSUDgu8>&U64FGh1fm>D{@%|cx&aX6Yc!o1KxbnCJMG}Q)fgn6Od+V6>g8)3)Y znHkM#VMb`E&7HbCI+_Q=e9*CO2L462-8^zXiSeKp%xKTWJ=gZM-^z>QKHGj~f#1ly+bq${|T2)I$EQpOy4h=3bqH_uDFnh3a2 zZq;Iun+UkkHM7y-D74$jp&`~d-QquHpc7ylpvZZuC>X>C&?FmANyQhC4=BH%{L zvpicZBH$+c;6c5!iwL;UI!)0o?TBEjO~)eT&l3SR+FtHmu{{xRqurEVwa*X%Hzt3_ zG^ry1b(!F4sY@>q05^sWEO}%E5pZL8`ZE3BCIW7Zs9%1~&qTnDk&p8i$Uy|$7&W9| z_Wne`jnNtMmg7XgjWP9db)HEC`(y7+tD}g38{>M9uQY=QxUp-urJWK1C}D$+$M5YV z0B$Txb8K5pBH+gI(fdZ9BLZ%$ID2qrTO#1b$}Y>;pG3foRVh~`{E!H^v3k_Ticg7v z8*9$3sojhSxUshV3tb))!JfLr3CcGo0&d)%ExA%fAa=DJrf06VW9t9{;Ks2Go9^Z$ z0&W~1vTN2XBH+e}$9u*IBH+f!`tuVECjxGqO1~&bMg-hAJ#c27^+dppGf!t!dW#6S zakfRaa_fkI8|Pll+PMG`mz;89v})kqeEqF_YrP2Tm=C2JBM%9Lin*W=#6B`nY^PK%R=cQIxikci&&;OeX zv9wOq*^rC>=5o=2yV3Sr$mM@?b#(vMXcI2v>c6=zd322Q?bH9wjXW`Hy65Je zO*9EO^bwrK`3buDzG~}gh3dDf zC7=?bgpvj*p;C&3BB69B@tgPA^L*dmv)}u}Kf4^CnVmVV+1WW~uj@YRCbwDOXg5}$ zY+09V{05G8WBt{}v&r^v;Al5CPp_HLjoXKIV_R~+Y?7hGe^7f&f>3sNf>)4qrRh>|1U-+LfZ?ge-h(NVF?up~-?L zrtm%P^vopHfQz$tbMw%y)F-cut-~DcO4IuM`t!`uuC&EZoNLM)?MgTNv7;xLqh0AQ zJiNOlbF`~g^fFB4;YpYXfjdpeX-lj8KnWNqC%iq;AlkKTE(r8!9M>mYV!sXGfR^Lz0^c-`v ztM%*|(KpO9Xur94e$!6O(XO_S%&GSWbF{17&YNq$%pC1%zu<Yt9mSe4N4swEce82|#?mfo*5tEm8KYen-`r}*ROV>c ziMKlJzLPoHb@JafM?YqccAaXGw|73~XxHf*ay>eLIofq*Xx8GDnWJ52FJ-GRhdJ7H zuKfy8iFpQo{H>p7na>>U`gLEA*+baFGuuZ!dJKJ-G1~QUo}MdyVUBja7~Sp23(V23 zm-{>KI>j9AdeyVQWA&J$U9WTJe|-aUwCl}?LgjlgN4wr0Dqid;bF}MSxA(GkXO4Eg z&oWycWA4*&T%2?DZE-^5bfghSGxHXw#b^v>Lu@j)$J2(&VjLG%*L!-leUCYq59`Ys z-74L}9L&d&6rFRw#T?AXk(yUvO0qE=hm-Wk2v5=7`4=SH!-0eOI5JfWUCqk%gZVhJ zy7-pS%)xvdxtZ^6&Bh$e$C2MUd;bXLFs|cxYpx0u3dCq9jutm*_X?HHz>(Aew4hDh+~Z7;#hx~YijahB5<@Dn`g3auFCbJ z-Pqof?aFrMXg7ABuQ*VHIogf=H=l3Y#vJX6T3LHjAcj$FETZvs+$U5Th0IjF&2`Mtu254-T%tKI#yBn(`p?qe(wrGiU~CKXQu)b!xqh@O z>@{WQ7i5lhg&R9&A;G) zd~@b#SBee|Tkm9!cBL%*e8b0>qg|;6R{DqooN z788@|Ge^794STyU&H3SnGw|t2e{skndS-;5p~JkZrZw43L3jpto{F{osNFYv!DxO9J@MxKlhYF%+YT6 z+fG@wJkK2MYWZ09tDBgkU9Em-+O{oow5#>3mQP<|omzQ9k+(XOf8;WOHRit*|ux?J+|LMn}MTx{YYrM@f09PJu$V}(hR zn4?`IUnny#7jv|0)IYWThs@Eg(Qntf_CDrl*O;5CHtf$lgZ|m(HCo@u9PJu+`k!uZ zFh{$__q$xAAd{l);u`F-wF!5 ztN;I7q1f;FVK!F}aa`=9?#q9Bov!ZDu8;Gs{NYdLXxEETOZz;*9PN5}VBx}b%+an_ zy|&-kk~!M-I@h+FPcTQj-i+K;oi+htk9NI1xVI~Xv9U+H-gRx(hBm-rk9NJ!+M*F{ z;lv)|IPvbLE<=;niL`kU4J%RmAA(>uBuo9U%E69|GvM(r6}Mkx4(3Cu)Q4pMYRtiW zNSGSF;+P`AX(%)xv}q`I>~zJ1KWd`PEyf8z(9W)9{!e}Rs7M9vko-x{u;|(PbEn}WRdC|DcCs~-I z-B^~b&~FrTv>U6Xwenl$XgAiYYnP;5Z-H@~h)ts!FWt)J(Qa(ZSH1sj=4dx|D=$A+ zggM%c{ic6j9Ksy!iYDdD{YV&lFQSRL2OMXNc13(F&ypU@(XJ5Pvj4G$c?R{%ELnbP zz#Q!gHK}FQGtAMh&_|ngZpIwPt}vZj-~Kamv@2}(jyFEa9PJ7>`uppCWsY`*-~ZFS zk1$8OvS8Vw_Y=n6$>@!IOXo30yD~moa9~B|Xjh8%#eVF=9PLVZYvJDuFwfxns(-I8 zpE5_gQlIKx?RMs9SDJQRZWzxT?Mhpu>uuLFN4wI!_04@#n4?|k|NZv*oXpX#8eC`4 z0m9gN6@6E)WIM)aSH+*z&wrUY+EwEA=kI@rIoeh7$%Zfez#Q!=^>yV#otbCQKK(_d z9S51CU1jbN{TeezyUISHKiS9}?JD=xkPjO%N4v^@^7i;G%+apCQgc~1!q_?;ZLgW{ zF2-nA$J6VS_?$V~)uLkcPC1yPT`lWWd*B`BXjiLkrMFAwXjkhQB@WJHo&E5sb_YCnI>2g8`7UDK-CpDZ7*ZlbG!Q)!Ijgoy_==|Zcy zaFX5u5k};1a~rM3!U=NOvcnCuniB@?8g(JpKw6CngLaJ`o_8s&=7d4J#uUi*JFUiq z$)JC>Y1SWUH75+(HSXl+-_U9-b_**qo3Hv#9{O0jjehUhnIZ{+ zah$O6yFIE>Boa>G>Q?0Jd?Q5?BD|bSea}Hy`Fgk8DG~`M$klK8mrx`jEZVhtOo0Iu zi3p2!t+~GVPqfq*7VTR5YN0X)c3xoNJLn)>-zk=mM0!8 z?V_d!J5UrX9L@|bE^weRMbW~cT_=VgxxN;SeTR0Pym0)!z0A?BQ!NM2>CGJNI-P&m zgu~3yt}}xMenU~PaA?=rtD*l~9_>2U@xI?F3Kkg03FiyBA1R6sCop#X`AcVb+J`tU zB;S4Y>hrXN6HZ`k`S|(^9cc$hc(m)qn3MPBqSkw~>*a6fuA9Uh?Rxe4h>7a~I(m<>sMKe#%qA>#~q{Og64o0)_8ki_@FlYe(*4(3BD-}v0G z`%={ose0_4jNaV3cOql7E8{aej^|~LcBOc6&*w{-qg^R)+5J{Y z<{4aH^;=b#=Dd)j71AbU&J3A~k(;)QUR}WTqg`o>E_v87N4wI!eeF{;=ZD*;|h}+BK%&`nQfU&!B&{$(ql5F-N<`o#_4kG3IF3_<^sz+nqVuwbwp>n1XDH zwcF^Am4&8KX^i8fjo(>b=33@x*NSTwzVjq~FqU>vlTz90 zGRAq=#RW^M^~}+(6T`~1=*b-II(f0eQ%9JiU8h>rTKx!fwCi-f+9!Txj&_|HT;u%~ z%+ap1SE{~ynmO8auEXWGnlaD7k1zbs=VzFsUH{h~>xvVGI4-1xeyxhm&lv6c__~VO z2Qo*yUW}DbR$-2Iz1*i>n9CgPdeyt`$(+p5uGiN#STlt=+V$q0#_!z09PN7hN2B)^ zGe^7Lz4H4Pw=+k(-e>yl?N6CQ9H(BL^CQjbD!rai(a+tF(!j&-?}YTy^B0cWo75am z(yNt_WO~=q4{9+7^C8uAwUrlX|00}Vd8w9fqWzDMeHyu%v-errKL|Ofk?Xy?+SC3+ z$V83YRNq#G*0LcVHF7Jr=k3ky2lF8X_4Ic6Xe}L1u)N>iz67nMLsn|A8-LVmH{G@h z?ZnY1b3dF!SI6&FsyO~?-uPV1Gbk^fp1G0sFG99zXk9GJO}jw*A0cNoa`o~07is?> zWUfZ8*RFYv_8&t2YUJj*jd?!g_MzR_-nFIjeaz8r>^^&`B(0@GMr*9!fA>m$T1$t# z)?indu0v?jRoHtGO&##h^K^CmUS-I$cDV1?o0y|rAvz73*oAop^~)T?=N@8?c7+;u z^xg-Vqg|o@K6c&r%+aneFCTioEpxOhZ07wPFEK~E!oB-kXWGLEf1CmA3jh1lRcQ|+ z{HZgrDydK=bF?dE@lg|IFwfxn zs_%*GuVIdMr9OB5zIU0UU1>UA>2No5v@30)i;cfvj&`LR^k6mG!w6Zm!LIaIgFh~h zcGa&1E6@X1Ve3`2Y0q;zsdNVOvWi#ke(^EpXjh5STPvSsj&_x-yCY9?=4e-`&5Opb zW}ZR&^ztPi)@P1(l_|0605!KjxnG#+5Ur&{ zc5WEE`h$GMX}O}XbvpWfL;G>mdW_>jy6)MlpS_Mb+STITO?f|Ij&`+tWOJ336>-Xl5E66;9_8UKA6Rl-KHgB+_?W5B#(pom;^hWM}DROWQ*N=9!|M1rB zm6@Ym6BmAz7MmG-wkG=b$bHXHX^i8PiQhVY{ddgKt`RpMZ2Ssyv}Br`yH@16+R-payH@tN@cets(XLg8CVici zIoh>)#KeC_Ge^7O72Jrqw1=SZeVnqk=je&FhY(J{u64)m8c>GoN4wTfEVqPLyZAm% zxgXze{wNj4(k^Pz=7t@N(XNYc=x|##=4jW6p{=@HWR7;7yws#xN9JhPsb<-J{FXV| zb^7M4zdy|!?K<;TuBH2! zD|58#&FK6C-e-<>z1?46iDiyukxDC9*)ZffLjK$Bv~j*i@l02kI)8zMmurzX`8S97o*)c{<6bq{&4X3bs=^7Gp(l4{zW)}*2nUmCL?J7 zBVzSinQK91s?jH7DL{qB2)0#?S9H&M6XSMgPFvm|vh?i=ftj;`x`sFos z)@)^tc7+;W_DP!aLRNOLBlKV8U!XZNg=o$X zzv#jG{U77X&@&_aE>KMm`H1MkmU-K$+w5vp^eD%L% zj&_x-SFq)i%+an=Te_S+&pd1AENYRVk#DpRu8h~3Q5uCn#JPpiWm?JBo^-Kssz z(XR4~z8qeYIoj2Wa+l|c4O^$9AF}88h)QSh)YI{-EXFfOyIS0ryVaY_(XN&c=V@>= zbF{0~kFCF4!W`{t{eH(&C75T>e&bqBpTHdLYWrBzk$IS-UG0ATW!PlqXjl6WcCO6L z9POGiSzhO99j|Vpf66@7o=RgJ7gEu`U7^K4%+an9HbF^#ygunL;W{!5s0ggpa07o?K*wanzvqLj&_}S zd*j}pn4?{1|Jic<0p@7exwbEzpuJ^{->cNl7ky>*gIpf%`ek1<;{_eWaoR_{7Ctha zO5^t`wU2WxeN8h*yIzc3S!OtMwCm-;<%J3}N4s8i-~0Aw%+apbd3Swr4|BBZ&8Th1 z#xO^_-X7S#ClhnD>s^<2Yu;s!qc0>zYJc(?<`Bo3F*BBqNT^A#Crost_F!7f;+$)u zXo0HzF_pzQ&P4G}+4D~3VBSRH{)*+MG6(Y}lD10GYng+26R8gy4WgaHaFSjvnMnWA zcp2>+h7;uGp$2~r;QGP5iELDNXFld&-bC)bzt{F<4(3hde>!}oAafYkCZ1etIfWAV zO8^r`ODps`MWyk5oQdPrW$L`Z9PP%UQOP2|Fh{$wEMMxLhnb_@SS_r&{3vs@8|w`< z2KQo)c4O1H_Rkxbqutn+t#!B_bF>?~WW{!4azxnjK`pnU;$X4|*1ycAs3L~OP zS<1&$8sj)4;-lG%&1Q~vh3K5;v7*e;u8`Stzdn>X+7)VI$0KE!qg|nowcfRoIocJb zThpQ0n4?``v$R~n_ci{#N+aBeowKrXd9*A1pmC`F*533xcEy{ z8sj)4<5R^d{LUQhO3|)h?`N5#T`7y?d-QAOXjiH?y6$en9PLVdw#(70%+ani?YniwLTIoeg~J6&f1^9?Un9%gE`t&_Q}dc zZeoshm0SB}=KjpluJRxEi*IC(cJ+i(y=V&se*tB5w5{5*b5t7RIHThk)dw|Uj&`-E zQfKEb=4e;Tnl=Bd%^dA&wY_}LznP<5t*4i*{Tg$$t4+lcMZaQ>cD1cjy4+LD(XMuT z$3<(Iqh0OiOwIH-bF^!+R=9^Yknnw#G0}w%M`<+{PT-kr;^A#}(`wFS@KiLSK$9V~ z8Z#Np>qa(h^%XX*Fgt>Yr_yt2eFY z48D&u#-01LIjzRR3D)oXE`EYma|VAOXKdFpMQHmjgW7HMTen>lNf?adjE%>1K0=X5 zIDxBMaea>!6iFETeVnoK)t*Bs5(y_*Uj1IE7)25We;;S8ez$meibTQ*mNz*IyiSpX z!S`{-+Sl_xMv+K3!Se3!1&>oCVet2H#`N68;J)lJoEu`FmEGiuD)_Fb1-is^})pbv~w6v(&NrX z`qxQYY3DGUAU6+=esdAm59VzY{!H)3rI~|y8@cz(Em*=F?t2^gAMct{f;o(98~3X_ zs4QW$6Guyq7hJ~}?Z)w%Bc-2UoNyyrT@fGI(DfbWXjh0XoA11fc?R{%Y@2TWoH^PRYQp?JH0N1-A7_O=K4&4# znc)QNhHs5d|4nlJXjj;*Gk%!K9PJ7>^44!@&bRnJ&I*69$R&DagcFzW9qh0-GmD&+uY@Lp__T9RM zG1}Gf%mMowFh{#uR385Eapq`O%UXlp?7wJ*wYMW}kL-A4PCF095F?b>+k!ah5gqg^YmTmC~w z=4jW-*H->^kvZD6>W^JrX{l~8j%}`2Fa5?G z?Rxn~m2Y2Rj&{9ztx==x%+apb*EMcal{wn=W^4m@fjQdsc3<6`9hjqC?_T=*=Bvzc z^m%{H;XB(fhd9o4Dl&n_hD)1wQP(XqUP_AMxOGu9V@^{V7vUs*SGp+PHoqQ?i*SNm z)SQutuZ`cUbdjtwJ)&_LPO!Y1zF{Wa7Y^e%7wPS*N7H=~PO!YGvuQ2emkz&I=^|Tk z^BKA?!wHsmb2|M=?_3;yuhK<+@5?*solQ6ayK&Zyqkbp5XeW-oSX7tBg~Ru8E{?xm z(uBrEIDyv1;)zu;jZ25$t8}rvV_9Yzm*E7E5UxX7ZZ=T$G zmhMZ3->Y=7y?w_zx-Y{CmUkbw{h8jmIQ)H_i~ZUc57RrFZ~}J4gf;6J!(Ht}G-guP zN+ciO$2k%IHc`%Jj&_B3bwsm*%rmH8UO)QrzRc0CQ171Ha2s=&HJs4DpF95vbF?eW z>sM!8#~fx2Cv46OW5zK@yTXloXy~=fVb*ZM|9$_8sm#%?Jih)ce>m7X84VvOx>IR< zALnFzp-+}$%+anCO$R?wpE=rA)AP*H zu694=D7u+B+SUHO%;g#~N4sY7yv#KScsl{7~`73#Qo2o`;a->HKNFs**7vr zyGFLVIJQ4?v}@Ga$yrJJ>Q!#+O_edV;c@LN4r*JKQ!iX=4jW-&iiMrWsY{O zIyOMIVvcsLp4d0*N#X!eQXzVwClRx zbPq3yu(XTXcR9Lm*O1A~(XNwcyMLCOIofrqU9k!uFh{#i z7b#r)Ugl`mng016?Z+JLI(w>M?;Dt-UFVv8(ELs28Tj!xfAqx7%+apTdt(t^kU|{i zeAKP=t`k%m<2dKzEFF)uWR7;dnAmc~9_DD*%VSN4)?|)$z3QC3*q_YNuGiVKlz*K$ z+Vy5q?$_5bN4wr0&GXn3%+aoQJwAT?OXg_T`#hgCdy+ZCaX!bbb0}5<^FDfU`{+0+ zie`Nj&1{`}7-N_58yO^Wh_?B%mXA?#{ zarAkYidz_?-8lZHYpzDjGbk?}@Ab?9=4dyTcXn^rjXBzl)rw+Qeq@ezWBqO6&G$1$ zyRmsH-_(DYqutovR`8?t%+YS_7Jsn#GIO*W`*k0^(T+LV6>si3Qj8 z?TUC`qec^$qg^3htDCa~^9<^j*EMiUn4?{x-jVZZ&hz*_&I|pAnoM(MI7zRiy)eD2 zeCxS>v@7hj6)%0n9PJ7>zW;AD=X-n~=Y{|4jUVWl5l+%kvX^J}t{qC%WA9`%tVY8+ zjM1))FIH{6n>pH*qDd{^lsVd!vS96N&oj^9`f6~QNi^qqd>`kfzEWWx&6(i@>`K$B z)a6&XezYrXzLMYl#T@NQ*JsN2H0OJKALpe%KYkxQGr|ejRc)8oD?`=eou{HT*)|Vh zjCNJLB@zvi#gg=vSHq-5p%Sw)cQ8LMl#Q!eR@%cirJZ?U1iF&YPX6x z+Ew=XCeM^*j&_yX^6P`knB&Z#{IZ{4F3%k8>Y^)~t|W}D)6vhxuYSWA?dtfWLR+3= zj&`*$`KSEB9PMiPK!F9Fn4?{-e(hXoD|57~^#|Q@S7)9<`%T>Q=_}09uC@>MXxo}O z+STsIwXa-cj&`+w|Eq^OGDo|n-_j@>VZ6GDPK$Xt8KYej_tTT6Fh{#a6s`Q-9n8_L zk!>qoUd$Zr8g=IR*7=#EU8CP@*l-|ov}?>Q^{%bLJcIt(7wh}E%+aoKm;bz@B6GBB z{E)wHn#UaN+S3a&jUbG*+vrHy$&WBbyEdL&e*Uk_(XJKQN?&Th9PL`!rNpzSinYqHjKJ(;6jYrEFTd4zce{dWgX-Q0sY+O>Y<**lLjN4u{1 z^+kCxjHO-l;-bC-s5HiL-o>{pS(J}C+I8a1Rr}^JN4rj*S@wNp=4jWcwtH$!WsY{8 zF1owhwan43GyS&S@iB9>>+IwI@{{%4>L!* z-b~(f^C{+N*V`kT?|gwd+V!q`CwG=P+Vwu~%Q>4f_c=pOkpEdXbZp`8y#1~+qEgQ) z`~Cw(DkFi2o;>nEa$OOnRvIC7+{(J|luXLOo47!uWqGFl%@|9^n;N{glyExxct|T( zxADnj%PJhhm1XHhYmyDCkYEm|T3+QzM>VJa{$@xhx4GgA3y7xws!K>G*FfrHD=jIFPrz9I@9WB1`ZWgjOB9%KK@*uraxf=BV$!4k=RCZM+>daLEB=}Ac( z%_9D{>6e-)cod>p>yg8WV$JgAj?)Vh1&=}v&22s-3Lb^NlqbhMM8TslEwVQlLlis; zdqb91nTUc%;okV7`@2NJqwr@xt9cDk@F>4KkT03f1oTHnqkDa|KPf3;Jj!@~_tRa8 zf=4NObeXo9D0q}IPuCF*iQ;wDhLn4;L(`OZAOeBiaoL~cbJxqD0npP_a9az zYs!$tEqxCe|L%{&ll6Sa;g(((*{_Ffrw<6~wb9A^Iet&BYQdw82NW=;YshYKpn!&hC|}DdXpw5n>yh*H4BBb zYo#X6O}2ExF;KPa>b=)f&1u6S%i5L;*NrEdHY_7k|K#(x6HOZq>DEe~>-Z_rwBe9! zZP6p0?;x5s98#^Vf3)giqUn(;;ek3LazYjyFC- z6gV*1tm$pTQqs1ecIGscZdNiqA#a+$DcK$m$7zp34cYW;GDn3xYC!en&Fz!9r@% zKB^f!N}FqUWt!o_ao7XO^MCt5Av@ZH7fL2`aySO+k2lYs%;zCbTH2$kIq8!msK6Gh zX!n)-Zb;e$k&aaHyo=ZOBMKgs&?g$-OB6=Bl2y-l_<$&QRBF%2e{&NBk4n!S_043W z;87Vdc|vKT;8EEc6Xq-;3Lcf)ap%$!M8Tu-({3BMgeZ8_RVMaJ!Y}-6b@b(->z+tT zLZlOtp?$jl-gQL5qZaoZ?f5!T@Tlc8$DaR_D0tLr-Jq{)5(ST1FCPBS9-`n;o4W_h zZAlb7YWr;8i6@AHNA147cR*93;8FV(e#v>F;L*$&u_Ips;i@M3bK`UG5QOKoiKlFN zF$YobXv8(^D}PQDJR14Znml(A1&>DkHFNx6qTtc!@zXvmP82*ElWFdSnMAQi_LX@X zB~kEb+#fgYm_`&l8vo8shZIrpXt#|%G$4UcuZ>P@YySg5@Mzio(NHWCGoR`*#xt}#*YXiedTn+_5MkJffrda*N6?613$^WXuZ z;L-ZQ*KhAe6g;|WcU?#}KX7Js(JQrHDwdRlNJqLjQ|f=6fntZ?WHqTtcF*WS61?0HHD9-Y5# z?8aojR66kJRk@AHHW7UHA*3zqQ1^l7ldD3cBYj-BL8nbb!J`*_8kguo6g+x)zEQqI zM8Tt1ZL0KtkSKWcx>&_ezb6VFy%{JEwj~N4y*;6}Um^+~y=yXbQ?lnNJ$UrK;9D1y z{Zi@Ek&fD0d|a}!QK=`nin1*HegA(h3D4?~k?ga*$3_xN-|-}74WG9+79I}x; zx$W&$MAL>tF0$LVe^Hib+HlB3_Qk?7Lx`pghdg9oFD+DrXxebdLiWVUM`jUC8&;9M zWBF?l(X`=^f$WnUFC?qdaGY8Zh5Tb*WqvYQm4@TA$GF?lAG;64S-MTJthnW^B-#%d$bhODa|PV{EHTdpKDgsu(=RZhq0Hl2vKQI|eoTUAMGIR;3~9SlXjFwsK(d$^dq{ zm52r%9hN^a1(8ljICeFZB?=yeXnE-GDn!AfkooujI+rMT6l(Z@nmLJrN1-qD?LLJl zcoe4Tpu6uN3Lb?mF#OiVM8TtQ1EL&B)E+X7r9BFN(wZb{5BbH?9_7U4OOux?@J>L< z=-m^m>Lyo(NGGHg`~Cdz^+ds=6t7?Tr6*DFC}qxze;pwT9;F&HxyK_!!K2i_O{n!N zQSd0ut0Tj(Y1+P93njs$wAYWi>oigDDBaY%Z4$MITw-aD(jUI(nj~sh5a}qDeeZ3_ zv4bD)3+I=)pD1|LqU?;<{viq;wQM~7k@iHvqgJao z?AbyTJZimk^|3}o!J{_iH+_+;4nvNxv}Hvt(ytLbH>CI`}BB|D0nm?%iy~_QSfMFw_&$@L=-$4 zb@a&aJVe2x(UXp^nm`mh8k7CtUrF>FvVo;d8Qb~5FG=(q1};#y|JkofQ_V0QjUWBY z=|x1rqn)(9%L567dTn%eQuikbf=3&_F|p>CM8TsKw~n~=Rifb0%J!r0{);GhwCdE! zm34@MN2~jv8@8J$c(kU-)nA(u1&`LYyYTmUqS#;e??b0r5(SUezjgnYCy0Vam#1Ba z?-K|mUDP>S%ZUWRql>d=t)G`DcywY?u4|SO1&>Z1&1*{%1&>a3YcuT)qTtc#EFDJN zL=-$aGqKfI3y6Y8XOA^G?TCU$=X(D7*M~&WhR^-;FZU4zkG@Cy#z{~C`4&TBuXe>7 z>`F>Pq@#RXq)@BcM8TsM{qvi1M8Tt%rwZg~LKHlD)xPt{uZV(2uW#)({VAg0(VI7V zp8A_8c=Yyck1t;%3Ld>{vG&))M8Tu?H+=PXZ=&f)N6#$uLAC!dotimybnD!orY9&p z7DLXi6~zZ6k#jgs%`+im*SCe1BvE%b1}dJ)|3?zBhumF2%b)H4#(*>#iDkuACrhZWa$DbE9Lz? ziS|Q|E}&{=n=VP@95QqP)muAMOQPe=4Y>}A9U?v zh=RuunD@HvEu!F2$p^dNa0^lJsMOEJ1}-NG9+m#6@X`uI!DCoX=KFm#QShkj0|kG~ zN)$XQ_rnL@j35dgm7n#|zuAa_N8N7C?e`=Q+N7iPtvmll5XPg97j>-q3Q_Q=MVXd2 zen%8MYWaNA+n*r{9<^GNeaQu);8E)(Sq5|%*D0y2_6283_QfE!5n*S4`;L(VzweG);D0noo zYwefD5e1J%9Vv6D5>fDI^yCUV<`V^v#$+qiuOLzEk?m6QlfFd3qj3kOe3*|YcrP!Aq$GTMEhNc$vr&n&M8TsKx76+Q7*X(O<%e_TDH;L-ZG-}qnyQSj() ze(8ByZb3;Gby=ThUQ!a?>4bDx+16C9NEAFeF?r*QeTafbCy#7-t^iT+=v3Fa8$Trq z9-YoQ@51dw!J{)1W_~!HD0p=C__XoY69tdX^(xvg+4BtfuhP$%@E7)O`6MTm1&_Yj zD;;TJn*R1$NPX3IZ z9#c90#f$%`*QB20A&u35eoYdTe#Qj0ypg+15NW)*an5!`(}qKOs|F1k{XjHrI3%|! zQMb)QMAL>tHmjBDQnKe2j#DEh9Ds@Hx_5In~5Pc{5qM6s-Rw07%v zh=Rvh`dSTh5CxC1nqA?$WDnLrq+??JeVNP29&9*HdyLH^CFc#Mn!#gi?=3Z{I8pEz zy9ML>4kHR4W509CqC!N$qxfaeq1p+A-il~w$ERN>2p&azsdd{wiGoKVS~RWn1W~M6 z-q14lI-=lFsJF6AX-O2uqtO3mUvPpbcoe2to~t#9f=6L*&b?(1QSc~S|4+8oAqpOa zKlSnc-9*8o{BZb=T?vH#$Y?~@cJ~qlk1{^g<(UtNf=4NOcF%PKQSc~b?p_u95yk7O z(S<)MO%yyzy}#JhMMS})G(8G#&P^0NN}DI&mB~cGqjZx#I*^Abc$EI=2iqnP1&_)e zz5lfYLW@ z{IxF=1&_+8`VGG&3LcfMRr@{ERidtq^mhWGO*&dz z{@Y~)VLa;icQt5ha+_(6YWc?npNC6dv3T#Q%;30<;P}wp6)nxr1GDrbcC%d&x z*5Dz36i_{&^V7-tJ!FppYKr&Bovgt_?kJ$PbBAt*UD#yS#niB<&PD~xPZ6{Ii=;YzS2Oc8| z9-Vsm_{+zLf=8z_A9=7lQSj)@xB~@O69tdX{(Z1?eWKvex!0emn8bu3LsZ(M^Esc* zmBf=FKUCVIf8^=P$yOcwz5I|Is`bP-lRb)XoUYf$#V387>~ENm?)SgHxox}{Fna&E zq{SXQdUw?d^)zIA+IVtS zvgZ|!ftq?3&R?aP(}qLZr&3opv>}={9FjgQxqsIMqG`h+<vOJ|t#rlUM zW0O7DkmCucdHBHkl~gl$jO~2~&y^tx9%J{>Ge?&b1&^`+`PtnSh=ND4VdVC_354E? zXvqBNVS?aM#Fyt}`h_TX6yk;H^NCk474kWyO$q(ahPBE3 z7P2`3HO)7V?nO0&M`3T;H1jA?@F-lrmzO1Ta>(NZHT%<@1|{=($l{c~ujRr~2PY;K zLw{s6a!KZE34%u%A6yhqB?=y;=(ViLokYQ7_)XBNx*rq8>#9+^*ClgQ$lR3fkJJbD zoK5Dpkhck_>Aqt|WvUrGN}G4sE9<)w1&>PYp76v^M6p$R-sI*F z5CxCQ=uzTdqTo^4sw1rV$fo{1>7+1xWc>t{s4 zqqffslJ^h=kJ^1>%YQ)>JZk@WRP1h|;L)tUlJBMj!c|T5*P64n34%uxk6*uT7g6wN zM5ZmHnh*t#M!vFf#yO(k(WpP?#ZM6hk48_Kn>h(8EJQlim}{oheT~XukL*h`oBT}_ zJQ}y}mR`Myf=A=W7OiubD0s9>FP3;Cfl#lF&hJ`F4hAq+_ipynQAu zs36j@R(9AnnwAL=>4dCGCzfWS4;LbxkW*>k!ibv;T)GHN=>?R81(dkTO?x{l*JUTPJWUuo?VLUqfSE)KpiGoMxdXKN0 z?0JS9O6liJh;F7dN%l)a2Bq|K#@BryZ~vs?^gEr9KB-NkKi?z>9(`P_@y?rxf=4d~ zHW>5~QSj*HiMq=?QSj(h2UTtYQSj(>VOcZ}QSj(ZpNh4Y5CxCkp0Cog1X1wlU8^^m zCVQS1A|2~}zW()+{nBupigeuW54T_WPrWW>*hLGnf0zs?h`K{&q>ne;oP6rxIQ7Ye zyhv+T-s;TWiRZd2D}v^n_pTF8lX*XHWUXbZnUR3na?|@HcUQ=Z1k`;|WOA~zAF?6=^)E1)PQK$iv zK5kDGJPLhs;+y{v1&_kC9BhvHa}WiO(*I_zyh9W`st;SgNK+uRSVce0 z{r+@P65i>AWJd4J+t-39cnm+Snz<<1aSoY`()Wmxk522G>{f?7MnKi~>u)+jHDjyv z>@|1vBnlptxp!l?^+ds=vX5-3QI{xqRBq=>4c8C_kIFB2rF8?M;8Fk5@i}@}K$~>5 zdEsx1lalZ|oshO@`O+WmAPOF}D6z876r$i!%LdCACcD)R{`Q*FYSZ33=TcelsP)QS zH&-EwZ8oL1RUb$cJZf8ad)NF#!J~Go+qLOK6g+Cbw0)xjM8TtZsAV2{tin}IbZGM8 zPDx34r{hdKV!|7L5CxA$0Iildv$eyDPd_xq*qfz@$yxf{7cr<$S`3GtB0g;X~ zCeM|E)u}A@$o9BcdMi=zXxz~UD^?>49*v*$K&~A`!K00uR;C3A)N7+l1Ky01k`U=Q z8xQUK@hqa?(TW=e?Jq(UJX+ae_|_pr!J}3G9(|}RQSfN>TgP5mMHI%PH8&qBm7OSf zw6@v)0walHf8D9)a%Uk59x- z;L*!}^IXkF6g+y>qIKKZM8TuiH*|bDCJG+C8QL=UP@>?`+e=L=6(tHDy=(tVskexN zNAGXlS>P6;=}5=VTW~O$iPDCBw6xf`WHAiy#X=&X#o1q9{hvC+aq4*+(g>}~@>qSM zX~Q9j(6hOV9V41H98w6~ou_*6xhC||@&wn|8ifGz!NE`Ic&J8aR1&{IS!XG4I1H3`>aWuEsqkWT- z5b5|h-cz`DL89O>7B%u!m`@Zu#!?h4UWq7pjMcm@L&gyWkFnm}^|R}Ug2&iY?RE4M zqTn&My8G_ih=Rx1&0M$sW1`?O_FKO^cPCNsC>rIvm;_kxuGNdEk1Sd>DG8B|7x8&j zZaY!%C`5-UJv$Huk3tr%So;D|@F>*4#ygWaD&z!8-v{B3Xf^sXncqT2AfTpA-DR&( z&EQekVhslUMie{>H|+4}WKIsbfS_i7@$Z?*d>%3Zr6NWz%NN{~grXk$Bct(U?#Y^5 z6(Sukd(pi7IOap zH80f~Y^Y}NDD5@1m%m38JW4nA^o(Rq4q1PoX20*usAN75Ie*d~)xs{%E=eke7OUvT zHlM8}2=k+g-|sNwaiZWc{E3NHyAKluk4iq)E0Rz$(0az8H?=ZJzw^_k*C!J`*%Z<;xiD0uYppUtBsQSj*13v)8PLliuE zedGKn2T}0o&5#*QKPL(vy}dlW-d#k&qjxVBsl9?Ic=Z03TYHux>a&M$>;2EX-szPV zo#JWT+CTjt5a$U0^*`wY|MTC`>l*C4p66{skUph;s%)%v}*6nX$)-&x*5%uB|{)o>r&8fYcVJpJpuavmKT4Mzt~?? zYf3>JeLOAm{mfGG;&|=M_(z86lH$2}O_IALj^dQOSl%_aZgPVKn35N(PqwT}-em-s zk{9c*Hl9shWdxX#7n`To%;?4qO390Dx%HzCFigpd-50M7PUfLdZAxD3zwW&}nZg20 z$rFzkdhAK6HPuTIjT`=W5(CANNXZlN--DVZ3;O_5@`QM~uS}x(0MpgV%mcC}A!>jr zc|uLyKPFjM1elU1^x;FZl80r0DS5(lI(9x;AqSX}Cv1+R85zS$I6llsqX5O%^;; zgulD?UsHut11`?q&Fx9alltV9v2_@x+r&97%ynO4rkqlGvlqj)#R9S{8c}h0eGh-FQ zlsu(2EsKjVOgBicT$OnU!<0N_N-wD!F-*x*w(g=Pvlyo2DYyE%UY21>p7Kktt@A#^ zlsx@y&t3d^rJAFopEj?pPUUn;o{rz!bY?5Vl)Ug~HdfC}_UGbAr{rn*=!Vf(xMWJ6 zR^Lz0^c=&KJgsNXh`wQ%Zmqd@e$!42Q}VQZWKO+57^dWDxAW%OFEdQZ(|*AXJ@+w8 z$um!N8_IV=>WU`1bYSd2DrZviOg!}9?EDN<@{G9Q__?_ZQ}T>#ab$fJhADYQ{X6W5 zDGXEcjDBly^PCJ*@{GB;k66qw-4)wxV3s==rsNrS$`${VVM?Cy{qL)AJHwPbdt2uf zJjbLewb7vqv({5No04ba5m(35Wtfs@MecJOjxbEgv$E&O^F0}+rsP>Y zdc@<8Figp_CeOsIrx~W?S=(chY{4+yPj~e8@@E*Pl&!)XhADZ@wO=7BF-%MFw|<^wKEsqezwYZXdx-t#&BFgJl6};p z$IyqFrR4cIPtO&#^0tha#PqWB*FNtIF% zB$PA=19oy}2!k*P1%oh1=`K-9P#OWH8zn?KM8Y7HG(agyr6rXviU0gQd++z}z0dJ@ z&a=GE%+9`ccJ|IZ&wH+jl~)*36|K&J|37h!)px*#*jR zvqG$E`o=+oyrg`yP3AWcGI>Hh*1YsD2$?*gzsj~C`Eo%BHuVs~e4lwr^4&s!OrEfB zq~BZ~t7Y;;{CI|2+YvH(!hQSA;YtXZJmG)(cH1_DOrETgz9-Be*aS<+Xk0b71j@0g zhmhj$%6C18kjay>ONBaP5i)sFrK#BbZiGyp)RPO}fU!rgsfUo}M6sV?)Cvc*TT0um z)3KjLLnduUo zQL$%Qi7PdHBZN$zl5J`?+l`RPQz~D*+Akqw@|5mh<`)=ygkq1aGH1))gi%WG|LMBhSp+kDZ7#semjNTLFuMJuRqjT6>gN}K6YW=-18{|)kj<)2P zBH@f|>LIjPo_)b|giM~6Me^M$h>*$Cs%F8>0}wKKT5s(7#$yPXJZ)C>X!#vNCQsX< zol4({kjc|=jgFZ=LCEB3x8~LFzEjCv{K?M(=oJfnYW(4qxGCeN79o4kGlA#bgHs%@5M5i)ru ze!f|mUlB5S#_jsO=rahJJmVMqn)`QzOrEXU=4)8N5bTULx>@ArLMX?k9>R)2MSm%Z zkjb+$XQ_!35i)sJy#pMnnY`GHtnPXuWb$I0txof`2${S%?pn9b%Lw^(x|0Vh z{(z9li~XcyonAx8 z-hB`OVS}^@`Nf; zs?SJ-OrFros^o|fGI_$Ru37|^G9{aONMY+#YWDz^Wb#B@vi#FS5Hfkft-SeGeuPY( z@SCo;9)ys|ll4nJu@e}(N*PVa((ep1CQpjfS(kQ0$mB`cIpbez5b|nOx^%zSLdfJv zJ+VpoiwK!KX-+lh&=4W(dD3=h_RubbOrDH0w$1q>LMBhTQNP~(CqgDq`a{1z`2s>F zPvt35;Avp&LR9oY_GNRCF?mW{%G19zLMBhiRt0zVLdfJP^+3L3xe@Yn>b^ZXeua?9 zQ|5fv3J)P<@|11a@!oL=nLHKe@ATk32$?+PKKkjY$q1P|<^TQpp1Tk-d3s>A0f&LH zw(97Y8bw+nWAe24x@L}>2$?)BAFB8Ca|oF{tzNC&?l*)?p4LB>J=y^wZ;<(}%#I@n znLKSDmVN3XWb$cR88943jvO_OrF(OGxvuuMOr4$nqk?NL4YGIlV@%2496f$k(OUd z+#vl<2ymoj@~k`e?N1PKq9RT@Xfx_5v8S+c2zjKwG2 z%0p-!4yd14-qj%|1jsTR{P)XT8LRg@KLnw5IABm8%&`;#WEnGgu^FAaAB5I2X7XZt zPodu-l$9})7sv1B{R={C8T0FO$7f!L09nRNUhJoQ`XhwaGG_8zuYcF#V8F_`sKJpo zFg25oJ+@BdK3o?jb<#0;P7XVHPZj91j>&WC>Y1nZBV_WNZZdFoPlQaKGdYHiKaP;e zb9O-gpI~Yx9h2wc+u=H~B$MY{+oz7f)J(Fehjc!#+X<7pa6p4(f91wS9A@d%L;C35 z+x1{kARN#x;>A5z+rn~!^h}Bm z0!ecev8tjmWqN!Ago;y9^u_I=wUKcs{=B+!GeT-rMe^Bm|8zn~t*R*e=;O~r2&vUD zD_JnSGAt;C19JH)GP_1~g$1Q>z@U9*;$y9`TG}O56x#_8{ELuURgwGr;X7I&q|c>_ z{BIB08whEaRN_F{YEJ-T3PiN%$f|tEm^>lYA09RgAulQGo&3WfWb%Y6e&+9K2$?*g zmk#U(OPPwT=PF^=46QO6OEP)F*6x4*;|Q5N5tr)q=+_9DJmFS76+ecM$rFB~yYoAQ zOrCttyMy+r_d!Do%XTR_dR?FncIK#a6-b2XbNjLJ|w_%J`>{|~d{o(swh0!@2 z(8pHYEgw${wXzFQ(T7|2Pe8`xDRFVfnQRD|JSAK2`*s;ZCQqsR_kL6aAwQ??vnn5q zJ&Jwnp_I9>yabF|;eg6hw#A}%=VP@@o{9@BecmBt@{}8r=`|Q*75mmhDS!R$#xOdE z11e8{Tyw=#sFe*dI{JA;l{b+wd0KokvfGacnLI5Yob+frgiM}RuZ_R|7(ymb>z{57 ztAvm@$b5Tk)i#7op0~k_nLMMn9$lFMA(Lm! zj6*|*Bjl~MWl#NC93hiu;wq>AS&5LzGj8uI=Sv}E@{FJR>h~)UGI_RV?OiY}QS6L1 zx;3N3A5e}>J(Lv#r{;SeA(Lli_Bqd=N66$^)pYK=ZzE*#tiHDK%m#!^o;5?a?5}~4 z$+I@k`j1W_57oai!;?b4ii4bDvhJRR_2`o z<=E6i#p2@?#WNvf@?x23;d@^qWb$IwbJ+`zAY}4lePG+Zeh8Vo*o@hJIy*upFSd8@ z|86!yCNGZP+cTsrLVlg@MB|U=A!PDmKe^$jWe_rXZrp)O1A(z}E~;NNLv>_q%yS}7 z5xpKEljr2n;*Gl_Wb&N4R_e8r2$?*mn^sx<0zxLwne0{1{)v#ub9P{*&l)3S@?3nY z{3jO>GI`Fmx%p8;gj|Bpcm3Oo2$?+p_h0J@0i#n7<)in?nHeX@xnLMu!=(pz}Wb(Z3S^eBy2$?)@GSyl$86lJB?Z=i|TXe7YDRljmKR zW8Xc5kje8t&A}mGA*53e{qF3YFpz5U8rIRSuBV{$u-_NxD4Mr$?Ea*}a6m!^9f>{5 zzNmtbTGf%Pu<{yAmBRrmsq~6XFv-^JZ#{IRXYYR#rmf+Cl{EYJw1Nq*W>XIx*^1lB zK^PYfSjkvz&(;%LOReh2O>3DQLdS5xO8V`s3Pb3q+249-v7pCZc(^oEAfhF6W=(`M zvL9S&A%2)UE;B-2QocT8BTSVw`zewZ>WQgWVUisVXj_H;a(!!-FG_><3pNmEZB+KW;+E$O10=GI`Q{@>U0!E{8vTruRhpW3QKo>2lZsipo>Zbo?4# z2%24piuz6ZsUVz@{oqO~ac=y-0}wKKN;VrYTOwrglqxiG{B(r;oVxefd+tQY zXw$y8c0)NnAZoF4Z|j#3GI?4S+gkP#LMBhE>N~PFLdfK4y?N2N)d+cm%!;M6Y9eIv zv@N{qDui)bGkH3$wR|IlaauEZ+HJUV2ZWAVv!18@!Zb%Abkv&3Ghbvc1TS~Znq#70 zH?$fHZDmsrZN$vgZ{CfN$usiFP1%+pWb%x9VRN|$5i)s3|33SR-UyjIV?LcXHV;DH zTI;88f-p`SCeOq#PP+zSoHk6Jaew4LG8?O9@{FJL!1l5TnLI1_sccg9?giM}Q4G*_Dh>*#%`bPhM%Ohm+tQp+vr|k%tJZtYA zI3A|1A;~Enxoq5c=xms}YWA&%w(gQx)*P#4@~r>B_P>pg$%~&Y*z-?d?2g3Ip>ucN zkBo&Xu^4sv$&U~+d9loTyRAjYsh#?9~D zfsDy>;@&n7RzS$)IXSp#$7=|gJg08duh14Dljn593_E{D$mBV5U;1OOBV_WN{V4OY z0|;5qbMe2~`oDva$#bsd;_-hW^$@3yh z=bY&fGI?GO?_Pd6LMG3vqun}`M9AcM-L24HLl81~-ek@D`vVA>Ja0$k==T{yCeO!* zaxaY$GI`#0m^J5fgiN0I8Rt)k5YnlKxo6%n=&^>xFeX~lU=K{a*vrC1(cBK_A+in! zb9EK{+<{FjAbT@aZjtOuLlrDxa%}kXNhjthQz=Le}%79#`U37<)p> zd#XU1ze~OiqgKdx&!Fv6%)EuwGI=siQ#j*K2$?+TKAw^n#@O(4cUGRE2!#Bcx|eJW zV^7F)PcKB7E9NyAwL+SE25pEY~1_+rvZHx36z84{rr{kJkr&dGAEnLOqijc`O z=Ciiv3nS#MwXVsu@d%kb6Tj48L{@}Mo^gNvF?13_CeQdUcCXBWkjb;f)4h))DLbQ$ zt`~o;6_jIBkC4cINU6rx5i)sJ<|?Y6M#$t@)u>2@od}sct8Z5Nq%%S$&zeEyKRb+& z$+I?Rm9upa@=J-|uDW^)LMG3;ix>9SMabk?|KY_m8xb;jalMQ$;X8^|8b^oK+fP6_ z_N_VB#bGI_BXQRm~S2${T)*rfWJ zya<`RIPOvF+)#x4I^CJ$rwbuu@?t;!pM3)nGI?&x*DvFnn3Z$U+jHJM1m)Ndu0qQC zoO7S=gpkQ|a?p%?n-Ma3PTibZqBcS%&*?_%zx@{>ljlsXH6OJ`$mBUYWaIwd5i)r$ zzP{zmGYFYH=UTKo3+sm=Sv`GWIA5U4>SwVeljoOz*AQbII`s&N>U%7FVH%WUKe!5c z>N78U&md&-yd1Hz_%MV_o>xa!14^1RQ`>fD_O>D0rHp1y2&f-HFr+vsG~fe^>B@h&8%pRcQa z38mQ7!$xAa>i!5qYBi*$f4cM&QxHzE@{Q2Dk~sD zWa{=K*;2YxI~rXH4rj+!mYS?X*EJ7Px#Fj*40GFkHb8G{Xp5u zXkxmO0?M(ehn3<~#)2~uGI>&V$of(NgiM}P8MC}U7$K7<^@O%3iz8(6q&eMe&q{<$ zp0u4C49e>t9T@`3QM~%-6DqL&)T5`%sz3KS0Rj z>G;*M`Ewy;^0Zs~VcNb3nLO>k>?3j_Wb(}TqCH@S&i(+vnrK^v<(Huxn|fFyrdJ$L zA0d-xWVvd)_aJ2QjH+DuL{)@Lp3&P&cK-(?+r$mAJYx^RIX5Hfitu2$@c z*AOy!#_b;)twqS>89#eUnwJqWc{Y8i$6+0UeN_yS^LL3vNNZy-Pp>Eszy?{*&yp>@b6&!9e*w;%+_A(=db=95AtA+!#; zB+Jwguo949Zsq%IuLGdelo@XRq-E0gEc?Ql+nOrFyXdZn9+kjZl< zcYmo7GI`Dp8`=n_W|sY}hjsDQfv>=%E*!Adx@Pu8n3{#`@eKMxq9;u1Lh^Vj&yV<6 zybkT5Q;(1>{{72Wt|MbTFZ_AOxeculGI?Hpa&`1hgiM}S$8LZAG(slN>vt!p!w8u? zZ|<3xzB585&)dS9Sc^@tBn z>kq+EOsvMyg<8v3C56}{8?wbOxLmXkGHNv>i{EuF_q_9w)Rzr69FD4v<1*LF6t}rBr|8wG2SWpTF4BBT$eYgm#rB*|lcrpCTVhE|#kR|@J zC+06jNUeq>@jD-#To@tklCkJheLx9dOo51&p2@Qg8Ivc(nv=y|LC8zW+K1mcfsn}) zs??EZ-$%&g3B9P-Hdx9G`QfP_Vb=FQ3`?0IJv@W9-q6Kov05fi#Ki}G&;%iqC){#7 zYm65wXzFQQJ=k?pF_svDRE(6rQZ-Tc}lj}k*g&_CQqpX zTOYoOke^e3xO5SWJs}f3y%1$CF6s@VR!9WTpl!YU*A7@Mlc(bQR~a+;H2!ceWK5nG-%i?=8zGaYO0jVWd4tS1XY<^Hkjc~b!E4Wcg^Awnk4xP7m`egPqqXZ)PES~Nz; z?n&RH}PA(Llio~Z{)B4qNcsz2k`gl$48S*J>s~vdLlp6 z$|{Yc1IrdxK*r?7V$8x`I}kE?vAlc5Z*37Wd9iwL<-uzRnY>v4wWkvVq#@Hg9e-l; z@&1Ys>W4({4BAZF?|TQUW%A;<=e9= z2yZ6qu`X&_YVJpHMmF_`oycB%;{6DjJSPVh`Q=lDOrBG>ir%ycnLMZKmv1%-A(Q7! zo=UaTBV_WN9a=Th3WQ9ai?3Djr4TZC&NaXIFsvVjbnf(?IA8EWE?9XC+1#l-KWK0o zd`#%nBcyWgsVDscXJp@c#9m}l^WH$nXw+J3=PU zo4e~aFOQJP^L9)vcNHO%=i>v_?`ngP$@8w=Klj~6$mDr{=kZ6HBcxLg*FOJvxY3SS zby24+)7vG5Sg#8y+o#WN05>@tkXscpwr`tP18#CSU{F?`o(9jyesC4?wU?U~LB|RQ ztfZc{VFq+&$EF@3Tl@Caqo6Z~16I;j+q4!Qppdzp!MOD1i|_!2#O(~a+3iojVtB~g z&Y<7_&Td#nckBmOPNd&B@)*ct3PkkXqUvyyL(+CysSrOdtq(UjHm59n<)sHJauOMCA7{iL`B2;%dSw4ed`gDvtR9%?leLsPss)YU#W?Z z$x|x#&_=5f@^k6|CoeZb$mA(=JG2%dlc#KxBV%4d$mFRw$Kjc~5i)to^?vi~=Mge_ z%3pqC&>skyJl%0>eHbVmYpae{ub=)Mlw&`*3K`j#tr__kLMBhkk{j2-f^EpiP9FrV z>TJ0LOT8f>JA-<|ochzTTHYYDaBg*pkjc}w#Ei595Hfi>t~*r}M9Ac6x4FRM!w@og z+ON1je?Ejvp2;wyI=q4$YmSL_?;mvo%CT=fLK^n@duFsi$mAIrZ@c&_LMG3sr?;o^dAdr97V|F88d5HgN_J!Yi+bL+JunFGx2jP($qu9* z#wv}YQ>U)+PdNQizEQIn-w~%Q6%h zeV#))^{s8oO0ciPK%IQ z4Vl!dW|{vDLTWW6QZJKr@)HQD)sRPh&X*s4i;!9kY1H>E`SMYOOrE%Z+l-mOm;w=f z+p+W(WK5nAKXuAn2O%#hU+(e7VT4ScP>*zN*%={|C-gT3Z|y|LLU8Ri1)NP&>8Ivc) zfjV`@BV_WVe6RXlg%R><)!nt+QiM#N)E}#PF!qGB=_EP~iPHbllVH>e2MpSt<$m^9 zEt4nXOr>uuK*;1tH?HqN7-K`qbXKkZ`-9(LbPgHQ$-K?0i~H9OhFaN$sAy=V+SQOT zc}iR>-)t{JCQr%wReS@4OrBDCs%E-^ke^czEItv&o{%e@UWhWcO3j5)E2K(i&^9f4 z^Ifc#$y0ImBES5NkjYc7*W_Pej176xS+)MkxC1achcxL_o^G+CMscW>wN*!JGHf1% zjLFktY5H6F5i)sNmdd=~bA(KuR<*NDi4Zb*TCZ=Oc?3e5K@OJZ+0NZMg~| zlc(c)_1`Fgkjc|-%b(9KN66%9zkJs_B@r@trohSuD}k}*m}pm_+dm;=@{CxJcgtG{ znLHzHj>&%^Wb%x9CindI2$?*i|Ljm^D?%pEm@hhKsfdub){5@0-$KabnfTdmEt(-@ z@{HTLw#zkyOrG(d{qS5{giN08vnP) zR<$T|b1_0D&+3cyn&m*qEfOnLzslNXCgCFlK#kjaZ>hGI7wBV_Vo)v@r;7Z5Ue zu|89=PIZJ#UTnr!XucjHlNa0cm0fp)OkNy!s&>~&g#0?)k@NR;L&)UCe#E6m&LCv+ zT%&smU(M9Ad1_`;4{k0507oNLq~(-MSSg3s03KZuaY^SSPNViYhs z^$5w%JI!9W0U7IgUZkJbs|G?Q&&%=Ce>;Vc$@A*Ww1YhmGI?Hi+|cO{giM|{8CF+( z9wC$G?W9fjokz&z`S|4KN8U!r0gvd(}aOWOJ^N@*@eAH4V7a;P-Eg{24VV&^< zK;*|~A-~7M!Fh86B3}-L>>e9FD&7|mnbk@0(3$qh&6M5zt;@7I5 zTn5BS$rmcEPS!P~pqhkwvi#X(RYMA@N$5|DHhvE(q?&~Jy~t~Dy9CuF?2E-SyauJH zCK3Bm`UgN%lW?;>F7YxTs!8}i#^hTIh-#8wA1RzX5RzV%j6P~|ep*t7^+GBBZScJT zL^Vm-u-S-VfLMjaGN?$}bQO*2@x!5{jG_;n5{J58=mdyrQnFje zsha^&O-g0$G`uz-b}l_U`}mf^qTT^Bx) z0Z6L~CPVU$l$vVNdVh^il6R}rtQlr@&Cil|veZz5YgQDSteBGRP63E&vht1$H3tKt znyhM{zDWTJG`+AA5Y=SO*!n*d2Shblo3{Dz;ec4<<9FIj%>amMvhLV#tCCnM zq}Cw)-TF^<4ohOxkXVC63-OObw}Ww>$$+S) z*bn$+WqCkUlY3okf-XqQxaggM_m@b@II78sv_l_V1&C^La%}%@839pEPW{uX$_PMI zlhf^w{1E}7nw+`g@ZX;UqMDqYa&lFEKva{9kDnPf2oTle+`BIie*h4>WPi_VQ-=Ve zn*0;?8a$39&=1Km+75rLSyIMRO+NJ1G$ndnyteNB!6f5V04sVbqv7Az2RypZ_}` zgj^U!C(KD!1HxyD^zz$J-UAg9dm;73maBJ<14Qhp$ksep?;$|MUdVw_ExBK=mm@D^MOCrO3KjSTnX{p(0|teVx{Es z{bwgLe?^0HCDc>B#wYXs@R?{5`m-bVya*LiO~ULv{M0T$RFkkToov$(5Y;4Ncc$(| zKva`(U%XtQF(9f*_&;CkcmWXAB&XfZk%T{r-h3GinLc)8QicZSN{Z{#zDNg%YLc?i z>}$!orlQflk}B7{P06Ze_)IiOJ!sRL$ygDRTToEVXymF{JCZZ8=TV8n z%g0p!L^UbdbEU6~_0-bzN8ki>!_+hbqW z15hE=q~gqb%fc9=s3zsczg;9514H@>TA@GFC`U4KhU67Qlde2*NfH*b_UUNvt*7oy z8bO0|rN!K9_w)foHEC(i)_oEX?VehdztZLlKva{~`$qhm1rXJw&76@xO#(zUX)7m< zF9wKe(s8BnvljuPnzY;T$g;wKs3z^FKG=ULAgalfo6skjBCva9qVJF1{Yp}X2InC| z#aD-(TnC71GV<|LZQln(H5v8B>3SysQB6j#8}MUgKva`4i-%p`2Z(Ai_OX6*ngF7j zO#Ei=31;tW7iL>I^`v@o|^A8xS(`&(CXso?6y|F0il{=(SQ4!&jF&EVp(v1wwr*c zrdT!KQ?4Z-swvi2R(`P&5Y-f$UMt4d1w=K)Hs8WcM*vYxaolFvwGM#Tb-7!29XSk$ zYKr~9d$xB5L^Zhzk6ult(QG_(QI{(13MOS}aITz4Q}yWqfT$)X$5qN90Z~m({awDu zbU;*-)9s2bxf2l8%7g z$81bi!<3_%ynbS1vZO%2whg%?+Ejn0UUDWHoGUN#)oQ;95Y^;mueyah0-~C{x>6_m zQ9x9a*UiiIeHIYaXP4|Mp!Px7S+>jZNPA`oAwg9i>}%?9l&CL*H^CO~lvxPmcgXp2d(c z;+H-9G6Ev@LW+o2w+&eZh}a7WA|BfQT?s(MUPuq|-NNF701lQ_)Ij3&dYXoO6sE* zETU=a_iqJ4H3_kO&6$dTSSeX;DiV7E(bF zP4e{0{>c(9U7FM~8gOc8j>H8G&O;W6+o2R7s!7TwNB=1Yh-#85$Du#x0HT_t9@ek& zU4W=2X|DF}IvEhvByEEMk39^CYLap8VGk?@L^Vm*FS;X{AcnLLM3eOAVw+45L(&JL zNljR>ELqN`%d}cWpPXG)JvkE%&O^S3V^@Z)2Sha~`Tngxx&xw`l)CHM-zNc4O-hfR z)a?a8RFg6X$5;6i5Y?pYyCcHgZ_ui9W38wr72h-R(F=g6Cgr9)7AF(Lkl=x6QvUek zcP10Wkn4eH(i!(Zm<(og$y)2^k3}WMBxhp7mKL*?=1U8RY6_ij)$`u~qMEdNZuz@U z0HT_--o5Y4P(W0ZHuLxH&kKlZ3Y+C`{U`~HLOKU>!8(3=$G1sf6p}emJ9t00HT_VS+@Fg9Y9o*u_ZTsmjp&3aRa$v6W7^1BngZ{+6Ic; zruH8vL71j1=i2y{?|hmBVc|2;WKV6Y@O#nbLVtxMH%4&e9rr7koJajK0swuYlZ~xf<5Y-gNEwBD_1rWO~_wRG( zn*gGkV*k<8-=76UHMy)U+x(gkTE;~kGBlY0glcjkWBQue08vd&PRxAgGC)+5Q>U`U zMF3GvPIqoT^#edulQZes49^9KYI1f$(;wyoqMBTMy8Zm!aXW8|~ zQ-G)@->p^MBrK-cXF|4xmW68VNy^aRTzio}Z_}!Ps3tG_=CGFmQB7W*&wWRIKva|0 ztvZbO0T9*X%>$jMy#|PC^7e!7=l=mjHTn2bx9{HrL^XNWcByzrSK`C5*@ zlF4C6s6df=F!%4tlrp4Kph$n)p?or73&|8HGCy_hkWBMJDg}z{%iSMJCWj%B0!8s7 z-Et;V%8*8ZBKOtWyGO$n6MG?T!uLNsnFbK)WJa`_`AL$ZTy>A(`fdqzM$+ zDoyWA#*vURfugu<{Rfk0$I#&12sii7r;>MB_)JUry}Ry7#_y0hf%GPMX~xoFi6(kg zG8$OoNKqiPH%W1;|A9Y2$2XL>+XlXky-@zZcXRFn3fFZeeDAgalAL|@O;&ZP+HNc$s>$k;#gCQ&L^WA6snm{nfT$*G zGZgKU2M}v~+_A`#-hikk>kdzzl^qb(Wc|o-t=71l%YRs z37HE%tW)?BAgU>r_g8QK5+JH6R;_D2vl|fA6zdCWdsjeIQ*8R^Bc}mTO|dOd?yH)B zsHQk>QM&JHKPqMDrUG-u;ifT$*C($Bs65Fo0_+3_=GjRQnAx%kY~ zarXeCnw;xVpii={6_OH=A>8@Q_b<5%N>NR|VVAZLhmoJPgnR@o_Ln=8l%c_S$VO0L zPquD=s3tG_YIXB+tOxrhE4GW+DyK-*J=D2q4f7RLlz5qjdfqs4JC;v`{ z?*LZRaW{c5BsSN*t0f>}FXSAkRjbZ#fQY@2aiDPZ=Fb5l_Ckt*mHI}q6cRrF_r`|= z1Dn*($x=x8Op&c#deTOykk|{E28xxNR~Hbm7xD}&`(WV~K%|p}ECZ|i_NoJjbTTWR z>%HW1Lg)pH=+p9X&3;&C=l9 z3iE66o5{jj_)Ihj`$Cbq1EE5yNyJYUomdDE)g;{falMBEqMC%?J$X@HKva|bW5Cg> z387adqrq)oe;)|dB*l$pElvQUnxt&ppv)_PScU4|CRx@2qMD@sDBa{HfM{=$=HHC- z&jO;Fq-~h>c4a_RlZ@}nvSlA2s!6)OOSV=6L^Vl&{>wvq0Z~nA*03FW5<;7Yvy|aQB6u^=~225Aa*W2D&K-)fT$*A4i%iT2oTkzY_~j{ zvjC!+RGc;Ytx15WCgmnBIGhy_)ujBXFSd;bL^bL7)YI=Jgtkvd`^`-l&1i5QvIESP zzqExyRFjsf%)}jls3xr{mYrJx5Y?php4$EZAl3{suU@8i08vfa>YBBG21GUKxI&F) zuLGi*wA=P~=br#kP1;XCQRyu}RFir7bi02NLK|VCwIzRE4utk5Bfczgqa+}z$;d|w z&l?GdYBK7rVw2JVqMD5Up~79C0iv3WSyIu(fLIId6V;jz0Yo*K`1Q(l9soo&8TaF* zih}@AO~!wBzEgfcRFmC(w8(1-p=YwuKk26=!9dvZpNySWjLonh2_eFU{}joz*={Al zK-lh|BK1z@%}EFmHv6YYA8+$U5)4>NHCZ#IdCMe(2pj#=QudDerITPFZ1Yc1+`eh% zB!mc?{8Qu(?#hz{17VAQiu~w5izOjM*x;W81o8YsIethq(dy#pa^Cq#Xc;#5C)LHG zccIBiKpM98r%2|_y(I}P!^Zv;skS+8Cjn{L)}JDMu5*hdv<#d2Q)K#ecs&V7!1f*drf1=4f`Bv6sje@Cl zQTtwtFC}N9!FkxY|Bn72Gy+65IXPwMw%vfJCZ~=MJp2+Ms>$hh&b)IP5Y^;N+LOkJ zbTikmV}G*=A12e@@R?NSMWKmbCX;3BsU|P`jXE?oX`iQ>ygE00>)n8;Ca>FGer_Hh zs>z$Y=em>uL^XNa`)bi-+G{=4!~L1nmnB)nf8Va`ssD~9M6?WCd-xv z=W&(Xo04}{{9k)Lk>Tmv0V4LoCj6@|k4~0C!sq{9QP_fiQ{A->&*#;2lWU;6_zVEVx&Wj%(5jNQ;E~L4>VQn(Pge~?dvW+&6>H!r}O)}24X~ro) zRFial-dUcEfnjrfTA{zten2vEhOPCaQ~1rXJw zWRK5^yLyQDtaJ{|45GG;Rns!5AEXJ^(1 zL^WwCuU$R_h-%WR(yjHK08vd^?;Zcj?|@h{%-l(ho&iKPX=_Hxe*sZVIxasVU28y8 zlXhDlFLVPC)ujE5$4a#TL^YW@SDu-l5ZVY6tvmhsFd$Tu5sOca&IgEUGV-xQ8>RuG znv8n$=oJHqYBKu!-VH|sqMD5Ps^80L0I?R@$A_i=8W7cF;x`7U#{p4I#{CqR{0EU08vd=rr9#GJ|L>esxBL+Uj{@q zS$$%zcnuKMWXcC4 zx<9)t4csgmoQI9>`!9^JfwVWpw&02eFksN&JdWe$D{H_*60_@a*V0yf3a*%Hiv7?# zy3YbcHMwHXmdlwCTE<1~%J2Ff2<=Ty+*#@PtAJ>4a&mIj6@LSwy~(K)RR+Ebi1sF@ zyOhem7ZB}D&ZH^+cr`#&le6QB^tb|u_9hqqU9?&QKva`+J;zl~*0sXM_GAcmKGWp- z$!b{G)}9REzWOuS`X=>}mGiKtee*geJ_JHFc~P+L?)v~yO!Rf)t7rf zRFl_j^b_L&QBB_DQw6dDqME$zRl4d@Kva{DuaxUv7!cLuUDFR5B?_kn3!)IE`zCL|n@)(5e z>M4q!nAR(K48msh6uIy6PfC_t!&dba`5zyc_evs({^s0?-RZ_bE9eD_X!gE`lUo=z zsVCJ5v43x_M4>Hw^mBJOW2^EBBge8PP$9jo}MB-Z_$CIYlh9~DKdMO{+e{n zur)nJw!*63$zu>Url%;@%NHe&LD-g_A~*fcxyh1i*p!|kzb(zgWT7@}Nl$u{d^*cU zxFhtcWYqgym(Gb=`kV8x8~x?W&m9ItHA&g_cJ4ZWScNL@)uLMfQB6|!oA_lbKva`7 z=O%o39T3$dZL{H9p9VxV$+*y{LpuRcP0|f}^!9Urs3z&JKC$IDKvYv$W4;Q#lQvyN zV^2P-lQQ%-=V9OZf6lx!6%f^=)6ws9e!Y;C zp)2QMhxt$E9%u}RY6`#npRp)eE)3htlP;~*i&J|ii;Q7&d5ZL}>vNri3RyGE%ry^p z2Shb#`{c&X>j655+bZ@4L^YYX`u0va08vfGt!~-87a*$1_+_o?P$iqJYqP@xLLucP<28e31X4I8u;T27Tb7yVV zTX`x%Dc1P7+qGg_0Z~oXoqD!(1wd4j^%I}TyaN!`6h{rp!dseF7e_bxeHbNWXmIXg zF}U}ap97+rVtMa?L-_$wO|fb`Z0jIER8y?~J@s4(KvYv~K04iH6(HK1Vte1wq8R~E zO>x}tQ0@_c*mb${Z)Hgbh-!*`-`C5G1VlBtPyZ;gFCnyyi@Ft@avKN@2%N~8Z$WcF zRFjjVvftVUh-z}`P@c_|08vd&cklSdF+fz4Gg&&dYzK&Ha&~x+(mw*Cnp}LeYvwlr zQBBTuUYF+=K<@}n$U zh6AFSy!tom?F@jZCa)VeYcUfL)#S~+ZC@9Fs3va*H_0*>5Y^=48x6`707Nx;*Xoa= z9|5A8ynkSK?)w3e;M~uhe-ID>ycZ_vn`kVR!e%8Q)+1zpBFL|Guxn-tOzt zy;Ktru@`oBe=|$L(}0M*u&?`LSxa^WMC^rK-M?z~&2B)%Uf9$9`?iB$0z~YE9o^q( zvgZ;YVlV9H{&<6vjQ|mQVK?`0|G07<5V05ba{p=fhPMGxO=5MvFOt}oe$nDZG^gN; zy^}IDIQK&A%hxjxAgW2oO4&=z14K0mCG!+21Bhx8dTz%-V*yc3!tCwz_1%D|CSl9> zIJE>2)g)rmb?<|Ks3zfNtXuyjAgW3Dt>0gM1Q69E>tw%{%me6`PF_a6RDtqI85*2> zDX!=zwgaM?q-;~JdmBJhlT`UiSG@{|YLdEt-QCGp5jJWkJvZ#vQRhT5!h~(wDYDJ0 zFYf{sQcW^0SZlyRKva`-LywP2#=x*aJFU=P`)5Wna)#~M|2v2As$`x`$(+d3rmJXN z@yF9AXQIKmSK{wd`9}ewnw0EPv|1@ZRFhI^iu70kh@DGMu5>&ZE5erSr0L3>D8DNi zVZw&&6xnuF23n|)Y6?5FR9*2YAgW2ZF&CyMV_?{domS`%TpXEZ*l^nnv5*c{baIS=;>C9-l&@0_9Tmp;WN37MsFQQsx4M0?r)rV#NdVr{=@Lh{}d=nt5$=YmXdvpZE z8XtEpQ|%}qs>!;OeX4f?L^WAI>BIVm08veGiLTkQCWKZOM>i{;xDyD~6pKL>c7Fnh zYKmpf$^#w)L^Z|g?P|-v14K2&`g(~c3Id{|iEMk<)dfT~IXQCc$RmKL zCZ`VXnBD;p)#P-KMdCL=RFgBAm!^FV5Y^=Dh*i~Z0-~B+d}Mk3mVl@x=Q`cl<2oRA z$v%CWYOMfKO@8zDg|j4t1m|I2^hO)j4g^9qd68@N#X^9nCNGC$oOXQ%lX5Y^;O&Uw)tfT$*K2TgDAEg-7N$2X_dcoY!T|ydFiADg1NR|DfpG+$EDGV-ZFF zud;0ah0*L4QQD-=urqoJ7W-GG8G~4)U|Dfljij-mZkd9WTG$}D%z#x2*7LSsOqL4+ z)+yNR*|s)WG7Q+HU|V6&D2+|CDLB^qXG}#Lr(id|;qs&-gvwnC_S+f{NV-tKo+}kU zd6BZaEel)muy*= zEK~=aqEwh4HeN~=s{>9^D(vfPrgz4sr6?8g6YEDFMx3HlxbNN@m<&Oo@)V`Q|JZXy zGI#}?qEx<|_oY{%?v$&R(b!=xCsPIy$rPnh{4=0o5-JCrqEyOvdaGpm8gOdmDs8{? z$+Rut6s1y6IW#&62?9=0D$VhupC_-vfK!x8+y3;GBs>i`MX8MMIJF@O%K}bOD&6Q; z_bfsyDN3b3_{zy*h*OlRM+&|FJ@bjsB0OC{^O}h*$7_s1&72wjG}ip8%Dj zRH?j^B#t@ZcR6g(QM%u?nR~HiDN2<&cWX>F#3@RZZFXh-6~rk@Rb1%ohpV>AWl)Lzv*@o>rQF6jyCVOlo8766s20M*t%{6;uNJ?7T!Cu1mYB> zTGiS&eHG#qrCM)VF7hKzZJJrRD(xV|DN401wzPVLI7O+Bt1qhmIpP$h+O593M~pZ{ zsrJhxL40e*0V%pQj{9? z;)YSTuyl%2qko;2<}Ji2N{yL0J^Bf8YRm1D^BS~AoTAjkFU+p-7vdD9#_hhZ>N|*2 zlo~((-tGqwrzo|rbsmh*Ps*8XbmQ=t{!rGYD79knk(oIVrzo}Z-ZPiyAWl(gRpXQE z%OOrtYW2TEUzv~S zsZtLiPEi^^*x?%-u~MpzqoY?pUk_#D6s55ketUFv#3@Q+ndS0^lZaE4#;W_dE8P*N zD2?@@Q4Rk@oT4-~qlUly0^$^UFzQ50$)#I7MmfCq7)T zA>tIJZt*XlR|c9=)9M$>c+lW(?dUdG7p7V%PlzQDg_e(Vp zrzrI%OOE$9AWl*0?eM%Mdmv6x>f@t@3Z6ooqSU+2pQi7MI7O-V>1L|ai2s+E>pv4K zao6exMFItz1&C-ywn7j$3a&HE8s>C+-9pKfiB*Vz_jPaaDPnFr#DbMNm$@G?w=E(i zJ7oC?F}E!uJvaB{WT{t#1J)73t8j0Q`N?u`z}&WoZ27#m(_`h_wuoX=Xv-+X+_s3^ zj8C^_K+J86$Zwr_XgFftfrZ$bxfDd|B1MXb7T0gt1h*Ojb|I@eIwjoYYDyyXL33H>A>7ZNSb2(48Q)oA zULM3LN~Ie!dLfMALZ&E{{=g@_;0+*Tic(cJ!;3H*O__oV74@o_=@Oh-rHl(oT&dw3 zAx=@MWSiQ}b|X$vs#LyuwO>MeHp8EqSI$ICILx zN@%e>`-16+QDQQL4?V9xcB^oT60QqMb_L zi8w{6j%#$x{0ZU|rP{6eIZqnIDN40p`ctvdh*OlB9ZgRy1)9<@6Yb18za!F=UTVZ= zStcJvoT4=BoRw}%J;W(Wje05L?M;YNlp6h8gBC3irzkb%^Cqv~K%Clg`&8R3&mvAy zYU1acmH8EMic;ft{a*AL#3@RRU+`=0-w~%MwN=}E4Qo9j<;*s^S>)zID4Q~|5>^Z< z`b$y7DN3!(S!&`$#3@Rxdb{}C%!pHzT7A8$pM^L@sWn5YWO@p5ic)KHm9O0waq5M| zjVd+Ei8w{6br-I8{s3``QtSKNtdt9Jiqg1D^Fgq>CQ_=7qa&q%J_lt}CRQRABg)-q zia14SEHmqQ`w^!oja3gdsS4s0rLjI->#pO7QmjAx=^1=Ik7k1!ziH7d4vt>yt=R9(*Tq%{cG{;uNJ$ z4w<{?Uc@O%ow`1!cOS$lN}YatORi#wQy6|94#}N?m+&eWgiQ zIz_2-t>3GS&#p_=;_vU-4EtgDBag6R`JeAlgx~kuOstzvtb~uct~mHUoY|)+^&;EK z-%cP-QR?N$WxZZOoTAjL!wVO#L!6@2>mJ)5X@WRKsW+Lo-FFsoic)V!?5PNIaN$#w z`uNEHPB5PoK1Hc_of3W|g%p`<7%B`pmSN|(etU-z8f^>U8a=bvY<`pAl2r zA-nI)QtQl>PH*B3gr6frL?>J?p- z3Ncd}%VH(_j6}?o#%f8G>=rRo8tYY63&G~GV1bp0O`S^Z^I&PFG`1zmm-`qoQyRM! zx9a9a%#_A{+~TS1wbf z`n?unrc|hjP0L?G%#;d!x?#sgh}kF=repI*b|Gd;g-zS;{udE5rNWK+_1-@aGo`{G z{{5*J5HqDRXR&Qz~Vi+{beu=I2&@ zdvy8+F;gn_g{~DILClm&)2h>b;}A2Y(&p{_a8|@jsdOK0dU`Tqrd0ZWe$JW!F;l7r zRvT~xC~LQheyLHY6;h^D#oyMEs>CDp%RPsfDOK{-+UNVOT(g1BPYOw)esEOBklqsB0Pf!vaberqt-6S(d=^OBklqm>g-2 z!vabe-Z9%S^-frR3B!~ccmBIguz(6_j#t>SJR;;d>L9q--| zmidAOR>H=^x|N3@Aso;%TalsT{jkgzAdb)tn;;k!jwyBaPUtt5W=frF_w;cHMgdi|y zjnlv?WKw?jPJP%i2?ul_`8eyfcCg_hJX7k$=<`oygm!zT)XPIx?wyF3DfR08;S-)l z%#?baVbq*ih?!DvhE2QyTPA`9R>Irk6MljXmvF#3=H7nvci1u!EU*&Zr+nlDY`BDj z4B_(o|DIUM{^HJ%+DJK$|G2hdGh%8xWLiO zOl^nE%%4A+t|el6U5D(N3`=hg2DjrJ2&$)b9V#lZctp*p}?|__v6e z(%7wh+CG7pDUJOGm;QUiOsRNJ=N$u-^;$&ZH=fyplqnVQ+0FZ^B4$d3=)C^pD~NgJ zGWDwO8z5#%g_<E&Yn#-m_-K^az`guf^H<2=>DqcRa+fRs@QY9Xq z^mu#3OsSHujeqDkVy0B7O}B?uLd=_{zq`J28)BwZna9rk)dDe7s_e^G58Oe_lq&bb z^FOvl%#ryb>d?xAu`pArMd^NdV#G|T zmQ{N{F%vOUs@2wGE7BllO0}MGc*tYs!_H>94wJNA(_InAt*v@~L- z)QvlMc_2_$)TyV&805RSxBuz%#=DYq-c}wh?!C+ua|u76k?{-sb*DHy?~f0 zbvj$sbAKRaN}U;4>9Zz?nNnwOmmhW!F;nVX+gl$sLd>=J+&8|vgqSJyf1Oxc04NQt zLKf`z%IfS$nNlC$TPkgT#7wCdW8|x45i_M;9#n75LClnT)wBBf42YRhukWd~dNN|B z)SFN0eex(`rqtULbw2$HF;nW@JIB9&1TjRgsj=~7L47WR2vS+lo2v#?^*Ii6~xqb$evwc#dX*?2nVdRRLeHPwnIp#&9Iui z|4j(qLrQIi_5MArA@mPvwHY=Qx0QuOXGpEhu$9}h^u(4^+aZ(ov{u<*!5a=(X}`U7 zL0IsH)Y?>O{AG{5aA7JYC5{%&nKcp4%w7dlas1=laTyWw(&F_Q8(`xgq}rwpi)Hbt z*I?Trq}yg#eZ8&?1n(i`Hp6<&>bem6hqT)ao4Ok^&%&lLrLlc{OPQw;Go`WnwtXR3 z@P-uJtaAU$JK15u8`5x7rNVR`1QV%Zy%y1wemCmFnc1s=Ar;%n-al;+$zMpX?oe>Sv3(erOFgsc?}kwA^kU%qHL|D8(`5HQh+n;)~DYA z3*L|hoMFEp#W7g$hE(8mlyi+^Ty^x%v)~!^o_9S45`AY6m4Ieb{!U-Aze7b z?m*t7v$1lfRQp+ZwwFQ7l$tpAi?A@{U#5xvIra1#P?iN&%ETX?&H4*srqqZBkJft! zF;i+}qa$q(A!bUAy4nAq@`#yIqX+lev>h>1YRr8D$3p-gQeskP#x@x;8v^){TAX2b zIbPBND`!fL|G@OWgP19`&*ty>2T)eEjSio`_aUTg5oP01SDyL^F;i+q<~!{SVy4u} zZrAF6ikK<2>e$4eQX^(ctsXw%#wf&0sWq8~&w&75u_so_+U}z!KmZ>Os8Z|BJl?M; zR?d`KKcVT-W7#7wCd!@HMXikK<&@>sWyB@i>E zUUe&Q;$y^2sn?lv{hkLgQ|isA?EOAN%#?b2ILBg(m?`zH9};~*T+cE|F`2E$?7A*4!YSbdcyJp}I|T{^>hZE6dle@L0muz53M)5*yg3u)6C zwohcKnE^3V8oT9RRT+nv4&abA{pWAGWkt-CijwV?W4~ChMKrnMC(WTO3#_z=|E%!o zZN%&g5u$zN^A!>E%4Pa$tG6O%N`)F%>{Xb{LTYs?Md-guyah91NUzSYd8df3i8mDrNqf=SCpr=T^N$6PU|F8g|+(sjuqSU?vQy*cmo$%cRMR zl{2N%<}R%UA!bTLzN9|IU=9yy*;(cO_J_IQ4iHkaQ>AKM=KOg5uy(6xLw@@goS6ky zAxrzp-09y%%#G*yo#7NO)u^IV?)GDsWOFn4Bv~G zDOI*+*QwPIGo{L{Tf1@}Vy0C2g+C0fjF>6a3p16#8H+VcN57>__a&6&cP|}(p2~Q{ zOsN)6XKMB#Vy0Bf=QG!O5HVA#)z0QWEJnxt`2Zl^S%#>>X#qJfE5HqEwXsY*do@Qq@(T$?7wT7}RunO7VKQ7ti24bevhzAO* za)_ByBO4b=vlB5>YSgVt!@3}5N{t>={<9;9nNnl!uX3&qV%{tc%{7^X_3N%U%TxS>^Aa`+R4_OsNxtX5`+Cm??Gg*3@FP5i_Mu zHD34KKZu!9ryp4TQ5(cesWTsM*#A3XrqtORTh2a%m??FxW&3lmPa9Is)7z8t`QBOe zES6?U{nGCn;R=ifRw4U*j|DGGgR<;Zz>tAH+=!V{ zue$F4_*=wGsn=QdeE%e3rqr8}+s=$e%#?b2Wc$7ph?!FFI<;Co3^7yceVW$i(<7#V zl^H#K$?yakvKTPYsj35E)yd}3kga~cs`@pQWPz25;@z_QV~DBkkhQ*Csp3-*Q`;eP zy)B#X9>mmk$XGwC&H&h!3kPHng{<`l>MezBxp2U+d9K!<{jhRsJ7ltts_)K*nA#56 z>_7c`O>e~1cF1V|`^if=5z_%|;z?DO!8)J4hHm0$Ny#1;pe%c0W#V{M(Q0oYW=dmG zr%>Jlh?&w@mMHw>^N5+!SS={O>@;GgG}h}Y4eWuKDUD6Ns=L-BW=dmQtjft6h?&yZ zExUAi6=J3|_M0!Rt%;Z^6=}+!gy@F7x^F}@F;$5e%Cf-9i1>8c{4)_Vr9yPf{8B!| zOsSA*Grd0;F;gnkgm$NjB4$d3KGS^93dBsQFkKoBPJ@^!6*g7V<#^$;{|ab?8@~JV z)L5D+75>C_03(NJ(OjEm67p<{3VYgW=f@Km9yuYh?!C; z^JaVTC&WytR3CKS+X69DD)r?~r|%$UN~LMt_1j8_nNn%<^cb`aF&m}Q4gO(FdBjYq z^f%Ye+>V$jRc8xTI{}onTSY(B9=HT4-BB?r{=WXQLWr4CB_6GLaspzeRLNIr?8<_e zDOKthU2Q&M-ZcHK=;07ErOG@~`pFLvGo{MDS|;xUh?!F5)_j<sPTYPuJTRzK578)uvRzd_N*)O0}(4r1)!ynNscckB!zKW=gf6Jtf7A8QZyNUf3VwRom!Lmp!n|H!QF+HXhya6a)$3fS%cktlgHwGT*S@SQ#td?LHWS zgmA!0tK+%y!!qBn-&h%|hZQIRK|(lSrA>w$@53_RuqRf=+V``+1VKVLV5QyP^IwK# zzG1(yGS-is)d+%wa6s3k?&z45Kf_pHUx$mDo?QpQs9}MXadGxjS0I!P2ee^M3^+1# z4A#vqd?)XOb0cO-ood)C)l|gn!go4Hf1waFrOpf;(innK!+v9BoV_;i6$oX+0c*Ky zZeE07)Ud$HIA0*{388E_pi2FSPvY01Wi+rdK6?Mk)f-6Jh414G=hwGJ%#?aD?Aqv^ zh?!C^kKg&c9Ac)_t9K{JBZ!$&ud_}}-32jI>doj;FK5HV9K^3~Zr0x0XXh$gM7m<}mZD&kY?JAZ(;<1q*%tg$UD*5t=uSy|iN|pNgYLRh>dDHarb2+mjW=fTL`1-ToAZALHeeL$! zk054Bm0SBvtFI8V3t#!Ko_*s{#7wFFux!-`C~KIGw)Wn-8Yxq%;~D)9*FwycYEfqB z*JlwkrCL@Q@L@N^OsQ5o&g9sIm?_nI>gmESBIYeOr4PSz2{BWuZPjDXHA2jkYPaw8 z*DoSwO0}O;w`CK=OsRQDwqh;F1LukTRt<9f)XMi(9J-E}DYfdv zp3bnCwk)u+R)4a;B7`sDfX=1X+_U|`x3O}j)Y_ig9y^Gb_uKu|+QFuXWr3Boeq5^z zuniRs=v?YOIR4Go{X6ui{H0W=frFap_Umr?o7wvd-tf_yFwk zh6Ae94;q{TFE1KcSs(ROlYW6SvnN*8$M?v2Zy;t$z4)Zm%|nQpQZG-G{rMflOsQAz z)v2=`F;nXGz4cm@N6eIZGp3fihL|b!_F(l4Z4oo2-nIYx!8?fANb^4Z$;VnCrh%2~ zkas)`1xIYVsPmTT?URCRgt{o2KD!|dgK$7DJQu~==GA~<5Dpj?m8Yk`^Ria~T_np+ zi(m+b16Ep1TR#Jazhi-wi}dzYqhR=l16JBp+qecUJ;z=JbdfE!`4U`u;eeHPvpf6= zk0~5`70^X~|J%FaAxAi%O5@ZUMji*HOiCPmzpy$C1IM0Nxj6o5aRV3z;ea+Q7O$*~ zVF)_*Dxizyqf1l55DW*bwEA}6NErT(1y(NBzwDh3!#^Cb(&p8zm*LWL>{UP)+edb+ zg-b6Su+r}9mb>6Fg=4?5a_wRguFJd|;IAJqf z8$A{=Q!3op=LX+{n9d1K_`l07pMsbvm1oyo#_xl5OGZQci>^?XJ+X2!zSb+%8N^Jf z6b%QyQWG&#DrJr#jaMP&=T-wwU1^M%DV6$8XgOl0RGOwo$Gn7?DU~++k(s*@Go{k? ze)HSs5i_OIUwLED0mMwH>NK?hOu~+}TScqZP5&OsvR46}ikGY&`2=F7REZKB*20## zV?Q%=O4iwO88+O*0loN?T0f`3bgY~=O)r>RT_9#kl_@qOkqvRzw>SGo{)-w=6|{#7wDnzo*N$ z88K6;{ii95*G9~gnn`n0Rszb-Y@!S2);*1s-SJJ__v)2dh?!C&^4^|#KVqiT$X3_K z^hL~+8g+S6s=|nwQlmc@FBTwXN{z`gqH#vVykoZY$X6yJW=f5_`NWG^5HqF54}P-e z1jJ0KJvArZ29#B8qtmCa_C(5*+IZrb^(PTCrBRh8Q8h?nGYw-_$`O1TcnNpwS{=B$$q=A+5QJ3a> z&OuogSUDf3YImwBVy4uK2~C&pL(G(Vd8Xmu%7~d#uR5m9|0iOm)a$gVO1zJlDfMPz zruWw(W=g$1o%y9#5HqFTb^H3|9}qL8-e+Fa=vBltu=45h%!UYt+V)YK?W5wPAk*oi zXh!o)Ly*!r!AJ4dcBOM6rnY?~sx)mm6*0B#BUz^5>k2Wo?ISfc?Upf!scj$W9jWf5 zL`-e_$W+ZVe>q}m+efx^=E=noQ`r`qBQl>PHH+9Zf2Qe=#UheV65yVVsEFbIIstaPKG*-*=-`D2+O||N7uJxC4X(603T7Y5$tRP&ezAjD}RIT@5Ky zD&y_NzN}H|FFMlCsN~P;H`B#|3J$qv1rN27vAlw1M0adD6E~`-#>SkA$idLuDJP0XM zs^Z0|Z|6nKlqyj&3iU5kt(5c8(#g>6fvMa+~cQ?yyDm57;A zW$QP1qZneQRJkpGJi8PzyYQ7?y6f!{h?!EIZ$-luKv~0dw5!0KO-Pwi9ec#Yf@Ks-4QdT)^@Iz;S^%tZ+G;mgS=hTDlx2aHckx4u7iL4ulsfU@%7e2JGo?;mTKa1l#7wDEE%#NLf|x0F zI^W(d_aJ6Uo$0gn(XSCRrOsa5@xWta@P|0mMwH*J)N&d>%1V z>dmB$4_-jblzMw=^J8xzW=g&5+QD5$%#?bcT1tAvokYDM)=3MHlJv#J22rJ1Mvjg*y*H`e-e2~d)fIAph~ zUhjOeHxxe8v&tfMRwp|{AR(fkaH@#qz20=aM}j5mR}w`m}J9_n=xTFV?>odJVqLn97UI zi$&AC1|_My*!q&{M}Smb>}GvZ>}7ydUhEHy$-M?3l_$PES}?hk1ns4WK5BYlT2hXU zS`q)#@COZ$$`hhd^ASSY=1l_&h=Z!4zXrU|lUsnGZB@~pGOs0fJ)x*P70)Z(B)MY>Mdc|G z{*t?G_(Y~PB`fGO$(>p#Do?5XHHIa3d7)T?^z53SB`XA>s61t4{d39cLMSRv*^0GS zB`Xu5s66GipV*%>@uR3b<)%n`v^hHZvB+C+uhCSVju#bt6(&nf zm@V{c1`7JwWji?@MUT$PwTI%eS(c;tu>EVT%D{Vgr@Sety|%I zvIY^F%G2(bn`de!el(S*{kQ+@TLqBHGv5z93a6xJG||b7c4kt}P3zag%$v*JZo}woVE=hm1k|cE+Z-dWc_q^ zf&_q6p7jHMSy3J!mFHfMH^Pvm)kUkQ-Pb7E}&ZfOBhc~1V_ ztI7y~RGw2Ejvj~rQh83NJM!1(0I57@rkq-t7a*1A?8&o32LYt=oO}1>;duZu2|nv< zQ$Ges<@w^~I$Y8ulnLpe+6{lAc~Z_(c|OiHD$i|zRGt^TCswWqkjnG&%7m_40aAHh zHNW$B2Y^(b*9ETqaRMNf=S{!!L+S#g^1MBNWyL0dRGxPYUzqkfKq}As950Ue86b&F zWYun2-u@TKyRT;%RXg84xz*BTO2`4Va`=rLNvVifl~KH5)K9$t600HqQ;mrelHHT= z`R?f@WPd6=VNSAl5 zl@!nSpPlTJNE&*`SU%lre6mXtJ`;Jd`s`@d7ol1zFV;JcJiQAbl^2_rPPJ_Wkjjg# zJ6rD(Kq@bGU%XtQ2|y|@_J6$A=^{WXPfWX$Jz1(rIz&YD@$|7HlX5ilkRrY@?Tb_Z zsXQSX&%U1QE=wBGNFg7Xw=vmU4xfoUp$2VyGnsorIwuPCt<9~ISt}%SqR_mxzU*hv zASzGT`&VVL0I59TK72o0GRKBAPPAHo>D_|K>>QFfk;_(Yzg8$&%1YW18I4>yYe#ZM zc1_Fp$kK5Y08)8UbX~Y1*+-V_JCQQW;%n`oBs-@Xv9(Py_k_evq$5%v-BCB0wL;n^ z3QdoFnGZs>R9;A_vbPM(TawC?Zv0z?k~uacY@*frvyHPSvvWw-MC7T;6Bi{*eb!bL z?Y;fP$ zRGu`AeE=wj>nc11W4s6KlS1Mivd!3y6l8LNhrj=OC9}i?A}+B zay0Y^IhnpW{M1^2RGt=3o^JO(Kq^nmH_p`m6CjnR)!G3+RR&1qY5mpE8~XrKdD=YD zZ%$KyRGzkP_MUJKAeE=x&rkJh2$0Itez{+K6(E&orVrndEn)PGCi-(j-A{ngyRC^Q zZ)uYbAeCoC`gLW#14!i=*?x8A#{p7#M*THo+(3X-p3&o`&ME+q$}=X#oNF@xvaZ;7 z=5CMxsXXIO+`nTgKq}ApPaZg?08)8&+o)sx5=JYv(YbA{e*;G4*|`7qH=hGY?zv{AeCp;)fHcC07&Io-D}y{dH|_BYjQ8xcoZO&XKmXh*E<4a{dBi8 z96bV%%Cml8*6m#YQhBbz>o-; zcw@4EC_RQty6xkY0EyL*66ulc-xmW&tcHY0-!CXS2q3W<(jon{Bv)R5#A--}^va4C zJ_ksws>nXN?7aveu^JK~ElT%RvUCie?_R@%G)OaVneM=bldXd6^t=n*m;9ltts9Y88iiuZ3H=njy|lQP5gzfJ+9@}wF)soM(x zsXVC0%8SZqvkMm8a;c-G?FDFIS>VbHF8emOuYPs!(&zFQn1m8aD1 zeP@RNr1F%WzjuEwfK*;c__Fn*WEmF{7!d`^mfP`NvWyGqizsxzw)`wvI)>y$6#AKM zh9yhKkh+MBJl%26eaS6|K7>&^T0cA2&ZMn0^axpt7R)PJ4j`4MMX~Ac-2h1CX<2XD z3#|cCd0MSnzi$gbDo^VrtIpH`Nabl$V&nJ8GA^VlA_}yvv-#s>85a^2QRp^y_#|06 zD*DtzX}{v_Pm`r%_)O%P)7vWip0tOa(L`tZS9uT^yWRMqQh8QgIKQGAKq}AbzE_6q1xV#tllRUa4FOVl*0#F#_f>$bpYEUME;I#5N^~ zO#J{LmFIM-w!;A`Ivj8#)KGUuPPXna#e7Dy1 zlBGV)O$(E8In3~J@S%g2>1dmuD;tv1%seov;{@mp^li4{W0wVI{2UGjyPHLqckAkl zo{UfEW9I=>N`55P2%^y8Dw2F8Yd&R?#1w zDtwxpk%k^2+s_xBI~jmfUI>YMJp2(rDo@F0yWaN@Kq^nEUHSVj14!j5{blYYB>_@- zVbz!I_$Yu>p0dy6+?g66m8aZqUu+rYePwU00`n3hfT5C#X`h7b< zDo@+mnNO4lNabm_;p-dQ08)9{uUzy~C4f|(xv-%l+-c|;O?0+Wp@)-l^o6UC!DoE= z?27T}&+0zv=ox@io;CT(ep3@5m1k|sQhiqeWc_qE z`i`vukjk_E;}5=A50J`p54Nult0P*@MV;1Vo|}}TPd!3bo;0h=lmbZQIWcKNn_d8^ zJSR_Wsha~JmFHCFIUBwKNaZ=5dhWGH08)9*jGr-U96&11*|Sr}Wd%s(IoBg!pJe|q zq~Req4CgaGv?v3Vr1E^DciO?~k-TsfGVrwAU-oQLj)oo~|4zO=S-JtF^1SG??akEy zsXQ+)Zf{)+AeHA;n+4Y|0i^Q0erUAp$! z4>f)2p}oI9W#44yF?_!JsfQ_({rdHPD>ZlDhC@1?etjAwgxo9wT2{y10>)sxxn71= z0EyL*Wv5oHI==xVRzr@Rg4J6*2as3|Np)7Jo5_Y!_py)>-nwf-L~akPLZsR`u;w2Ot@eCVsB>qALlb3gYO~^7niP zjLM7S-z)jY0kV?f#j4Fe0Z8S=(pRaS4j`2mtC=N#NwzW#4LwY(e=T||*~$!`iM-gn zP-yNzsFuo$?Nfy(763@)#cuw%-a`OVd9mL;d0{SqRGv66;8@j!(O!yZaJ$#v2S(+I z_-6B#e*&cPglN*R^eX^awer5EnbrcN@`U;*)#RoC>BtlMpS1JO0i^PTX_Wa+Wq?$k zun%V1vJW7YCtTk}TdM)2@`S(e_2In$sXRGr=#D)JqaBga@XoEC0!HP@_*kbmz5qz& zNzuJ)#`^$Lc~WNTQK}C>c1|@a_m@QgQh8Dz&Oc=#Kq^m~ZaFt+0!Zaan>pLB%2R4jZGR9TYmlB-|DLx2QhCa#nzerhNaZP8p+@u90aAI&ZTqXsCV*6) z^3(sUR2LwXr^}sb|98S@b9A((#Lr8C(UGU)uZ!I*0g%em;_-s>MgpYrw5(fXQYwH{ zo>o6r$nY6JDo^W06`cjhT5F0|Ywbkjm5Ur^^)w0i^P@|NcVfya1^@ zv-?<~*Ahn0XrjMUPf3<3A;}GypG`bA&6mjnC#1NcP^8RqJ6WcL1UD4Qw=-@|7C0fj z4Tb7t+c%PBiZN84(NkKqN)|XFwGAz4(lscREK@>a8wzcQW*L(OPDpD*p*yrIXR=HQ zNo^?fqyH$9EO0_f8?scf^ABhLF_A|rwb7Ma^OFT~NM}PTwQ=tPlar-&NM=K!$dzMD zvOo^0Y$%lNvfoLT)*+D%h3b5lmdOG+q_Lq;_v`q2va}9KY$!AZx@Afh$RULdg|>P3 z(&M0;te@`MkA;&3a!6l8OZuT}awJRZki3S-b5GUHob2#0=`QNf>#NJj8ENPdQq`pE z|3PDbRGt%4hHTpnkjiuNQhRd4t)9yB^8E0v_X4EyylQvlxp@GoJg;+| zf2TA+D$kqV*9s?_0>)E$-d?$rBiT+cp33vCX}L_vra(wZL%ZU$KUX^0P6*j)?hZX{ zl^h$Bd#1hHs*Q?G`liso4W*ZC$VIbs!plDaBUVEunk^F>y$O(54S8s4j*`a#600E# zO`+kb+XEz4L)w{@S4JlrO5yX}Jq$@_Hk|)F*-#3fDKs^%UA+U<600HmOyN80TL2_h zL++Wy<@Q_yNUVm;GpnCF)fONbk~Xe4zVDKRQ3Y}IS+B-}fl+yJytDr+`2ez#;-w*} zW&))0;*jB8!~m(hSbctWX0nwTlFbl5vHtDUm}Dz6q?)17Jbz@}3aFOKi|x}#uM`DH z<;CvHH%>1DNae+T*PDAw0;KZ9`Vrf+B#ib_M1$r<&jX|KM0{&@iURA8~{KLe1;Q$~*z{{W=&lr29ZRU3d*o^o5C zEN~Mbm8blSCrY*iNag7|SD%@mFxng)tv&PkP+(M^j=wrRIyXQnPm3oGub&2x%G2`A zV^=jmDo?8)dN&#kkjm5gn|?2+0LWTvo*bI`TYyxawr>oOPXeU!wA*A$d=HSy)Bd|C z{}TYIJhSR{wg(bM&uF5*R$s0PjLI|dxOHpy0HpGaNU>#P1AtVXk?(Anegz}#{l}^42kd8bjCRbhd7eG4locy!Oz;^-Ck>}JqCG+kD zNJpO2DT+Q>4Iq{0%(y~5t^%YZ&)L5US8E86%5$#gxa!INVMygdZWtkX%j5>h&SOaA zLT(to`ZHPjCbg2M9wCcMi#mUP2#m_}asGO{9|TC{dC|YtfG+`3d0w8YzSIMx^1N!R zijN0K<$0Z3=F1F_%JZgIsj7W*;<6_75!$J*@ZH`qoHx9>eFm zLl3ui*7n=~R_gAy>Z18+XC;G&2D~9b%h&54Oupps`R=bOq-R;P>apZY4xcF$bvJfS z29~2QT!qvuk8iG+46N{(mQ;&o|Ay@$Rzq5rALkuP26OmKOPbfGFGwz+kf4P^TYOrt zhlCn@JDsHWm46Klng+eKJbV&wQNXSB=nz!&^GMGa;77F#A#lI$lIV5AD&{SC2 zJGp>DDi#W@TDmZ~fI=b`3f=VdbCb>RkcNdqzb(bYWE(vsVId<=l*_aMzAxHK5%oU* zPM1VF{jEpHzjEcub4LJDc|x?ilcNqmR;|o+t?(9rRGv`%CVt%-AeATd`3WE107&Ht z(|q{Wasa73VGE2pyb~amC*07-?>q;P$`k%t@h!grr1HY^fNL{y&;_nh3eOJ51fK(S%dV<)sJ=uNaZQ>)P^qW z08)9%zObcIb%0c!a=Y8tUJa1SQ-1zC&1(Uq^7Mgrbz$D8&C${31&6*$%F*9?gzPEH zmi+c8Kq^m*f-8DW21w;;S!>yXWUJTF|8V8B+PMF*IZ%?y(|X082g?Fvtu;lqRqPLt z%G0*`_RiS>QhC~~YSp3_Kq^oBC9Uh^07&JT=bC1Q`yxG~iH=SBszXwaKJ{=W9zOnq z69B0^BQlNHRu>?ZXJq%0M>YYZBhRSA=iY7(kjgW9)YWHU(Lh5FXH4eXIV(a*))m|B zdXcRFsXXIOKU=B-Kq}ApiO*!*0g%eGQNuE@ZlIOg=w`nUqof=SJ)DgP_x}2GfK;9p z_YF9l7a)~qWs{*>2LYt=torBlbHxBsc~*aP=AD%Q>BzI@!DEHf0;KY+ZFD%t2!O1g z?n2#6sQ^-W*7tqA^hkhIp8NDbp?wLXH0w9&=#Hehyw*jQ` zoIIR!b0vUOo>SdBy>T2MmFIM(&aK)5r1G2@-lNn{0I57@k9E!XCO|6Bxh`vS{sNFm z@Tq<%@&-UE&;PzVd*y_Y&?DqIX_R*U3Sd;8j~`4mxhOy?&x?;TZ5a-b%JcG{%y-fN zr1HFK(!AwNfK;B>_qBUH21w<3Gq`D{!2qc|Z*Mj%oev2mY^WFE$kj-Rms+Vd4BvwN%lQ%Qv zKLe0h4Vg@y$Xuc;Kw>rIG5MzX^4$Q5)sV&HhjxQs0!XZe942ox-E$cru^KX%JlXJ6 zV}QhJ$Y1i^fvXn)600G5$)?@w-vUVG#jA3Ek*wzF<1`;fbMn8~J1Iv)4RGv`%>+MeFo{*Y^jP~%~i|YKD%vvEa35BLb^`-AXwN#$4`D+b0 z1dz%TZpg_|$s8L}lF(}X^}lB%vvWvDa(CS3Wr>^{lW5S>j>u?S(I-YdqNJpgpv;3}P)(R;|C^YS>3^Y(J zl_zcbs>?nFNaaa4=Hm2Zjt$92Xtn;}rIE?(98!-Ed1^tYHy0R=w*OZUid!&&Gw!INaZQ{QiIby0aAHN{g$TZet@h&`t#H;Rsl%mDf4v30_Ooz zdCESYrDQXJRGxCXz7kgeQhCaM`Aw<@0I58^zH{q>38T%?(Uy*b`vRl#biB07^7{c& zd0G_eek$3p_4J?XdM#^q+mmedhR@`S)@nn6mnK5BRG!u=bG@GtAZx8DoV~;XfK;Bg zHFD%H43NsxZuP9xO8`=N+Ap3j3jw6^OuZgu$0v-Q(L_hCp)w>x%7Ky4o>-RGx9C z`c&@>kjgWD(uWNW1Elh7v94J%CyZ8VqgxgKOb?98v+>q3qX4NqCy(ry-Vq>`=Twh{ z@oxaBypR-jamwcaQhCmdSXuoRKq}AKqe~mK0!ZaK*ExNU8vvOEpE^ag)&QwIzxjuP znG!}qkB|wZ@%l9bfl+xreqhz50syHzFFxKlV+KGf&&wN|M@fKGo>yHmmG{52ADvq?X|K9> zkEeEN{rbP`Ujv!`KN7va{=VcUog_{F-|J{tDvG|#Q6kybkE7^+rLx>bquI-%lu5NA z4+!V+{uL?4Adfj06_?aVnjESZoJ+Z&Vbb5gCFg41_Djh&ZQzP?y=U8+WT!T8&AF+t zXOzPB8P2WRKVvF#%ekB0XlXJ)Lj8_&zpcrDWON1oZvvaU?8ck2=l=_;=XT=g>!~Tr zA?9}Dc+HG>C-VO&FY3;1kX%i16mz?=e0)yzVnk z;M{I(USB=E3$~BjjcxIDBaa~Gc4PPbdjpfXE7Z^J#{Q?C%aVyNaBf$;oa?1mp?cnX z5se-CauWK)k>GYk{P%!H$?86EZdZu6d&?wf5Bxvsmnr+DPU6zQxm}^A93Gu4Ljvb^ zg+6)g^W;t&IJYZIhcj1`1$5xtuCVD&uTR#9fpfdUjed2{Le#_U3V-O8Q$>(-yYjID z@BaXp_fAH=M>hUH%~DGwBmMEylm^*5y-h+B?|5xSqwS1t7NTx(^n$rc9q(=G|r3sKia2PtV}rw zIk&4!k;T;`U-yH^xn1*Gm%(^x@w1!g=8-Y|p|s(4 zO+5JM%QTdAegQeRYfa_}sV^eucCGC;Q8q#T zAN_ZyA1QGOIk#*5#7FZtLeA~FuYUQwGGJcXMU5JKdk`XjkaN3U9`3m3 z0&;HGtL`~os)3x_^*U4b_tzumcD)&%t3(gv+^)CB3gkbHoZI!T%crTkBIkC!Pc>7X zLH=(H{6B{ntHl{sJscAr@ zd{|#r>Qee4>HnkK^K4|B!J?@ zq~2T}>*sc3`(&Ei+mUm-vHNcMkxIz9-PmvXZre8G+^(pSx+er;Jc^A)G_IOk1f_W# z7mN6>${8L;&g}~EPK7#Sk#oC3rl{EBUgX@aP?HPZggGzfaa=6)pGAI#IWruvu`Nvd z5|esk{oJmw>5I+FiJaRNZp`QfFz3gD+ZF!cuwHP_hy}MR%cOY`!eG7!5X-1n&3i7x z*(IMjWPG)TZ;YJVm7;C!=DU$|yHe(^U;8EG+^$spOaB6MUM%^{AoaO2w_whUCATY0 z3q5xg*3a!qn_o<IVa^Xfc4Ob3^w$S1gnLH#KOFztal32YRZ|=6g&_Pt zt~?cO$vH*9*?Alnt9WU)FQ+5tc9kfU`*wch+^&)}^KTx2oZD4uL)SN+K+f$dy|PEE z?~!x6$`tNgDm`*;SJ@h!G7dw|?JBqW=bR~!bGynf-c)2Xa&A}eXm)xrVBR_%?aVyC z6JkDgb^KYT$;Xg$yWv0orP@*-Ik&6jOKIt9_?N=Sxm_dfFFA1{a&Fhiw~Efqh@9Is>PA&R3puxI^v6~1c^WylYs>@XYxhO| zAN{k9D>c6#Ik#)v#T#8dK+f$N-{)4P2at2Swsea@u>BYF+HG{S)X(RkG>_wA8;>Y^ zvl()3*NTj4-hSlVu9ZFHq$aHhpZr7SDby}=J&h1*;wO*Z< zk^e{k-Kj$re?-phT0iM{=hu*PyKc_TF_{4K(k^N|_1C8m^O?cL56n3D1#)iJiI3+l zybn3I>*S3&z55{NcAa`_%L7G_bGuI8zv0n^$hlo-2CeS$f8FDDoxQcL(j+X;?K;=y zz1nzbyZ`XxAL`j02WR*PZ1k??_x&~#t7mas?4z#B4!sX&_uQ_Jv#j{-PvqRL7bBPS zdIdSR>*bLJ3)Uj%cD?Gc{jsLVxm~X_ZhP?+jsV?73a< zIyY(o8(^{LcD+yCqz-K1#Gb`*V%Ww`gOkg-zS;{uhyRyTXn7 z_1-^_bGyPH{{5*JkaN2-XR&!L`efs;-JB}jfc9nTl^r?rO+g0`zy=Vh+ZdbV<2hFO3oZD4?(Z}PqAm?`V z?aE8L0OqaJ(e}#O9!JdW>UdhUg5M$McC{!~u|qoK+^&|@Dm?QEa&A|vZAG?Aal*#Kx|N4WBpfh%R%GaSKSUBDJe(u%IS4D?@A3#lBH@5@btwB{ zh$MvNcC8+rqaQ>f!g9OTWG(PJEcJ!ucCCFk*9nM3g#C~HyOT3-KqMh7w`={BPk(|) zL|AUu_4;QGPCUG{iy9tn3sJCed}eTQjwAITiWZLBbzYiFO{kDS|e zs_DSlJ&|*}PG=u7{v>j4*O>wRH$fCE9JlN2ozQ!gBROXU7Rgyaa?T)GP-Fp>Bv}O|Nd^g$b4J2_&*`@g z`^~d>Jm>rFy~7_fy}#`+zkZ+Us-9WO!f~ACdKujVJc_0T7)^Nh9^KZ>K8)im&-nOh zeZGT33s760nRve)-vP2bwCh>pZ@eDENAICs&;I>x!c@)Ct|vYnJ>?C}(XJ<>#w}i` zIokEq=Tjf@9UKeCahB)&F=Z#;0ip$be{R>@zwsR$3&(Mm=LNrVj_&}`!au$f|Br3x z&*SVrc9qWRADta&-Zmx_OZt`}n6(XK!P5~nHBN<_C+@G?r8$_l4XfOZYn?R*^R_`> zzdCVBb1+Y%ye~&r;5&!35NO6W`OYCNAXjfrEzm}9AL`mR95voaMLc1?5Z>5(HfPi1|p{>9@`bF^#P z`L0}@qdD3&#i}7a_^!8&p99@7LC9j&@CV{TogJ z&C#yuZFeKT*BtFypU5l|7~{LQjLADLAJ!P{TIQ8qM=NQLb}g&(w$JWrp33&^&<)== z&>ZbrV%pq%+~(Q%d7N#@i*qt^n@I~$yO!#-=u^#5x4m^ zejaCA-szX#=W7NnK+9k!SOtT5^Z4#;qu-umQ#3}qwt0L1m3Yn3u5GnB`dzZ-XxFwg z9~qug^HlaP`mfK#ZJv#v$Jw^LwK_kynX~|QZPjwc$IJBg(XQ=eUG3(e84aH`!q{#0|cE8I~)(>>u1+UlH+pm9{izkE}3Eua0)5RczqPrsil@+Lihg z_(F5EE5)9(>%ufgyHd_OHDa{psT{v5e(6L$&C#xODqg;}PII&?-6QYaETTEumEMx~ zH?7ee?W$fi5Arb2#=R@!$-Ml>d2JlW*~%O;JJVa5qg`dCS^U;b&C#y1n=SeHBhAsS z5)Zat*`_(#RdU4cV^uXryGmu)I{cF6sk}d@!N%{BG)KG2z20lqWzEs9@&+%tLA9ewMOJ?(o=J^>)H>Bd~i{7wCjpy6*s)CIofq) zno8GCXpVMWHKgpUCYqyN*LhOv^IMvuUDs{1jA0yS zd&VavWLk~Uu4g6`4IiXA+V!jn_WLC?N4uVVQhc;nbF}M;UR7>HX^wV18C_%Jbj{JO zr$*Hqm0NSP>pACY&HPGpwClOu{`meC&C#ys1^xcnH=4sZPJF!R0JrKQus$vxDz{2>kmvVTeV? z>hS#oTEO+?vBRzT{)51AoG?_Gy(PFWOAENZWABUarH>!X3&Wk$Dh>CgX#v;w_O;Hz zeQAMzA1BN&la6p~1==xVam-L>+<~vTEO*H-R*G;_3@!y(|K)o@i#O_yQaIgeNOI6 z3;g>yVR}DzOUr#}T7cTMROg}G=nAYY%a}3nVSU~^ey@^d)jZ#4=MK%$u4Q!?Jf)N7 zschel7`gbY=4jUv6ED49PIIhXOJ2L2@QdbX*HT^2)^Djf+O?eEQ|%sUj&?2g^ST}R z9tQol5^(k`?~mH0_#OuRk9T0#ezVhBzGMk}ceXKb+K<_I@A$n+VVgH5|2bH5v};?< z$1JimN4vJ2eeC49ny0dV(dT+%q~>VXmUr*H@ww(`*H-PGv@M`H+O?gG59)odIoh?| z!R1QxJq&^4IAMEF$)8>y?TQl_itq(j;L!_X$I-fnc*tKHP2F+7BKDlPq!s?o%U7>QUzt4SHA<(XHYOLPQeOV#Uu5h+q+8S(j4td@!O&WeKbeAQqEjDF@xr*9KZ7B?%=+xkZ4yr@636?eOV#Vu5^!Q zIkQM_AMHwS;miAqYmRo6ndu$wY^HK>l^)&(s|4I^yiqouQheT}O_d@^GByXqP^M8@-tCAqf0DPB>1_aZ~sn0xf`j$G!a8 zzZ&=~b=_89O+Kf2 zDt^2y-z@5&Io7T>YyNu&8N)bEct+2TuZ-0g?b1hZU0w{)9PN76=$@rkYmRn3`)rSn zg*8XJp6HSN+-I7jT~Eek{Ox7U(XOY)r5!j+bF}L@r_!%-G)KFh+i~II`I@6$&kJ8R z#n2qaaWZk~A6)q|u(u|SjSUa;5Erj#(lC~EyvYL*S_n*)G|VGiHt{f!7LZ$|dXC}F zpu};UG;G;p4tK?A0oNCcvn#$JOB}~ZLmtf-!QCNR!1Yzxv^BUY!`r{C0yhh^0Cvq)O)v5d0*Q7^Rh*(%Yv}?JeW@O|xpZ=l;x9^>sn2)a+^glO1?bN4qrtTmSsr=1UyMN!xqUFB4xgXaVest#R3Ov@m(X*q+U~%6rFg zoD}BzOpzaJj&_BWCr$NDnxkD|SIyA$ea+FX5W73wx~F+6$0t{J{-L4fXjiD5Nu!Tw zj&_Apz5DFSnxkFeZr!r}sODI^!dtOvWChL9u3QmYST}5Vbkg`WJmO1UJC&=RH0OsX zPjj>@tv6zu_0t^fO8c$28ZT*%cBMGbeA6n;(XN!U+TF~dc`C=R+@^CTYmRoM^KQd2 zahjuD>7F<~Vw&b?S9%K$t&7zh?W%kspXz2E_pXeG`95gPYvVXhD)X}cxuJYiXX!`*S|3v|+dAXji$n zZyl?rIoefTzuQ-~YmRoE`r+^Diz;s2F;3eb72>t=^Em04W5xU5YL0dtE51Z{u;ysj zvAY+Q6EsJ=jyO|qrqCSiI&w^{QL{BiyN-&fvN5CPXxDL)YTOv1c`EPUy>k9?cFobQ z<4wMHbcp6?*WJCgn(k8I+OF}@;*U@9+W5Un>6$Mtd8@PLXxFud&dan*bF}N)k7wtv zsX5wpMdPjC{i!+Hb!EDZ!`o<%c3t(^_G7ic#IokEyPOUb6t~pkp=Y_St z5ve(h+mfxR(h$m8`^^PNLlK(5NyxG+#}AIvMmF{&I&qdAyY zhCB1x=02K(d1ZLNoxhzybEs=&POG?vhZ6Xg0LnCyizMCTwej;fWttoERsKkGv};ZcW-mY6)s%wsRt*}z(c_4*4{QoMI<&Hiy zKUA-eb}jGh@$us{N4s{XvIlt>i+_cpY-2#C?Cp7N9LFi!yqT@YADW|G+iI1eR~^mK zu5D*Y^UhAq(XK7}c0STVbF^#AJDn~))g0~Gs&)6ZWi>~;w)1k*(7l>t?b`0JO%qCK zj&^PD(UvdvX^wWqm7JB&F~*}8#?G2Uk~M}qJW80~*I$!UbF?e0+||!d(H!jx`@O0M z<26USLi{W%FVj4gcQeDdT?IOWYfy1-K?Fb7YGn+?!LWTosj-zQIuLjj2@Hb!9hfww8NyDwUQ= ziKldYeDwJDaZ1U-VcEGiM+LEOBti?YcgITX@hy)e6#jjj za_oAA=W)t$K27^Bk3?tz z*LSZitHvVDa z(_dpS?-=I68|h!r9Lzh0_2&KZ3p5Awj$x~(-Coxm%sYlyFy$2AIi!Wax^oP9V(K2g zb4UxwRk?BfR_N`6dB>ptruXZ-nuB@AaAy@-wn}rzy<>O>UYnjnbEs>_>|bSYe#U6W zG*(^7utj6EYnmG`=6z4|RMxj@p6+@xm-gm1&%w{*97|qVw4B>a zS^&HBt|b1d&nmPdTepap0d?3VQ|g!AU{ z-PuO}BVEdCjCO7F*3q)RYL0eotL6T5tu#lwww-lP?#G&^vVYNU)e3I&9Q-`avE}U* zeYnk}1+Z(YHmiT>sJD-HZ71{klV>zXyS6(t=rOnX4t^fz*xuvFpZS_W3t(5=F4}WG zZywt?FyCaLAh@?ah&6{u?uD4&EwV`<7D#k(i)>($DFXd_kPXM zu45&v`L&(qXxFhnS@-(`&C#wS&K>T|U3CY?agHNL9V^2FC0c-X*HO{?Uh1m1k9HlW z*WOo8YM#pbcdxc~`4)|X<2c9hCbf#<8)UQq?XLTHxS>xzR=#UAE3#xb?;Xc+j%%jL zH#M{7XxFud@fdzG$XmzZO4UD5)sFXn8U$GLEE9OoKx--dCV3tGVS zRplL#O(o+~Hih_`OQkbY9uNg>y*@xW4;!%Y*!xi;I6B=bGN;Hs|>>8(ILn z*5r*_G=@~WmN9;6Xfc+LpU1hD`TG=mspe?cvOXT&D1+vyY~M~CSFMlcXx9>--`JK* zb7(bOOa5{9-b&5UuBASGIzK^kXf<5RiMl_2qULDVawnD_7Ogq78m{GCEBWmV&C#xX zW$PV%a`4^R#>hcdcU~Jmk8^GFe(#XWnxkFYYB=P*>YAfn+fF~C@dnLP*}oWk@or9uyyYDyf%KX(iLX%#<2x7N4vr*ynPGb^LO!YM_pmp+I@#VGoS@XrV!f}H<+up zpUUyczGq7 zZ#%C!+I6h(v*W92j&>cpyRK-Jt;hn+(XK1AWXk@!=4jVd1Jbrv z>$;5=H14N)Dt^3|zI^W`&C#wG|6&&1A%$_A>ls~|AHL3O<2cUs%#d~$n`(}BJ!?wS zZ;xt@c0K!Y!(kOPN4uWr7@qBd=4jWG;UR@T)g0}5YHIAKTQouQ@cN4uUEx3baun!`BGi+FhvkCni@XSCTj&h%GBvz}qhYaTm7V`w!z!`#!ZSVqmk zyk}Syo3@;-IhgkhyLiLeLUS$3fGn`^^ z(+g=1<~_q*{B^(YGzasZ;T>K1^=q1=T{H9Ed0#L_JErknr=q(xM!Tl@W9OJ!ny0e9 zRW0em)0(4Q(|)ykt1gn5_OZ>wc=Pz~Y-2>(nw2$1ySDkDRP!U6qg~r- zP|<6sIoh@D43(nqX`agd#gKecxy|$N^El6zPl_zzHj@^>uC1EodHk{7KH9aNG&z61 zsyW)V-QLrG;WppH&*MDXyEo}1Uo&U{?249as^;U(WAcQtF>KdRjnS?!SA{;wqB+_X zR*{%5=WC93g{?iJv@5f&Yq*Xv9-TA}W`Fvl#%NcXUuN80S97#0EtPiqInB|o zwBJm>tb^ugSBeuIi|x@I?Mk_zOKcgU)+ z(wp@|`F5J4UDZF?2xE+USH>-CNtDKDSDF3g)aja|U1eo0{&Q~4(XO&v7JK}a=4e-m z+x45L)g0|A*{^2JL7Jmor7~BIE}?lU@6Tyd-CL|V+Ewo33%QGGj&_wd^lG{#nxkFk z*7BgyjB)FZaWVh2w>3t)jybLH(i57aUB?Q``>2WLXxFhj<@o8A=4jUuSIX3?qB+`i z)N+h|599YwCjqNN6XI89PPR?>ya+e znxkD;_1}~GYt7NF>)hI(?p4jvuIn~#8NE{TRQ!19+IYD%N4s9Ss6yiy!#K|KjLwUe zZ_^lS*E2(x_O7Zq+V!l-bAP?0IokE?D|3EN(j4u2qSLm{$2CX0o($Vi<}J<9uBWE$ zc^2~4J(>p4yagD;(eF59Qgll(u8umihi_=Y2g3rLet6Buw^=?fqyGVlgZ@} z;eD9~egl=Jl3PA3WHUJev0FrLOz8rYsXYnqp?2CZF(@!J{dE%lQGHUK2c;>YaRH zAMm>1(R92b;s>VS(R3G%%3qBscr?A^6EbaP3LdSsXL9&Cv*6uY#_*;$=lE-4HCyJN z4L3=q;L)-gH6JsQDQ?+*sok7ROu?fihQ+G2Ou?fiAH_u!WC|WF)g-*ec&6ada$X2& z7Q_@hT5jL(yMN9UJX+qJwG|?nf=B!3(`o$n%!2o08{?9GIOVTtLp|E&sqVKrGX;;f z)uYqwT};8FZO3&UU6U#9SBy>*`6pBGXv?!1oHk6sqpf;oYx)~g@JRoAcc$uZG6j#e zJ7s>wUzmbN+q*ogM>(e8QM@;3fPdYv;E08BxcCSDCDRh%QJ71MHSsUFmH>~!qJRG7 znqCFAfWj^>yD63ckHRTabA#W> zu>^P&?!I%!GWezhcog28lUMpM1&=a!?{EC;vjxW_jUV!U#1|vj#FF4qI(19m@VhgX z1dr1F`O)R-z9|VFrMLFaqZ^olNA>-XGrU{at1`~VIA8c{D)6Yx8S#-OQ}C#)h_LFz zn1V-TcL;5ol__{s;##v?>zIN^B_}rcAsxh`T*EJ!J|W9oeVH%@$0-qoXo*oU@lHcyye0UB;AUitpEbN;a5+ zN5>od^SV+@!J}K-+`**`Yr00)A(`|0Yr5dkHG@aIww@_?bghYldW16tkFI^Kcf~PG z!J{iWoH=eV1&^+bIDK_KQ}F1j85h@QVG15y=lqqCLz#j{*ZsKK=$DzI4KMM7*`F~5 zk6t15fI}ZRcA|OM+Kn#I++WiJkDi%v+{;gxf=AElGqpk)rr^=D?@sByhbegUMDwTD zIxq!~p3Hv##5tzm(NhC&jHt^LJbKQJyX$^r3LZVT;oEa+GX;;Hm;RkGKQRp)3E7o; z#CQECvjY-u8o=KL$3o` zA)0nA&y>afEggCdRIGmbdLnN*&@j!iw)=jQ|c?Gft6~TMemmMyEQhrFpc>`eyriI2_qfbG=ClO=SHTu zuJzWSMgHv@8%8>|X}{5XvVZf2UI%(K#jG=l@9>tvqbU!Ze&Zlh@Mx-cFSc#O6g--a zd!^oOrr^;lk`FK}rzkQ=w)PSmwww0L0 zM+F`&=fw^24pZ=Gx&1y(hzyMHU(#xrEw@vH8)<8^VrV$7a4ej7zIrUkwqTb|ip*Kglw zzBHgJ>1f6G*0Wc=QK}Rphr<* z>Pml5fk!NiBTwFV!9OM#>Da}{;8D2yUro-z6g&!VcCJCIn1V-HVoHC1_=V?I z8k^20yyve8BORIy?VD4tZ(#}^rB(1!yHA;dM`?d}x&8&F;8BV#gLhV73Ld5WYUIPC zOu?g61qLo|$`m|Gr%sHhTkz=ll0qx8P@R^4L?9@X5@`_uRk_Nt5v+v|>E z2-j<6PT$=of+=`ZR^-;=-!TP`%5J|g?lq?1QHiVbCJkW<9+jLldtr8_;8Cfd#rNki z#dqX%Te96|3Lcev?#2DHnSw{Xc^+ z9vv&&vG~VK!J}ihI9#F?Q}F1Bd+Qc#X9^x2*?Y~zdQ8EiqcSbuafT^)bey)y4>~f% z_v=21I&+#ScyzoWiTk=R1&?m&*Y5i_Kd@zWjcyg&XY<#Dk&f+}L6u4lW(po%Yf{;? z7E|!(+E+{EoXZqEx}tram61%rqbnnG_WztIcy!hDeESP91&^+Cp~%_qnSw{x{bbaA z|DLDqf=Aa&n6TZyUuwJH(G!KX`?rbUyAL#NS=%aa*7x@cBOTi_Gu7y@gDH6Qtlsr< zbYcn~J^Nm*G-sKDM^CgUF`yh%@aV~GMZfulDR}hMAp1;9rr^hTzdO_iemVhAFOVRo?!cKVqjj$O4baw2ST@?vM6q1~Q;x?&3`T$eHFJ z11k3}$?uQ4Y4$Oos=~aF{obK4!K3MvnEjUDI}|2(G~K0HKk$2{H18N}**l!MiQg-w zS;qoBT9?-i@;@?wJKe%E245PH);9$s9hz|LDXq&CJX%)Mv)4*61&@}U_SA{ROu?fi zMh>hH#S}bR@_wK0)0u)tOEnx^AU9L+XgTRezWfza@MyUMjR=3#PBV-JdbGS7j`BzC zG{0D&M|;YeWdFkz_)I|9#^=}9SMm1>BORJr?2mgRw=xBfw)N?g<2{*zN865iaP=Zn z@Mw$i(|Wwk6g=AU_sJDcFa?jc`gjbLO~ck*ng|;_+D_uw*KRQdkG4Cbfa8zaX)duq zkG6NdV5C24r)k9kJ&N#Sx%_4oK0Oh_IKCqP1b?sCdJ1#ls!YL5!6Oy(`nSGi3Lb@B ze)Y$Nn1V+k4jsKRf+=_ua@mn%8JU7dno?=caKCp*GlvDb5KhVc-}${mnl}un`%BAN zey@~f4Fl?Z(dKi%S4wk+1$vYn55M4Fyx?;tA&qT|G9K`c7)ClYUD)!a`ARYckJ8FN z_mhWA!K1Y6&3U^uQ}8IohHXc8GX;-QCU3Z0iz#@Ns_>5Q{oWzX5fEHZ=g1su^@}P<@F@)=kGN<(J(T^#3 zR9484*F2`+QQ2KaWd4#VcvRxj#gTDL!K0E>udJWU6g(;we&(t_dZyXH0!=BW;$R_u;J-Ou?fwbzi49eF)oij5|}izt0dnI%eM~6*e&ikB;^7 z=$Aic3LYK1^|%68nSw`0+`O@_GE?yA$N_gp9AOF`9hK$jiH1zUqvN!?f9)Pqe829W z9?uj!y4KW~$YiGA(X}td zJ2{zxM^|)dF}p8Q@aW2rwxiQA1&^+p((H$2Ou?h;TyAj7WeOf$x95qg3z?z~FZST^ zH<*G)uSe^8{-6TpTcnA-T4k?s*k2PyI>IxvWNcQ6DR}g(0cq7;rr^=DZ>EoEz!W@s zqIJhHKQIN4o_x8>oDZ0SM^E+bdGi`m@aQ>rdTjcHDR}hUCYw*3X9^xY?}Z<(^drvtVNo$>)Y%D{gE@h4rpg+#;%_-uJT9S^ctx3LE3Zv zh@Ix{0&3?<|C>MBr@<+}zZx>McW0#KpsGDZ(0;;O@e8nHJ)0|yEomYFj z=#Tbk#x9`lH=7g2vy4E)G(*>>A6^e)8d#~)Y`x^1^dYz~joD$z{>WLvXFAd}_lFMh zN8R)~@R&@iQp^E=#7?tx0kw<8|KX4JX^t+SVqS|*{>YhT=mIMDv@PwAx@mqcpsHfC z7yKrSX6FLx6mO8r52l2Xjx^mRC*JTc$MhQ3^^P1&^e-|rNmpP!T6gBH8tHq4cWW6# z@}J4W5bDt~pA_D=oGEy;tY&!!#4yDz+i7xqGnFZLw8V%qIs7(?=Hd!`KbCw@I*s4H z(M()GRf7s|zQ|h!kCv06a)$v-p&l)F(9PC3&2Ifh*hBF0^!Ymln;8y-G6j#q{dK{QqnUz7;m!Z@&oHLoQMTInN`sr^Kz;8BT-`OX$&3Lcf5R%HKDrr=Sjusr=U zFvWM|bjrE14^!}{+|$z+reO*ml{a?Mq=8JqqcdQ4DZZG)b{*q(y)=#eHR0Q9G)Y#! zS~>181&@xExk`t3nSw{hZd2pULrlS=BW~IIx-$ijj_fbaTxJR$9hJ4jH`SSfN5^Sd zbif9t_d^)(X}t`uA81IcyvYQ#oNDO3Lae-Nb`T!K3HgUbEmG zrr^PgU10MxmV~DM( z>FrarWrh%2J%<_F! z-t{54FpZg|qQ7GZ9!>MNvfgV=vfr(GtT$rZ;5@^=QdI!_w;L%n+GVF?F z3Lb4IF3po^Ou?h=PW|$998>UUdzTjMoy-(G3g=SEPkabRER19FF}JHQ(xHi{7Fj>H z<5j_N1;UZnm;iGkHRTkwRvr( z;8D1HuXg#7DR>m#+zVyvG6j#a&-}Xbl&!ZH0RB}d(R(^kp=9GeURYZfLexHYClmhB>Xcpu5muNmI zpziMnGx&WTnoSC*H~vIkzrRFtNd@+MoMord{@{Cr?K;NYjLZB!IL#v!xLwEWlYP41 zf2UcbfLa;T@AmuPG=~&WyItC+e*c|jkOC@hbZP1L!D;>|pmJcx+J66?W{(1@viFGf z``|Qp6i}ym&tjAK?c)1&@Bfg;?}O94QLwHza&vmW|4y?;1$uN}uN&vzfWkB28XbCn zb;sW;jC5#@sE9#*8#4uut~F!C-a|~mqidfZa{67S;L#ObuXMf46g;{z_+q*4Ou?h8 zCZ5i)fhl-&ooi?ER%Z$xUH8)ui~3_injtFCqw7W0iSfsiG(S|JNAK<0#r<1#@c-r0 z>iU;Fnr6ixU0Kfk%HJZs>%Qxp9o_Q0cO-x$3ofhlCV^T#rP@x0fre=csNGW=)nOWFm?nU# zKF+g>D?|ISdfOkr>J#D`+-@oTYuYszn_wPOBEe9H= zX`k{u-PVF>pkbQyX;sO?_n8J7rYWB`mcQ7RX<(%~X1&P+l6?p+Ok-B>#={tbN7Fno z=)J5=ab4@(5g}hN1&^lf4zWz8;L#NGuYBR(gQZ!X0!^9n*NYSUd$2Ue6HxWm>8uSGErg@xT%igUHgZ=iLW^oG4wY_}onJNB;;r-agm{q~i48fyq zo>^heUD`|CuSDAuG`kSEjRla75`xRr4Z1LMDnzV9!25n9bA9IaKyqmdT+um zhTu_{i?4rClPP!32*kzw=?aUNB3UOred%rQoqmoOeHGY#Rcoa&Gwf)xBod>3j%#RmRngcPcRikIJ01b<1I<;89sYyT>+Q3Lcf+ zZTsB2Ou?fP7nYbGFa?iFPG21C4=NlO={QP7&aUzauZ!=-&Cq2iE_ znL<6fqFa$HN0>rAx-uwV!OBd*qpK$6OuEMu>d|$s=BeC}DR^|3`&958L!Hl@dNw~2Y#kQ(K%HCDR}g(K{W<{$rL<# z_Vp^OJ*ME%6KzGI$xOkcCo|buZX~IzTWndf7a=Bz?q|Ykv4C5 z)j#X>8mLuwM`ypQooeIGwvGzSt;N334qr;BDl0_x6Y7dV7PW`kxBW>_>t1XqAlJ z&ZiIFt!4DN(XESr2H@{>XyT*0cgvq<3LY)1-P827nBtc0jQ8{GW(poHF>vbFt(k&H zOWv5$?;%t0XsPC-_mpG`9xW&PxKjt1f=A08`P$R+Ou?h&-7mEJSEk^RKIOR2g%0n> zHYQ#yC;Ts&3jDU(?wx+wtTgIc3Uu?|X zlPP!<>h${anp?>VO6QCYEL_SR(z9+lm5?CBqwLOm*R>U!7aOu?g);L(x8FLztd6zb7YFP+U3&J;X4 zPNP%l$1uhB>)xyz8^RPkI^KZV#l|uPkM7LlIgk1f)^v>?*=9Ut2>l+{jLY<83#Q=F zwZ^4+vX?1%bnR0ac9mrc9$nG1(}#aB1&=gyTIW{nnSw`GjZP}MlPP#~owMCz>M#Y5 zuG?ixhM$?D4KHL<-Vd3ANAI^oX)E|JaHK;M3^fX0wvHiq^vst+rsrb{9zAP#?C#M_ z!J}vY8TT}dDR}fmljbeIU434Im*>4tx9lU@g0Z)qB#Eg|n# zXBudjCK0L=o9!~wK*KbJP=UC@-I)d&rU`_;Y5wgYrh$fO`k+nihP}%)&@fFN^kLJ( zcbEnmrm2GpHoVxFX`o@6IOx0M_ii!`G)&V5{dj2GM@+$^xgpa6f7k$@AbO^;INLjY z{54^u1EIRZ4}M^6Zn2?d7;!n zzkQ>*e}Jm?6^AI^GI+F|$VzKwG6j#eJK@$`zfGoDe_+er$=hT7_MGPY3G^tIcdE0> z-!L4pFb=d>yO|-hkHVbQc4#%G;F11|iDpO6GX;;rez(D;UQEHG5Wj}?I>r=_O3n{` zry^7EDAXG<*>5lfkHUE?zDP5s;8C~-zq0N!1&_k}@|%zbOu?hv*12^KAHp$7V|T}4 z0~mrwX|C?_?TbvoqqK7Nyy)L?_TYb+_M~0C$6^0&HN6fz^HOZj{_a%XGI*47ea25? znBp<3JZTFrX9^yrQ#F0IJWRo(bT=*xO=b!nrMGICos%heRP~ZdO!gt{RT*dMA1ucZ zJSuZc&GWx71&{QNwW@1cGKG3nc2e~rkC=i-B~Dve>N5q8^dX`wxPvKpR4TrBQYWVP zj-2krDxYNv9+i8sf0fQm!K3n~^=oj7DR^}9caM+rA#B$%9+$Zg$q+m`=FrlIK4%IZ z9qYvkg9|VPkBdqrr^<$pB2tJm??O4RJy#CET-VmaT@37IhQHE zU-#CGhLS0GbiDo(tIlBx9^Ext(oFLqtmzuvkKWnN5InkO{E;p7n1V;w8oOug8K&UT zwNLM#+mR`FbVbq%^H-+e(Umc)g3B`nkFFZCzRF{!;L&x?tZvYXDR^|<&XGwEnW7CZ zG^lcGrr^=rwJAre4+BR!GzCxNZJUQM1dpDXZo}>DOu?gPeYRuXJf`5$vmfpnXEOzl zp7>}{&?u(h(UUJOH6oaTM^6o%+weQ4;L&p)&#C$vQ}F1yZL(DQmMM7jyv#56EX35K zFYEo!M^tjIs0Lv{LD9*%CbpvI=RMjbt!;KQ^m#NWs8iC~3y+D16rQnh0c|I3a;^=% zQ*5tW(%KIX5PvB-SMd+&=L?6FE~i3h@}C-seQ$*a1@%Z;d+c}OnUZtmC_sGakkU_H zBfjWfSlwB)+fS3$4*G$3*5q8vyAWS7r1Yxn#FyO*3m;CrU((v>-NbVy=Q>$|cyh~6 zyT2J66!guI(rwex@4cFwtIn(R^Yk?mmz}2ltr=1}Pc;2})xEF}8q)e--3wdVm7a$u ztu6jGZRh(TrH?(;sO$1QqZn6$Q2 zLMr=dSm(ll7s$iXq_uZ45r3mbV)`lMO+T8{xiEejt*;-ahm<~0iI&UMNW6H6ey>HH z3tR7};OB;v9`PnEzf~i#*K*=*X!}`c`~T^qN{vLdo7V3_#~=AO*6&WgSMYEA-hX-= z&-wje%F~7VVL`!^r%(POPr;O@N`H~3V9Hb2lhB}G%F~mDv^~nxtQ2_)raYCTJOxvp zKBPPaQ=SS^o`NY)A5oryDNpw)Pr;O@V9HZ4RpkT_=V9HZ4<*5hdDVXx~6XhwG@)SpT z3Z^^_q&x*vo_bQAf+Bqs8r)8<+DVXv!k@6Hwd20CoLY{&tPx~lO!IY=4+w@F%s+mfjf+Gt$zc?qUGeMfl;rab*bc?zaHEl824V9L`1%2P1qX)5I@nDTV{ zzsOTC<>`IOQ!wQzEb8CqB$)EF{r|%}J^zgVPZC1)lu#1eu@K5bxqn<={*N9ZR7dNm zjzZ`>m!R_;LiK;~AJ>zBM+lwg_5Wx$|I0^6`Q%*3NB#Hf|ChTB3ZcAo|Ht*`Z$3h( zeJVfy-ydfPwI9991_y;uUT#ueLWwQny_`xZj&)0|<{&7iy-o=nTsPG+ZeGKAW-$JN;{GQs!5Ndb( z_NMJo`*t&meolFLxQmXD^0J7oKOxjkzCrC|2(@pEsC^5e_V`I{`h9A5>ruNKLhY9G z3LP)CZ)zuTYTtfzXqnpGtJLm>Q2V&49Ia37*9mIBLa2Rwh1$muYIkQ-yBk97X)?8^ zA=I8GcB5x%zyF*`&(!{A`$vD~zkGyJUJ9j>mr%+}zW>9#gi>B+P+meQFS#f$p_G@X zGtbVuKKxM1%i;e;UP37^kNhI4TA%WgCV_rVdAV7Mo+&T= zC@-Oum-CdDP|C}3>VJe%UUtr;XUa>76nP1w->*~g`R5^we*a*~-zVQA=z1MS=ezKU zf8RcaQNAjrk}qBFVRU|u)AONjDPqMM)ff8?O@t}jl^}-{)Ex_ zu1DuPjLy$Hbbi8$CFlBRd{|Hzcz7Dneg+eG>~ou3FgKVeh{JE&h5Mm}=Rre`|e z@6h=Uqw_R^`fXw4s{^$!`VmIwyB_r?!rIokFd{EKQ(gUegP!TUm3uFra*>HMe76-a4& zRA0BKzQX9duZerc!>GNAq4p|_>Z;2>#()3IM;O)DTVrZ+6caN~M_riYt>G{7EMmZ}(ISadRFYJ?K{Jk*B)dso_J?{}l?QtZv z$6=Y1a|uPyl)t&J(X*(LSZY4`r1p3SwZ~zUJMoY4cEBT?@>Ak3@)J(^xtAh8;gp{x zl%H_QPny4xpK!`gtG~!kIOS(^D)|Yg{8Xa+gj0T&r^rt@gj0Sx zQ+~oJKP6M-C!F%rlkyWz`B_5w38(y=r2K?aei~4I!YMz^f23#1PX)?PIOS&|r;MSqWpwYeg;r}!YMzwDL>(qpShHuaLUhn zspKbu@{{c^@)JS%nVcd&5tN^5l%EL7&&|Kl--w|64E>AzL{NSfrIMcr%1=wmPXy(s zQHuOTP=1D24yOIn{W{7|1m!0eGXeir^kej+J9qf*IFB;{u?6AKarH5S13P`l%Fb;pGeBjhVk@s%Fp|hpGeBjM9NPjUIKarH5A5!EelJe7*@)JqrmLIT0EZMfVvW9(*>wjG}rST{So;ipF~{(ReS4?lX?~G6kpeS)a~l z;3JCae?HZJ6x|QKdy}5&y#7w-HHxm2t7TYF6xH|L&Cj-@*N>ulZ1n51delGH&rx(f z(=EmQ{fpD}KK>`Y^z1o``d{g({}n}f-cEUrqU-%PtDf=y?W-udA9}X{ZJ+WdKMo3t zqVrd+U3gFw^(W7dqy16dj#1wJ?jwrMXE8dTQFQ+|g6{uDQGa6RJlY=Be+1Qk6rIoS zsXdIM`@ic-5~uqfxl`Qth@$!oYDkY+?3^$<<S;|qNyHkw+Ro5rh3>x^$<<v=zkrh3Rh^$<<< zaJeG+qk33E^$<<fv{4ucE0Q`cOSYQ$3VP zQ4i5n54ZoK9`td=(D}3I{Ke4qq&r*R#oVJ&U2byGwN! zL)WWM>3S7I^?c&uv+c#u`LpQrjTrKjCxxdN>TgV@{(21gnoPc8s6Vlj`V%o!XYHun ziJ|^R`aHxbH;O*5iJ|jVl=2%x^_3y{+0n*OZgx^`V(9$sEJpjG^LCTYTMX6n!Zr5)A$dCxJ_|7l6%?-;7bJT%_@Zyzz#@7P5Bj^{mM zsa~p3y~I+zjHP;srFsdcx{RfIxfu5R{l-$g%%*yYrFvQaH|iyp>cwsK{CbI{dTIX` z^%6_Lr%yr4-dmEY-_us+U-*m*1#fVyRvxQ@zAey*x}&FR@fF>8M^} zsa~Q|)JrVYOAo4-SgM!RR4=hqFXyRVVyRwUr+SH{dXZEwu~aV^sa|5KUUE>q#8SPy zm7-o^sb0=gy~I+zoTqw;rFw~@&;4SlUVd9l%TzDbQq)T<)k~dJ>Lr%y<=}r&FR@fF zm8f1~sa_)g7xfZL^|Bz9dWkEVoa@odu%I~FerswU|Iay>apWxzd5fbuoJn=~yhj|>)8Q83L2-1R-cE7-h@<}VUg|&VM;!IPU#HKP z;;2ruw4&`(|M>&DuO3JBRE_E>j{4v4WTj=Qzv+4Cna=BtZ)lEVs;{S1Uvbp$&z9ra z@&0SOR9}7Q`Wi?5{sYwSkE6OeoT9GcsGi=T`~7j$-a;l3X&lv4 zN2;ed>hBfoitjUy>M3IR^YikrSuL|`iZCd$wBoKPxbSd+PQeD zpY~Kg@l-#8>L;G+r#012Jk`&4R6p@lKe?!%5>NGWhUzDt>L)Cf`iZCdnM(B&PxW)1 z>L;G+=erd36HoQin(8N>>L-cnC!Xr33e`_M)lXKcpLnXD_o;s3seUS^Qa|xjKe_)# z{lruK)LrZkc%)K5IsPsjhFe&VTq%A`_12{e9MNcX=J z==@Jg<$9E`G-+*t%*4shBJz_!b?`FPK?0rsMt^bs1M541#+StwV-Df>$WOHteiEoI zYE!$CKy|b@MIGt&5~v>DOQqkMK=rYM>LY>1mrr`r`c#kEnmx-;0@X)Fs*eO3Uv~e; z_wWK92~>Y0sQ&aLf$E|=)y3a^Bv8FgPf>4ry#&hFY|2*xjn7wAr2SKU%%4ZkRF5xE zJtk1TYE!-vsQ#8w{UuQT7E%86BZ2NqoU0rZltA@Yis~(l+4?(}_v1ggVQDeCZFzeo3PEc(7w0_E>86!n!z^;L}OE0OA}dn)ynNcGh}mHJAg`dUWyl}PnT5%jXZ4au z_0^Q>E0OAJQY!V8NcGh|MSUeweSP>J)K? z>dT<|N~HSgO!bvW^)-s>E0OAJdy4u>r21M#^_58V)qv_Nk?QL<-JjEsM5?b1De5bc z>TCHdtgl3>uTfNAiBw+=sJ;@Zz8a)bUx`#-m8iZFslM)0eI-(T4Ws%>r23lqH|k3t zSDK3(R~BCR{Bd%cs~cBl+wd3s=EjxjCp_b6ZWoSubNsXSo#yVwl@W7-gVIdc*!mEC z|1VARtNqVB4GT&$abxQiAJFo{jVnj=qvsPpM~~e@&yNbnq)ke({*#R>dyP!7{h1qE zx5-Vv7gQwX(u@?p*Qxl!zC*)<(#+l1`m^H1!-~ZG-Z%yCR{Y`Qp2QbyY~A5jP*9qv zA~Dy!w3Qt71APbIvh>&S#}hi+zi4c4b5duh zXZyr~z9olLw*Mr5$_&P191R0~eeY6V_9u0TPqTh(oWmVFS8%-Mq|R@R_5BZBN6tWB z<)@bWAIdil^cmK$f9I3B1M}E_Mx4V9TiE|gS%ZnUQ(wgceRp+}e|X!(du|dqcr|OV ze2fw|Acwf7__%mvSupa)5#UCg9*^EC<__IUs#|eKn z;g1vk81csme~zj6cd7W}gg*^DFL%P9XYj`ff1L5h z34a>##|eLOhT)GB{@kkJj}!iQ;*S&loW~z0{E5aNC;V~7A1D0T(?dS~tQGum!k-CO z;Exmjd?Wbdgg={1yy1@?f1L1Vj^K|I{v5?0C;X`w{Bh>GyM^m^#-H=ADg2w4voro2 z|ErDU47WGHtuxoTgzJ=(GyYW5{y4*J9NaqN&#wf3obe|If1Kgl2;a`|`w#ea#+R9! z_#S_n@y8jy+u_?8et&>pXFRH)9dgE>V*GK2+vKTi7fjsTW-B>!yrn& z@_w9sw)ZbN#`WUG3KcJ$1Ge|iIYa)wZ4Kvc8z%AOguSX$NAZw)E)0q_T$_+&?lYeeDQ5G z^M#y=BU6YY=E)h~F5#Oq?L=y!X&E^#Byoh~f^V|~-(2wRpoVWQ_-0Fc<$`ZVf?49* z2EjKMd`rYP7kul+Hy3<6^|(@wF8J1tZ!Y*2gl{hRwgTT=@NM!szQ?x=d~?CK8TjUc zZ+ab{@hukLT<~oyzPaF=_Y}6nw`P2E!M9?3bHTS+_~wFd^YP6E-;9E9F8Ha`in+v`@pyHbgzOBSJ7kt~Q;+qS; zy{6%t3%-4fZ!Y*YTf;XOd^@P(n+v|h;hPJ-{iNcX3%)g|_~wFdw)p0PZ&SvyJ-(g2 z0^eNlZN1=|3%1+W$0rNym8_I8Plseq5s*SNBZTpk7Zt@n8$_KnH*0Rru2p57yEC>hO2mO3UNv zhzG0bzv{l_KJXcq@E9cUs3RUMVLROg?gL+936GBOC?_59;1cn`TG9~@28ahbINYkz z57NQk8u-&u506t1a?-)y(FbfL9sIq0Bl+<6$6@fNBksLQ+|$9Iv%sGY{(9k0N8I~o zCi(DpX^O(l&~b@(ub-799r57sU*X|}Z4-;^SRWo|b3djd-hJxKXZYIyf96R?+}nL0 z+r!;@fjb@X)RuUvBmUj`CHchl{E4@}74Ey`e0ixOE;o%+IFrA(wsfVuXH(v;T<;>T*A))S;Lw%xxR3L2 zrMxBD8CR}%3)kz)@fQonccr}NP~NVTvm52?3V-(S=gN7caUQOex1$T+<7F#ey24)! z{JBz|N9Wl~u9Wwyv)G<;m_|9c;^!#*bfr8qC{I_)dnx7Z%JnC5{jPBM2^_lOsm~wpXHzLJJP*Qi zIl1EZQT%o#-jow>T=8(D;GrviJ3OMSFK_ohy>ul$Y$rZkZE__(oW0rl_O8SSKcRhc zB|eQOKDpxOEBNV3JQ%AnKI2M!IG_?A+^Ekv8ujT$eNNM;PdDoGu)w7o=ku5lFWjil zz0{{0=ld<^>vrVbYa(iFB{z7koX=d}z;8FaeO2(* z4ga>|pBsMPNj!DKS8pM1xZ&TyyUE9I7y6HG_-iAyt8%@$;dd^6yWwq*O1tWY-%Dq* zec-oeMjd2-c$<#5a&p7(Yw_C+9#(4b;D*1?;IEq(<6rH{_Cx!DhkBvCkiVCgZt#!; z4{pS_Xq9%>?Q!~NKAaz17*)7%dxG)H(c}}~I*D&?@G(u`!wvtVw{w2Pw{qf}8}an4 z(0;fPKMxD-g?wJ-OE==%9F=xfUeAsAc8vJu=0g19{cJblrWZp6*iLVrr$pBwRT67leI$sKt*%S$xKyHG)6x__Imy#~pw6 z3I4d_PYC|FMnmjIjz44Z#~puyRQz$rpH2AVjz78h&IXZYieKLz;Xjz2D& zSmMuE{Bg&hohtsg}K;zZb2ycbD_dJl})s-?`XU^1zQXJa_Zp{5EiY9*ifB z=e*9*ie8A7aVOuV*secd3*Y8C;$%Fgl_ouQRTt5NV z9@Kw3_3uIXI^9Y>;|kNZa$I=66P`WjzbvBv;=#DV%MbCH{?2NlzvDswWlbdcjHB9* zW54usmaJt-`EDjYd2oMyw4QwUPF3OCgZt~jLd)?ixm-L<&pRe5`5wfNI_A53a6ekl z{iuiOc}Fk%XWajrhReq}dB5_~1AnJ~!1jzk&s(RoUmiS9xOhpqACuQNUwY8Lo;N}{ zt_SzKQ}xPzEY^qTaCr7$d}H7kC*d3;WOtK z1>Fu7^)g`1}YyJ>hdZe0suXqY9s%@ad&8{xej*rsoL)pPuj;1fQPpnF61l zrsoOp=?S0PuYylc_}t3-;-2t1QQ*@PK0SF}e7WQapX~ylp76Oj$~vE(@L33-p76O( zgHKQR92EHUgwJ9XK0V>nS%Xh6>eq|wIO z+>>=r>n#tFRpJj*XKq3PN9Ci;C*Ac5_jeOc~QUfsb4SZx291! zz8CG?AnlzOybqqbd_Bqglb2rfTYfL}`-i@#{*Lk9trz{4-_URIqFvukyY2=5zkz=* z>iY}o+Y29;VSW8~; z?g8&!c=ThNQcqs^cv$e!i}*2z_#r1ReB6tVa`Ga6I5VE$MSsxHCwvd@27z}k`h#J^ zxIgy7pA-A3AKLdY#udCwzXUikEu;M36YiAT1?BQN66Z;3uimEdXcb<);dMK_dc$iyyn4gylVwUd z%je+@uWR7d8(sqhUcKSf{ivPf4X^G(e3rMn+|nCfr>OAi4X@FWBP4HlT?DV*@VXLS zz2S8cyn4gyMtJpx*V*vu4Xsi`6Z+P_*c=d+Y27y;^cr^}#S8sTY zKS2G!Ycaffo1XK)t2ex^R^infUXOaSKD?F+y!voo)FV9qu~t$(Y`>lD<>W*CKQGk3 z54<0|-d^(I{%#J>w|poc1LfmG{RQ(J$p_w@;oXP(yJ^qzJ>?T7l#dU*>xs|Sk`KI( z<@t+`X*3v6@`3l=@a{wTJj!z- zA9z{~Pd?QDPU4ld=Tzh4sk9?FNO^YAp>dC* zwf*>Te;sp9qW-o`eAkw8!S4h3?ZbVh!za`$_t$MYK69V{yL&ia{N9P*KHO)1&wZvZ z{LUN(zrOIhQsCDYe%}`O^@ZO-`1OU~3GnL+zh3a`3%_~r>kGeU4dlb`BKY-%U!%aU zFZ`Bk@aqe|M)>tLje83B>AvuL6a4zZ?=txHHH~{twUvC~_lUr+FZ?dj;MW&^PrkGf*;nx>_ zjVk>5!fzt{`oix76@Go;_dN}Mec|^|JjaLMSt|Vc!mk_8y@#88;djNquAEkGfGd+j7Y#;5)sVk7y%*-6?t zIr+g$5WM&?KJ{gmrQd$=k|Er0_;LQbc<z*^;gR@QPy22y`4Jx{5Fh>F{YiNDhxd`f`z8MHZVT`J@a~muxqp9n zZx?v?hj+(+vOc^!z`H-Z?}K-Lcz+1q{o(y0KKjFZBE0*bI zEb++SG~TVkyFa{p@qVGdX}lZW{o(x`c=v~QJ9zhp_i}jmhxage_cx8V!n;4bFM)S| zc%K6A{_s8p-u>ZyyuiCZyjR1!KfKp#@a_-qGczG`8hkk5Rz=Pr(r0+@I4VU=b52QZ)Z<`2kc-o>s$B|jjR->FOGJeW_L z`Z@W`FRlGV$Y=h8&slpZfZr2%Y=?4u^YfKI2lU;u>FK@7dI8J_9R0L%opAKCF5wrAeR zu1MB<i|m=CxyfbW@4bRXkp0nBIlb&ZhEe58ACpxl`kxN9`)F`wnL6d|Aa zxe*RRJun~d%@84S zTr`0BH+{nIN|>*2?Q?+B8$s3gD&=jyUI6orLRvKPneVtR(t3U%e#GEMAb!}_S*{m| z9}f$D1med+!H+=v2o?MY#E%8|5r`jWRs0CVkJSMx2*i(Nf**nSku;I>z>kF*egxvj%I6h+ z0`cSX1xh&v;>Rrf2*i(RDt-jwhb?{t;>Uda2*i(re^hEY5I@G^M<9OeRPiGaKh82= zKM+6G;YT2T>=gV6#E*UW5r`kl@gopFmJ5Ca;>R-l2*i(d_z{R7>jggo@xv(i5r`iH zA;S6L$3nr6K>QewAA$I>4?hC&<1o(`%u^tKypA7%_)&x(f%viVD*OnnL6mP9UGKy&&StuZb@~%%^;13+09X{rE4ZAmYp4g!mGK|H=3tM0|9-OYu60_;M@p zB?$jFuT{1m+7Ep1hwmW#e-i(L@K}e(LCi1x?oCMwBEFouo$c}eSP)D6y@|zd+r)O!})0y&V%sx5dH@Fy%Dth zx2#WlFq-y2PQmbRJYXXQ!+(tm|H1HoqXz%M@IMazgW>-Y{0GCoP9+Wo!~d_~KN$W` z69>&xF#Jc{W&W-5IT-%C;XfGu$Eoli4F9*me=z)KY49Hm|2hr+gWvfeIvD=Hh5umq-vs}`@E;5R!SKHW{)6HFrAcfL z|2yG782*o{@E;8SBLmqE{^L~m4~G9(_z#AEJ^Tm5{|xvKhW`Ti4~GBIv|qvSf303Q zf0>+M_}>oy!SH`C{0GDT*TQv%!2Kx|?&a-6;D4(M{~^@hE|vO|w=-Xc!2bz@<@tqB zZbuB3aY5cbg!-FK{e@6|y(;AwLj8U3I6?}c9BYXqAzm2+DIYr z`z!bj!NVjx3?Y6@{Xl8|L*UotNAmITg=+Gtzq!<(oI>FDU+^2UmEV^tAs-+8h5kqg z{5}i6A$a(oV0kNL83MmQ!EXrub?Aiei9f4|KXM9z-z1@bIP^Wg7w~+xaz66+=F1TH zoq3D0UI_6g{)n=Fi}mqmFoN}YaA*~r;L9%Bix7Ut;Q^iHenRo* znsUqUS zvBaMl_!Ejh=ZQn1_!ERbq4;BqKcV=u`a{K^Q2hCx_#cWt2k|Eqesp@h232oHhIj#h+)qDR2B)YHQt}Q2Z&zpHTcs7=}Ng_;X6bpHTeyM#G;_{F$fW zPbmJx;!i04e2PDz_)~{Jq4+b1KcV>Z0{(>J&n<#KVO;Op0!weg;G~c^8&=kG!;ZD~ zQW%{4^JcmBBq0V!XTyIO{x87)Fv@4X&>szh|4-mQjJUE^ zXqUs_^A`9Fga4=CKa99?NhPj?!T(3_FQ+j4&%*z(6Y~muvRJ}@3jBw`Q8XNd;s0jZ z&#<^Rf-;O;H*sbSaV8A@e|(qk@xSmn)`Q=3!uY|^{_#Hq|IJev{GXq|_W0k4|6#X%AsipP@gW=^9->~t@!=>wgyX~U zKd~M@98mEg93PA-K7`}LpKoP*e8|CvaD4dsDtrjXht(=RgyTc;xS#JsI6myu@F5%@ zW~lfOjt?K=LpVNsj}PJauoNG{@nJbWgyX|2_z;c{M^$_X$A{DS5RMPi@WEP&pno!L z@s-Ds2+pU3^NC>Gc9hDvZ3OL;JMB{hqWrx4aY}F5!~Nie3RqDgXF+x z?hl`Lx0NE``8IftV0=-D*Qyk&qWvV z;peM|Si;X<4SpiwXFku3BjG1ZrGFL)KQF;gB>YstPbB;-7x;;UpV=DxM8eM|6@DV& z=N$1c5`MfGABlvYqr{U)_*n=)k?=DIej?%L=%eJrPd)rZ!p}PRiG-h<;3pD(*27OE z{B*-lB>cP$KaucL4L_0a;|xEM@bmr{^5JLFCoJJ-g1}EC{P+s|M8c0V{6xaf4ETwJ zAKNS7ClY=>(cmW%epdZBe17!At8v`-=&64v>R(SeO{JXVq$ghOAYSPypFNb19uBIA zBYHR}CywZ8Cnjq2`{n(}OFie;VPoAFJ>@e|xc|_@$3*zh!%+(y>1i*^epKRy9&Uau zaHFTayyipB6FzQ*4?P@C?y`P7`Mk`RdiYx*@TZ51KDf}+4&1hi^PoMq2r9}!{1iommWU$_1H*y+V=?VqxJZegW@Vu?0T#@O3@+kC#h&`0Et-)6)-nQ@C%`)1L3AJ(rW7_|ks`@g)kr%Hb;t zzM|nP3clI}zM|mkFnmS9*D3glg0DpQih{2w_=MP(Y3cma#>*W;%U)2I%QSkK; zd_}?6T=Z){iHboB1*dzP1Q_MZwpz@D&AL zPr_FedB66a`;70$)+^wLsu23cfZ7d_}?6zpj8U z^W#U;f1Uoa62C0+>A!w>!Fs;=`ts*!>bbw&UW(@W@8kNT>F$HAG|2roJ-W-T8G4M7Y z-eTY_4Blek?Qd;1QVhJk0B^G+t1)F2Hpk*-eTbGehuDY;H?baV&Ls0yv4xVGAa>jmCox!-)9@yA%o<4=@FEZ1>JXg_5RVyWLk>Nl2h`IK^r zrQCN??pK;(Dc_qfTHh{~`aMVe$|;ueJwf@#!o_1M_ldF8dxA>6$8!De(|*P>{<7d+ zwukrcTP!({k7K@!h3^QV{fwo2N6~)DDVFh!_l5E8Sjsnu@{OhbU!eYD;X4h!WAR~U zi!weRi!ZD2B^JK>Rrro&9OWY8D6#lbi7&B?w?v;|3Gb!w9?Lk@KX^VJ%Q#lstt=UD zSSjJgargD6IHsW(E-0z3`Sn9t_qaBT9ykcf5`-ksiDtr%JFXKh6XW1V_=0wlYdUfP5V=PBtsPy#;f$EO5*I)qON z_;hgw`S|n{J|*DOczjC0rvq@8fKNN{DFL7M3qB>_Q=8yZ0zOT^rv!W&{3FM~r^oOq z0iWF8<1;>m;Zp)Wotw}8@M)FcQvyCs!lwj$+Ko>M_*5hKlz>lqd`iHlsrZzDPyZ2o zO2DVw8`&P8W(z(g;8Sl7=Ydbf_>_Q8BV*VPJ}nk}O28){d`iHldcmgzeEMJ*J|*B& z-!I@(0zP$W_>_Q8kKj`RJ|(O8lz>km{{^2C@F{2*J|*DOs9lsRK22BgDFL7C_n5Uno8$;8U-PPYL*xj86&pG*iW=1bhm?rv!W|Rq-hSpH|^h0zO5n_#}Iq z$nmcej-SYUq1ESw8}DBBP12Ood1WBZr3O?=3WC0;&?mvTzP$8()*kB>JAJ`UyMlxl0UQ_@y_i9@`;~~^heB7B0hdQp8XT=uFn>@Bp%pZK|Dw# z-o?HuY>&So_XwI z#zsoQpJWw(lJIA);7=0%jGn3RnuI?(!Jj1jxsLWM34gW<{v_eg3j9gJpFaFa!k-C( zKS}tLg+EF7vj=~Y@W)QYpCtU*gFi|5(}_Px__I#%CkcP9?_qoVDOK?&34d1MPZIv@ z(C{Y-e-^0tlY~FJ@Fxj>rfT?;gg;YF=ZQZz;!hI(9Gj%@mxMp#H2g`zpT+o-gg=j| z_>+V`H|-lCCE-sS{v_d#ZaM4YPa*y!;ZGy}B;n6#{7J%}KEaV+N%(VY-0=ISN%-R=_>+V`dBgB034a!= z_>)ZeoumAc;aB%N8z~w7^58ES{_lkUWXda<@=Au^Ti`bt{+^^imrQ+KM|~yZ!;Scm zOu3B{$}JiG3xxI~nflpvP`U18_`jF-BN_f5;{GU^`U%<1{;03U-F$}sCsN3#JQvfS zOU8pA@F1D_s{7yPGyFerMv{`5FYIE=deqMm>L(c=I`APG{#{4%J>35t?vtsXTIxqm z$?!j)``2Xnj}rJ#rhabDU_bDm@HzSL8O3p|rDW!tc3w|;;K2{I9EbY3Pw3AjGvBl} z-Lig@;V&fC@_TE?x!UiOnQy(Q-}-Tq;cv$k@OQcMNM?TVUtU+@p1giC{67x=mrKd; z+^NBHGVy2}@hBOdrwcqM3u*mCAmSiAQ$)zJ4<8!A6z-Tnawz$EOs0ipQrEeCoxg6nuIFpHlFt5uZ}< zsS=-3@M$JKrQlN}KBeH(B7920r$WJ}6nyFxd`iKm&jp`S@M#@BrQp*fd`iKmU*S^< zKHdFyKI78_d`iKmsrZzFPaZe&JwBborxbiz59cZPWW=Wwe0mF?Qt&AUpHlGYxIN$F z(+PY^!KYsfKBeH(v$wH5K8+T9O2H>xI{U#VTYO5vr&m8VuNCEU3O;4wQwlyE#-|i~ zx>4{c1)nCX_>_WAcK-#RQt+w#HS0d5;M1yK(5Do9TCU<#3Oi1u4*m`J6C(IyfbUP?+W`05 zXh#hAV=MF*4DkI1d>i2XadT5d-mZ1@Y2wE5GwCJRdR;Poz=Sj(X+a z8}PRee+}!i1{b%p#E%e-{_o|EV8D<4_+c#>@S{(-Uo;T^w-Nshv=`~L7Y6*;q0#>} z(B33B+DfT#I~{JVrBrwgg4fHXROIc}j)1LLr`{;>TctQjV$c=E(h8Djxd_9?Sbl zg|~6=mWs#k;<21kso!zbZz}aWmHJJEx9^4fwxRXltqx z{QD69%u^cuuPHAp`_XHZ`{>MH|Y?uVBQ zc$}1@@RdP)e4F@~LH(Vh{xYc7e(F_D8N|nw8?DDrc|B{(4BF9d?kBG_Wl+E8g!+}Y z&!BxiBlHI|@SzAFz2Qlml?Esg)0C5E(893>nTs_Cy)Bc zz>ANYRrZhHKHPs=OBwj>$o*#q^?EDyDyIzkH{-bf%rK3|<97x;ZBdCw8K&`gelITr z4_+5M$e?|`b0p`7-<$C}6JCz#Y@|$h*~I@<&xDr~@RA8H@4-tZyu5$ABxSNRTlgIqs033BY&VDDW@#X zFFBZec#PpWLKgk|&xHHaEY5F*&|b;g$;&Lx@1k&iS@2xHMk%i>cs?VvS6QaH<4^wF`<&;JFJ}Z=O7C!94hb;Qfw+Q3q^7_MFW|_vjdEO$YEPVP`{|G4y zUrr0YWKn+;X+N^yvgc0ayyWfVWfuMO1^;4u)3~a*xG4+&lea5zAq(CmcsEbk z@O0E*BW1%=mI_bV@boS`Wy8~Cc*=&S-SCtRPkR0jlbo{QNe55a@N|jxFdLqB3Or@Q z(^d_hvf=5Fz*9CnIlQ2pZ#FzR^8SLHvQ6VtDm-PwQ@_AdHatBd@RSWtTLqr7;b|>A zWy4d&&)_NBG~O%llnqZC;3*rPM(tDTEgPQZ!BaLoE!5yC8=hX<&-SKqU4f@;cq$cm z%7&-Y+)ro2(>Zv`hNt>$mhf~hJY~bvMtI7Gr(57D8=k7+DI1>FTmetn@T3!X%7&)^ zfu|fkZ{`0lm2daJmqj5A8{Qy@YO5ul|%VoPx~mR9QevAWdEk|!{=ha*JAj}VSMV-Tb1j~!HfUkg`9Gz4;}5b zoO1BuA6L--%`uHnF5o!ulBm*u&7nS~3iTl$$9$Os|5@;#!?@2!6O{6h*UN!_=^Xj^ zc62SrgPXB%^Gl~3(|8;2>*U}=2tMR6KD3GFBIYTF@udLf%jV#Rjo?Qvd>w(WT=-h5 z!dEVQ&4jO9_!?BIWIZo!dHZi@EN|61-^3OYsD4tm1`RRfv;Tns!`!97rq*Y!B;MPwZc~}d~H_Y zD;K_w5nprRYuSq&559{22l&c0jnBbXE_~&|S1x>gpu$%!eA&TQE_}tqSDx;TphxQM zq&(*HF0vmX*-{4Sk8OMS}? z>t?fL{@P2|NK)RM&XeoY*-y+HLAUTePTsK#cJ?316j(0vPO#lu%J+G-%s<|v#K}Cj z+h#5ZR^pJd=Fj^pkN5jKuCw%4-fP}V{Jw`f=U>eHiN~z3C$Ini`ZDht<~@E%J@LMJ z^?9XyFL&N~%)9*NPs&<(%)gxdsZt+#%=doc5h0&>sWXl%`_E%u_$O18eDlBZHe9g# zT^HQg#6DfNiSq7b|6dCG-*mz5(OPAzyk7Xy!58z5zC6d0`SPd7vfY*ocIlbyXB@wK zdly_YU-j3LQg3<8AF}r+|9O7@BaVFLpH1QS-}0EZdiN*nhxh&0r}261yn;>emdE_d zmHvFsy!jtbuw=f~gc!bOUQ<9f$78rMBebV;er^|7 zF&`iI<6}NPZpFuZeDpZU5+D2UF&`gu@i8AC&+z*d`S`dLAM^3?^B6nHNcn6L%Ez2f zBR)K<;e!z$Dg+-awx@iGDIX(T7OQY+qE5NrY8om|a+swDD z9aWMF@NFEv72w-VD!vuq+k1j<1^Cv@e0n(*;M?hYl+W__=F0+n^U(0E0N?rq-wN=p z58n##Elb6>0(_e+_*Q^#r}3=--&PkmkRLh=0`X` zd|QTZ1^8BuZw2^vFTNGv+bVo3z_&(xE5Nrey=g0Y zTLHdB<68l~)e629;9HsCTLHf9(eSMR-!^LaR)BAF*ID{j$b8!I8||e+c=Cd$Le9gM z^C+Z#e@p+gkomM3Pmm8!?a6%J#P7ylOFs4cAL_S|^D5`O3OWAq^EOi97W%UipDC{m z+mv>rkmDcUKtA3rSY#&^QopMnWqam3pPsA46-y%iyWbSTZ;a4CGCz7D+`j*N^KX^U zg>d@JoyuqPzY5{CtXq-_;dL-Xr9R-csm^>W<#QqY{s_P3sgSs_M2H)OaGDRNh4^;M zjh5wG2(KrF_sa^23)z2FSr6Y1;hUTai5Ip)yeK4II}oo6@$R|?Wj%R)d0B{Wb2WS` z#J4?DEVnNtUJnwl3-Rq@iAp~4TJH)+_!jL8S7C1iZH{0`90{8Xx5Vr6d)c0LBcFDo zka)h)#3^y}UgBmUeow&fLgMw{cP#Pl7~T~UFKvjIh4>wW--X0$J@LAb_T}Vu_Cx%< zf%qw>Li{cl{4OMZKFWPx5q_28R}p@t;a3rU`QTR(es$tk5q`DeR}p??;a3rU9l@_6 z{L(SMqX@s^@v8{GeDJ#nzxLo)5q>Qd{3^n)e#Qlh@askVD#EW%7qLBlT|3XR+~o8A zZ!e4R>*6ZwM=ipykt+Q#d5a?anyleh5q{0UuOj^VG0f7hBK!)`@T&;FVzcC$W z7q66UF@8Nr`&GQ_g53vREb;3NmG^Cn@ylzL;$1O*?fhKnmlWeyG3{3|ewAzZRg7Qd z8h#by*FqJ)it%d;?N>2=Wzb(VPsR8(9>0q5YXW{1T|4t|wTJ`t2p2_Aj9)bjjGIG=T#PYLHYiSsj0CHS=w zze?EeboOhWN+_R0lurp>oy05iR6_ZbQa*AjA-?EmDcqG%KI0y^{QX!7<+FnFDWQDc zqkPO$3FUL0a6eW8kNpCV=Ifh(E+PI*CH|C9FR|2%c`70PNW`BK>g71~QUb5%w%bW1 z@OTU!OQ;XM(4Vl_p87aSeOOB+#Je!!op~xD9&{5AN{DxdRqn@1@atOqDj_c1q7oNM z;IVHW=SRHjc%1VlK8zOPLkWKUK>JlfT=1aXl~W1v;R4V7O7QAdyec6s%poq6z}sSY zlT!)t;1S|M34TRs_*H^mFX2}Sy!XL-32}cvaleGPa4C-M@yi>(O5lAmyq6FcMtSo+ zabhEJq6EL@;a3T9WgT&)1l}+DSm&mMcH{)@h@49B=54%@QwibF=T1@{_l_+_IGM@dvI| z&fk3fQpU&a86PiY9Kx1yh*Ivm|MY;eRw?5Tn|Lp?l=eT5_P><-@2W|XR7$;i)+_6o z|5Zx;Zc~Z(=IhCyOR3LysZTkT!r$_TDF>cUOwzHWUVlTpms2S`?1P6=#(R(QdrNXE zC0^|zUX{Ybe0V6uw*>xga4GR*CGn(`@!w_9d=HP)1RhI?C)@j!{mA>1m!xFnyO8A3{_j3C0rHl)#KA_~6;@49CZ)_>!0qwj; zTS`35R*9!W=RrLEd_CpQ_|D2m#rsm8x19aEvR`@q;Vw(@xm9Q{%i!?{JeI-Z8RB6X zJU$1HW$;)6k7e+fcNILA!Q(9&JeI*@06dn#K7gU23tEQ7~c z@K^?qCp7vWW$<|R3V1Ao$0r3I%iuAO_Xf(~(b0t^JgydaEQ805@K^?qy96G~;PEAR zEQ7~Y@K^?qliubtJodq389esGV;MZQ3OtsN5mcgSN?RyzK_6af_^gB&nQwy(H9YIlf-P*K+FP0pdeB^*VuiEr*YF@L`_Hsb4?p zw;W#GcwQ!_a{TOW~~o-h60ZD&Xx9yj8%P$41H% z-hQXSTLru|UIA~FlxGIzDW^)#_W|-a+|R!p8*qf0gv7&z16-a=EpS?cpVe=TDXJy(66O=}(9Fv7}s_Za3eX_0N^u zPY!ZFX)RUa+che_RpP~QPs#z`UiGxxx)Pqsh4@to*R=xImE2!_!TnVwT-oy6zYksFephlo;5CB%!jm4JD)FS8 zxLk=((`cV7xgXfa{Xiu=9sP+Vp7?#B9A7?OC4TI~k4o+*9Y(TWyt@nU%u^-41>##J zetYqJPfnHiF+sx*dA;E-tKhL!;IRrGH^O5TJievEV--C1Y4BJDj|<_k3Lf`s@K^

D|=tKhK!9;@Kdcm+IG!Q&wf9;@K-!&uAG zs)EP49+U$-9#!G73LfJH9;@InLEy0p9{0dw6+HUDV--9e5qPYE$36`ntKe}iJXXQu zv+!62k1oX9DtPpP$0~S~;IRrG9pSMG9&driDtK&#$0~R{4Ubjuco80};89ZHu?il~ zz+)9WM!;hgJeI2PSOt&u@K^<+&O&)sGwu@j-3Y0g`}XC+ zbJU^r8FzV>@#bpEqnq-m<~rQDj%v>LFy~u+?1J6Ryk}ZXdAJept0~V!%Cnm5`s=HF zPkpcC{p;!z7wm4>A@B9_%WC|YCiqiLJ#QE4`SN@CKi81X~JcoL=ma3`e zqe4I5e7$PwdkOVjjc32YvueiMP6+b=)x?KT-sh{vzvu9;nsJ+(dH+&Q)x?MAh!55HZyUi9 z4{PwSn)u{=p5rs_bAyAt_LVQI@pqQsZ#CnG>v*4|8ow_6#1d}}cw5c*-3Ff5R5N~f zSQtO7W_;naUiYv6T<2Cp^n+7GWa@Ot0@t{Yz6iY)sVHSp>KuQl*$(@ik)XZf!7`IS_7}6 zXisY3HCTn$8hHJa2Cp^n>Hx1b@H!1%Yv9!&@LB_}+u^kaUIXE^243%j*BW>|M|)ZW zuX#U%*BW>oC-7PWuaCfM4ZI$K*BW@Oh1VK*-4Cxd@On|;wFX}I2)xR~)E<8Kn$wJD zn5WuE<^l75XD#y#7Vc#ESk_<;?+@25&Kj&{oX$Me{vm6y*^&IES%c$8D*3g{uWJ^5 z@5^F4=G&&dr+8zYZ~nP@<@Xw< z{EhXUPU^}Z752mY-|ueV`_H%akA2^AKefycFRy3&liT`(URBBG_Y(ednEbD{^*g`G zex_s%F1(-mWWLWV#*xia?X;}H)u#Gp{^F8*Su$U6{S(T5Ynh+D@=wZVi|v`8J@$T% zlfmzh{Kc|fYx%u~`iHG=Z@#}P{alBi@2R{`Ux%O1;Ab6v-hrQW`00nAb@&;p;%6Ox zPQ%YS{G22BS%;tL8h+N{XVVqslzOSd&uaXv!_OT2ti#V&@v{y;`|-05KR0Xm zS%;sG;Ab6vp1(=qR<5@?{9J&ab@;hV@Usp-X_}=v{A|R}I{X}1uWVn3pQG`!4nI5b zvkpIZYxr4*pGyQk>+sVLKkM)_8b9msGehvR4nH5k&pP}J$hXd89e%FH&pP~U#?LzZ zoFMpFho2#ie2<^ARQ#;N&xBgG$Ip=}e%9e<#J|bM&shAd!_P^k`ozy}6+i3nb0>b* z;pYbYti#VGf}eHxxkbazI{Ylg&pP}(t>R}Leop!y@Ux!!J4OA~bA5ZczIx7M8RtFa-5|-58!L@8pJx>$rxgJjc0H<=Q$4?*ptf&58 zr2gyS_a^wQr{3mMZ}s?j=^x}%|9_?a>*4nc__dbmssGc55hH&6A%pC!bfdiX7a zUpdv|=U)8$rBXfd$FEt5FYhU=PpUqP}@o*jSupXcF zDn8c}KezIHsve))ReYBBSC7x-_*_q1JfIR6>xqXM#KU_0ogw%;v_0{#n*K^XK2K2b zxdERXI&7r|d|HQ34fr$*pBnJVh))gp^f5j);M2&d`66`vaL zX#+kr;L~`)rv`lb@)z`}0iP}j?YUf^4fqs{PYw9gtm0DxK280K^TVf636v{7HRDqQ zJ{?r?sR5tXsrb}@Pj~-S6fQfKPrZJ~iOeLVRk#r(zYK8t`e9JN1Z9IT}7S z;L~_~YQU!rDn2#fQ#(F2;8VAXPYw7Ki%$*sG)KiJxtx|c=uB4nBaQf! z$h<5$HNx8#c(axo;q3stnWsj4I*L!$QX@X?pnZ~4BlGDKk6Om5p?rAjhPOt3k6=;- zOXk-H4RHQ=bszV6jqo;&`@2Sdk6=zVOMIH}Jm16H<^etvFP;$kiH-Q=d%tpg`8e{j zk@|8Rt;F|6>UV(pZ6t1V5jPs~Y1A9a_Kn1gD2@JOBk>}gc+p6`E}>oqXvwBk|6Yc-M#@ zNAaVPxVc;CKQEek5x7K|6EyKI_-rNIP?Mf%SalOs?*) zCj8ulpH29gm!zC`6Mni1em3Ffat%M5@bkQipH2ArwuYZg`1z^eXA^$9Y53WMpE>y1 zgrC*;*@T}t_}PS?PvK`1elEe!Cj4BDpH28#g`Z9MX?KRt_&HPXvk5;Z&sO|0FU2PO zY`sC@y9qz{;%5_n=HX`(eufHuHsNQfhM!IN>4Tq5_&E(foAC4Wcwzte=|MZ*gr8o* z{aq7&{supr@bfqL*@T~y@Usa&yYaILKX>D26MnvjpH28#j-O5Vxl6^*Cj4|0{A|L{ zgY-w5@N)%zHsR+c!OtfAtQ>})P5620yd*W@=T7`=!p~EJpH29A3w}1?=YWczP5Ak4 z3ESi6G!;La@N*)5HsR-T{A|L{0s3oA_<2#q&nEmV#?L1F48zYR{G2cN*@U0f8h$q6 z=K&2roAC3}|A3#(T>oz2`kSf$J=DLPnz{Z>T>s@#GuQ3FbvMKPzXa~(?V7p%!BynL z&mZBZnK-c6pu~Y@u3t|-SWeAwA1w4|o4I~3`oYccb2IV18UDX{M&Y@c`l%*x+b z;emN-hMzyePct6J&Qz{bUauMc-xT_@^7rP;X8dhd={L#iH4{H45qcoXexGx771 z5I>vocdsx$(@gwya}vq{e<$E?Gx76zmH64r?=3Z_aJ|IO`NYrW!}(KojAn_~=ehrA z=J%E+g_DoR?SjY6#LwkIf4Z5txrw;hOq^TcI6`VBer_OsHsh_sBKAZ4j3<6tOU=a3 z-_BxtelKYWzmImMshRj`6ym46eKYZ6jY|J_D4+PTOeKEE`7QXfPVlD%e_j^+X~CZZ z!|H7!5;(uwBSz~ z{A(}F)A;7<$woNyW;wcyVI{As}-J^r-dkBf#sE%>vW|8v}eKhv(lpBDV- zQSqk*e->)^(}F)6RQzdW{PwO~8>toEJ`&pdR_>oS+@{1Q*)w_B%Kh`+E_Xlocs}QBy&UB0l9z4pd|2SQ4W6HX=Qemg zL;KVQ&u9K%o#!@qo&?Wr@Z7D!a~nMGgy%MR-k`#B8$8>n@Z1K^6X3ZGo-a-2df|Dx z2G4Eqe4O^V4W11G&u#GhEAGd8G=^ZSXt|p4&{%D~G{z8$6#+wqB2I@LUbgZSXt~p4;Ggp1^Y(Jnx0)Hh5kl z@Z1K^(eT^`&tdS~&iO7>IbZqu+Bx6V!uhtd{S7MPo$Z|O0nWFb^8BgCMrwzv9PT6K z)Xw=1gelk4PXDH*O!23k^Nr>HyPfuJ742I)?dCM@C)@EO5I@>E-(u7G(!Ra*0Qt0| z2Wdy!;j@Hz+irTkq2gaVzJ7qO?WX4&_&40t4o7YRN0-~K9X|gd@Y#+JTkxSBzFXnD z9ba!3?kC&v;Q&6g!<8Pc+UY;}msxt&PCOVS9<VU6F8hmxY*G_@2 z4){9zqGdUBz}E)&>VU6UeC>cQKY_0f_!^B5a_WGu2!XE-_!VPk|-;obrgYeY>UwZiJFpc-C@YMldBl9>P(|G?C@YMld=l`YnA)mLk zWe0q%7x?OcujTO70biN$)d6421-?4q>pXmQz*jkZb-0UEq)yZG2cCCz!vCBsmfSyX=l;<=b;7@4qT*L4_rEiEUe*bJYrbL$|JB_} zzWMgnK6k=brRliT_hIV06aU{7+Lumv3$Z01&rkE8#!RL>}S2=aF|M~3STIy#1GuVGO=k++} zWuCg>XSB{<>V~%^@MfO6;U^4!y7AyIc+gFG9%`|S6W#FBtin$><6GC>&35oJ34Xfa z^B?fp4S#Fkubc6$j~L(ThM#rRleN?hKRe;48y=3qLpMHj2<>S%{G6wK?}pF6(7%vV zH{*4u81L?epV2D(bTeLeP#Ev;hKEx^``8V~3+y=_{P^+y-qogVc*x;>vu-$h49>bu za-Z3(>fz}tEi-g@9|fOypdZ_`wG>w&jo zcw&yw&lP@YVxw3pIG_ zfw$xE)&pz}o?M>w&jZ0&hL=win)d;4M~#w;p&qFYwj_Z@(h`$f*b3 zZnEQe@ODY)Kgs#SUG~76AH4Oz+je;CF^#VZy!F7_GoJY53cU5e z+iZC2fw$!-94KI5|#<1HJs7#75q? z?`8hOiE~PMn{RiepL>}X^VLCRt6t_i^c_|9-^=ea*u~pQz08;BXI@h;^Mz-OQR26} zWiRt3l7;!lz0B9SzsjSWKJ!gR zwUf_$zXuNs`OMGjXMSEU^Gy~-lh1rV|IgSz^Mx0D!S`LACpVvE$vi#XPc~97^UdAg zA)nuisARrDFY}MT-^};SiyT?P_RKf0c#rRyr!sRYpKt0s8O?LuUgkwcgz%a9KxsVx z@6|IO?@pG?54r9;C0_M1pK#Y!zF)xat~IkC=7+5HX34ztCv(_7x8;V@!n|+!I;|~x znJ>6`q;kFTdcDloX?&UEFh6}}BKv2)=ts}Vxt7ab=8M)WRKD+JzRq=fh5BT^_S^hE zK<{gvMYG|_JoPeP`{^6WXMWEPqj0^<*SYz6u8ZFbxr_M(=BbzYI;q|Y7rlSIVE5Y$ z^R?KoZS2!`@jhKI^L@Swu*{dg>1BS;J2%Lm{`<=@_?Hl)_%{as4p3fW@NXpJoWE2W zgMWS>D|;J*f8z!J#^9eH{*A%Ez4$i<|DI6sZw&r<3I2`2Kj-r!q%ruHfq!H0Z!i9h z!N2Qg+DT*Z&r$Gi4F1i>zcKiCvWwXNP|qm$4oE%fLT7{0qlFJN(;s4gT5TU%85ZcKA07|LpLu z82{|>&$>}*FT=m=@NcZ(pB?^nf2{CghkqCF&kp|zk0{^E+p)txFZ{E^zh=QdJN%2m zKRf(uR`Jgc|HcUZ+2P+j760t;&-N|0gMVZ1l7H1Wx5GdEpWvSc{#L-BsicAbRH6Sd zwVbK%HSo6^{xr6fxkKMr(r(p*~#SiYlpuY+Vi(Jtt1V;zKgFK z;>R{2e#qO=;OnVll!x&i>N6_a$JcLnseBLbkHfnLU!(EWFlmTKw!|Y1+<%{LSju|% z`>bxDq+!14n+bfz*N~;`2OiktfrfZAm3X9qdkY6;eGTz<_QMvE2HyV(?;7IL9^#Uz zq#-UXB`#^;{#F(4HN=_C#2F3o)P{JfA%0kGXTR_ubv)ZAEcK2+2dP% zKYX*tx8do^aT^^6zIB`N%^u(O2)^0l+h%;T$G739jpM4(_VG=FZ}#|hM8!9Id|Qog z_V_mRK5~4Uhi~@ywqu=@WRGuM_-2o9)hfQ(^TRiLd^`S_vK{#^dweUwH+y_rf^YWtHUZ!4@omd^wvTW2 z_-2o9bMeg{-%i}ma`={mZ}#~15x&{u+went#m6g?bHqxdX>D>M3P?x!i&J^`L%DB?o-9I%z36P``VrUpYC@ z-g6$ckR0%JC%!sRzYD2f2kIk_`fz~5TZQus2Yg+hN;&mxOFcWl-z}N))+EURU-R(Q zf%h+59zsq%FC?xxz+G|x<@h=iUmd(wopP8&j(4ee=Rm!OalYn2{chV%4u4O;p98+V zcZG5|wRhk%zRtW(>3oDW>x z9`XJ@Gx6Smc;H4naA30SKi;Muh#yk0ar|^39;BIx2M)x8k;DU2$pQa9!aoP%f<#;} zl^i*qTls${eI-ZusaJ{HjvP-i$Kwb;weaIeea`PR-ku};%z__B{7JwcM~=^v<8y?A zMR4E2n7*bxs_jyGMaj(BijDBGd^SBEI=??`;w z(U16KdducZ9EndS7*7}`N8+a!@zas`bd31q2tS?h<48OmOFVTXe$FF)%E^)Vv^+)G zj^hhGRyIfEQjUxGw2t`Xhz~1Od~hT_jU+xf5})P~pB(Wa2Os)Mj>JnH@zRm_G*XC9 zhU+;JpN2oI!~w(dKkKs-@1I#%Jy3FDe|i|-IKkofaOebgK5*v*uTSEi6Z^A?c~nvs^mkpLfaO)(LK%;BD1=l+zBAX$L3TD~Q0w0)=sb6Y=4w5Fea~e`|&7A5O%#`9go^M7+9}cx5U%5%=CF?#aoC_%Mg@ixYAD zh7RhV_*X&vlamwiLE?QCPV@)M=ntIeKQ_@HIuQq765^l}@u4S)?a(j$OuyhnJp5oB z{1N{viT_T-hZV#JC*J>)#QT4o=pSC9e{iBdX{0}KB0l&KADrkHmeDUb(H|7jA2`u( zETZ3VqCa?%{=kWTU=;m;6a7Zl{Z^7QKDyzfGd>>1M`wIIfRE1jxEvpy@o^kJI^*LZ ze00XgXCAYVobj<3AD!{>-}vZ^kJIqc86W?)h~@C{`Yq)6_&h#3<737)%JI<}AD!{B z86TbTadbF2KF-8PXMCKGkIwjbX*TQQZKjE|e}(HS3m%=qYxkF)X786QvoqyIiS zsA%10S97G39so=!}nzf{)Jlct!Bh86TqrAD!`Wir}L&KCUq1 zqcc9{2|haG;{<$k#>aha)IUD%7kqTa$Fcb6jE_F}=!}nF;G;7>ju(7%#>e7Bwu6s{ z@zEI{d+^a2A2;KpGd}9@(HS4>@zEI{J@C;PA3wlHXM9|OkIwiw8y}tVF#;c*@$o@? zbjC*)e00Xg8Tjaoj~DULh4b-OI3IVxgSQ@%qe$Pm3*O$%|ATO0e-^SorjiTehcoxE z9A4$&l?&Ig=kt7(3myhVD89Jh?_OaZfywO=KZX%MT!;$`pR$l#xE^84^*XSfA_7!-eO(T=4xKe0SmeyERkUFL^&)h!01I4=$X4 z&*6HP3)g$jW^g==i+T-HmgjnP=8(SoNf-S6?Pj)1 zJlT5<@x%pR8u7)&`jwk{ep2?|h4FJ9=btWI|45H#|KQ~WyvWG~p6_*(KN;sP@NBWv z_>g71~;>z)~b3Crp z|EJXdpOsu`->tN-oLu2yIXsw3uC)6;+T9h74{_h!6|U=f&cT)X9mxBnOeI%*vsJnN zB`@y^-_zjRFuCH}V3q&BA}{BP4}W{YLUM)gdiXX>uJ|z)KV0EE1HN7HE%R;amG-~j zz;WQqVSI6=Jy+A7uC!-6?db~dtKr=h4raiCEA6?G_H@PP0DN|Z?|Ja;3SV2{tFPpW zAKm!jiXT_-!^Rf`n+=zc~6aVDo#yH`aaQ*4(_w3IwGx5xg{h7@9kQ?o7HQ2bGuWpC>U9w5| zJ;pmeD(6FPwD-ZPfsz~Jo#~tpxxu>x?{2i$e%?1JCpX$>GWWmSsE08^JaePHchcT& z#Ir%A%Ko@9?x|$l<3@Y?(B5wF{ErXitx1v_yjQ}z8|`aF`?}%NI(%}2YrD6F5vVy3xK_D)G#X{&1#neZ!6ZIY79M=0~!)OXg*SJ2J1y|$4sS!@%^ltz7kG1rw`1_;4sVw;l>Kpsw-|VHhqoE<<_>S? z1>W4@Ee77);cWuExx-s7yt%{Mbb&W_c#DBIcX*p2@a7J0NBhB>JG`~Sn>)N|Rd{oU zx0eOp+~Mt6fj4(}TO#o04sXZc%^luW!J9k0jnR?Ao8Am>?(nw!{r>aj4sWMb`cJu? z-Qles-aHu3=9#&ldV~6@G_tdztVYfd}K+iDNik`1dFNd(f`Aw5yyvXy2tm`+C6R;kVc>?er4& zdHS0?;D0^*d(iGaLc8~OeR(+##_a{%5A(ny5BgaT;=zynzd;YqFD`R_;Q{{+@bAI6 z{U^rl9{9BizdSe(O%?8!dJs?kK|Jx`Jh;cqbzu+AgJZZ~^m~&Bek~OI@*uu!vsT*8 zgLu;QmJ(0o-|>XMb!PbUgue~&=Lvu3IG^`~zZ@0*JmGH^{CUFPJoxj3zyHFYC;Ux= zKTr6Z1b?3Jw;KLD;m;cWJmGJU8U8%s?~n?Ap78e~{CUFPR`~OTzmYsY;t7A};Lj8O z_6huX!rvhH^Mt?U@aGACK>~lC@Hb1~&lCR6HnLs#i@65=JmK#E{CUD(#vkC%6aK>B z&lCO<;Lj8OCeR;w!r#~M=LvuF;Lj8OhJHg1e{u zKfs?S{5c8ydBWe=Aqsz<@K>_lm_M0AFV2VO|7|T0g@N% zuYQx9_UokmylDSvD);BTsQ1=CQ14!}Ul8plC$Hl?Z^-j&UU2{0Zuwi|+>7In;`qJb zx=c8Kl$Z44{M}`$GXC(Q{U+0Xa`J-Thv3(X^ZApU&l@JMw(!lKJWt^Tx39yk7uNyi ztyJQL7w7-`r}915Eq>ZWj)!aT(2I5myHRNe!|nF<*$e+y;lE+>!jJY(21;J|@k6{} zDfPwmkq3A_&kO$-;=ifnh5siRUl=AYuBUv-b1q)^AEx5J7uQq%ei!BNoetk##Fr@I zix=@=R0sP-{HY}Vc;QVZaobe#;(FiweJs!Qnv=P#PaGIU9Pq+B8{ztcH+(w5r#F0V zfKPAu)WN4Wd_J^dfaDFIcda9b&m8#lhR>T+`1FR)J%4~tZ}`*;e0sxY6nuKa=X900 zEFZTwd=|r}H+*`i@aYYoiv&Ks;nP~+(;Ge)!lyTUT8=a3(;GgQz^6BS&V^5J_`Fx( z(;Gg2fKPAu+^~@xK11Ns8$J&YHEw@-zx&F);WI>qPjC3#xnrQ@4WHvx;0y5z2Wl`{gpR-z6YP)@VU(l zpWg5}1wOsuGbM-h;qy}Oalz*jfls-fe0X0$_1(tpD*Y;A_ zxPg)n^F7jT?SK9IFrVOZ7wa>xqRHNPJnX}K0*AdyT$1<8F!#xQr(t^U{xCmn%qz<8 z`!IiQ>QU3{`!K(_w4U{Nf5Jy;10)~jp_B;kPw-)WvELHo^?jJ{v0KCOGCyoXuu?xh zyf5M`pM99maWtNs`6+?HZ1?ZoaqHXp9p*AT_`%-fiw6~1TwTshRn z8|?@4t!+w_av$DHaL1jj&-|60SNI*~TOV!Y_-=pD`a$k@%E^cMCvVR-ZpW+T%sZ+1 zS(1F1Kb*w;bf4j;MieFqzr*~K4{zps<`0iJNPc4b`na!^?fEdT&iZlI=lu~jybr}N z`7odE%ah8tKD_T@Zw&3k{Fp5ntn7;%78I_~PeF z`00zErTFQKpRL#6r!Riq{w3?5HEy@xd2AHwk|F;^+96jQ#Y*&qn<8#m{bcrG9+zb6AL_y~@v{Iw zeetv1jGw;v>4TrX_&L0Y96vV+e){5P3V!E{SRX%^s`%-PpUr}wzW8|tKYj7@2!8tF z=VHN6U;I3dpT79n!+lv_{2VO!>5HE>X8iQU&piC}#m@sOe){5P=U?cj7C#T)qwpvD zsm0H7{FIXxKeyoLwI(fo?x6oPm9+R--VZ;u_&Mn_D@lu=tMF6Hyv~C^T1Z;_JogRX zKgI8yR(?mzd`TPLbEd^l4Ss6zb6~m>|Fq1PT<=Lae%`)JS6YJyW&pe+XCoO)yX2wq~em;kvTKrs$pIZD}A^54q&wGdSJ$^2v z|1?Zm{Iq?F_3_h3#ZN7Mh6;Xa@pDKl?TD8%@lwlt&NAVBFIxP}!cQ%JcHyU%dAB|9 zaa?$MlJ}Ts@iWbgpIZFv#7`}LzNO-)7C-a;LO=cR^TH&ho#guR!_RjFKmG9Y$nWse z4?n*#S`00nAqwv!YKO^ze4?iae ze){3(LHzW?&s{H@_R|kPSHrm~{5*o6e)yS$ zpMLmR*t_5OnWW;UAAXJ!`cFUnY!v+T!%qo6{qVC(@Y4@Jr<(E84?q3!(+@vA@Y4@J zZ3I94@Uw^h(+@wVtN7`MpUa+Ref&&Q@zW1K7Ycs*;pbHR^uy0l`00nAn*~4p@Us;^ z{qS=We){2O7k>KT=WhJ;!%rVGe){3(Ui|dK&q5VH{qQsBPxRBD_WOqRlaoLDzk>aj zlRx$UU!M1plRx`^yv9QEzsPu_ot*mr_jyV3r`_J6-Tc}A!R)`B{HgyNsed{7v;Vf) z!e{Egp69&`lRy4n#(#h6KmU|rsqtrj;zySdKjbC+@y7>$`bz%1&!F*sevkN(B*YJY z{4ZV3_q@-bbs;(Na>*n@o5x3`Qy(vUvkKP*3=?34i>FAMd?l96$W=XAS=JH~AAk z<`6#&lRy5968teN@9$@S;)f0WyFc%Du=(EjcT8=^A5R(?Z~7BIpAzDyspb3n-k-QR zfVk;T{A*cBPX9A}vQkg}#LriVpZ>&;9^!{TelNstIr$Smzv*3%_z_3^@aH`h9gcj) z??U|cCk}qVd4xalqmlUGPdvRNTJVYZnacTxocxI!mw67-pZNEM5dY+I!#sff@nC-f zsLy58X8`rNOsLNQ>MM!*3SfVN*dN0b0H;H1l;brlzt+zI)ZeuG_&wTZ)gNe|0P1_h zM;1~5?K58J*W~>PpuSzG?*Q6oHtiEYJQy=xIlchudnffBK>J*#eFA8gy|jyA3ZTA^ zQr`iz%N6=L!xTWg^C#W~&`ym)I|UFAl86TZ#Jk63c(8?d5I{WGOFRf5 z-c2Ol1;D`tI0zt~?hxW>0P$`-@h*V&9yFGmI9V?AZ$|q=yxk|n+W_KM7V#^9`1UjL zEr57ClK#$A3LxHoJxqxk@^S&hDI4Nc0P(Gk_!dCieQ%~*Dp9}hG$e8TH-Px|8SyQE zxa&mR4Ir*n64wHVU(<EO0*G_5!gYoK;#U{(D}ea+ z0P!t=IJsJglL5r9ImE93;+x0vzT-szK2LI1X;0!;56^7|5Z~@riEr}$%6R~B_Jj~; z1YQhp&jJ!zRE!%foM%J=?|OZ}C_Mmz)gByCLa_k6PlR4v$~LV;#IMhF3Z1@Y5APb@+L^C*|;J z1Fxo%4qk(V@v07gAHiQa>EN{zUUj4Sy;s>DUPt4#4u2~If92)nT!+8OjF)u8zisq4 zI(Y4ZSKTd)54nG(!(VsCOFH7;7Gb=q<9$w}c%PFFZ@Xr(KJo9U%6L_Ww~q?rB^~`o z8vTckIJlEIXqa@wzwyLB9sS1&`VSrbiHmT(M@Re{V`jXnBmT`H{^=fn(E4fa@9OaS zX~An9@y|*a?-~C7AN?GNpHBB1w__lFJ}CGZh@Y#j!OuYa?73;66o{YhDt-py=X(4M z#LpZ1;b$OzzSU+W1>)yr!OuYa{F@m+1M%}G{0ziTz2IjcevbW^-^b74JWp;a1>$D} ze90*gKlkHjAby?@{0zj;Q@#6#pKs%5Ab$F*_!)?wv+y$zKc~ZcAb#47W_kSV!OuYa z+=-uo_<7V<_+9+W#Lqzd^uo_T{PYq048+eu{0zj;%k-at_&LgqpMm&UDEJwOpALeb zf%qAPpMm%}6F&p-)1h~N@zckQpMm(f2|okz^9X(h;^!vnFAzT)@iP!VZSgY@KPv@4 z1M$-aKLhdeM;q41&&w))2IA*n!OuYaY{kz&{A^V5GY~&*@iP!VZSXS?Kj#a62I8k= z#?L_fT!o*3_&E_j1MzdB;AbFyp8ONfCxd8*T-qUs>j;^=FH=rIjF+Y|UNV(}xPB5e zZ=e)J`@c+l4ubDPAMu&-MAtF4$Muu*9(?Ay%Lw7ROAyy(rv6BI@BKkL%HjJgd0oOEV*v? z8vDuho_ZVhkMaG^Gwd(d6@F+|>d)}^g5diodZSelVHygJ5{y0q?=^t`&F>hWCYLcn^m674RMm z@3Y`N7~V7X@EP9E!+S8ikKlS?FuY$B-bWh@@5A6dxc7b=ya&Vkuuw}WxcB}V_rHVT zebZ}JQZT$<;<=V!c)w*-|9O>ii_yuSzU z!SFs4-h<)&G|%}2!@G+a-h<)2M}_xbc(?2a@4@iC+YIl)@IFf5Js95i!+S8i@9zii z!SG%R@4>zImsNNVhIdzZ4~BPFcn^m6O=frxhIfB~_h5KmA@Cjy?}Onzg#LXQ*Ed4o zeVM>}2)uj1dkFpe`@(s&+&-p?nB^xuL|#1w+HVD@E!v1 zh43E2xbhzUU$kKgf%g#t?;-FWAn+an?=J8j3hy@X9t!U^0`H;lo(Aus@P15%_t4(^ zK?3ig@P6e3mWTJL@E+QGKM3AK;r(5B4~6%4;XM@IyWl+(-cJ(WL*d;S-b3Mi4!noL z`!<31P6yBe^2Hr#A{iq7>q3~V^@1gKsPCJLf`@Wgf8@%7a^~z9q zA63ft@SX(kq40h{;5`)H7Ye+G!uuGWPcTfO@V*V+L*YFM-b3O27vg&;ymtz`hr;`0 zGrWhw`z96ML*ac*KX?y?cS(i!Pgfh4(XN zcn^j58h8(d_aJx=h4-Z@yobX3Oo8`Mc+U}d4~6&n@Ghq?cpt!hz_8x?AKd>8gZI`K z$T{EbQaRrZgZCbR_b_!{B`}@jVRwcRfMJqq5#;e9f^hr@f~bCMJe?@{m`4(|y9@8R$sc#Cm88P=Pr z&*AWX1H6a#UdK1XdwB2leRvOt_wi_i%XE3A~5H`^g;B{)WSQkP7eN@V?ay@8R&?2=C$W zzCwlfaCo5_kRrIwI*aFCBdCvsD)kXTd$bDe5kY;7qCO(<#T8#7xW4Bq zJa-vEedJLea*DviMZ$HX2mx$i`9l~=u5%@LhaV37q zzZWr^^?C1A1b)1V9}&c#O5#rhUZ&t>1o6Xi7&+H#)9WZF{x}eSBJg|;o=4#GWPI*# zionMdeEhRg1fG}Sc?9twjCc@%*SqmLqW8WC&mlzMlN~-q^xmJ~dGQFm9;M>7ygw0K z*Ip~!PmaL%7wnDsm6w+(9|`}h@E-~Pw+a6*M&6#BN5X%S z8U7>Te+>Lb!vD+g9|`{+D*Q*nf0n?1B>eXb9VkV@|3dhWg#U#C|B>*&Uf@3x{vQ(f zkA(l-@E-~PU++*jkA(k|v)Lc`AHw;KVTy$R>F^&3|Al7wkA(lF@E-~P+u%PE{y&BP zNcjI0{v+Yv1^y%9f1V2ek-hg51pXu8zxy#`{v+Xk2>eIFzb*Vn!v7ZdkA(jQ_>YAD z`YADM);5Hz3%}3 zk?2YPfDBcHh`g>zO z40*ZM&r!2CPUtMLlA>f}YMuim@Xt^V&>K8pF; zi;f$|+o*?m4}&ldF3MtB#oM1N_1)JIMDe~EO%L_M`(hS#8*g7;KZ^IoMBFw&iemnp z_eFBA@Xc?0PtJQ_ydJWYq8xZX*BWx>FaKysJ7{_DdJpBT$)Cd#KsB!Jii{r`!b#!KsoQ(`En0A?{_%#i-i<5iuXJ{Ncj%l zOEQJ^wqCUQnCGjbc)x>NDdl4}PB`gJd20CPnL5^6Xmi8b-u>To(dy1yS&sMXgu9V5 zUwYU)mgD^qpGWeU_f9;JPtLsa_-68W-nTPnfE0}{E#v#&4stz5lK zFHt}E;$g;@Xna|NFVXn26knq88NY!k1`#$;Fpwd|8Dr(fD#6U!w8lEWSkJ%cuAfjW5sQOEkV*!Ix-!v0>a1jW5^v zP>wIYfvkrwBYKYyUl!p@G`_UsOEkXx_#DgOOD(=cCGSrFo!k5kX z5`!-t^e-{^vIk#c@I`|!G5E3>Ut;hj9baN@-Z)`F0-y2av=z(Y%Pf3}!52$>iNTkf zJ|f4LYJ7>omub@}XWsUbJ!}tezQCIpeA!n{IlfH7ml%B6%zUR9do4>r7GKU?&;H?ykBTp`_|p8p$d_1r$$G4BUt;m){CB3uOR^ua_!4%o|GvcHi~AKx zip7^kGrq*)%S3#M#g`C#iN%-k_!5gR=bBg_Un<|=GrsKaBFC2*_!5gRL+D>(@g)sk zV)11-zQp275B*ClzWhM{5{oZg_!5gRd+;R|UykETEWSkHODw*W;7csNXz?W$U+%}3 zSbW)!FR}Qt3}0gLWjwyb;>#3#iN%*K_!5gRH{nYxzWCrvEWXs^ODw*WmQju`&*4if zzFhQYJ$yO*IXS*`;!7;Pl;TS)zPyPqvG|gMFR}PC%$*!x2H{IAzGUG`9G(orlQ_!X zc9o{8a{FIv9)~9xf+umj@4;!lY0lz!AIaD! zOqXBl@BdZH$KlJNe)tlHFKvgF`tECg;_zkpS=td_4t;OB9^&xj0=~rI%SZSUhcAcl zB@SP<<4YXA=$!bBCokej9G+z0NgTYluOBGI;mZkpiNlwRhbhOGdGs%F__7pV;_&4H zzQo~6Fuug$%N2Zy z`grp#-o)Wc9=^ok+hh0h8IQ8(kmJv(W#srW>FEJdJia`LFY)+t8eih^6?}=umxX40iN_b4{|9`D$Cpk0@FgB!PXDw2zQp6ptLJD( zd^vB%mw0^X!k2h_>BN_KeA$98@%WO3FY)+t(3#Kp;({;n_|k|k@%XYAU*hp)H@?K< z%MpBu$Cs)25|1yB;7dHdjF`ap_>zY&@%S=o80Gj9hA;8>vJ7A1@#Rf?iN}|t_!5sV zrT7w$FE=fx9AA#(OFX_5;!8ZfB*A+;z9ivGJife$FY)+t@*%#*7au*}<4Y{Q#N&%& z7~kVdE55|zOKk7&;mh!m94Ed6;7dHd491sueA$67@%XX{U*hrQS$s)gd(~_&f%bT~ z%XE7ru)XDMFM)cs=w`hMyvKPlpZVSL-tFMsfpL7MUSp}(zET49yPf)#Qv&|&5d2G^ zexIR!6X2;Bo)YkJlHg+k^?Minx2cpseb!K)eWe6=8qRnn0U!4WJ|p&-`3GTnMw(G){JL;r9}4sV4;PS z2=^breIhV^0sZ%@t>i7%^(FNyd+ z9sd*IxC@Tult_FTM0`oa|5E%;_`3#w6XE{wKoyr*NML->LAOh`%oQYnT#=|J!{i$J?cNYnYO_-nyw+X~!gf&q;_ENnCer z`^<1H<#Q7Cx|w=S;=1gzt;+ZEUrE%b7xkG0FH2Q;NuqySLjPteCDHGmXf|Cgub0I6 zygTRfhAD~jxN@G$P2zn1d!C~;m6AA5%!*galN`BT|A}#ZCehxjg?OI?&yUhyCUHHr z-~u`A?L~Vh(Jm)xmn3+88=jN6o_f0_pJ|Vyv_}%xwU?h{zwmm{H{^^DuRL!lB@urb z7gLUBb$FIUdq>dTNsJFS*iw$4{}ud9V*HT7_(4udj7KsVkC;kHj7M%}JYtxVxW2G? zhH*V7aXrV2=X_12B=~KI-z2VI&!`+ICBsvZz*90jMF~75!&8_Uo|56I6rPgdDM5v& zWOz!0r(}5YQ{gEYo@SciDH)!u7{~RMlHqBw3Qx)Kv<04$;c4)#3NMD+ONJ+3fv046 zD&23)Q!+fw5qL_5r!nx93{PS3lnhU4@RSTsAHq{IJRO9mWO$khPs#A~8$2b$lRrEq z!_!1~N`|N7@RSTs$?%j6PlwLJeg01Sq`>h; z6^>J=pDPazlv0TM>xla)@UleUMc!fxaef--6NV{;`thQEQfQy|&9qMn?en}!`^ei% z!M6yZUrd1)H+V^*|Edn>_~2*;=dCGtHv{kFlmb7e;U@*(G6dgJ7=OI5TwY3&Qs`f2 zjAA=@Hv#Wb7=QdUM<}O%o#D!M@FRwK1%@ewe${#g<#?yTJHwPhzq*5dHHGn4IpePs z;?D&k{unNA>T@c5jfSsO_*$dFS1NoRfUi{eYK5;<_<97sQsJw!AAF_4m&4b_?I`b0 zDttx3S1NohhObokS_faL@b$GyyiA2JH~310uahc#rNUR`w1HA8eC5JdDttYx!dEJM zy{N)hDts-5uT=PIHp5pcd}++^l?q>L1in(?>jn5qg|BZSI6n9q1z)M~wOHUQ6~3&v z9+C=QJ_28<@HJZCD;2&*z*j1K&4I5}_?q~%T&t3l3SU`nYzMxMer(KFDtw(~+?EPo zQ7U|;!q*Y_N`&279LUQV=4tTxh0q_RQEz(Udm7{0U5szj z;PX|XKT4zi7U5?aoOY!c|6Uq=%&-|KrNNgk@hgq`*>F&)pEURzaB+Z?#&~ui_o>q0 z?DqGRy-I_>c=$_$lV%}KD@*);|4M_uAzX(yOlj~jSKuQJ{|4b-8sqbSFg{O%zq{ct z4L&a9v%mN^693W|pFhWS`!x8g`-~h;4u3{D=SNSwb3B|Mc}W&h8eDiWeoW*1ZblWK z;cqJZr7?alXZ)VV`BqVsvR7&NcLM+9lm-_gxew4+O5^-4>~Z7c>uWv3@-*VfJ~Qzo z9e!^89sH!j&lZ87bodE_pLF;+4nOJe<1g@&4nHpNlMX)@RrpDVpM$doO6l-32Y%Aw zXT%|epLF;+N&HHOp9kP49ex%F{G`KACH$nrPg+0tNr#{BRpM7V{7e-1Nr#__@RJTd zjqsBWKZD^X9ezd-U((^HXF8wZXAS(M!_PMONr#^;@RJTdTKGwaA74ugDII>&;3pk^ zq)I-+&yDbt4nJGqCmnuVqLfxlhaam)6@Jp;XJ$Y6Nr#`3KY^cg_&Fo+lL0SUGrVNb zexIp~UotqKT*C7h8Px9}>NkV-3l-WggY!x2OBPZF^|~1^GvK5cPBJ*3PdTn|kU_nU z6ymp1Q~&c<2KC!a{bq38$h_?4H;pneY%w=>}PZh>F9T+TD# zXP>}N2E1PSRM}5?Ia7HCJU>P}G)x)PxAiH0AD-WV=M1iw{^us+IFP~hiX!f3X5if( zyvyKxK7{*^8SuJi2szg)?%;Wu47`iMy9}-~?cqL02D~mI-etgD_uu&p&#%IB2G^$+ z%~Sauc>X6m8>S3=)jeVy$1>oxS%`Or>&c%p@bm~Q-GT~_=JY~XD7d&Oc)7$Wr2~QpZPnq!4gQuoaCOqwir%ZU-pu$roJZ*!g zOn6FA;VBcIdf+J&o}M(rQzkrh2s~xN(~8gf&r>EmjaA_(6P_l+Qzkqef~QP)T4ZJ1 z&YAEO0Z*Cm)Co_S@Kgv-necQHo-*NS7CdFbQz<-U!qX{u%7mwx@RSKppTJWlJUt0d znegSwOg&EES>G8Y8~awJDikCIr9Un7Lqfceo`IlGcR*?G5N34teXNA z53(NWj+^#{Qjb}@=b-Km)?@z3%giIldZasU%x<>N{HQYr$d5d&UBY>IR?T*g_aA1t zn@^3{m%(;No*HqOxRKSc-D5k?+hsAI_Q6D!W1foL8|2J;U4Mf0Tef?AZq0hkr+sK1 z-~TqvI<}ac`L7%5g#BQ?)U>H=hxaDj;7U2~JFuI^`giPDKj{?fGY|Il?~G~B8nwl=S~tRX`*Zbs-cR8cLHqN5hAUsFlQ6@*lUa_u_fREatiCg#Qn5bw7UIzJ7HB z<;-)l-e4SevhisVK4s&R8$M;@(`bCk#-}a#l#Nfz@F^RgcHmPsKF!6aY<$w4R~8jVlc_>?C2l#Nd}3qEDzlMbJ<@yQ3DvhnFLe9FeB>v=9O8=vm~p6~JLJU(UP zQ#U?kPtW30Ha>ldPucinbpz|+(+GUZ#-~br%EqT6!KZ9| zx^)WM!Kcail#Ng8d0sRdpRDmI8=ulte9FeB8TgcqPiF+5vhm4R@F^RgLd^J-jZX*u zLZ7nnsg>uU`bydOAI(_s~# zvhgVdpR)1E@*S1+@u?o4vhisNK4s&RUd5+udrOd&*>mH&UUbAkeKB)hC>fbQs(Ed|s{~YRHPyOfM z+h}~tf#0X#H-~stLA;Vv4sq;CgtDC++W#=^FQ**hSa%BR5wE@@UggmKyJ&yIltcV@ zm-r#49NK-4(C$~iCl1|99Lgb{cE6&s9pcr`36v8jLx_`d%E7mCe9IwTxf8E)h^J1( zQ^S-)oO+Hp)mO?PUS<$4<&;A_>psbG60h=wcy+a$c=q*qrJm&P4fCAwKb>7uZ`{x3 z&=34ZKk$1~4*kLN^anYWycc+?vYj0IqXqOwIrIb0^aHv075sy8{JHoQgI~G$b(a3X zFy-P`?IlYo7r!)uU%B{o2Y%(^msapA7r(x{$8amk=Un`n_MIf<;@1mi{K~~IKQn&i z;@1>2e&yoV68y@=uL8lZT>LsclHHq$e&yoVI{eDTuSfAK7r%z#S1x|V2!7?_*D=AbT>RQB z_?3%a)~T$AU%T)t7r#!Z_?3%ahXucK@vB+IuU!1H#;;ub8p?RZFy-RcaK;0<__b2- zD;K}s!LMBWT7+M@_?4jIS1x{O`r%hDezgmJ<>J=?{L00zbt-=4;@72q_?3%ajSa?r z<>J@=Kj2p`evQJfT>NSj{L00zDfpF(UwZt?qh9K3Eu}p8a_Ue#%%lF!4j3rqG5_=I zB`YZpzLvkk@@&^`3_10&C{t;VJox$=zVg^^^h|QNc`;k@HV?k`!B-x?ZynEe@i(0Q zF%Ld%#`F95nvJh{_&p(zayZ)sXL)dRC(mK!!OKN>$%C&A#P2-3-E+4g4QvlztB#Sw z&G)O7-BeKVM@En%jFTTs?Ef!JmOWBnRs<|JItr`eVyN7 z-g0=e@qXtKujUZ1@|gd;+0~fiJmS<`;#3~QU@e7li&m5*;X z{%R%VRAsAKy;ndp^D`!nb^U z8#9jI$G2_xmXB|9@hu~-}3Qo?OpQGD|tS??Z>x#eA}<$ zTRy&ZneixXao_*RK;`S`X*#kYKXdmZ2M@vT_Jw|soN2jBAX?Hs=4 zkn6ij^CoW|Wm-N4KUYDfwb9&y#F?SB-#HR@2lm0i(>+;Bnmjj5G zdc410@Lo@xyqh>_nDqG3j30WuZ^e5(aq^!k=k0ppG5NM89(&IrwHPcp7(w1`-Yr2Fo-yy#~Uxa z(G#D%h);Un_p$Xqst8{zs4h z8G`?Y>&u_@#K~CVq@MSEeCKIgZ}M`6c>z8)3qBU$;}2(*{Vl*pOZuwY1^D=UKYT2}$Kzuxr2>4sAe;vo?ss3G3-EC>J{I8PY!x31@bMTv7U1I|6(0-m zaX3B};GIfR9U5d@R7n6nree$4q=I zz{ge<9}Dnt>=0wF3h;3@J{I8Pd-zy@kCAsuQUN|jenF0pPWV`WkN?KU0({h}_*j6C zQ_T2SfR7q{EWpQg_*j6CcjIFLK4#!!0Y0YSV*x%M#>WDDd;=c~@bNG{7U1I`&PNOI zap(gI--a9&;Nz|5jeRV@$7%RjfR8Qx@UZ|Nm*QgqKCU<8V-kpPA1j1ocQ`KOdcS+Evj2s>_s3M8b2U6}`Ew!HKNfKPLr#TU z|47(mAr*2Q@f=4X*FQ9ag>tTc=yviw*-dj|zz!Uc`+;#*;s3 z*}vZVTX$0q|4+hyA>*&NRw-N+QvWFzSq}bBzbO0;{O=O@FNA+f_?J^5{BOt;w!?UB zw#s;|kob0h_*TexZ3*MGYfXh*KjW2E5xrC@v9JD67Z$4_dZh)IoHn`HVN$s|9bc@#4o4M<+YW(5dIV3|IbQA z@HtC`&m#EzMHqhbw__XIasv`K@Bk)-SpUncFMesR6;Ijxm4+wk~!RKfdK8xVg zPwnsEvj{$WD%gMc%z)1#_&n33@KXezMJjw2!KXER7QttT3ZF&r83vz4@L3O^MZNcJ z;IjxmcM5zK!Dk437QyF>D$mIl!Dsa!;IkNh>fooC{<)m~xtR7}MEe)>dow?kq~hND zBk)m7|C~YpTul2vN&CyGxcB~uz=!<3oEO8#VfZMfUz`%DjBASF=M$c@DTa@8@KFpe z4+!VS#k7Bh(Ei2n(F`BO^cR+Iu^#OoLHifeFNf1F7sJPnrzoc$H&Bm;sTe*&RQQmW zD~698_$Y?^dxid`7=8+c@mn$DDW3_(+rPRU{9dl&_tkRvzlHPTVmNw+I9Cim8{nsy z{xCzhK5}(=+J80eFQ;Ps9xXiARZROY6579*@#g}rPZqpe@_zs zMl0>3K-eoK1qKbYaS1b#Qe zZwdTb3;dSA?}RcdsRVw9Pox}v7YO{8z^_)|w*-Du;I{;R$H8w2{1(G+3H%nrZwdS! zgWnSPJ*2{K3H+`!!*2=vu7lqa`0epfcqxJ3P=VhP_+5IJX?{!KS7(Ob68N19za{WH z34Tl9_cQn{f!}r&eoNqYuE1{z{9Y3JrxN&e6ZkEG-zo500>3FL{FcCPOh5Q7f!{@D z_$`6oEch*fUzaj|4}J&0ZwdTPhu;$T9r_44{JQ;#{FcD)YXZNe@VQ6me`T)ayc9l< z34E4Pe=G67l=GX{U$l@);d3<4Ym~y8RQqGSe zxnESuc~srJ-g5r$`c%G$PdB0eEv5dt-&Z&(h0hrHETz2%)83`dyjNa0|15>i%`T?n zN-2CESZS`D|I_lZeANBVOW`nI=#NX`ZIZy7;g(9_ZKS}PVY%VwQaHRrxZhez`vnQ@ zXS6&#?XRJp@c4p?$A-(xpG!Hvj@U~(aGo`#p8Cb(4+W1)IluPmqMY-swudPv{(O~7 z&iUA8;l5!h@$Ulhuaxt%dd|;E@mb$NPMrLPI9W>kJ3;&_#p7RIQ{sfYA7${^Y=*}& zc(lA;X~!~noDGj<@Hh<~%i!@6@v;maXTxI|JU$JNW$;)Ek7e+<1Rl%aQ3H=<@K_3u zW$>5;k7e+<03OTWae=^N89WXZd@h5>YIrPz$9v$h3?6?Lcr1g*VlzCJ!Q(kIJeJXa zoZ2JTO8@gRcnlJFEQ3dDGdz~T`3eH~F5 zJZj*v3?6TU$1-><7I-X!#}s%hgGcER%HeT(J~=!V3p|#=<6(F#gU4O)SO$+9;IRxI zFT-OQJX&(UzYHEH3+Dy${*`n6W6g9)DyKd6?)&2CKq`mV6nHJ?I>cUe5KW67I*#sT^Kcsyy#j&hfe0P5>94GOhn0Qdm^`k{RZ&(iBU!NAr;rCmWeqG*vIoChK zu9)7xtIH7&+J*i}{@ySzhyOXeceI?iatCqcuTJIoxK8k~0)C&LCP@|WyKxse{8qzn z1^mu}-wOD3gWn4HJq^DV@Vf_oE8zEre_Kfv@Vf(kE8sT)ek2gTtAXDN_+75TZw34w zhTn?b>y9e?R>1EJ_^p6nTllSj-@Wi#0l(M5Z$Q=fZCV{2mbat$^PU_^p6nH~6i9-%A3&74Un-48IldyYsic`K^H8ef{9K z0)CUKsWaE0%C}hm7INm#e3v6hmCUd4Y$fOY1%og1J@b3U4QvBJeuwuPxHGS!lKC&8J><;$sqA4tm|yoS&oflc+c@E1HowpOmktYZ z=KJ3G3cthr78fg)XMS(&6MSa=i}w*eGhgqMHUS%J_FGc`ro-^HmH}CG&f>R4B(;$$YADx3ir)H%=I#HD107f8I6YPZj;ZNOQ3#kf!X5mj2 z{#?SJD*SmJf2#1O7=Nnp$J&GM@h1U)s_D*jaA z&r1BM!k+~Eslp#U{#4;l0{&FtkFJp%f8N5MD*VaApDO%$5`U`jX950H;m;EMsluNW z{Hem9a>1V}{K>$dD*Tyf#-A$uiNK#K{Mm&+Rrpga_)~>H>s9=z!XLMO_)~>H_ZJy^ zR)s$^RQ##JpPjpmxsu!8R9=NY;b#1)!kDP_)~>HAFKFN zg+FB~{#4=5VZom&{ISBHD*XA`S2@mVxH&UW;lG;skH^;zl&az4AY4>$Bp&-pQZ?=D zNjq1=LksUusD|I_L6pNyW0TTvR#P9P)JHX({0mMDQ#E|N2Op+VHT5x{`lx1pWLF$H zeB{H2oT{mhV(PfcHb?Dy!sGl7e2#$6 zYWOdL|7!f7=)?N(k*~r>HU2+~|Awg={*RR^{8!_@E&l)BRE__&_%Ekw=5Ox4pZz3W z49_v$9`f?0@@nFzt(o{)O}w0VkMZ%z+p8vSj^O!=YUbnKegpMKT)cs}D5q-T<#W04 zO8oR7ej27~;^nv)%JDyRkCjw|4}*xKa;m|HUHDLg51a9!1|QP!p#~pp@u3DEj^INL zK2%syjt_^M6(4Hw;T?Ra!H4xKKGfjDSu;M=;Da+h)ZoKKe5k<(2f>FLdds@P=gPX9$-K5;iv51??VkfEH~pr4L(?#@u3DErr|>k zKFq|28ho%7e5k>P5%^Gp4@*>hsKJND_)vonPjWxJcGAWPN%xYoou(GqioWw&+OJa> zFV(`!Wr3HzxA6OxsD+m@sq6XTXz;$Ii>uNEKA z^8SKa;>6>hkrS_Nh}X4Slb<^!EczHOrs&eI34R#+N#LiNTjTe6jokzSQB%WB5{sFGc?se5u2i zjDGl1hcBBx>c7u*__B7Gg;a+x>&*C4hcE5;Qim_+@TCr4&f!ZPzMLzdp7ABdm(Tbz z8DHw~Y!IwIGId&)I__7IK>hNVO zzSQB%UHDRmFT?3i>+od)zSQB%0eq>$m$&hy4qp!7OC7$9$Co;M8H+D<__7aQ>hR@u z=DpY9%T|1;!>2Pvc7+zF4l{GrmmW zytEEqy0gjgRjY~OJC zdfN9>YnIzhyu8z})b!_i;(?xc@Ox7|{?y`6J-&K>V!WMt{7J(fQ>h+*!tkfBRF6Ma zD&rw}y?XqK!=HNM@;1gX^{4p%5^qr-#LG{2k`o`=hgwMWc;taca;nGM@pxO0KSyho z@8$9acwZv$-T?0r0`Cp*{+$`#8{qvSKATDn@UDaR26%r(h4%({*TQ=Pygv@_4e&l$ z;JpFfpY&&acprez4e*`|?+x&NP9lf*!|>h!@3+Bw1H3PW_Xc=B3GWT?9t7_V@Lmt^ z4e*`@?+x(YP27@G1H8Ws?+x%St)U#==L@_y!23{mZ-DnocyECB)0_`9z1H2biD!(u9hnzRSdnLR#!28)T^0$5G4e%cL2Y7FQ_pR{W0Pi#Q8MkW# zylc(y-T?3Ve(>G^@27{G_M!pa%i+BN-oJ+T26$fs?+x%?3GWT?-ZI2OYJm480`Cp* z{uI17!29reh4)6zM*^*lk4LW2My^jiKgvpKq`#R=f78hL!B%Db&`AFjMgL@&8mX^S zpPJsjynG|q4+|e<`&|F5;koTbcz*`o8@W!Gukw8I)%7_32RVK@HFCa}cefH}8#zB8 z8)y9cjrh5g{-}}bcjsSI${V>px0mO#8|j}*KIHfDQ>Wsm{ClSIM)+SU@ZX4^v+%PK zz8BZBUB-11HdtMRy zhj;JcT_Zeq_Jijp_{uTER}*|qg|8;~ng?G^@O7sOUrq4kq{3Gde6{XZ+D|?{!@LQ; zLg1?jzP7?w6MU_PuO|4~WrnXN_`0%Q*;*5PotvoCdlP&;p~6=ad_4kRP4M-(z*iG| zorbR__}TzpP4JcfJNRmXFRj2=6MQX$uO|4qrH1Xo*J$`^g0B$xYJ#utCs7Vxhv2IT zzN{J7Ho=#Z3SUj|HC*7U3BC>s@w*AWF6k8Co8W6C_xt741YaY0zfTiTeYF*G#SzoW6_wBsYRV>A8q&uhr(j~`}y-wb~< zR+*_MeE1R{es5}qvzc(#41Ya~*bmz8b)jEwhPN&7)(n3Oh;MRg#&-vNms2x7e2Nc- zshRPFFXQKC#v$E|L*&#~)2wu_HldF2ry4Un zwZPNlL#DPONiFa+3!YlwY2yeBsRf=&1)f^qsTiJG;ORAZYJsN>0#7aQG|voAE%0=x zA3U|d(<*psfv0Zae7FUkPT*k+JZa#m1)fgBQwuyTho=^JdIO$X;ORGbYJn#!cxr*C zoomS9$pM~P;OQ1KJhi~n`9Hu@3p_1?rxtj+jd<1qPg4b+THvV~o?77Peu1YJc#08t zYJsOSDm=Bo({gxffu~Odo?75(5g`>Cc#rHJnb;U zQ!6~ZYKEs)csjC)?ZA<{z)>sXy21RvmsWTh1y8N;v@O9(YMr-n!eZ|Cv@*UM(n>iz z`7?fKg|qPjXYzix!c(IV-{tS+yp{3dVaAKC@bn!#wQ~N@!1ZZ4wZhXJ;!`U;y$MgP zoIgaz$fZi&3QxuKkFD_Z8uw=mQ!70AnBl1vo^IlP`0q`v@Knfq=v(1wGx4dF^Od8V zue8F|uXjjNE9W~coZq*?)e7!=w8B*u{bVaVbw0@VobPN7JlU9uPp$CuEIhTr)3;`LYJ;Z~Gd#7y(g4W3>y!&4hPP5A>nwZYRmcxr>E zFX5>To_4@f8$9iSr#5)Xf~PikdJ>-6;K>)B+Tdv)Jhj2oX?SXbrv>oT22Y<}15a)6 zvrmMA%3>w%k}uu4*#9--;N(w@T2`C^~7_n za%zWv4;B8~;k^ys<0G;=+yo$CWj zxli8C^{a_0&rh}E&no zv~xYFj_cuaYUg^@60LH)^7nGy0dK#;TL-+Az*`5rO@_A)c#DF!4tR5cw+?vg=m&2d z@TM1d>wvfRdzE^Wf49GR2fURFymi3a33%&(H)~;h(E)Gc;H?ARtW|jHfVZszZyoS< zM&PXj-kubA>wvdu@YVruU2k&y@U{=$I^gXacWw+?vQE%4R>Z%YK; zI^b;|ymi3aYJs;7c$3wvdC@YVruUMjqGz}q%~w+?uFScSJv=2!3LcRFjgd))byh1ALXxrqS- zrOrUxOCzGmnWvoDqa1H1&o684QaJ8x-R^N0*Nr=wA2;epbVw)V%x76u%W)LgUOIR)$ItWi$FC!2e(0L5lrx{@BX7!?CmC^*rPRsu^<58B z&iwYcQsdw6WM1f*FqY$acK1-$zp#D%S*|N~GSAbWd7hojAB@Zt>Wk;^7e*=D>1?#U zA-x}ww?5E-W$)7u!A37vUS-+F# z?`?UmzLWW{KQRAQPMth|ze<=NDt~X7caHn%?4_?v$G1-Aqj@h->e+DpzCL#{Kh*Db zF`eyzo?VEkH(U+e!*b#ERYMUg-5cMgz6M061mk&TFmh=?q@sE7!N zh=`~xMnptJL}d{Xk!2BC77-B<5s_6y7Fk3WktCDJeJ7Lqn%vjqp2>Zmlk4})>+|Y# zeP?=x0Pgqm{nbC6nd-CZdEQlBJ=MW45B!S6FAw~hk6#}6wQVE$`1Q>QXUPMcPfnO)Cz%LK{I;G;5 z2YyxHmj`~W!Y@z!8r7xv;)!2t@yiphY6Y)kLp<@T={8|~{EEOYPy8|j+H&uSUx6d! zt!oNTDWPaGN)$9k)77Lzv;#cBJl6I)#IJ4@zdV`$HQR~( z;@5Hf^2E36g!hMe;#c`bg%|7kls|jo*J7dl^29G!{PM)FJC|R4dwD4jg*@XHIo?izq!UicNJ;+GeGZFAt47k;h8 zFE9LR6xuH@{92D+Uij63UtajN8o#{otA+N<3%|VZ%L~7D;FlME>EGfreytg<_~nIP z*(!c{;a38FdEwVA{PM!D!{N@JoYV75JsWuV%q74SwZ+s9c8zzgiY@9?WlceS{@` zO~Wq@z8%Lm4W32dnTGkz9dDA4XMuPoCk=jGjb9qP8YXzvpO0T(;+F=$YTT9W`t$KC zk9pl1{Fj~TK`nPBP^OP9d>*>$OuY1PWZm+?wg?p(7{CZ-lljMzGpZ2;) z-uN{Kzr68l4}N*$*LD@Zyz$FCM)AuVzwXB`Z~Xd6#V>FC`eC?yR<=uT{Q4b!dE?h> z_~ngXaSr_Q#;=c6{PM=H6bF8Jll7{zOyZFZ~QudU*7m-z%Ot7T8m%a_!W;|-uN}*kWilZ)gk!h zjbG{b<&9ruvsvQTNc{4~uR#3r#;+p$^2V>xZ;_8*#|6K<@vDyC!Fl7?6cxX`@#_M9 zdE-}tieKLN6^vir__b2SFK_(XgzpMm zobMyEq))t zZ!PbyC@bQ6;q3^#X^CUc634Xo`V78mdEbV|TekJ4C0^|o`rF!n(LbD|?7#o~h*xhC zue7`$Wc|}B`S?A9{;9pBC0>1_60hX-wZyCO#49;zi8oV-H*(SvuijA^*Ov3Gms;XV zK5<1(TH@4rt+N0A?};-zgg7IAFE6#kn*=XqKl1nTQcJv55Cn9&wcPs!ebwNJ16+& zgKrD*%?IDM;hPV>WeUFe;9EAn`QY0meDlG#Dfs4tZ+d=r>w|A&-gl9F@GTPGeDG}= zzWLx=D8BjNTlO%P__i3|eDLj^NcM+s@9?{1AAHl`n-9Jn!#5v%+p6N5559E}pY0_d zeDinUn-9Ltx&+^R@GXbmh0DnY-*&0^=7Vo*gx|UQ;M*2_^T9W%nCr#23;5=PZ*Sw9 z55DcfHy?Z}Qt{0P-@X@o^TD?S72kaD?J0co!MA-XzWLzWNC&?8;M)fdeDlG#M+Dz| z@NKc+n-9K?<9GH0O+NV6E%@exZ-Z2P^TD^34t(>$w;Bh&`QY0y72kaDEg9c@@NGyU z<&SU21>bz|ZT~HBh;Iq_=7Vo@@XZI`y18HVr9RIQKdqB5J`MYVeA;J;_Su(l`Do$2 zYx4TOj34Z~aggL|d2W%c#0y_|Nrx9-%k!ndEaBiC?f9;|FOqr7~HA9HB0<>X8M zYa91BzKkDyvWoSIA8UpFpfBSGiy1$-Sn_2&;E=?A@J!upKa=iZU@h?m8 z&zJk@Pw!OL^M$XmYp4g}*r0njFP^jf`(YuU_%vZF$9o?yc;4bmT)Rr;|LV!dv$yoc z&p^RXU*g%sVB58QxzG6Ztc&DJKmTC|{e0Uk|K}S&c#Bct%@5vch~IwjR#xXE`N7-I zD!lo@+gW(?vy9^ly!pYK2HyNE&s*Tl58j%Lw)N#_dENqVe(?5)z?&buEfaY2gSP|l z<_B-P;mr@;4hX#Y!P`O=-u&Rr7vB8fZ7sa{!P{5x<_B*R1m67M?HIiI!P^RW^Mkit z0&jlsHe~?3`N7+}@a6|^cep6L`N3O<3U7Y!<}dK(2XFoYZ+`GL3*P+TEt7WB58k#4 zy!pYKE4=x^Tg!(m;qCs*;LQ)-as=M|;O&u6TWb8^Edk#A;BEM2@a7K>6L|jSPdP28 zocxJ<$Ax}|KRk@)`P;>kKRj%J2Y=c-9qpYzT(rLFB>BVh7I^lDhYJ3`hCf^$hf9C> zb>cauKRnMQe)_{hBs}=j54l^oUzCsQ4-dZZ;13UX!Gm@3w>%#Zc<_hE8i7ZD;=4Qd z$8z$gU-TLMB7f@lZtB;c{*;FPlt2BQee`$y;dCvW`cuCyvxV(w|AOA<`l#P!#1nt| z7l{t~7yh)L=Y;lCKECzRpZaa_Q@-_w&(i{*{?zZcms7v^H5~N&ozPa)ksnR{Dcbp1i;UR%it#feqOu+egZ7dPXvAf;Ab%7 zZ~^c$_eN(ako$uXpF2r`l=l?MI}o1M!c!pS_6ptw(m&fh(M1ZRpZ_G!4FciGeE>WK z!qXIZvX=tk=`8Umka7zb=EVhC#(U{M2f~vpJO$EUUdZ#-K+5m$HIfua|2T#IaUeW( zyg@$qFU35cvQB~Ulpw?-IX@7dX9?|lAm#B8KFKK%o)6K!Tc<$F_^?XNgPIz9wE@ECs^#Fn{~&1>$GE(C-K&F1Ygi zClK!*ds*2okbd@EPb#179XF8szleQG`N-=9!uJOF4ur2EJf921!^bB(NkQ<`!u@6t zJPm`VAb6S&PeJfB9iD>V>A~Awq#$@&4^Khxv~2)91;NuW2RsGAQw%%>!Bd`(e3p_F zWEtOtryzJLdzd9W-33oU@boM^1;JA-JO#niXyQ{4Jb9|{6a-J_RCo%4CwGCTAb4^Y zcnX52Eh;<(S;j*h@DyYj4;6R{f~PI;6a-J>;3){6E~xMn1W&iZQxH7q2EbDgJWX=I zQxH6L3p@qElZkdF2%bg>JO#niiUIHx1W#vFcnX52eee_nPm|y&2%fgWQ!xCThM!=L z8^>|2Q!xA-8vsAS@KbXK`=MMOpj?6(AJ|gGlKOL{{(|9WJ^Tcx@?LkI0|#^6V^f@^ zVCpZ1`U{4i`NX4O%Ktse-#P`u&rJ9UW_;k;WLGJe@-fj*3TB+ekLORp@N+=mCm26A z;72gyE|2p2Bzq~Ca;T(zlT$GDl}@`Rr(nDa5&B2{-@{8fyj&~=Gfwp3aAALpGfm<> zsB#KsyvXe)iSy_GF?g_Kd}+@vu8aDbA+&G7_*95b_ENBAJWu8KTfy)eS4VkS#`9F3 zAK2TU{CzOI#0c$MF#f$K_!j~{d*CMoetuEmCj@?c2f$AV{3O6n2>h&vpAh&t1V170 z;|4z=@UstoLg1$ZenQ}97W{<3&pG%Bfu9NR69PZ(D*S}Nk30N?z|Vc~69PZJ@Dl<* z$KfXgel`gFguu^Y_z8iZRSx(GfuGTnmHG^Up9X=S5coNy!cPeNXa>Mf2>dJ+_z8iZ zVFEuP@bdt_lMjKPpZL9a2>hIcpAh(2D)18mKNlSE6Ji;kQsE~AeoXKaVi}*h41Pl3 zr$XQ-1bz++{DhA68NZ9)YuQVoZoF^uDJ4IY`P!xRiqD}Fe8z7ZFXV4#z6!sW3oYS2 ze@m3{A$k9yTR-rBH=BIpC-V-^Q}RQZ2Y76zs}#EZ1OL`i@}Fj2WS+AW%J137m{>AD zQrn>T9?JZkyfEcFDW*!vEgZbBEy4e3u?XI!MB`K8O_uurZd~`l<;JvXEIL&SnE_j4T)%9HtP+P%Vh^O$$R?>j@8kEU6#l$Ubkf&CTAeA!9NleJEv%>TW? zQl8BJcXzkVm)8$v-hcXZ`}tw`@x22-!tmox!H+Qfh!Ok^17k-4{N2rP) zVff*WA7S`$PVgfPKYZ{b3_nJck&hn<_z{L5-A0!9aZK@MRPZAVKhpmuY>yxA_z{L5=kOy8KMaB&VfYa$_z{L55`Ki? zM<{-T;m0|_k1+g*6Z{CnkCTEQVfe8HKf>^1J${7Y#{t2QF#OnwA7S`0Q}81UKlXg2 z_z{L5Zuk*~A75R8A7S|MI~6~|@MD&xyz%3G2Y!U%hqnVil#;%*zrx{p1w6|s9DbVw ze&z44bQ#Y4u;nLQrEvH?Lwpa%yEAwf&O9)eR7nbFKG%N&l%s^>-73L5CFlRGzryi< zV!q;)e0JgRTk<2<$^5y)CzSXr=MQul4j<7gDIfedG^iX0U+?;keEeTP`xs8#`-Qj{ zj*r7c%48P&{KhclR#Jzind*OJf!^3cVPQ&ML{5Rl#IC1YM;$ApDp25d(58m@S zo+bVd$NzBR-d)7KaC}^akKxRR8_xJnIDD^z?{MPXzlnF@_*jRJ;drwZZ^H3^EB=QQ z@9rSph2!HHd<=*Cqi`P%-$&s)9B=30Z8&~+;deMbKD&qYi2rkm|KY^D>1)`Yd2ae` ze8%HyezzY^{NGFb4<}wc_XMAbFHaI*!ihsa5QoBv|5_pb548NP*NecHzY4xY;7j^{ z!Iubp`4hfG;7cUFMBvK-e2KspeTpPS;7h*XO9Z}b`~QJ25%@Ah#g_agR;K0N0F3kD&-nUIh@<=Dn&X^{r;9Vr5%c-Tnj1J zNUk^kdGe{B&D2jMzRkwBNXqpU$~BU5Sjzvwi==)gQ$LZ^?`OorNUpz|>yM;--==&c zsh_zwkx%{INBu=|{gGUMB<0(fKtAvH~p4K;>B3vMI`l;Vc9SBQ}=h)3u$+~ z{V}CfBB{qW?od8k{}oBSU8gdCA`+k0;8P^?ymu}icsmn`Z%6504KzjK(4MUJMH0_86VD>?O^0uh_;i^2GdV>P z&n64;EE1o>@#%6?BtCiKQzY}e!=nd&-Isg3i)|N$PYr@kQTTL|1D~SsX@&!zqVTB( zpQ7;TGkl7|rxi?Lms1ozMdDKwK6T+!6h67*QxrbA;Zqbo?ZBrfe7YB(qVQ=xK1Jcv zN_>jKr<*tOJw7eNrzm`ygilfUv<{!5@af<96opU2h*wegG;%ok_;m7d^6}~Sw~>!e z+wdt0pN8O56h4h1?nL3!2z-jdC#RKck54W56opTV@hJ+Qx-I*`r-$$<3ZDkkKZwGo zK*o8Z@M*c=QxrbUbl_7IKFw9}DGHzF<5Ltqx&3+IoJ8T%woC9S3ZFvpDGHxXtN0X! zPh0RQ3ZG8nQxra}6nu)pr=^~XFH!j9j!#kebQYhY@M(+SQxrbU7krAsr=lzHDGHxb z@F@zPivAz)Ne6$RG~{;ZLrT zZmQ4tjq{ZK>G0_cKIwQ5Mcps9+~}z9b=0>GuU6reoOJkc*APk4QQt|_x6b#?4<~GQ zl63er9KUq>ifJGpYcjZJEg=>xd@-w9l8D zbi|X{!tYIW#Gi4(_=&y!>hNhcKIwSh$4@9c6Sw80BOa{~;*pN{9ZY}7UW&$_QsTG06pcU6sQ43& zKf4_G6OBI`@FyC7#^O&j{`BBaH2!=h_!Esk2k<8ve}F2 zZ;_;E{JDTX(fD)64ksxZe_H5|N8?X0{zT)?FmFENPZ9n^<4=s>Pc;6_#Gh#VnT0>m z_;X(HCmMfF<4-jHOv0aN{OPUdGyeDr{zT)C`%d!lM~gqv__MHr&-gR!YChx7D*TDY zA2aQ7H2$nt@h2L8-0>$Ge?0Lg8homf1>f{ zZ^Cn*X#ANq0Dq$KX9E62ec1np5Y{zwn8#Gkw?@FyC7;vM)CjXw`K@FyC7 znuYPTX#ANa_!Esk1rGd)Vf=Tp(OHV&{?Gqbmh`{wY;=`kxSutAH%N-%zP%t>l47|3 z+syB&tWyl!?T5PK1n+k1Baf(aXH1{T_WCDrx@;|M{iTgP0o)Ye$H{w|A--eZc>S#{o50_#%)#V ztN(lYp@W}M>aG8K;^#EtXAJQ-m-rhCf5YG}7XD7cUo8BsYjTxh;V%^aV&U%`{KdlG z0TupY;cpZC#loK!{$kT)CcD_$w0l zi-o_J1^!~;F9rT$;cq7V#lqie_=|-b4m;p47XFG<_=|$BTpizrlYT?blJ2_A3tVZ-e`Rra0>FsBr%tM}K20 z{q;Eb{RDpF@Nh34$|;WW9lhB}ii6*)xsQv(!`XNkM}K50_sMazKL>^WTO9uW(9C+2 zfB18Qq&V822f6=_qyBT>XNe!p_+c-_;h$$2$HV_q_#a1mrl&oN!<%pM=3*(1{>?|+ zAH?Cm9{=NrFYAaea*CsWa~1CkiGzctxttGiU>$KF4qt9wA+JSw<3|L3#NmIq;C~$b znRwxTAP)bx3inNM#Gh5fpE&wC?+W+nam1Is#24!nhyQz3{Ex#w9sb3`=LLb!c=-Gd zKI7r@7<|UV=U9F(_3}n6+YwP^BjC8aQ>$_{{-q~JkJjjDBt^PT%`ocXF1P965!)sJoibUUUF61 z?*#aGLU|il<+PV_O2D5$q5V#vd>*8H65z`;nB!9ae-?f}(0@GmdV}^mfqL!{>N$b((d+D= zr@Z~emI>VdOl{;m@GBg@5~%N1cgl_XE^nT5$SHyQu9tXTl0f}`H;*Ov52ycO+pkNY zehypehx>MEKiW>nbI}-%R*RguhMj zmk56?D*PqF-*$n&MEL8(k3{%uz)yQA5&lBqFA@IM!e1i%Y2Ysr{>C!DR!)iVHz$bm zg1`0fmk57(0)L6{=MH~~@RtC8iSRc;g}+4j+XH`z@HbWX{YoPIHT(wrCBokb_)CPp z)9{xFf06K)2!DG7{u1HuAp9l5UlRN!!rwwU0)L6{cf%$pDG~lA3H&9(U*&6@C;W}Plk0%LX^SWy_)8P`lj~VtCNVysV|*Zq z>l?-OB~jmIp}yty{#(l=>Tj>mJ}1HdBk*r8B{6={#PgdZJTMKQolc_u#tH4Syx%1F zy@mEU2@mUNpOYBhdeuXck|^I{{2n+7e!qv`Bs@H$;$i=O@b}j`)}y=^@q1x8B~ky& zssAMW@KNz2iSes3{9ZN*|EdaE;{O=@Pr{d%1z(aFAA5a=yjR<0660dmzRr2$|03F- zB;t#K_>#o^>{BV0d`FOtw@00MQ4_~ZP661r* z`F&gxaiI4c>k)r)i9bn-g#W{Z_BpA9aq_3xAO5Yv zKkJkXpGEMQ44>2BGZ{W_I;g}`>-B%@&&lw)Uf?qsK8HKtGZ{Y9;4>LMM+LMYnVTh44*ZN6+V;U z^F8=XhR?k!d?v$Z#m5{6K79==;j^3mV={am68KDp&-)$lnGBzE;4>LMr@&`2e1^hj zGJLkeXEJ<#1)s_A*$tn`@HtV1&t&*q37^UEIR!qG;d2`ON$ZphpIhKF89v9-P9?)< z0DLCHXPYp7n+%@~w109+hR^fxnGBy(1U{4DvxV{1WcZvf@R`EAfY!e{Nh!>`JI#2G zbxL90<;1Yd=QM@+%5~|kQc3~u^L~Tld^^iL{6VGuQto5kT%5B1l<#MmCr%k8r7(}d zY*gBF`Mgs85U#0eVn3&6nT<-#OE^Ui#0EdF{9N$)6-Ch52AVC$T@?Q}FBo&gVbO ztG-IvK85+S-{g{iL*>XNIpqJ!djo{=yA-B%DGRI<7Yr9^QZl>y*m%26Mfsc()AiQn~KyBb}vGc)pOQT)*{pm-{)D z__1#=`^T?M_;tA{64-xOwEmiw<5=@#`FZrQ)0M4n7l)4hr$8|9JT3j&G^NBO~!hPN~G71;n3J z;`8r_FV)+^^K%T#<@DflMmq!M>}UuJvab1LyU74P2QcaCyOCH|Zu{-hG0Yl+XP zc()DjQt|5uex(wBj6U``N+tg2i9e~taRYHYl{o#AN}Nt5o)2y&pZNR&@i~=vbV7(n zsl=bT#Gh2+^RTCEIqyGT;^`wiAGJ=Y#GTE=omArUy~JnhluF#0O5BlCD(#_;_V7}Z z9v^l1XfNsU(SVP7e2iYN>|c+M9S)v{+uPs8zSrYp{_mV6Jw8TXfscB8+>DRx!le5fix3a{?aN@TfA3wrJJwE=3k8;xEW0Q)H{rUJ|T!Iu4pZFqUK zaGv;Rz(+kkYVc7%y>8E*LY8=`IVDMYe4LAqdVD;Jk9vIk5g+xf6(f`0;(L5tKz~4w zpYIFpU;lODV-r5=@ldDYVgGvgw-f*L_&5t6t&<)fOMWCDAAkRZZN9v|9v??%sqlc0 zhw)L5k4sd1)Z^oDeAF{fb(}E2Ti)+LmudKT#(|G%_}Gh&Y53SC_?U)|a|}v680hgX zwq6=Oh7NI-((v);EATN5AD81}8a|fbV;VkQFZh^-k2~=(4Ijs-w2x`{xD_AM@Ntof zk7@XL0w2@xaTPwM;p2IHOvA@ud`!bfEk35<<2U%2hL3}3AJg!06#w@z4Id5on1+w1 z@G%V^w+TL`;bW!)AJg!$7a!B`(TI;}_&6UQ)9`VLijQgdsKv)Ld^{ugn1+vg@i7e_ z=iy@-J|0u?F%2Jk@i7e_4+=h};bSm9rs3n)_?U)|uj6AHK7M;rlG5;T4L+vfqZdA= z;bSH~rs3mxd`!c~iTId?k1JGsOvA_P|4cqUmOAh;4IfwGV;VmGLB+>3d~~LLOvA@@ zi|qTDhL4RIwmzoeqbEM5;p1B>KBnPgT{ijn*qdbEi*)wyY*6-}j$h;OE1hyL7VZbq z@oQh9vY&Lyzf$O*q*Gs>p{`Oo^U?FPE>b$>Z9HS&u5`+MkV(l;r@q!?*e{24>T7SF@uc`QzPCR^rc$f~~&hM~3apXV5k#zi8fM4lw zy%(sR|bBSpW-_3YqYbol!0Hp_?3ZQZTOXeU!jkak6#-FzcTP^mEczf ze(CTl1HW|mm4RP-@hbzrg77N?zq$_#<&Iy&Rs71pubudnfnWCver4cSD1K$&*KqvG zz^`NYm4RR5@hbzrV(=>izt-Vb27av*{K~+uJ@}P@U(c%em4RQ!1iv!yYn+N-8Thpt zzcTPET*a>p{8~Q%zcTQvNyV=W{F;eh8TfVnGW^QGuN`5wer4d-m@Dur1HUEl<)%#hnS?)?yhr7&7o4O__*?>?nQ(D6 zTx85A9zMZ*+K*dCfgTf4_wF;e8&wXTneQ)08Xz zY{s8VIKR#T=b8A^dzAGC)9*RO5`KK(CzJQ|yxGI`;LjTT$%OYm!+R$D#H;Yre_q7r ze+cn8llbRM{L93n>*;r85}!X6`ahY(zos1W@#r8PWfGr9|3}`NBxMr+#tY-2nZ)N; zh4`FF{JWn1dnSA@p#8|?y)iDlHzpJAUEn^G_&1mMmr47viuS`gWfK3U6aVa`Oyb`f zmG~#GpGo|CR)~M{_rLWr3qO~-DxAps&%)2-EATT5Kd;8mEd2ZiKeO<2Dt>0+=Y#l} zg`al_erDn4-b?T^3qPM9fS+0T`5*ku!q30qXBK`Y;b#_p8t^jXBK|8<7XCr9(3Sm7JdffXBK{bmdg@9XA6F2 z;in!yv+%RTfuC9Uc?LhT@U!hH*2m9@_?d;Df%ut)pNH`?3qPki@G}cP4fvUbpGo+c zg`fNIGYdaI#?LJL+>M`E__-NBv+&dGz|SoFTqgLLg`Y;=UzdfS)A2J4Km7$iv+#2` z{gW*G+=!o9_}O<>rcaWx@N=->XBK`Q7W~Y@&#CyCg`X4gGYdbBKXRVGABFe}Qvl%}gW8|~td7tpR;%x3mS8+d@&G_-qCh{%M`yN%| zSvL3UnFi%NvKcRVhVhbY{0YXNZ2Z0kzpYa?{$#q7Z+T7>&i>*52Hqbbr)>P)BKVsP z|Mf!uOI}Z2X5;TF{FPHS{O7>`rKW7gCmy>|<$U39(XYzhvZ?=Xg!<2hzc7KnZ0dhM z^`Ff+#c;+ctx!cPwTJf*@<4*c{^B_Dno%5Bdh2Y!~rPY(Pnfu9`s*#kd0@Y7q) zXUp>$fu9`snGHWV@G}v9axBkhgy+RM@Z%@&lLJ39;3o%uPQXtN{LE6}CkKAo;3voO zyp89KIq=g#{FGA;{CpwslLJ4)K2q}K_2gv^{5&e~lLJ42m%&dC{A^L-CkKA+6@GV{ z13x@-Chp-{2hLB;KwN3*T~l=FLU5W?|`2i z_?am1lLJ2o;U@=v&cRQv<#~YHASsvgIL+@faw-4mlz%SypIh>&|A{=GvzKzI|Jg$Q z=fcmw;3pS;o+bX|avj-RM=tfhllqraF8s`eA35bx|3|6+fu>yApGPG25BF`|N}Q0l z&!zo&ocs4&>d!QaB^=%ehq<)tH*RG~`(5s!{qEln?Y4n-JJ<5OK&ZF=eA@51+>hEz zxwQXFi#Q(qeeZz3T-q;B+OJ&vIgUTM^rwys;|002U#Dola_L9yq#u<_J#M5Pf2)*> z|9SW?r(FD*fIqqLvlo7HX}5>-`-fc1^95l%Gne>tg7{;da_Rpi(f`Y}JdYy&$SD{9 ze|6_N=>Hk%|K(Z6zllG2@K+Pe68@frzdZQc41am>H`oDxdGNPN;4csUg5fU@{_cmr zJopQMzdZO`CGeLAf2#!k^59Q<3H;^3-w622gTLRa#N9mj+X{bq@HdM1kq3X~otzi^ zndw*N!JqR9`|(RIuRQn*hQB<^__zvxdGNQ!0e^Y$cV{8TgTLJl_{)R8qwtppe;>kM z9{hbM@RtXF!SI&{f8WAi9{er&4f)H1zjFeAdGL1?{N=&lI{3?jzdHJjd6x0?M1}i2 z_zPCyFAx6C!(X1|`HR3`9{d^LFAx3(->q;dm$$skhmY;>k#89nr9Yd``Ba^AmGa@M z*;}c%e9liddXSXQc^L#R^0~ioI^rVbbAE=Q%5n1Hyv1Ai%=w+5f0xgF#Xe8lIF^qe zTKveTd>;5K=Yt=c@gpCeqKIet__0pKk9_*marCG2@na``)ex!UR*MkQKh5lE*<$1|=^0|NgW&ppp%g0Bj(YD8t&nqAP7xBEx zUdo62sc@f9eEKu-X`m^ec(k8*WG@xK+fjHcu#6MJTLHW!!dn5njUb-MsQ}*2z*_;l z1q-|tz*`yjvjype zc^3M}KB++P;J+sFsF9(da*@K#`XegJO; z@bHpDxHk07#MGP^%>TOx7`A7 z@^P+oX@IjO0%w=IKLfn|pmM(=@6P~lBUE@ZSjKO8{$YT(x__~Rx0~V3aQ8h!s~P_^ zSjKOK-?Pg5la~f~3wOX-l;jPdz-Ux4n@MeIwLU&R&w88LMJjst z1OGp|x=KY(Q@?+wl;sPn+^2H?Q^b1&KIHf3Ma&bMvyAVVhn3GftRm)>Y5&e==I^e# zm*wAh|7r#6F&}H?9pp1#ZE5`=sfhWz8s_g7F(1Lie3>HV3+LZNKJzNx|5}oYc)vjA zljJiW;bz_^Q^dUBgc0O3|6?q_lPMa-yoL()b35<({sZgrK8e7q$Y(xt4f7+4m~Roi zf%TXVR`U2-~_t2<}JK1jWKi?EFKiT(;;H2R%im_rqK}xD`iSlEi}C3d z2R;?!)6hoS?c}p6#-}0pRE$rf@u?V}W|XkRrasTiNW!KY$;x)q;_@u>};it#A} zpNjG6@Ay=VPs8x37@y|iQ!zem#HV6>`U;R7O5N^_tiZK0Sa>#rTwoPsR9@gipozbQGV8@#)vyY>!VvpX4(>HJP2H zVtg96lh63{3qBR&lkrhL<5T!y^6_apJ{99r2|g9$Q?LV{it*`w6`zXn=@dQ{2xg7YOE^p0Zd|HQ3CGaL)=PZ?QKG~d43G>m)j=M@F z@ajSQFM+oPc(YC=@cInAmQZfTDYp`M%Z4{Ol~7(YHA?i0RezT2c&Wk2%vzxA>N{(t}d#rIl*w>RLey;Op4pVEHWOC|7j7T!wm zdwi87mEiB4e__dd$<$GlJN`a|zxGlI{(9rDbt)m=93bA5;O||6za_*+S0O%@;BTyo zza_-I_9EMUNeOYTg*aD2yqQ#>)I$lrhvIt)anYH$XfKrzA14qW?WGdpV-xYwUMj)& z)%adQoE*8>cD)jOFT(c{;^P?Nqjf65_v~lb{x#k=9?z2Yz>oe_3G-yD^^`llZ^QQz z;`)gDSkfN6KzmR^yd3%<*NyLo@V$h%K8d(qLVMs%dr-oB(<#h1Eg_ztC7zcs57zxQ zmc+9i#Iq9GgGQ5!REke7-`Sp*e4eG8$JjLa)1@ycd_tCH_n%{*)4ba*02s#J>dMUny~6 zK5?Oxcy~YX&N`Li*KPP!N?a&AYMsk=#DhTMK`HU>zFjKYl|Dy4ye)#aQuz7?zDkJ`8;BF7_%#o|N{I`5RN_J@ex>18DRE&XaiJ8>C&Rg% zO7ZHCcvVVVI7VD3h4bNXZk3@{co(yHaL@Du3OT4pArNsN8#QRd(lVG7ev00DyWYr<1Jt-v~8Hh(^9DfGw zSs6SX7kDb8KL192mT~+E9N#*XQNO>&lfQ@enrE=&_@_C38P~U(>nr2>_R#*9F>iau z&n!89%mL;2Wz=ucKBa!k;Hg`sziNHl|JLU+xP8w-dsc==ZP##}aOo;lIm7KUcgo+gKECPJDcqF7?OmgleEF|3;>Bwf%6k3Zt)Zw3eLSsA`fd)xN7{l_O>pR3_Gc=xPIdsargoS2=#2#;m+^|@oPPP8S!g1ei^sY{~X2=zaGOcBYwH#mvPSX*Y5gi zkYwcjFC(V%J${wrmy!36q^Iyber>`pBYsW5FC%`1;+LF^__cOF`QgOn7MA$6ZZF$E z&ifYkyGlm<3jJU7%ZOiD4*W9W*Z3C70l%Jc;Fl4X@XLr_qe>*n$a_?# zJ|Ds#CuMs%zk>eZ%k&TJ zr3&t^@8SNsg8S($G3=lF^}z95ALIRh<#**3`2WC3mc+y3#6x?jf_Qj1Z-}&3y-un9rcE*dN_d+n z@Kyw@P?(QsJ!<-cG3SRtaz09q?8OZzJK2#{Jsm@KyB!c>4_AD&b8p@KyTySJ6Ltc%E{!D*RfBUvjFVf8r(dPpa^% zL-4B#-)p{A%AtQdcyog{>r_Si9bmo4u;Qa_>A}s_^gIc^VPQJDW7*Wd};+gtKsvAz-KjlE)w{xhR>8xNveiV z%?#W9R>P+OKC9t#4189@XO*yCHGCe1&uaL*0H4+Hc}n238b0&jvl>1PDtuPM=R@#W z4WFkfm3pa$PxE6+z4l)pd=9%Hf3jUx!)I`VlT;0#E8w#lKKEY&pVjbr4nC{lb3c4m z!{;~^KC9t#D|}YNXBGd)yc#}dz-Kjl4iosShR^K+pVjbLCh%DepSc2`)$n;W?U8k= zhR@k5d{)Efn=exS@aZ4S5KoWRKxyE>9$;0@3)5j z>_VR3)lhHiXW7=9_4@MX8hFl!=Njs(L1?dQxPO^cCrLH%yb_*kxSk_iPYvaBhVrSw zt35({T|<4HRcWtl;QtezJIbks`bwa_A@=hVRS!wz`v-=6!g z7?tOHHSjr0;IoGNsp&l5l2Z-$eV_8Y+g_@{mr1owQVl#u!*dPs;WY7~2A&J}eEz&ui^e^?6s61d~TqBXfM^mTb<56Z?*8Y0p4oiEnxt>)xz7I@Ky_N zx`&new0C^@`&xJlhPPUH8~&ZLT`j!bUL#4h@U|V^YT@lXyw$>66ui~K+nWM!weY4> z;jI?l`~}`>;qAP@TP?hufVWzBn+|Wa@HQLXYT@l8fwx+CTMlov@U{xxYT<37z*{Z6 z4R_E#tcAB-@Ky_NN$^$+Z?oa87Tz`vfVWzBGpg`b3vVj~-fH3P8G*N2cw6Csw_12x zQRgJp!rM7`tA)3{@Ky_N!|DIk!rM-GtA)2D72ay$?Ll~}g}43iR>yttOZ*PZI@Q73 zp(tmmj&Y`2_#Ids_wPqley_l<_Wf@mUS2HK!P$gI6kaa6KI2_WojESNorJeKc+(Ky z>Pi?d<9T`=e65DBI(Tyv+Uq)aTMut_@a8MC2-dNJ-prNfVVn$dkx;|;Ozpu)xlc>{eU`n z`(EI!j`5;huPgDXj`6M>2fy#BgSTz)R>!#4)BLVZPIZh2&f)i~_EH_=fh}J;Npf!Bic&mrEBLZ*r@U}*U zw|aP+?|`>@cv}Q-_3(BW-s<6PDDkZx-ckhK>f!Bsfwy{iJL`bAdU!huZ}srDOW>^@ z-X=QWtsdSce&r<9!&_T5OL!}Rw|aOBhqroo+jI%M)x(?4L3=85*}(jy$bEyP2Ie13 z5$0F*=QBUaZJTX9_U9kHbL}|(zwpIU!|S~7&&B?Jkul`e6uYvhzn|#RUy#Z-2E8ok< zZ(#n}j6m{-+%xoAp2IaTKXma!%Jnrc|JCh#@|lP6^;~892Ih-Qn@D*x|MlEi@|mAe z{W{w-KO=1kpP4T*f&a_Z!2JJ|z4qf%|9SL=X}0jYmImgZmEO*h`64x&*`9f6zcMdP zP7TaIYrNNXe+|rUtN74aYGD3ZLWrx>z&zyUZh0$Z*}(i`r{Ae;&wS#OtJt3Ty+_j6 zp7$8c{aDzJ`NRu1kk35dZXNmebDUI`ny?L@6iZSA?<3f!CBL&`nXmCY?_aS_ z4b0CNQNi`i<~TfOYjCO@*|1#c4>m9#Xl)z$%m*5KihSmS%;Ehsa%y1yN5v29pLswp zj#t_bIbU8jFuys!ROR^0qdfE(+cVE`hL`R34a_G_$r6soe4vQ;ZOcnOj=g0gK9=EQ zBR)Q%;$tH|N)CK%#K+7bw&!nsyhePyV*oxj;^S8Se`q5Q^dd^~`UjriE0;$tH|j>gAEd~D*mT_Zkj5PWRJ$65H;h>se< z$3}b{fsc*&cnBXG@i7S>8}ZTkX+GoQe0*%g$C?57*ocqr_}GY#*H2`LkHg+&dwe`B z_}GY#;VM2h;^UBA&Qc>jR^nqLK929Pu02U=#7F%QTfQ3cF%%yg@zDbx8}aeQPn7K& z@o^G9HsWI;J~raxulU%Ak1O!85g*UuVYl{+@j)RBR)>S$3}d7A0Hd>F-7pP z5g!-gVxPd8B?E2xhq=C4gQu!PTH@Y%%t!m;Vnwa+({0&R$ z!|8VW=g~xbId~P@;c>>hEa7z$yf!hv@!!0UqKW!2d$NE0{Z8;#KEAwc!sBaJvp)5E zu$U$D2bc0biYEA74Zm_~g4ccU+Jx5`D(~lMg4Y;$ZNlr91h1R$_dVLvCVakqJ^P2x z3j&|={^ey8d`beJO~l8Y#K$H)eiV?`-Aof2(CPexC2OJHA#?RCE*^Hml@Us~|rwM*GpUwCgsp4ldes&kJ9)1?#XET0Iz|UsKEd)~tJ&Zj?XFST&JCx!lO3-jB*dr|Sb1)h6d z?Z+E=zb){P;yg%dVSal`r>!3?uZ3x(b|qK{E%+7k#l`n4@85da0xwJ9r3KIa zq0*miftUPj^6@N7@T`UR5xCyWdBTeXFD-cXIG$Oj7I-;Lf3^j$zQwB+;!%RopKXDc zL-5jqSGNgX$@^)6m$5&Qk6-W8pKZan8}Y5>I*zxQ&+sw_UM`ke;KdbQTJY=+?k`)2 zM;nMoEqHd@GUdEm@a>oB%Kln-AHmX>SdaL#p8Lxd;Rg{AnRRZz4Xo;9Fu5-{V#3uUsebXNZr|pHQ~^ zKmTeW{%j)tv=E=&RN`|B@w|oK1-1~M&k~B98{^FR`{skcayEuOUM`U z-s~@1;eP`Bw^C0hiX5y5FYm#NoLb?(2L4;&X&*ebQooYQeM$fN@Nzr6Sf^HangdU* z@bVnIv=U#mE(*u3@N^cQ?4?%tKLG!&a5V|8TH(b7URvQJ8$RUJ3jfpKzZG7h;H4G5 zmJ01#EBq9}Pb<6x-b_BcjQv76-&S~92v4o>GgIJ4-oLzTg_lp^rIon3QHYzZ@YFy% zW-qnE)20FNWN-UR{l1lWo9`grwi3U+h~KTmZ!eYj-AbH2Nt|uPpYJP$`XZhW?tnky ztiKRvTZzvbh|jIGKWk}!T8Xo3g*e+vd=4c(x6=OXqWx(lJ}w|Wwi0i55^r0H&tp~E z^H$>R9O7*&@p&KdSx&9Q$K}LF>(s{ijqgfZSZYW`Q2^XPxMpxJ!czyKMmh)ynko*IhOb_1wYz& zU(KxbEaBVx4}4GkcdFEX8~$x{c9Ghs-zTWwHvB$|-*Re$?~U-?M*Uv*jZ*J&zPxOs ze*aGWw!z0_BTGCl68fqA>*4p1zc@>6)blGU{na+SUWnIi*W5GIZ!!C!zAGPP2_Jq> zQ9i_158`VZK2H;TmiI3&+wglAez);HnoYcqrVYM_(qCXKFj-&muY!gm{S`RyX|;d=vox8cto{AuHTIzOG^Jc+wM(XO`9zPQuAwBgTc{As(5>*RT< zoZ4t#mI>{P{Jp$vqkY*Xv@i1a*2^~9mjv3Ewm$AFk1E$^z24=1ZpY8r%nPti?fCiY z75LeXpVfk&?f7|4@UtC1Kf44!+wt=xezxOhLydC2?f4n1;%7U4p2W{~{QML@+wt>7 z{A|b1#rWBdpDXdR9Y5FbyYzPa9L4>nb!x}YiB2w3JASUi&vyKb5&Ue&&u8$n9Y0%y z_-nHt{5*l5?fChOi6wqYw8Pe^9X~5*f7|hMzKWmi__-cG+wn6KKilzhF!8t@KP&LF z9Y2rYXFGn5aNuVezxPM4nN!RQ-`1J__+l?+wpS_ezxQ1Qo+x5{G92)&vyJgG5|l@@$&+Hw&SPU zf5FcV`ajv{Z8?!ks)O;G9R*4|*ugkQ`v)#k2Y!{|R|n(UWhpG-^GVu|4xTSO`GdlF z2lw+u3C>andp19|_}%MSXld&UfsIv7uWa;xHD2Y$KYR|n%sMZY5-K0k!d4({_K zqnxA;`ggZ|Mn2<7E2oi9eN3Z1I_RILy(65D<@w<4#K0`kB zbHPFVbm0FkVZKEN_2VkkPY3>YzpmVucksMvexp#1^j}j}D*4vsVDEDW<5a`HRJQAY z%eUcDP95~`R;u*x6|nPWYS(pPleIRp7G|KBYuwsS`f42f$}1e4d8SPWap|@YxBUx58&9e1^hj zCwv}<&rbL}3!k0vner+5@VNs%JK=K#e0IX8R^YSK^1KW_JK=K^e0IX8r@&{Y<#`!= zcEV?Z13o+9bG5){Cwx|XrEuN}pJQpC24gwG2CpPlfz2tGUEa}s=ZTArW5XD58V37?(txfnh>;q$g{B&icV&#Lg*37@`~ zz-K3XE_J}Cnfh2ueVE~F9MAd8v}0HQ(Md8>|Bq4saxzmNbEpq9<+Jk;OU`#a=WC{2 zyQWu?%+yC9@xlxb*D#(SCo|`JobxqPA0xQmHN#ole>e{9=1Yttm}&2IRV?8v_H!k^ z$lF^l&GZXa(=RYvo(ECBX8IQ;^e?QFneyF6`Ce%@g950O#7Zs`)`j!+gYB65`V0dnf~-##&gWXpH0LcGjVk;akY!> z*9-Ad-mZ)5NpWyJ@_GYZc2OUp)JGTfv5ETV!k;yQKV8&E8THY{dA)2oFV6R6&bN#5 zD5E^O@TP${+J%QscqpeX%kwA7qYGc=;ESBPD32+WN0;UKlS+AXQ6Cqmk1on%Ddo|H zx1&B+%B>3zO?cRa|9kMi3-9xV{%aRLZaqjoe$T@1F8p^B;<0=l^0MoxN6bsve;08e zpE%Hkmr;1xg+I^YPZ$0-sKnzgeBOo6U6$uZLcEmsD=)k7*&Uy|@MIdEbP<2{5P!Px z@>{&@B97EnE9KL_9r0(^CB(}vd>$dh%PxHUs}S$I@VUc5doCZ>dfA2V+4$at*V9zI z?!wy(c-uuh2qhkLS)L=|v%S=XKe_nRg+IB1KVA4fLh!vC-Wr7dg}lFR>S+h{)D3U$ z0&o5K)Ynq#tJ^X@&+|b!byNOblz%tnf0FXQSn8(y*HZr7@OB8^x~b<8JSXhtzB;at z@8NRlF^&UoSs#ui67m>lNme*lv6kLdprH9ZsNtu#EWk3XJ6*|sCDWlzFinV zeCsCuxDtQtrEcQSYMy_06Hkt)#P@FEPrge0k@we4{JB8<>1MoSj>_}*ZsN~r;!ijC z)!*}+);e_)Kf3(m3!$KkmLo*z=-xd)!7!E=x0`GCN44?G*- zxd)yD2f%X=JTHXj9(eu*o_pYV8a(&Fvr%FR&&H!12cGxrB_E!bO=Jnr$1Z{A9(ax% z0M9*^@oRYQf#(!~=N`-R0eJ3#=b7-_1JC;eo_pXq1D<=}`IHLJJ@9(;JF8$M+-dnz;oLRlqWpTh36i4o~y!h4?G_l z0M9+}Ji!6aJ@7mSo_pYVn+nf8@Z9Zy=N@=I1kXM2{E`aKJ@6a}&pq&*q{4F#JddV5 z?1ASZet+M~IMx$;6@Po#zKrdA;rK8d_cDH!%J@|;<(K-TupZloTDFI`6U<-jh2L!W z?X`@r^LxTxe3*p~y^Q}Q@%*Wm@xkM5x z&ot|GoG! zU+|@uIB~BKCwlS44PSa0KfU*9_Dj6z6yilMz8rVpOE11O;7c!YTT4Hr58kH;y!XMo z>$8KTK6rl*-uvLa4&M9Vy-wi058hA0dmp?nXFfN_rZJi z74Y6?8J~vtK6qcpe49Razn0&7_rbdX-uo=$)9~I0?}y;M58lrUy!XNT$P}&{-nXgn z-UshT;JpvtPYb;F!TX>~;Jpvti{QNv-sh3|97z6HSY3Pf*!(g&dQ1t|RhN`HXT44e@4 z0zeuBKr~3d0+c^WgM(!>JVsc00ObKTAojNWWSgKGW(nsTrM};}Hp~dlHx_!)X&q(^ z=bQBO$jk^cf%8p&ckA$lnZo&INlmp|!pz`&bJ12`r7&|i-=ftjU~`xSoNxH&*$eS7 zLr5xQV3<}M$xr~yyD%^v$Y(&}&j9iRkoYr!`~)QaEFj+ii9Z|2cR=FL0rCZq_;Z1L N1tk7_Ain{~2LQchn0WvI literal 0 HcmV?d00001 diff --git a/src/leveled_bst.beam b/src/leveled_bst.beam new file mode 100644 index 0000000000000000000000000000000000000000..09c8b2bc29096f50d8948b26ceed54fa5e2481c5 GIT binary patch literal 4012 zcmaJ^dvF`Y8Q=3t@t)$dS33C>+wnOkCw3Is=lBuqByh1KE97B}gQ3YlWZ619vaORV zof3xxP#%U*pg4sDu9G1qJSGFvP8v$eBz=S_B|Z|G}6sOZT~|C&u}j93(*yqgh1%jS=3JG5~v92`s-j)W14 z8^PhoxMs6hY(mrXVq^IcJz_*c<9IMOOeSIp=)`pU&UnPoNK`XI;gAvHlX_Sij_6w0 z9*W09yRfe96o%qj$k2k@wNO}#&(q@LA$`O?6d!6{UN{ufcW7WNJyw*|$Mo1vJqWr4 zfEU1|U^J45hK!-@uK%AK+@UntjCfMB4~NDRnoW=Cnp2BT7`rlHK|@OzMT4Pmb{~hv z$N9ls;7uZOyH*<4hGOw>Fs54}vtX@NHn!BRB_ZDV(O5VM>WDrZD;m*^U{=4z_buxi z72j}D_bER6j?j2gbA=KK2yzpPA~+lZD`C49kH_NWzOT8LS=i+Jwh_fw;mZj!v#Rwf zV=Y%1Yre|Z@~ezBU1d!1A=DKMYY64P5A{WX*k=L-fJBa(G|#n&a?*-cT!5$=K%yz- zOQ_wCJN#Jc#7OMKRQCGCK2^qcT8lWPP@*b!tAp57Y;;P+HZ?A)ZfvUg5^dLThn8OF z)nJXy8tKiKB-MtUHgmv6pK9;N9sO9H!*1`FJNjjbo>V$z)Z8i4uI?FI;CMg`GzYo@ zr~db+@+oXiB%i*NljBy2ndM9wcGKIwC6)drd@~$fw z6-WwLWpT@H^Bf?0e}mn|1sg&*c>&WK0Tx=R38wGAfK%S*Mb7)Y?1Ti^sX!&}r0Q^4 zPM=d9Y46XQ=hxY|OZ%{S2|S-QXTb|M z6$b)RK!(k9l)8EWwA3VYJ+x&eajtiBq#S}KE{K{eFo_6pmm!W+6e@_-7jat!<{3dI zDc*n#p~}OOEL3H(3_?21Af(uy$l?e!Ny!GJda%=Dl2TS~LM841nkzV4CzjluSnQFFx!Sll!FvH<>`)kkOIzvbf?!O6&sL?g4dM*$$YIcN#!~ZM;4l-3VbdHpQ{UL z+UJGfvmw;LzKN4Ujp$6FozO^*1yDyOsa838k6q~%n`3slnaaSORF2wZBq4c=9g2tB zVu#YH*=n~|SY-y8sD?;J223KcG)S!0CItyw;1U#R1%U4Y1%Vi9R1WX4OT8k1W0_*$ zPKu*;3?}e5n=r{Db~1>ZIU2U?lQA-35>F-p(mtC>YN1$3jY$@>R4g*d5=cQMq(HWg znF8L{*=TLM;DNk*umTqd9!Z&`9x&jL5!jbHwzAelu&@diE>>OGL_KtXBTFo0H7M&c z$_b9tr!lP&yZ{OiH88NOU1$Jv4R~hmUhxVI;&HFwlLM04hrzCo5~_4d zIN(?mko`Ch`vWQl*2~xcV-r}k#-UyTIiF5*q(Ke9t&{0ALL$IkE!b1wwuBz+idSgD z4i}iPf@=aJv|ECDUlhQ8kr|n0&@+sk%rWAzj4X$J1>D7e+w2uuutQ9H+>#4+jzOGf z>{1SxWP!CpxdPY{0BiFKE3l(94Ync|Y?eU~YH?={T!jU94V(jjs{^=puh4-5a6MS* z?8wC9Kdo5@LD*-0JszY zuk#9U<7D<*&BaqKpamAM$bl=f;MTwe0&rf5Bb{EM3v<9kXWGRsN<{0D!*I#vRuZYS zkgIZ#ODyEIaEk%*GH}!n;vPHg#n>x!V?N_?cdo}|p#@oOaVZD57=STCn~nHQ@O#by zq3fd)Ox^?J0D@5j*9FSU$RtbVGcuf8BmlAXML_KTggLn)BQJ!uDoZbYO=kFPE!@zE z^G{kngh6KZs+kUGX-0+y)d4X)_ALggv?PRNXv;G)TUVBmnIBFdhQs1w_L!X4ozz*p z9{{m#rjzNIoY6C%SpPbvV{%4cpV6;^mgR6x&-`Zd+kseK z=k!Z5`Zj3oS$Y;5o8JsHFH2vW(JRpAXX$G)dKL%EorS+KVnB@C32bVxxH1-@zXQ=zL;P9iTZKsrPlbTdmmlgbL9p1iQ8)qUU_K>AA9}Dkh751 zpFZ}{8@q>gqk(et?$EK9f1Y~r%uTm`L|%=2STcR_)v46p(?{kt)`x2gw;%od^%w67 z)_nO@`|Pz_dtMgGiu%JZwEgL=SK_Oe|8#YiGtaR}FBrX$8ocK(r=q7{IkWc2?#s%y z6|LQe3hq6&a^{1rhktYm38f!Q+&p~C3GvAC?%n0Pf=e&Q4vg%cREYoe04HC&t;Tos zomJX>$%d9Qk3WqI?|8;_Hhj44!#<(-^x?lofA)}X;$5NGUiQip6$Da$Hy}WmJYulN3Lald}k1m~AT>R#3-p?PKJ3t5L7@3S*)^G6huO84U2oYn zmR(obwUk{y*)`*4H3P)XaA8I!YYZa}a^3!MrrDksMtDa;w>mmS_iy{&+kgD?idJuj z{U76_!m82HuIu$-n0c=?-^S+W$0pc2Gcjs)G7rNsWN&Ry8d{jnW<;Xe_IZO)%^!vA zp^}goY6&&7k$H(&GCrh@q4vgpcz`7uqwubc=!s-h*W!)g!FViYG{URbdijP28x!%N z#;kX61H4p6!I?GiCWZ!Wgl8=@%)hyBJqRp+h+oV5`E~sD{Ca)^zlpz*{|>)}zm?y{ z5Ae6~V|)2&wrnPkUzx#fPaX8n4jh!;E(W+ I@Q)(&Ux8U#yZ`_I literal 0 HcmV?d00001 diff --git a/src/leveled_bst.erl b/src/leveled_bst.erl new file mode 100644 index 0000000..886705a --- /dev/null +++ b/src/leveled_bst.erl @@ -0,0 +1,156 @@ +%% +%% This module provides functions for managing bst files - a modified version +%% of sst files, to be used in leveleddb. +%% bst files are borken into the following sections: +%% - Header (fixed width 32 bytes - containing pointers and metadata) +%% - Blocks (variable length) +%% - Slots (variable length) +%% - Footer (variable length - contains slot index and helper metadata) +%% +%% The 32-byte header is made up of +%% - 1 byte version (major 5 bits, minor 3 bits) - default 0.1 +%% - 1 byte state bits (1 bit to indicate mutability, 1 for use of compression) +%% - 4 bytes footer position +%% - 4 bytes slot list length +%% - 4 bytes helper length +%% - 14 bytes spare for future options +%% - 4 bytes CRC (header) +%% +%% The Blocks is a series of blocks of: +%% - 4 byte block length +%% - variable-length compressed list of 32 keys & values +%% - 4 byte CRC for block +%% There will be up to 4000 blocks in a single bst file +%% +%% The slots is a series of references +%% - 4 byte bloom-filter length +%% - 4 byte key-helper length +%% - a variable-length compressed bloom filter for all keys in slot (approx 1KB) +%% - 32 ordered variable-length key helpers pointing to first key in each +%% block (in slot) of the form Key Length, Key, Block Position +%% - 4 byte CRC for the slot +%% +%% The slot index in the footer is made up of 128 keys and pointers at the +%% the start of each slot +%% - 128 Key Length (4 byte), Key, Position (4 byte) indexes +%% - 4 bytes CRC for the index +%% +%% The format of the file is intended to support quick lookups, whilst +%% allowing for a new file to be written incrementally (so that all keys and +%% values need not be retained in memory) - perhaps n blocks at a time + + +-module(leveled_bst). + +-export([start_file/1, convert_header/1]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(WORD_SIZE, 4). +-define(CURRENT_VERSION, {0,1}). +-define(SLOT_COUNT, 128). +-define(BLOCK_SIZE, 32). +-define(SLOT_SIZE, 32). + +-record(metadata, {version = ?CURRENT_VERSION :: tuple(), + mutable = false :: true | false, + compressed = true :: tre | false, + slot_list :: list(), + cache :: tuple(), + smallest_key :: tuple(), + largest_key :: tuple(), + smallest_sqn :: integer(), + largest_sqn :: integer() + }). + +%% Start a bare file with an initial header and no further details +%% Return the {Handle, metadata record} +start_file(FileName) when is_list(FileName) -> + {ok, Handle} = file:open(FileName, [binary, raw, read, write]), + start_file(Handle); +start_file(Handle) -> + Header = create_header(initial), + {ok, _} = file:position(Handle, bof), + file:write(Handle, Header), + {Version, {M, C}, _, _} = convert_header(Header), + FileMD = #metadata{version=Version, mutable=M, compressed=C}, + SlotArray = array:new(?SLOT_COUNT), + {Handle, FileMD, SlotArray}. + + +create_header(initial) -> + {Major, Minor} = ?CURRENT_VERSION, + Version = <>, + State = <<0:6, 1:1, 1:1>>, % Mutable and compressed + Lengths = <<0:32, 0:32, 0:32>>, + Options = <<0:112>>, + H1 = <>, + CRC32 = erlang:crc32(H1), + <

>. + + +convert_header(Header) -> + <> = Header, + case erlang:crc32(H1) of + CRC32 -> + <> = H1, + case {Major, Minor} of + {0, 1} -> + convert_header_v01(H1); + _ -> + unknown_version + end; + _ -> + crc_mismatch + end. + +convert_header_v01(Header) -> + <<_:8, 0:6, Mutable:1, Comp:1, + FooterP:32/integer, SlotLng:32/integer, HlpLng:32/integer, + _/binary>> = Header, + case Mutable of + 1 -> M = true; + 0 -> M = false + end, + case Comp of + 1 -> C = true; + 0 -> C = false + end, + {{0, 1}, {M, C}, {FooterP, SlotLng, HlpLng}, none}. + + + + +%%%%%%%%%%%%%%%% +% T E S T +%%%%%%%%%%%%%%% + +empty_header_test() -> + Header = create_header(initial), + ?assertMatch(32, byte_size(Header)), + <> = Header, + ?assertMatch({0, 1}, {Major, Minor}), + {Version, State, Lengths, Options} = convert_header(Header), + ?assertMatch({0, 1}, Version), + ?assertMatch({true, true}, State), + ?assertMatch({0, 0, 0}, Lengths), + ?assertMatch(none, Options). + +bad_header_test() -> + Header = create_header(initial), + <<_:1/binary, Rest/binary >> = Header, + HdrDetails1 = convert_header(<<0:5/integer, 2:3/integer, Rest/binary>>), + ?assertMatch(crc_mismatch, HdrDetails1), + <<_:1/binary, RestToCRC:27/binary, _:32/integer>> = Header, + NewHdr1 = <<0:5/integer, 2:3/integer, RestToCRC/binary>>, + CRC32 = erlang:crc32(NewHdr1), + NewHdr2 = <>, + ?assertMatch(unknown_version, convert_header(NewHdr2)). + +record_onstartfile_test() -> + {_, FileMD, _} = start_file("onstartfile.bst"), + ?assertMatch({0, 1}, FileMD#metadata.version). + + + + diff --git a/src/leveled_cdb.beam b/src/leveled_cdb.beam new file mode 100644 index 0000000000000000000000000000000000000000..301deaa84169f745feaa0f75d77ee0ef31f00efd GIT binary patch literal 21212 zcmb7M34Bw<)=#);J1I>`n-n&ZvWi7ObtX! zUJ%7y0hNbOp5g{B=o8SofQSzS6ct6l1s4?k{%7t@iGc6>z5blc+%sp+HfLteERr*N zY)_qT?1w|LGe%~F0+l*l>}s7(x4mhFZ>F!pS6Wa~T5K$<2~-x8`b$Cv`iThz3M%}; zP;^O+uPEd*l=&-shCsEis!3ovs}2PHA%CE%i9Zls7O1H#3PqJv1cE+8X>Db-AyiZA zi=wtcA?TYCRTiiyt%&y3R1{T}3&p`2y_17#%tWoAs3hc{=_?@ef-g|U(737u42A02 zP;~XI8h^-VEUpXr3WEMQKD|RI_l5Lbh2lV2Or<{<^n+wcZB10Ly22ldE~*B_(ior= zRQf8LQeNeo9h&Z|i>(S&25NjL1>*=jttdDxRO9nS71h)f)v@y80{RL@R|EpnYpbJX z7FE>x;vhjIVsveFDM=Odg$z}JDqpm}D(I^T#e;keR4AA}v%1J%Bb3yXM3og)1bs%7 z3T6eWrq{&>ONy!r0yBLzT*)Y3Wp$`7rqowfR9g`es(iByFmP!+@JpuoN~RZZ-a=7D z1uH44C@BD(75hV6eWAuz83lm57xfPHlGJ|h;HLy#3vA?RQrmjHiM5i*z7f#Ffo*PR z1`nOkLjLx?zDq42P>C_fZwDrs32KlZrOgaA+dNwMv&J zHV!fny(nL8l|R(9GEiCzu)nG-KrIU*zS}sjZ`#Ak9Sro!hEh_@) z(kNd|O`zt#v~oFHI^9;hlPsVJ%q`ZPxBJkrRhz4My6Or3N7zFwWF{=QzF-~PV-8bz<;e~o+{oqsn8 zH?rugpTC44w)z@1uC2aCi)5AeHPH5&;MXjzJ>1%K z^WW=TP9v?sJZ@-xy_I_O=2TqHcqc`LJ$h+GqtvJ<66kUw7|jfn`gFQp_oHY=qCZ8a z(=R}Ni{7l5naB5Rt0WjCi=bFz%sNT1NCqLwnr+s*6jzhPu~xkl@3KmkRLQHe zC{m2ooL(#Rs}-z61f6AwV757GEsG?t*&^8lNk~$1W31M6pU}^zVmN%LO|>MXN=brJ zXt0`U)5i(@#tBxl)eH&-P)N--G_gottDscJSY3_C^+28nEJ;YsP9+4FpsXbXgGEZU zC8Vx^+y+6}YH*p>QDG-DOICfZQaDu57b>~9PB))Ifua!<)qOH&Bw<-u(C=`BT8AMxSE(FEF-XQgEs|bPF2-1MCA~!wY~Yz-z_+VVxi~M% zTrDIBTv~&wPOhLZGl@&4f$l7U4D*K2?+wi`Z-9OhR1wO}3QGiqdP4%}rmld>V8jxM z6ZGbs@X1_j&HBBd1DzU_llVQUFiVrf*yd*a#lB~Rc$Zo77FraRM9P?23jL6-_L8Y( zL^q`>ESbbHr2s1>5-WwYCaDzG933z+-P8goEjUUNI?Eu%wot2Ea01Oy+dPs$bIT%! z$!tpki3G4oi7`tKQ=+hFpeaF*9tyW^iPmPxoGmmFvXkI*-fD1cMfT!0QCMqEJ{jbb zBgrQd`D9LBVQpf}TOFn(0UpMgwdKq#!XiCOFuQa)R>7O(H6*$$dX@-LU@`{#w52== zttn3iC(4^Eti>?oEuc4jr@%s#w}d?@Z>6wyuyspG&swLOTDkinS%yuPM3 zLY!16S&}4gJqCh7%1f0pJql~jyP_?WXd9_STgxJ|%avnwdp%wZ3_f%WwkXNe8syuU zBo|0|;0jV+l9Xv6H7s76!mI|f=>c+oxJ0TT*w>j+{gz9yT1WQjv>k;)ROtgud8*-lWlQ>3z;R6oa1f{LI8vbJiLZA_LI{FLwtPt_Raj@Jc#kAxDXa@hTNEZjnUf0zK@&T+J>(8SNs+rpS1{|U zGIN-^f)Q2O(QLJD(o-WJebzV7@(TUn|9??MYI$nTZ*9U zu}E1E6Mo~sxVT?o-AFQFp~Z~ZQ9vizys;g;u!MzE>c%N`0i`Zs`MOxpOU`*wx_VCn z#6*eOYo&%%41$F=%oLIVy_Ku5?lG>5>TJBiEGeuS>GiH_cU;`KI z!>#T?15eoM9w1|NV0~4uaHGzR)yGAvkA-@zCDWjskfyck~J>ZnqWvs8e-AehFHuVu5hYkfd{8^Vfuk#zewr((G1qlmW3g# zuzspkX+TPEM54J8dJc%UV(mu< z4uYT;+WfF-IB?k$xv7h8ElBc zWT?;+Dr8Eh-O-XMThE}wFakV`EcX5dvL$sHUfFBXQ_UKmFh?QT?%`z3f1z;~2EGL@#AWVDqV z+rs`imRPqG>rs0w7!L6% zAEB_((6cveI5LjB#K1c$H%7-=glFr%n62$PXqAiZO3{`sK1Ym3vr!5gLlj`X(eZ_v z2S`G`!p5pb7z;%ZDr%8tl9Z2O0msNAJ5Wr9l`|j2^Fh32x+x#n`Hk36JD<-J3L6J% z<6Wjap}%0tgWJ1Jxu;EI^emS;oyVSA1<8}YLV}yzgb+jv9Sq(i?y+MOHXgHfLlzBK zl*|nRIgnaf=qB5FXjYTlS)9bpw9HsSXgiG^W!RFdWOBo@N9c@U0l3zhT9rsJSG zecz<839w8*v!0EgYnou*D_FXiCS+LR&E}A4qF~kuwUimHW}RggO4iu^fbltFB9xu% zFx?FEoL1NOn^z#L_wIzqlVZ%Q74#PpbznXrP6zm2z$gC&K5-F1Q%Iai3Y!Ws20)C- z3M-f!WoE4fOg;sehFBX5%|25sdW98^GPAaNHbr4YsHtCQ0ZwrsN^ddSmPOfgRGZ5nVP&Wu0!Jx|)1hn1)ow0E zHy1~CbFtRV)6lRz8WsbUq%TpJp9n~LR_ZX7*kXIdTQSUTZP@QZu4J0Y|E3At0TNE|}w!*HDO^1_73(KX3rYb)kN+wZHU~mZM2I#39MNdr4hpF12#*Xw^EoC99c1xw$2W6rGMO)Q+ zHp64849}T0kc7+{gs_x{6jle-yzu4PI2Qs2iko9QpwBU#&Ee|Igvc`^%{-I35;nZ` zh7;@-uICEn`Fu(&hRO}AqEax34y zSZMnq+1yf|jV9F0B!$g`_SoXgfjn~>110SIY_wm&K7?jnw-&|ChC+27(=0@Xv{cMN zYhrw>!sZWk>)Bj|Er0++fi+K68%q~7*Y?t^p0>DPOO~L?bs@LJd~lv0X^Ht{2{-L! zQY})GObQ@#P4&=fKAI4}1qxe4d~)@yKFzd9h+E4&Cm}UMpxs3&1!b8Y%+Hbq+l(a& zTMX+VZh`TaV4zq%riFr(k5!O&swWc>JaJg7XA2=0ZDW@Ltpj%B3R{N7XSu?b(`49@ zCd1{J@=jW)FV#@_qS~t~z=CXAZY9AD2$xov>lJnz&I`gSlf&N5RbC907e}hR7%CfR zb>l9u60}myg}NM9o;QTA`tmf>Z9;}%bElh@+4><_huE-5wwd$NOw01(atkx^3|TZizzy7LFm%T? zlf+v3R@j}DQS~^tvP_7gdBvb+g2%KX%bd$YTaVY0ZBzXoU4p(}Vcx1$WM$h@1qy#P z90c}+aGS@p44lbTZ&%nlh@14ySizhkRcHc*vR?{+ohb z_jLkUV6BvgfhB(pO(vqUm35|(3Uv$uKX|y0s!G)uE(Q=bsJ09tzbqD(pcJ7zYCP&}LGwNG+42 zAlpOe&P~V=#IY_yo$gQkslkw&9oU7w7C`b$h;`&q+slTrjMe?L;+TD z;~3GjNzXQWOo|2E?p4@hXgUGhaA5JcO56A++4w$%J%Or;po$ZNZI~&l`Hr7Ttc}1u zajU|%17{MNKag(P3OCs*SmPzWKP7Kc*p9hTF4hra+LDSh3w)~sE~-|+99z(i>G4|ELtf3L?Dh_cwsD7*F=v@#(3 z%$CY1%hgUuQU_z|#3%}jJlfDyzp%6ki4t^wP+_~E*A$(eJ;WnTW`-G`0mFOBalG-* z;X2;w4=e0hpiKqZBUjOO6Pm*Qg=|ZCp~9Ziupd>}^S~|u_G5(2Hy3+2R<*)j(6Am? z*o(j_1lAKAD-N7r;#lu;oe1`9>8}Zs9-L^5X=&2XLulDCyO1U zXN4JEK)~d1bOyILs5MOH4b_o-Fx=3?N3&!mG-_tBl*4lm!8&w?_!Ly6q5CwdOz*b?Jf#`VS19YN#BcYw~vlc-5kPfX4 zLOE1EtXX51!rmn-JfpB9utF)U@K5qE%zj>$f;Jz4(#rC^3OlNK*Ru-y7??g_?ry|v z3Cv?0vq52>Xqf*}*nfao2F&LO6U)4pwF2gGj(JjHCp66G74|7G%YnIvFmb!!#r{oU zCpqRtg`LtcUr^X-U`_+(i-g&eFxvq0Gmgp33j16`eMwz6SB>AikG~pT(|-#=7MT8@-0uh&C0BqSs602!x=ppST!5 zg~h<3F17g?_iOx@b=7W)Xz$@=ozUVJI(G;kN3hO-{7U3_JWx)z7MG!e6*hVeyves- zSJ-dRr3$*d0b42T_Z#SPzXTK zj8=C*5gF=cpza{AYoUlnt?rN_#-gqUb?=b7da;%(VpB-cRuSX0r3}EU3s}ZA3z(p; zRlvb&RgTLZ(WKFSR}teuI|$khS7|p>X*c7CKk!HqS9E;A{9m}VRKx_0%X^BL2reOT zdH*UGi^?VGf1|C4$rQS_#yAv{5eU@dA~Iz906`LthLGX3$c8nj99LLkbB+H8i!6%R zfp)IyTOG$h29nj49l``bab)m4}2iqF>EUyg=--vLk?)%U5kt-*XL-SiPA z_>UrY;7RI98|wy?cI51I66U7O4EoPW!=Ss3_DDGTXGEsGx(z)7I!E}5X;4R657b$S z8;$MOir4`v(cEzaO~~Osf&`Qwg`AXstcV?<;7llZEWVnY#RUpBn7SP~vzPV6XDZj| z3mO51VVHWIbq4&;L@rck__A(BcwefB+I3aFBC6L_;*GAWO2I(AuKFCrK39o3OrL`g zRoOAxaAffr>OKn_%)_@4Hf&?~1knpb92_D|q!S{8i(e`wtkf%XAOWi)^ zWBC`mVIIGq&pBEAl-tYPAn622MeL5FrIT3;8*kV;;IxS3>dr}@0_js>kxywN_fSPX ziMo@#j^s_{tz2r=r>OcA+VC*efZ8P7;S1RwZgCH&_at>BI(-qc4FYz@=!xEd_X>Ay zn-!SzMfY$WBA9X-*d7!;bP1=3xEI3R^$GAe5f<$PMB@>Q*as5e%qfX)G9*Q$i=Xw1 zm{u9ZV1aA*lauRphzc~{>;u}kwSiIk(Ro`EjdYaE{F;uJ_%)poTRhUCKZb=f-E3PSC>zCawnX%1|piIPT>N}Q&_hMf{bR>Xn)O7}nTMRk`X4&t}kpYZ89 z4G4qbyRFJ`T`Nl3aQ;LQ2a&J;2dybT4%sL_0aH=_sUo^4KdFfPBJdP?m%dLcqJ#3! z;DwZb4o9W@j3V-j!!Iy0==)1W%)l6$jWP69yo<(AdU%u!!P@IEeb0rXkwn)wu5{CP zu*r7bNiL++$FW|I-lN9wgASrP&8T@z1^(6B+NwEn! z1tb@Nq~LHG1TQWLNlMXh1Cm*sOg8_`xt?UMN-_#v4w8#OGTPyc3F|GQ_HXp|a#};L zr#M=r$S{IHaS13I9nM$?DXT)p-hkvVPG&g&9dSKLuS&8h#xh7Q1<5#v(*%+uRg$I~ zkQ~LyjOM?&*OSavNycNY1IcANz1U23LUVWmk}x`~4yPH1_25ErrD(nZpD~>FSpJ)L zJs(NslYnIge3nD>ME(kanc*THm4yYG7f;V#_6Zr4M>j}D5f{2VefZ%N)n4${ZRg2x67>ONl8`&-0iRWNQ;j2`e z!`ae;unU6k9@u+aW`1$bc_Q0OK9bHUe&*wQgy1 z1L{*b#RC3Ycs=#zDs`;>3820b)Z20D2`csW5!BoNE%hQ!xA=PM2`Y6gA|_B@1?n9* zb(2cHqi{MbddC|OFX04L;-%LUH>t!iD;hz3H9R*}^;{o1#i(-V6d`}78?aEVQ^tvx zU(dp*vgnLm6IiSPi!LgQX&1$tmw9A>U7yEjgy=l2fQ_=2r>BaOkXFtIt zNiMu75pcDGSAw=w1e!v&8~S!}%6OrDGDecFe$^&$3zQGCXfUB{Eb>f|~+r zy#B!XgxkX}eZ=tw@HXdaZv;Kr7J;AUiv2v!{y4=DqHzYJ7yE-SRK;QbMvbFKFAl&# z4%E2^_8I7K4hq|6(Enqf+1K0WtZJXZICcU-1@@5~&WsM4!84#}9oT-ZVhzDL7O*w} zE7Rf3(rZ{*z?uWBQz}+A&c%SW8CW?Er?;nu~IbX(;5b> zxxo5F#lj1iqrln%tPu|9NZbo?M;HmLdB8fVV&QemVPM?{tkDiO)37E0YYDJkR4};!&8-I0nnBK?Ku^#5Xb&Ndl+a%4rlQMoZ5yli-EZunEzBUOBC^DU_Ro~ zi>2vKJaa&D;bICq0lH$#L|C7vh!f^T6`ow^rc*$Q4{qo=2Oda# z*>gDYxWn%BrRuSFrkw!2lv%I7(;4V2(~CYF8Pm3I62TR5GHudQ@eqPa)&>tw@&%h- zEW`aM1|5dp3Nto}Q&i-sf4QvG?neZ$!E0_sEWoHs#RnZ!h~M}u1{%hZUl9wT`=gRxyd};h(M>6iZrxB3;dBx9oW0YX)1g=7qYP(-_U_G=#II__ z=?>J#y7{HC1&{v>UVXMjtE&JRyfP9i6|oqM9*5^ur8_G@vQn_(DHn{4t(KWzXG1nT z`V&hK0`~K^B0%l6MXU!0B&n8x&RV@#!>`#13G6C3rCFdfE0WTztCV2KN}`nR z#N!pD2!26JD*k6LTaxxbEY+Jd&>7N;LG7_j>^A;}M=yrxP}%@Ds^zlrw~ZoyHRP-V z%{rB)!&ygX5IA0e)8W)uoC!tA)UzNa<+EWOa)CNUr1wAK92lAMTNN>YuGo%oHy6)E z6tNl?#`OL=a)<%SRmDRMr}j|8;iN|z>H*9QwYTP>b{>gCA#@E5Ap>p(Ntk(xID?qY z2O;cCba|ngeEdHt42l2AmDTuevI-x4kOU+Hk{&4 zn^qQ)x7EsagR`gybhV2O@>bgESn1 z57ywsFGDL6JX0$ZUpb;2?hC>hrolopo(aP^2pP`jWj%uy^Hu8X0ehYb(R(=C{g;stQ`E;$k7r9?6zl?mER(=tAxmMnT zyi6-UhaCSO8UNdj-%_poPvj+9`5(xOwepk5i?s4~YeIKn%b|$}1MWVj%h7=w*R+L*Jk$=&cY>q_u)NdxOY(yS~L^KJHa-MQ(tI($c z^(FBnSwUY<-}I+9*q2O3PI9B2o@`IFn`rRIsSFU=o9O8@_@h(=oP3w$4dauZM88z6 zZ>0Y)zCl-Cgq-9I^LvXmA=!ECRk&gO5W_wg}_jk21+W z7I`d{HTmx&9Q-#LIgO7n{uT{B0(o2n{9X+%Bj+~P=xo;DImqLwtid;F@J!^?Phq&C z!6oG82>3lHlbi#Q)BGKlbDaivBTtNg->JdLXZRei@m;IIoye1@tikU<8Grh&_(`Uc z246#Ez`Njw{3;AzO=ZAS@za7z8vZH*;=c~aQ>d)LS8DKf$jMj3@Y^*w`E9ER_-z`T zXi!}ke}x7o8QVm_muqmc0ga0={xX#5lWa`9!*VVqApWC1Y)@rP&LtY0e8d_7U#!8& z2T8Xu{vr(ypXTdZ7`{-0Bh%TatkIvZ!gW`WlkGTsgg=Ba7>zbbxwG$mA4y9g!oQlEHa<&NK@eowym&cPk;rr+AbxPH}Juhmt}oH_rMEnAf1D~nP- zO38{Ljfed|8^ zuF#?rPGtSE(sgO_zQF9#smYtyrMG)#_`y5h``qD4Ki>P}KJPC)IdiVP{oYLuRSwctk|FVMUC}H=kpu?wRy+cCCQx!w7&B|n(5%gdp=l_zH{)Sm2I9~dZ6#p zDHXxA(U#xt>hj>eT5I}*o1L!h9d~qXnst;7*n0C9-LslMo)MS-c}mZ^*5|s+{PsYb z6~_)%HFTW5X2*98+r$ryu9u$M_(bUCq_@wlI5uN@!bQ)EQx{G6a>}Pq_d8*^ePFZn z;5;_DV)-54ez@wUIsg1)S>eQ1&wWw6%lJKeEAxT1r$!!FwyM{OF_Q}9XAf@Qf6{%v zPgeK$@40jT>lfqp{y4~%I4b+qlrtHJpUYaf|KN_DYd22r&|%T8Q_H?v`_f~Hor_nU zzufPuE^8VNEF1Jv$^LIff3*6iFKW$~^)F9tI%U?5JsF2nHjH}m{Ow2ncyIoO)=zyh z=do|6H64(6Zs^XqZa;79c~kuF6W^P;V`s{NWp}x1<~n~A_H}!l?HGA%*O-R(6DBN3 zaoqaguC(M2Ud)zvRr6ZLmKbf8^++N@88y?v;n&&s+vZ9Y#Z z&D(L?#Fg(&XxOr+$EKsf$Bm(jb6Vt-KmN!swF`Z*)$LM(j?jI^-V2YtHDy|lW|NMe zxgsXmD%`K0-MV^y#|?c8f9$$z@7G^8teaM!@w9%#<0sPEJ^$2ugPL{PIPR9H+q*tn z)wM46jJ?yBW8QJpwL0VP^zPOH4}Cu2?(+2mmVe)-Z2#uH7Y#?+uAkEV)CN*pBOs!s|#u7;Hr5a&zkbeZBI6AtG-}-_MPKZnU>V0f9#BH7!W<} zk5|~9bwTmVYT@zGpT7AY|NX_g*#mDLI6B9_F@IO#Q?JiD+wIvanYHoWx|11|b@NSM zo9~#MGSK_bYp3hlUVg&xW$vZzLypWF9lLFGn^7;U>9TkHpU>}pW#6Vi-;?+BT=U#% zMfbqt4ZWsJEBWHbxEJ;hDqML-`yJhzf3#unl@+_anQdkeLB2!ZvDLDmrFO-D+ z%iY9xzZw)X{#5hbcYn0y_rs(1PwDbd53^Udn{d~;F`+}_23TBw=*EqjRhtpa zZr5)1$~8MW|N84dd-|yjPtIELVPI>kM}O#d=S>Ti)_&9cg?asr-<0RBu64Sfy(bXJU;gQbr|&+0>am;T*9ZRi+NGR- z{ocX<+l5!XNgLkvw_US;#PN53-nwUa%m-HKF0q`kV)BU;p}~^XPkzjr=ZT+w*e8fp7OLY;P(Ya_2)Ib^q~# zbU5{J%EqyVQv-kRYTwqwe0klmA9j3o@rNgOpBVIv=zhmmO{A+_K=SKc;2`t}NJI^>xGZS04E7!-4tpM*s9flfmzNH)~<*uBQ3t^xLJu z$%9LJ96i|0^ZkF~28{SA@xHD9er@#bM}B$muCzApUqC-=c`EYA z*zExgPVw4StxPdmWL$Qi2B%moGX9E;xgz7N$k-|}o{Eg2?glNa+4^+kcWLw~){cy) zB4en?xG6GLij0pUW1`47C^GhmjCUeqoXEH)GM0&qUm|0cC7R3>r!3aW6q`iGBZw!s z{gLS+yeK|f(^ON#vms;WSd;A zOnOtF5u9`yiWG%ZuayZt56O!}bY^R1g4ZJDAQ3+8EwYhFhUrK{kl=@W4@PCeFG7-# z2&VvPfL5N2yq{Jkoq8h?FOrMO)OI9NPb8vAe5gz~nMmEV@?hj$kO-e>w?!hHjz}gX zf;Y_wg=&zCi!1P}pVp*QC(I01@sBFAZsq8qUtSt<{M7PJR963*6hbM|PiGu%9AV5ijx*k59B;hYIKep4ILSEK zIL$cSSYfO&2934G<;Hc!yNq`m*Bdt&?=fyNZZ>W)-ezGHmXc-VNv__6W0@r3b|@wD-b@eAXZ#_&U6y{SFUZnnqUlkF+?*7i2`w)XaRhrOG>_I~z0_B8tt zdzO8ueS|&Jo@*at&$ExS-()Ybm)d>ya{Dy9-(G32w$HE!?X~vV_B#6<`$GF7`x5&q z`yKYR_B-u&+qc=b+jrQXwm)OvW#40e!TzHC75nSn@vzDXW0Pmm|dljODxE%N25^TqOtOYI%lSBZuUf@+`SdzEz$p&zI}vh4Nx~sk}^H zA>S^qlvm4ZL?ee(VCR`~(>LHQy1VfhjHQTZ|X33;2m zUEU!-DL*AYE$@{7AwMJkQ{F8i+{%_J@4{ literal 0 HcmV?d00001 diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl new file mode 100644 index 0000000..13c3062 --- /dev/null +++ b/src/leveled_cdb.erl @@ -0,0 +1,804 @@ +%% +%% This is a modified version of the cdb module provided by Tom Whitcomb. +%% +%% - https://github.com/thomaswhitcomb/erlang-cdb +%% +%% The primary differences are: +%% - Support for incrementally writing a CDB file while keeping the hash table +%% in memory +%% - Support for merging of multiple CDB files with a key-checking function to +%% allow for compaction +%% - Automatic adding of a helper object that will keep a small proportion of +%% keys to be used when checking to see if the cdb file is a candidate for +%% compaction +%% - The ability to scan a database and accumulate all the Key, Values to +%% rebuild in-memory tables on startup +%% +%% This is to be used in eleveledb, and in this context: +%% - Keys will be a Sequence Number +%% - Values will be a Checksum; Pointers (length * 3); Key; [Metadata]; [Value] +%% where the pointers can be used to extract just part of the value +%% (i.e. metadata only) +%% +%% This module provides functions to create and query a CDB (constant database). +%% A CDB implements a two-level hashtable which provides fast {key,value} +%% lookups that remain fairly constant in speed regardless of the CDBs size. +%% +%% The first level in the CDB occupies the first 255 doublewords in the file. +%% Each doubleword slot contains two values. The first is a file pointer to +%% the primary hashtable (at the end of the file) and the second value is the +%% number of entries in the hashtable. The first level table of 255 entries +%% is indexed with the lower eight bits of the hash of the input key. +%% +%% Following the 255 doublewords are the {key,value} tuples. The tuples are +%% packed in the file without regard to word boundaries. Each {key,value} +%% tuple is represented with a four byte key length, a four byte value length, +%% the actual key value followed by the actual value. +%% +%% Following the {key,value} tuples are the primary hash tables. There are +%% at most 255 hash tables. Each hash table is referenced by one of the 255 +%% doubleword entries at the top of the file. For efficiency reasons, each +%% hash table is allocated twice the number of entries that it will need. +%% Each entry in the hash table is a doubleword. +%% The first word is the corresponding hash value and the second word is a +%% file pointer to the actual {key,value} tuple higher in the file. +%% + +-module(leveled_cdb). + +-export([from_dict/2, + create/2, + dump/1, + get/2, + get_mem/3, + put/4, + open_active_file/1, + get_nextkey/1, + get_nextkey/2]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(DWORD_SIZE, 8). +-define(WORD_SIZE, 4). +-define(CRC_CHECK, true). + +%% +%% from_dict(FileName,ListOfKeyValueTuples) +%% Given a filename and a dictionary, create a cdb +%% using the key value pairs from the dict. +%% +%% @spec from_dict(filename(),dictionary()) -> ok +%% where +%% filename() = string(), +%% dictionary() = dict() +%% +from_dict(FileName,Dict) -> + KeyValueList = dict:to_list(Dict), + create(FileName, KeyValueList). + +%% +%% create(FileName,ListOfKeyValueTuples) -> ok +%% Given a filename and a list of {key,value} tuples, +%% this function creates a CDB +%% +create(FileName,KeyValueList) -> + {ok, Handle} = file:open(FileName, [write]), + {ok, _} = file:position(Handle, {bof, 2048}), + {BasePos, HashTree} = write_key_value_pairs(Handle, KeyValueList), + io:format("KVs has been written to base position ~w~n", [BasePos]), + L2 = write_hash_tables(Handle, HashTree), + io:format("Index list output of ~w~n", [L2]), + write_top_index_table(Handle, BasePos, L2), + file:close(Handle). + +%% +%% dump(FileName) -> List +%% Given a file name, this function returns a list +%% of {key,value} tuples from the CDB. +%% +%% +%% @spec dump(filename()) -> key_value_list() +%% where +%% filename() = string(), +%% key_value_list() = [{key,value}] +dump(FileName) -> + dump(FileName, ?CRC_CHECK). + +dump(FileName, CRCCheck) -> + {ok, Handle} = file:open(FileName, [binary,raw]), + Fn = fun(Index, Acc) -> + {ok, _} = file:position(Handle, ?DWORD_SIZE * Index), + {_, Count} = read_next_2_integers(Handle), + Acc + Count + end, + NumberOfPairs = lists:foldl(Fn, 0, lists:seq(0,255)) bsr 1, + io:format("Count of keys in db is ~w~n", [NumberOfPairs]), + + {ok, _} = file:position(Handle, {bof, 2048}), + Fn1 = fun(_I,Acc) -> + {KL,VL} = read_next_2_integers(Handle), + Key = read_next_string(Handle, KL), + case read_next_string(Handle, VL, crc, CRCCheck) of + {false, _} -> + {ok, CurrLoc} = file:position(Handle, cur), + Return = {crc_wonky, get(Handle, Key)}; + {_, Value} -> + {ok, CurrLoc} = file:position(Handle, cur), + Return = case get(Handle, Key) of + {Key,Value} -> {Key ,Value}; + X -> {wonky, X} + end + end, + {ok, _} = file:position(Handle, CurrLoc), + [Return | Acc] + end, + lists:foldr(Fn1,[],lists:seq(0,NumberOfPairs-1)). + +%% Open an active file - one for which it is assumed the hash tables have not +%% yet been written +%% +%% Needs to scan over file to incrementally produce the hash list, starting at +%% the end of the top index table. +%% +%% Should return a dictionary keyed by index containing a list of {Hash, Pos} +%% tuples as the write_key_value_pairs function, and the current position, and +%% the file handle +open_active_file(FileName) when is_list(FileName) -> + {ok, Handle} = file:open(FileName, [binary, raw, read, write]), + {ok, Position} = file:position(Handle, {bof, 256*?DWORD_SIZE}), + {LastPosition, HashTree} = scan_over_file(Handle, Position), + case file:position(Handle, eof) of + {ok, LastPosition} -> + ok = file:close(Handle); + {ok, _} -> + LogDetails = [LastPosition, file:position(Handle, eof)], + io:format("File to be truncated at last position of" + "~w with end of file at ~w~n", LogDetails), + {ok, LastPosition} = file:position(Handle, LastPosition), + ok = file:truncate(Handle), + ok = file:close(Handle) + end, + {LastPosition, HashTree}. + +%% put(Handle, Key, Value, {LastPosition, HashDict}) -> {NewPosition, KeyDict} +%% Append to an active file a new key/value pair returning an updated +%% dictionary of Keys and positions. Returns an updated Position +%% +put(FileName, Key, Value, {LastPosition, HashTree}) when is_list(FileName) -> + {ok, Handle} = file:open(FileName, + [binary, raw, read, write, delayed_write]), + put(Handle, Key, Value, {LastPosition, HashTree}); +put(Handle, Key, Value, {LastPosition, HashTree}) -> + Bin = key_value_to_record({Key, Value}), % create binary for Key and Value + ok = file:pwrite(Handle, LastPosition, Bin), + {LastPosition + byte_size(Bin), put_hashtree(Key, LastPosition, HashTree)}. + + +%% +%% get(FileName,Key) -> {key,value} +%% Given a filename and a key, returns a key and value tuple. +%% +get(FileNameOrHandle, Key) -> + get(FileNameOrHandle, Key, ?CRC_CHECK). + +get(FileName, Key, CRCCheck) when is_list(FileName), is_list(Key) -> + {ok,Handle} = file:open(FileName,[binary,raw]), + get(Handle,Key, CRCCheck); + +get(Handle, Key, CRCCheck) when is_tuple(Handle), is_list(Key) -> + Hash = hash(Key), + Index = hash_to_index(Hash), + {ok,_} = file:position(Handle, {bof, ?DWORD_SIZE * Index}), + % Get location of hashtable and number of entries in the hash + {HashTable, Count} = read_next_2_integers(Handle), + % If the count is 0 for that index - key must be missing + case Count of + 0 -> + missing; + _ -> + % Get starting slot in hashtable + {ok, FirstHashPosition} = file:position(Handle, {bof, HashTable}), + Slot = hash_to_slot(Hash, Count), + {ok, _} = file:position(Handle, {cur, Slot * ?DWORD_SIZE}), + LastHashPosition = HashTable + ((Count-1) * ?DWORD_SIZE), + LocList = lists:seq(FirstHashPosition, LastHashPosition, ?DWORD_SIZE), + % Split list around starting slot. + {L1, L2} = lists:split(Slot, LocList), + search_hash_table(Handle, lists:append(L2, L1), Hash, Key, CRCCheck) + end. + +%% Get a Key/Value pair from an active CDB file (with no hash table written) +%% This requires a key dictionary to be passed in (mapping keys to positions) +%% Will return {Key, Value} or missing +get_mem(Key, Filename, HashTree) when is_list(Filename) -> + {ok, Handle} = file:open(Filename, [binary, raw, read]), + get_mem(Key, Handle, HashTree); +get_mem(Key, Handle, HashTree) -> + extract_kvpair(Handle, get_hashtree(Key, HashTree), Key). + +%% Get the next key at a position in the file (or the first key if no position +%% is passed). Will return both a key and the next position +get_nextkey(Filename) when is_list(Filename) -> + {ok, Handle} = file:open(Filename, [binary, raw, read]), + get_nextkey(Handle); +get_nextkey(Handle) -> + {ok, _} = file:position(Handle, bof), + {FirstHashPosition, _} = read_next_2_integers(Handle), + get_nextkey(Handle, {256 * ?DWORD_SIZE, FirstHashPosition}). + +get_nextkey(Handle, {Position, FirstHashPosition}) -> + {ok, Position} = file:position(Handle, Position), + case read_next_2_integers(Handle) of + {KeyLength, ValueLength} -> + NextKey = read_next_string(Handle, KeyLength), + NextPosition = Position + KeyLength + ValueLength + ?DWORD_SIZE, + case NextPosition of + FirstHashPosition -> + {NextKey, nomorekeys}; + _ -> + {NextKey, Handle, {NextPosition, FirstHashPosition}} + end; + eof -> + nomorekeys + end. + + +%%%%%%%%%%%%%%%%%%%% +%% Internal functions +%%%%%%%%%%%%%%%%%%%% + +%% Fetch a list of positions by passing a key to the HashTree +get_hashtree(Key, HashTree) -> + Hash = hash(Key), + Index = hash_to_index(Hash), + Tree = array:get(Index, HashTree), + case gb_trees:lookup(Hash, Tree) of + {value, List} -> + List; + _ -> + [] + end. + +%% Add to hash tree - this is an array of 256 gb_trees that contains the Hash +%% and position of objects which have been added to an open CDB file +put_hashtree(Key, Position, HashTree) -> + Hash = hash(Key), + Index = hash_to_index(Hash), + Tree = array:get(Index, HashTree), + case gb_trees:lookup(Hash, Tree) of + none -> + array:set(Index, gb_trees:insert(Hash, [Position], Tree), HashTree); + {value, L} -> + array:set(Index, gb_trees:update(Hash, [Position|L], Tree), HashTree) + end. + +%% Function to extract a Key-Value pair given a file handle and a position +%% Will confirm that the key matches and do a CRC check when requested +extract_kvpair(Handle, Positions, Key) -> + extract_kvpair(Handle, Positions, Key, ?CRC_CHECK). + +extract_kvpair(_, [], _, _) -> + missing; +extract_kvpair(Handle, [Position|Rest], Key, Check) -> + {ok, _} = file:position(Handle, Position), + {KeyLength, ValueLength} = read_next_2_integers(Handle), + case read_next_string(Handle, KeyLength) of + Key -> % If same key as passed in, then found! + case read_next_string(Handle, ValueLength, crc, Check) of + {false, _} -> + crc_wonky; + {_, Value} -> + {Key,Value} + end; + _ -> + extract_kvpair(Handle, Rest, Key, Check) + end. + +%% Scan through the file until there is a failure to crc check an input, and +%% at that point return the position and the key dictionary scanned so far +scan_over_file(Handle, Position) -> + HashTree = array:new(256, {default, gb_trees:empty()}), + scan_over_file(Handle, Position, HashTree). + +scan_over_file(Handle, Position, HashTree) -> + case read_next_2_integers(Handle) of + {KeyLength, ValueLength} -> + Key = read_next_string(Handle, KeyLength), + {ok, ValueAsBin} = file:read(Handle, ValueLength), + case crccheck_value(ValueAsBin) of + true -> + NewPosition = Position + KeyLength + ValueLength + ?DWORD_SIZE, + scan_over_file(Handle, NewPosition, + put_hashtree(Key, Position, HashTree)); + false -> + io:format("CRC check returned false on key of ~w ~n", [Key]), + {Position, HashTree} + end; + eof -> + {Position, HashTree} + end. + +%% The first four bytes of the value are the crc check +crccheck_value(Value) when byte_size(Value) >4 -> + << Hash:32/integer, Tail/bitstring>> = Value, + case calc_crc(Tail) of + Hash -> + true; + _ -> + io:format("CRC check failed due to mismatch ~n"), + false + end; +crccheck_value(_) -> + io:format("CRC check failed due to size ~n"), + false. + +%% Run a crc check filling out any values which don't fit on byte boundary +calc_crc(Value) -> + case bit_size(Value) rem 8 of + 0 -> + erlang:crc32(Value); + N -> + M = 8 - N, + erlang:crc32(<>) + end. + +%% +%% to_dict(FileName) +%% Given a filename returns a dict containing +%% the key value pairs from the dict. +%% +%% @spec to_dict(filename()) -> dictionary() +%% where +%% filename() = string(), +%% dictionary() = dict() +%% +to_dict(FileName) -> + KeyValueList = dump(FileName), + dict:from_list(KeyValueList). + +read_next_string(Handle, Length) -> + {ok, Bin} = file:read(Handle, Length), + binary_to_list(Bin). + +%% Read next string where the string has a CRC prepended - stripping the crc +%% and checking if requested +read_next_string(Handle, Length, crc, Check) -> + case Check of + true -> + {ok, <>} = file:read(Handle, Length), + case calc_crc(Bin) of + CRC -> + {true, binary_to_list(Bin)}; + _ -> + {false, binary_to_list(Bin)} + end; + _ -> + {ok, _} = file:position(Handle, {cur, 4}), + {ok, Bin} = file:read(Handle, Length - 4), + {unchecked, binary_to_list(Bin)} + end. + + +%% Used for reading lengths +%% Note that the endian_flip is required to make the file format compatible +%% with CDB +read_next_2_integers(Handle) -> + case file:read(Handle,?DWORD_SIZE) of + {ok, <>} -> + {endian_flip(Int1), endian_flip(Int2)}; + MatchError + -> + MatchError + end. + +%% Seach the hash table for the matching hash and key. Be prepared for +%% multiple keys to have the same hash value. +search_hash_table(_Handle, [], _Hash, _Key, _CRCCHeck) -> + missing; +search_hash_table(Handle, [Entry|RestOfEntries], Hash, Key, CRCCheck) -> + {ok, _} = file:position(Handle, Entry), + {StoredHash, DataLoc} = read_next_2_integers(Handle), + io:format("looking in data location ~w~n", [DataLoc]), + case StoredHash of + Hash -> + KV = extract_kvpair(Handle, [DataLoc], Key, CRCCheck), + case KV of + missing -> + search_hash_table(Handle, RestOfEntries, Hash, Key, CRCCheck); + _ -> + KV + end; + 0 -> + % Hash is 0 so key must be missing as 0 found before Hash matched + missing; + _ -> + search_hash_table(Handle, RestOfEntries, Hash, Key, CRCCheck) + end. + +% Write Key and Value tuples into the CDB. Each tuple consists of a +% 4 byte key length, a 4 byte value length, the actual key followed +% by the value. +% +% Returns a dictionary that is keyed by +% the least significant 8 bits of each hash with the +% values being a list of the hash and the position of the +% key/value binary in the file. +write_key_value_pairs(Handle, KeyValueList) -> + {ok, Position} = file:position(Handle, cur), + HashTree = array:new(256, {default, gb_trees:empty()}), + write_key_value_pairs(Handle, KeyValueList, {Position, HashTree}). + +write_key_value_pairs(_, [], Acc) -> + Acc; +write_key_value_pairs(Handle, [HeadPair|TailList], Acc) -> + {Key, Value} = HeadPair, + {NewPosition, HashTree} = put(Handle, Key, Value, Acc), + write_key_value_pairs(Handle, TailList, {NewPosition, HashTree}). + +%% Write the actual hashtables at the bottom of the file. Each hash table +%% entry is a doubleword in length. The first word is the hash value +%% corresponding to a key and the second word is a file pointer to the +%% corresponding {key,value} tuple. +write_hash_tables(Handle, HashTree) -> + Seq = lists:seq(0, 255), + {ok, StartPos} = file:position(Handle, cur), + write_hash_tables(Seq, Handle, HashTree, StartPos, []). + +write_hash_tables([], Handle, _, StartPos, IndexList) -> + {ok, EndPos} = file:position(Handle, cur), + ok = file:advise(Handle, StartPos, EndPos - StartPos, will_need), + IndexList; +write_hash_tables([Index|Rest], Handle, HashTree, StartPos, IndexList) -> + Tree = array:get(Index, HashTree), + case gb_trees:keys(Tree) of + [] -> + write_hash_tables(Rest, Handle, HashTree, StartPos, IndexList); + _ -> + HashList = gb_trees:to_list(Tree), + BinList = build_binaryhashlist(HashList, []), + IndexLength = length(BinList) * 2, + SlotList = lists:duplicate(IndexLength, <<0:32, 0:32>>), + + Fn = fun({Hash, Binary}, AccSlotList) -> + Slot1 = find_open_slot(AccSlotList, Hash), + {L1, [<<0:32, 0:32>>|L2]} = lists:split(Slot1, AccSlotList), + lists:append(L1, [Binary|L2]) + end, + NewSlotList = lists:foldl(Fn, SlotList, BinList), + + {ok, CurrPos} = file:position(Handle, cur), + file:write(Handle, NewSlotList), + write_hash_tables(Rest, Handle, HashTree, StartPos, + [{Index, CurrPos, IndexLength}|IndexList]) + end. + +%% The list created from the original HashTree may have duplicate positions +%% e.g. {Key, [Value1, Value2]}. Before any writing is done it is necessary +%% to know the actual number of hashes - or the Slot may not be sized correctly +%% +%% This function creates {Hash, Binary} pairs on a list where there is a unique +%% entry for eveyr Key/Value +build_binaryhashlist([], BinList) -> + BinList; +build_binaryhashlist([{Hash, [Position|TailP]}|TailKV], BinList) -> + HashLE = endian_flip(Hash), + PosLE = endian_flip(Position), + NewBin = <>, + case TailP of + [] -> + build_binaryhashlist(TailKV, [{Hash, NewBin}|BinList]); + _ -> + build_binaryhashlist([{Hash, TailP}|TailKV], [{Hash, NewBin}|BinList]) + end. + +%% Slot is zero based because it comes from a REM +find_open_slot(List, Hash) -> + Len = length(List), + Slot = hash_to_slot(Hash, Len), + Seq = lists:seq(1, Len), + {CL1, CL2} = lists:split(Slot, Seq), + {L1, L2} = lists:split(Slot, List), + find_open_slot1(lists:append(CL2, CL1), lists:append(L2, L1)). + +find_open_slot1([Slot|_RestOfSlots], [<<0:32,0:32>>|_RestOfEntries]) -> + Slot - 1; +find_open_slot1([_|RestOfSlots], [_|RestOfEntries]) -> + find_open_slot1(RestOfSlots, RestOfEntries). + + +%% Write the top most 255 doubleword entries. First word is the +%% file pointer to a hashtable and the second word is the number of entries +%% in the hash table +%% The List passed in should be made up of {Index, Position, Count} tuples +write_top_index_table(Handle, BasePos, List) -> + % fold function to find any missing index tuples, and add one a replacement + % in this case with a count of 0. Also orders the list by index + FnMakeIndex = fun(I, Acc) -> + case lists:keysearch(I, 1, List) of + {value, Tuple} -> + [Tuple|Acc]; + false -> + [{I, BasePos, 0}|Acc] + end + end, + % Fold function to write the index entries + FnWriteIndex = fun({Index, Pos, Count}, CurrPos) -> + {ok, _} = file:position(Handle, ?DWORD_SIZE * Index), + case Count == 0 of + true -> + PosLE = endian_flip(CurrPos), + NextPos = CurrPos; + false -> + PosLE = endian_flip(Pos), + NextPos = Pos + (Count * ?DWORD_SIZE) + end, + CountLE = endian_flip(Count), + Bin = <>, + file:write(Handle, Bin), + NextPos + end, + + Seq = lists:seq(0, 255), + CompleteList = lists:keysort(1, lists:foldl(FnMakeIndex, [], Seq)), + lists:foldl(FnWriteIndex, BasePos, CompleteList), + ok = file:advise(Handle, 0, ?DWORD_SIZE * 256, will_need). + + +endian_flip(Int) -> + <> = <>, + X. + +hash(Key) -> + H = 5381, + hash1(H,Key) band 16#FFFFFFFF. + +hash1(H,[]) ->H; +hash1(H,[B|Rest]) -> + H1 = H * 33, + H2 = H1 bxor B, + hash1(H2,Rest). + +% Get the least significant 8 bits from the hash. +hash_to_index(Hash) -> + Hash band 255. + +hash_to_slot(Hash,L) -> + (Hash bsr 8) rem L. + +%% Create a binary of the LengthKeyLengthValue, adding a CRC check +%% at the front of the value +key_value_to_record({Key,Value}) -> + L1 = endian_flip(length(Key)), + L2 = endian_flip(length(Value) + 4), + LB1 = list_to_binary(Key), + LB2 = list_to_binary(Value), + CRC = calc_crc(LB2), + <>. + +%%%%%%%%%%%%%%%% +% T E S T +%%%%%%%%%%%%%%% + +hash_1_test() -> + Hash = hash("key1"), + ?assertMatch(Hash,2088047427). + +hash_to_index_1_test() -> + Hash = hash("key1"), + Index = hash_to_index(Hash), + ?assertMatch(Index,67). + +hash_to_index_2_test() -> + Hash = 256, + I = hash_to_index(Hash), + ?assertMatch(I,0). + +hash_to_index_3_test() -> + Hash = 268, + I = hash_to_index(Hash), + ?assertMatch(I,12). + +hash_to_index_4_test() -> + Hash = hash("key2"), + Index = hash_to_index(Hash), + ?assertMatch(Index,64). + +write_key_value_pairs_1_test() -> + {ok,Handle} = file:open("test.cdb",write), + {_, HashTree} = write_key_value_pairs(Handle,[{"key1","value1"},{"key2","value2"}]), + Hash1 = hash("key1"), + Index1 = hash_to_index(Hash1), + Hash2 = hash("key2"), + Index2 = hash_to_index(Hash2), + R0 = array:new(256, {default, gb_trees:empty()}), + R1 = array:set(Index1, gb_trees:insert(Hash1, [0], array:get(Index1, R0)), R0), + R2 = array:set(Index2, gb_trees:insert(Hash2, [22], array:get(Index2, R1)), R1), + ?assertMatch(R2, HashTree). + + +write_hash_tables_1_test() -> + {ok, Handle} = file:open("test.cdb",write), + R0 = array:new(256, {default, gb_trees:empty()}), + R1 = array:set(64, gb_trees:insert(6383014720, [18], array:get(64, R0)), R0), + R2 = array:set(67, gb_trees:insert(6383014723, [0], array:get(67, R1)), R1), + Result = write_hash_tables(Handle, R2), + io:format("write hash tables result of ~w ~n", [Result]), + ?assertMatch(Result,[{67,16,2},{64,0,2}]). + +find_open_slot_1_test() -> + List = [<<1:32,1:32>>,<<0:32,0:32>>,<<1:32,1:32>>,<<1:32,1:32>>], + Slot = find_open_slot(List,0), + ?assertMatch(Slot,1). + +find_open_slot_2_test() -> + List = [<<0:32,0:32>>,<<0:32,0:32>>,<<1:32,1:32>>,<<1:32,1:32>>], + Slot = find_open_slot(List,0), + ?assertMatch(Slot,0). + +find_open_slot_3_test() -> + List = [<<1:32,1:32>>,<<1:32,1:32>>,<<1:32,1:32>>,<<0:32,0:32>>], + Slot = find_open_slot(List,2), + ?assertMatch(Slot,3). + +find_open_slot_4_test() -> + List = [<<0:32,0:32>>,<<1:32,1:32>>,<<1:32,1:32>>,<<1:32,1:32>>], + Slot = find_open_slot(List,1), + ?assertMatch(Slot,0). + +find_open_slot_5_test() -> + List = [<<1:32,1:32>>,<<1:32,1:32>>,<<0:32,0:32>>,<<1:32,1:32>>], + Slot = find_open_slot(List,3), + ?assertMatch(Slot,2). + +full_1_test() -> + List1 = lists:sort([{"key1","value1"},{"key2","value2"}]), + create("simple.cdb",lists:sort([{"key1","value1"},{"key2","value2"}])), + List2 = lists:sort(dump("simple.cdb")), + ?assertMatch(List1,List2). + +full_2_test() -> + List1 = lists:sort([{lists:flatten(io_lib:format("~s~p",[Prefix,Plug])), + lists:flatten(io_lib:format("value~p",[Plug]))} + || Plug <- lists:seq(1,2000), + Prefix <- ["dsd","so39ds","oe9%#*(","020dkslsldclsldowlslf%$#", + "tiep4||","qweq"]]), + create("full.cdb",List1), + List2 = lists:sort(dump("full.cdb")), + ?assertMatch(List1,List2). + +from_dict_test() -> + D = dict:new(), + D1 = dict:store("a","b",D), + D2 = dict:store("c","d",D1), + ok = from_dict("from_dict_test.cdb",D2), + io:format("Store created ~n", []), + KVP = lists:sort(dump("from_dict_test.cdb")), + D3 = lists:sort(dict:to_list(D2)), + io:format("KVP is ~w~n", [KVP]), + io:format("D3 is ~w~n", [D3]), + ?assertMatch(KVP,D3). + +to_dict_test() -> + D = dict:new(), + D1 = dict:store("a","b",D), + D2 = dict:store("c","d",D1), + ok = from_dict("from_dict_test.cdb",D2), + Dict = to_dict("from_dict_test.cdb"), + D3 = lists:sort(dict:to_list(D2)), + D4 = lists:sort(dict:to_list(Dict)), + ?assertMatch(D4,D3). + +crccheck_emptyvalue_test() -> + ?assertMatch(false, crccheck_value(<<>>)). + +crccheck_shortvalue_test() -> + Value = <<128,128,32>>, + ?assertMatch(false, crccheck_value(Value)). + +crccheck_justshortvalue_test() -> + Value = <<128,128,32,64>>, + ?assertMatch(false, crccheck_value(Value)). + +crccheck_correctvalue_test() -> + Value = term_to_binary("some text as value"), + Hash = erlang:crc32(Value), + ValueOnDisk = <>, + ?assertMatch(true, crccheck_value(ValueOnDisk)). + +crccheck_wronghash_test() -> + Value = term_to_binary("some text as value"), + Hash = erlang:crc32(Value) + 1, + ValueOnDisk = <>, + ?assertMatch(false, crccheck_value(ValueOnDisk)). + +crccheck_truncatedvalue_test() -> + Value = term_to_binary("some text as value"), + Hash = erlang:crc32(Value), + ValueOnDisk = <>, + Size = bit_size(ValueOnDisk) - 1, + <> = ValueOnDisk, + ?assertMatch(false, crccheck_value(TruncatedValue)). + +activewrite_singlewrite_test() -> + Key = "0002", + Value = "some text as new value", + InitialD = dict:new(), + InitialD1 = dict:store("0001", "Initial value", InitialD), + ok = from_dict("test_mem.cdb", InitialD1), + io:format("New db file created ~n", []), + {LastPosition, KeyDict} = open_active_file("test_mem.cdb"), + io:format("File opened as new active file " + "with LastPosition=~w ~n", [LastPosition]), + {_, UpdKeyDict} = put("test_mem.cdb", Key, Value, {LastPosition, KeyDict}), + io:format("New key and value added to active file ~n", []), + ?assertMatch({Key, Value}, get_mem(Key, "test_mem.cdb", UpdKeyDict)). + +search_hash_table_findinslot_test() -> + Key1 = "key1", % this is in slot 3 if count is 8 + D = dict:from_list([{Key1, "value1"}, {"K2", "V2"}, {"K3", "V3"}, + {"K4", "V4"}, {"K5", "V5"}, {"K6", "V6"}, {"K7", "V7"}, + {"K8", "V8"}]), + ok = from_dict("hashtable1_test.cdb",D), + {ok, Handle} = file:open("hashtable1_test.cdb", [binary, raw, read, write]), + Hash = hash(Key1), + Index = hash_to_index(Hash), + {ok, _} = file:position(Handle, {bof, ?DWORD_SIZE*Index}), + {HashTable, Count} = read_next_2_integers(Handle), + io:format("Count of ~w~n", [Count]), + {ok, FirstHashPosition} = file:position(Handle, {bof, HashTable}), + Slot = hash_to_slot(Hash, Count), + io:format("Slot of ~w~n", [Slot]), + {ok, _} = file:position(Handle, {cur, Slot * ?DWORD_SIZE}), + {ReadH3, ReadP3} = read_next_2_integers(Handle), + {ReadH4, ReadP4} = read_next_2_integers(Handle), + io:format("Slot 1 has Hash ~w Position ~w~n", [ReadH3, ReadP3]), + io:format("Slot 2 has Hash ~w Position ~w~n", [ReadH4, ReadP4]), + ?assertMatch(0, ReadH4), + ?assertMatch({"key1", "value1"}, get(Handle, Key1)), + {ok, _} = file:position(Handle, FirstHashPosition), + FlipH3 = endian_flip(ReadH3), + FlipP3 = endian_flip(ReadP3), + RBin = <>, + io:format("Replacement binary of ~w~n", [RBin]), + {ok, OldBin} = file:pread(Handle, + FirstHashPosition + (Slot -1) * ?DWORD_SIZE, 16), + io:format("Bin to be replaced is ~w ~n", [OldBin]), + ok = file:pwrite(Handle, FirstHashPosition + (Slot -1) * ?DWORD_SIZE, RBin), + ok = file:close(Handle), + io:format("Find key following change to hash table~n"), + ?assertMatch(missing, get("hashtable1_test.cdb", Key1)). + +getnextkey_test() -> + L = [{"K9", "V9"}, {"K2", "V2"}, {"K3", "V3"}, + {"K4", "V4"}, {"K5", "V5"}, {"K6", "V6"}, {"K7", "V7"}, + {"K8", "V8"}, {"K1", "V1"}], + ok = create("hashtable1_test.cdb", L), + {FirstKey, Handle, P1} = get_nextkey("hashtable1_test.cdb"), + io:format("Next position details of ~w~n", [P1]), + ?assertMatch("K9", FirstKey), + {SecondKey, Handle, P2} = get_nextkey(Handle, P1), + ?assertMatch("K2", SecondKey), + {_, Handle, P3} = get_nextkey(Handle, P2), + {_, Handle, P4} = get_nextkey(Handle, P3), + {_, Handle, P5} = get_nextkey(Handle, P4), + {_, Handle, P6} = get_nextkey(Handle, P5), + {_, Handle, P7} = get_nextkey(Handle, P6), + {_, Handle, P8} = get_nextkey(Handle, P7), + {LastKey, Info} = get_nextkey(Handle, P8), + ?assertMatch(nomorekeys, Info), + ?assertMatch("K1", LastKey). + +newactivefile_test() -> + {LastPosition, _} = open_active_file("activefile_test.cdb"), + ?assertMatch(256 * ?DWORD_SIZE, LastPosition), + Response = get_nextkey("activefile_test.cdb"), + ?assertMatch(nomorekeys, Response). + + + + + + + + + diff --git a/src/leveled_internal.beam b/src/leveled_internal.beam new file mode 100644 index 0000000000000000000000000000000000000000..793f0132628249ea2c7df951c16c564741f8d93d GIT binary patch literal 3068 zcmb7GeQXrR72mztJMS8=*}GkTf3UD;^KsZ-b4P`pukqUF1q{KlokU41eCOM>edc_7 z?5)9d(g+oTP!fRyt=bLwAW$0hk4nUbrnD&{Nom!VNXRLb(pCyZAVJkqN*bY}B<-8^ zL9P5%R{HIG^P4wselzp-Zo0ShwNlige`)cK%^i+4PEo$K6h(!FjJdzH;n znJED?o`zS_v20)33JhEJc+z3A6PXO-*b}Bdlg>MN|F~(7nEv5pCT|9Irn6bo_Cr`| zv@BeN!z>^ zQ8HpWS##3KXVNK?am>84d|sew?J(`TQ{;E1ozbjSm}QWJ4KjJlcKqft;WusDvhO$fn){a1dfy||CMcRt%P=X*p9WU|kjDj(dMc#_SMY_F zC1^Po7kO32I2y)^gb@>Gd?MB<sW1(SVD4gto^~`&A@sh!Qm< zD3Pn;5abLFUUZf~_jcl(cr4NoRNx=gsI1qI9$vO9I59~=4 zBQLomVxu23WVJbZdcoKkY(%lsiDM$((@&%hC=fGEA8{|NH|P`Yu8oL3>z;u31^3eW zD1FJr;NNs_t`F0!;IdE??+Y{=Y~w?y9gv&PolqitNSp&!0alSUg^Fs5&&T2wxN39pS4x55cSd7`#%mEBNaC2LhJ6RYjlCkxps_q*NDT zYe;MjiLFV^-U}0TG#^&8g0B^kpbFixs2>GZB!g=ORVKT8R2QLD=du;+5zW@SY~^}r zFLBu_Se*!~liT>Eu)3uS3YK3g7EANVmtr5uw#;R#A-;^FS*28^u#REtYKjCEqwolY zku)N}FTe=YvCyrGx}-Cn4yfD_wV;f%J&JtyY}lnBn*`Y!a0t27qXMScMvtrna(RL5 zrP&qpB$U=l@)>)iLLTm1dhth)@QL zQLHCAu_9uhg7qBv${>hIG(`Aj=q|29Apu4<3#{mEW^uia;-U|cjV)fHGjS22xW0?L zrpfCF63G@9%Q}>8Ei2?eW{b<#fv{!ZssVuy(roKIqkz;k0hzC&+2}kI1||lKgy4s@ z-?0b4P#eZ`#2Nq=K)M)!4^RXk++9Doz%K`%22=o~f?ox;`YS#eOZcUPk577YVetKc zO2PqF7JM+&asbIko(4df=TVdbmgEW$+dYvbnx6%bJYm8CVgT7-#R>6{Sclnw^J3=Vp7vgIRZSq&;?G52Kr5(^f_d(du*!#pyxo>XxufO`R@psv=pMP)bodXX%_4M`kc299F?(OHq z;mXvXUV8h^gT}??TghEM^@Z`k#vhh9&IJUyLT>6#bgPHY$Nc+O<|eByso%Nth5l=< zZT}y;xsmCoFGmKXV_y^2uoI*{#C)k5b3KJls(n>WhZ?V)z7hFZ{Ek^KBp1E4{oCnP z{Myqq-|sqkY|qyZon5)E{r8tjXP1=*t9InDG#*l4UP7B7glphdaI3fo nr*dE88o3&-jC+WSbKP7Iw~5oaEnF}67`K%h<1EhR$f5if7ID~j literal 0 HcmV?d00001 diff --git a/src/leveled_internal.erl b/src/leveled_internal.erl new file mode 100644 index 0000000..874fe61 --- /dev/null +++ b/src/leveled_internal.erl @@ -0,0 +1,118 @@ +-module(leveled_internal). +-export([termiterator/6]). +-include_lib("eunit/include/eunit.hrl"). + + +%% We will have a sorted list of terms +%% Some terms will be dummy terms which are pointers to more terms which can be found +%% If a pointer is hit need to replenish the term list before proceeding +%% +%% Helper Functions should have free functions - FolderFun, CompareFun, PointerCheck} +%% FolderFun - function which takes the next item and the accumulator and returns an updated accunulator +%% CompareFun - function which should be able to compare two keys (which are not pointers) +%% PointerCheck - function for differentiating between keys and pointer + +termiterator(HeadItem, [], Acc, HelperFuns, _StartKey, _EndKey) -> + io:format("Reached empty list with head item of ~w~n", [HeadItem]), + case HeadItem of + null -> + Acc; + _ -> + {FolderFun, _, _} = HelperFuns, + FolderFun(Acc, HeadItem) + end; +termiterator(null, [NextItem|TailList], Acc, HelperFuns, StartKey, EndKey) -> + %% Check that the NextItem is not a pointer before promoting to HeadItem + %% Cannot now promote a HeadItem which is a pointer + {_, _, PointerCheck} = HelperFuns, + case PointerCheck(NextItem) of + {true, Pointer} -> + NewSlice = getnextslice(Pointer, EndKey), + ExtendedList = lists:merge(NewSlice, TailList), + termiterator(null, ExtendedList, Acc, HelperFuns, StartKey, EndKey); + false -> + termiterator(NextItem, TailList, Acc, HelperFuns, StartKey, EndKey) + end; +termiterator(HeadItem, [NextItem|TailList], Acc, HelperFuns, StartKey, EndKey) -> + io:format("Checking head item of ~w~n", [HeadItem]), + {FolderFun, CompareFun, PointerCheck} = HelperFuns, + %% HeadItem cannot be pointer, but NextItem might be, so check before comparison + case PointerCheck(NextItem) of + {true, Pointer} -> + NewSlice = getnextslice(Pointer, EndKey), + ExtendedList = lists:merge(NewSlice, [NextItem|TailList]), + termiterator(null, ExtendedList, Acc, HelperFuns, StartKey, EndKey); + false -> + %% Compare to see if Head and Next match, or if Head is a winner to be added + %% to accumulator + case CompareFun(HeadItem, NextItem) of + {match, StrongItem, _WeakItem} -> + %% Discard WeakItem + termiterator(StrongItem, TailList, Acc, HelperFuns, StartKey, EndKey); + {winner, HeadItem} -> + %% Add next item to accumulator, and proceed with next item + AccPlus = FolderFun(Acc, HeadItem), + termiterator(NextItem, TailList, AccPlus, HelperFuns, HeadItem, EndKey) + end + end. + + + +pointercheck_indexkey(IndexKey) -> + case IndexKey of + {i, _Bucket, _Index, _Term, _Key, _Sequence, {zpointer, Pointer}} -> + {true, Pointer}; + _ -> + false + end. + +folder_indexkey(Acc, IndexKey) -> + io:format("Folding index key of - ~w~n", [IndexKey]), + case IndexKey of + {i, _Bucket, _Index, _Term, _Key, _Sequence, tombstone} -> + Acc; + {i, _Bucket, _Index, _Term, Key, _Sequence, null} -> + io:format("Adding key ~s~n", [Key]), + lists:append(Acc, [Key]) + end. + +compare_indexkey(IndexKey1, IndexKey2) -> + {i, Bucket1, Index1, Term1, Key1, Sequence1, _Value1} = IndexKey1, + {i, Bucket2, Index2, Term2, Key2, Sequence2, _Value2} = IndexKey2, + case {Bucket1, Index1, Term1, Key1} of + {Bucket2, Index2, Term2, Key2} when Sequence1 >= Sequence2 -> + {match, IndexKey1, IndexKey2}; + {Bucket2, Index2, Term2, Key2} -> + {match, IndexKey2, IndexKey1}; + _ when IndexKey2 >= IndexKey1 -> + {winner, IndexKey1}; + _ -> + {winner, IndexKey2} + end. + + +getnextslice(Pointer, _EndKey) -> + case Pointer of + {test, NewList} -> + NewList; + _ -> + [] + end. + + +%% Unit tests + + +iterateoverindexkeyswithnopointer_test_() -> + Key1 = {i, "pdsRecord", "familyName_bin", "1972SMITH", "10001", 1, null}, + Key2 = {i, "pdsRecord", "familyName_bin", "1972SMITH", "10001", 2, tombstone}, + Key3 = {i, "pdsRecord", "familyName_bin", "1971SMITH", "10002", 2, null}, + Key4 = {i, "pdsRecord", "familyName_bin", "1972JONES", "10003", 2, null}, + KeyList = lists:sort([Key1, Key2, Key3, Key4]), + HelperFuns = {fun folder_indexkey/2, fun compare_indexkey/2, fun pointercheck_indexkey/1}, + ResultList = ["10002", "10003"], + ?_assertEqual(ResultList, termiterator(null, KeyList, [], HelperFuns, "1971", "1973")). + + + + diff --git a/src/onstartfile.bst b/src/onstartfile.bst new file mode 100644 index 0000000000000000000000000000000000000000..72153f261c71b68b435b2d014289c0010926cd82 GIT binary patch literal 32 QcmZQ%X21!I0+*Wu00Z3tW&i*H literal 0 HcmV?d00001 diff --git a/src/rice.erl b/src/rice.erl new file mode 100644 index 0000000..68dd78a --- /dev/null +++ b/src/rice.erl @@ -0,0 +1,155 @@ +-module(rice). +-export([encode/1, + encode/2, + checkforhash/2, + converttohash/1]). +-include_lib("eunit/include/eunit.hrl"). + +%% Factor is the power of 2 representing the expected normal gap size between +%% members of the hash, and therefore the size of the bitstring to represent the +%% remainder for the gap +%% +%% The encoded output should contain a single byte which is the Factor, followed +%% by a series of exponents and remainders. +%% +%% The exponent is n 1's followed by a 0, where n * (2 ^ Factor) + remainder +%% represents the gap to the next hash +%% +%% The size passed in should be the maximum possible value of the hash. +%% If this isn't provided - assumes 2^32 - the default for phash2 + +encode(HashList) -> + encode(HashList, 4 * 1024 * 1024 * 1024). + +encode(HashList, Size) -> + SortedHashList = lists:usort(HashList), + ExpectedGapSize = Size div length(SortedHashList), + Factor = findpowerundergap(ExpectedGapSize), + riceencode(SortedHashList, Factor). + +%% Outcome may be suboptimal if lists have not been de-duplicated +%% Will fail on an unsorted list + +riceencode(HashList, Factor) when Factor<256 -> + Divisor = powtwo(Factor), + riceencode(HashList, Factor, Divisor, <<>>, 0). + +riceencode([], Factor, _, BitStrAcc, _) -> + Prefix = binary:encode_unsigned(Factor), + <>; +riceencode([HeadHash|TailList], Factor, Divisor, BitStrAcc, LastHash) -> + HashGap = HeadHash - LastHash, + case HashGap of + 0 -> + riceencode(TailList, Factor, Divisor, BitStrAcc, HeadHash); + N when N > 0 -> + Exponent = buildexponent(HashGap div Divisor), + Remainder = HashGap rem Divisor, + ExpandedBitStrAcc = <>, + riceencode(TailList, Factor, Divisor, ExpandedBitStrAcc, HeadHash) + end. + + +%% Checking for a hash needs to roll through the compressed bloom, decoding until +%% the member is found (match!), passed (not matched) or the end of the encoded +%% bitstring has been reached (not matched) + +checkforhash(HashToCheck, BitStr) -> + <> = BitStr, + Divisor = powtwo(Factor), + checkforhash(HashToCheck, RiceEncodedBitStr, Factor, Divisor, 0). + +checkforhash(_, <<>>, _, _, _) -> + false; +checkforhash(HashToCheck, BitStr, Factor, Divisor, Acc) -> + [Exponent, BitStrTail] = findexponent(BitStr), + [Remainder, BitStrTail2] = findremainder(BitStrTail, Factor), + NextHash = Acc + Divisor * Exponent + Remainder, + case NextHash of + HashToCheck -> true; + N when N>HashToCheck -> false; + _ -> checkforhash(HashToCheck, BitStrTail2, Factor, Divisor, NextHash) + end. + + +%% Exported functions - currently used only in testing + +converttohash(ItemList) -> + converttohash(ItemList, []). + +converttohash([], HashList) -> + HashList; +converttohash([H|T], HashList) -> + converttohash(T, [erlang:phash2(H)|HashList]). + + + +%% Helper functions + +buildexponent(Exponent) -> + buildexponent(Exponent, <<0:1>>). + +buildexponent(0, OutputBits) -> + OutputBits; +buildexponent(Exponent, OutputBits) -> + buildexponent(Exponent - 1, <<1:1, OutputBits/bitstring>>). + + +findexponent(BitStr) -> + findexponent(BitStr, 0). + +findexponent(BitStr, Acc) -> + <> = BitStr, + case H of + <<1:1>> -> findexponent(T, Acc + 1); + <<0:1>> -> [Acc, T] + end. + + +findremainder(BitStr, Factor) -> + <> = BitStr, + [Remainder, BitStrTail]. + + +powtwo(N) -> powtwo(N, 1). + +powtwo(0, Acc) -> + Acc; +powtwo(N, Acc) -> + powtwo(N-1, Acc * 2). + +%% Helper method for finding the factor of two which provides the most +%% efficient compression given an average gap size + +findpowerundergap(GapSize) -> findpowerundergap(GapSize, 1, 0). + +findpowerundergap(GapSize, Acc, Counter) -> + case Acc of + N when N > GapSize -> Counter - 1; + _ -> findpowerundergap(GapSize, Acc * 2, Counter + 1) + end. + + +%% Unit tests + +findpowerundergap_test_() -> + [ + ?_assertEqual(9, findpowerundergap(700)), + ?_assertEqual(9, findpowerundergap(512)), + ?_assertEqual(8, findpowerundergap(511))]. + +encode_test_() -> + [ + ?_assertEqual(<<9, 6, 44, 4:5>>, encode([24,924], 1024)), + ?_assertEqual(<<9, 6, 44, 4:5>>, encode([24,24,924], 1024)), + ?_assertEqual(<<9, 6, 44, 4:5>>, encode([24,924,924], 1024)) + ]. + +check_test_() -> + [ + ?_assertEqual(true, checkforhash(924, <<9, 6, 44, 4:5>>)), + ?_assertEqual(true, checkforhash(24, <<9, 6, 44, 4:5>>)), + ?_assertEqual(false, checkforhash(23, <<9, 6, 44, 4:5>>)), + ?_assertEqual(false, checkforhash(923, <<9, 6, 44, 4:5>>)), + ?_assertEqual(false, checkforhash(925, <<9, 6, 44, 4:5>>)) + ]. diff --git a/src/simple.cdb b/src/simple.cdb new file mode 100644 index 0000000000000000000000000000000000000000..14a53c03f8ebf92bd0651a6e1cc5b1edd057163b GIT binary patch literal 2124 zcmdPlU;qQ1QF_FN0~65aHqiJ-ra|I9Q1MY3oIFOuV>CU06T)bI7|jo$5E$y|fCcCW xE+EcMtu$OzniEx)m{Xc+h$L!cU1R$IENTSuF9w+`5+nqB|ZY4 literal 0 HcmV?d00001 diff --git a/src/test_inconsole.cdb b/src/test_inconsole.cdb new file mode 100644 index 0000000..e69de29 diff --git a/src/test_mem.cdb b/src/test_mem.cdb new file mode 100644 index 0000000000000000000000000000000000000000..f6a008c371bf352c80186bc7bc4a60e05367b701 GIT binary patch literal 2115 zcmb2;U;qPIJTwzfg)UTnl!k=YXgG|911JPW)4^yu7)=MG`CzD)11!MAD+t5}1_p*P wQD5$O=4F;-Cgv!VCFYc-f+VFNl19}jYVpPSxv2^zsTCy(iNy+espSw=0CR*#jsO4v literal 0 HcmV?d00001 diff --git a/test/lookup_test.beam b/test/lookup_test.beam new file mode 100644 index 0000000000000000000000000000000000000000..3c8d76474f8d8ff741da4eecc8fc903230a9a869 GIT binary patch literal 4096 zcmZ`+du$wM5#ROR;~U49?A1zw(>CyZHQI zrMaD*Z)U!k-^}cHd*syFAxYZu{-Me7$HpByCrR>GB}pF~vbYei z3X947yf2l^XZ$P}l6!rrt9Bv853`Ovo6TpetHF7DHj_;|KK@bi*3*HUm9q=iBJ;^Y zDmib>rtQT=EA3?Me0|YLI@zL=O&0?Nc(rrEQa(Fp7jphZh>L-=olhqn-y9s**_ZuA z$1YfMzuzhp>_PzhSWZ!ZyciWQfpW}&Y`$m}9D>Gt$|+cum`}~}z38`cOU^a_mE>Z{ z3Y3;INyqZ#?Yy-as>wnjdCi{Vu!9+EE?HW1{GLpiw;ZKtIqHg1*=@^;fURbq&N>U} zWO~60B$t+~e8yMIK4bZFR$<<1@P22#jfuF! z1geH>)J4wfvbq1fHm*g7UGuu%R2Aw_+|%H6C90UiT0ozaPmk=?G&$iW>LSmo@{+sc zxuOpgO}^pYsEfR;$~PNS?B{cD1|`GvMNOaPH(pcaTkb6^8}jY=>zce-mE7Aw$%LNy zmI78its3ePU7PYvDW;}GR879qU>aecsf?J|U1>D6n0qIvCR9_MRO+@5;Sh&7YW{K3~R^)X&Ca|_w1`ia;amiY8VzWD$vnfH%V>npf}wzX4!EYcLE`i7{Rp zfNOL`%UZx#0!&&N6ZAw3+qVVvI}Dqz3Y*yG>1>Bir6|gj&r{tI)bF~Y{kmVel?L1PtXvmD|%{c9BM{3 zt#LT0@7$;@dSQct_O7#Ob452gLu_+J9H2yj&-N;EeVCmJ_VqDLM300*^#yeid5C8_ z^mn+TpG^@#dpxwWg6BXDo&y`=xpQ*@jWXj6)`+Q1xMFY}nveiZ!)uh7D~5b($(+$# zu?JCT<=zQHjz@3kCLVUruxB_@HQZY^^xE-}6NcxYFpOb2Vqba2Bm?OInRb6uGr zL7B#@XdkMU>F^5FTqZnv!?04O3D0n{YIvlkOj9eiUYU-1Q?eUn>aHzQcV$Y}mT8)& zWNn>}mGPZkg>M%S*&T=1^60Fs({b2@*E%{;c65B5qi$Cmj(tx zadM-1I_VXPkA+hkEuZ3Lue!tDsnqDb@P_$?*Qd)O?uygA4N~Zfhx#i;JyTQEGaDAQ ze>4;al5s_ESi<=hZVbn86lHJ})z0JDb%llmg+5oU*Gx^l&ac$V6?nK|W)Q0qM&ch? zS>iUp$SlJZ(0Ws{7|_K;-Mi-syR`eE+2346za1TMLQ50 zl*Is4)(xSjcCkEzvKYj1k3-899}B9YDIPMEX&iR5{O19qJ`)YkAQcv1&iS3?rRIu6 zIXK_E_^DZH{fYBV1$O1b53D~|J+UImT;Gkg53sE zKt51ig%8y5!3tlG`z8=ytl!{qWqvbw8MFn&{~wtTgCZ54Z8Z@4XMQV)eK0kFSl$L= z|5cvjs`A?_d^_$tL3}Y^)nUE^#QvCPedfDBRiDi70D|{4pCYE#DJ1RV5)KcLY zQ>yTEq?G5qSeI=K5XUhJ@-F4z&;NNM>#)uUD1P;obRv;RB@*`*em$3X*2xiH@h;#`>Lp5R=VXa9^B=fQpE2QkLgUSxUQxZ@OX?|yC(zw$y|MpCX6 z^WH;Vl}caQ_l?ia{Hrgzvy z@4uu`Bel~`>Yz^QqFuC`dMQqQWY7RTN(bm5nRJ*YXp)Z7G##VkG(+dD P-KW3PKj`1|ALQ{rJu=`t literal 0 HcmV?d00001 diff --git a/test/lookup_test.erl b/test/lookup_test.erl new file mode 100644 index 0000000..f8632f2 --- /dev/null +++ b/test/lookup_test.erl @@ -0,0 +1,241 @@ +-module(lookup_test). + +-export([go_dict/1, go_ets/1, go_gbtree/1, + go_arrayofdict/1, go_arrayofgbtree/1, go_arrayofdict_withcache/1]). + +-define(CACHE_SIZE, 512). + +hash(Key) -> + H = 5381, + hash1(H,Key) band 16#FFFFFFFF. + +hash1(H,[]) ->H; +hash1(H,[B|Rest]) -> + H1 = H * 33, + H2 = H1 bxor B, + hash1(H2,Rest). + +% Get the least significant 8 bits from the hash. +hash_to_index(Hash) -> + Hash band 255. + + +%% +%% Timings (microseconds): +%% +%% go_dict(200000) : 1569894 +%% go_dict(1000000) : 17191365 +%% go_dict(5000000) : forever + +go_dict(N) -> + go_dict(dict:new(), N, N). + +go_dict(_, 0, _) -> + {erlang:memory(), statistics(garbage_collection)}; +go_dict(D, N, M) -> + % Lookup a random key - which may not be present + LookupKey = lists:concat(["key-", random:uniform(M)]), + LookupHash = hash(LookupKey), + dict:find(LookupHash, D), + + % Add a new key - which may be present so value to be appended + Key = lists:concat(["key-", N]), + Hash = hash(Key), + case dict:find(Hash, D) of + error -> + go_dict(dict:store(Hash, [N], D), N-1, M); + {ok, List} -> + go_dict(dict:store(Hash, [N|List], D), N-1, M) + end. + + + +%% +%% Timings (microseconds): +%% +%% go_ets(200000) : 609119 +%% go_ets(1000000) : 3520757 +%% go_ets(5000000) : 19974562 + +go_ets(N) -> + go_ets(ets:new(ets_test, [private, bag]), N, N). + +go_ets(_, 0, _) -> + {erlang:memory(), statistics(garbage_collection)}; +go_ets(Ets, N, M) -> + % Lookup a random key - which may not be present + LookupKey = lists:concat(["key-", random:uniform(M)]), + LookupHash = hash(LookupKey), + ets:lookup(Ets, LookupHash), + + % Add a new key - which may be present so value to be appended + Key = lists:concat(["key-", N]), + Hash = hash(Key), + ets:insert(Ets, {Hash, N}), + go_ets(Ets, N - 1, M). + +%% +%% Timings (microseconds): +%% +%% go_gbtree(200000) : 1393936 +%% go_gbtree(1000000) : 8430997 +%% go_gbtree(5000000) : 45630810 + +go_gbtree(N) -> + go_gbtree(gb_trees:empty(), N, N). + +go_gbtree(_, 0, _) -> + {erlang:memory(), statistics(garbage_collection)}; +go_gbtree(Tree, N, M) -> + % Lookup a random key - which may not be present + LookupKey = lists:concat(["key-", random:uniform(M)]), + LookupHash = hash(LookupKey), + gb_trees:lookup(LookupHash, Tree), + + % Add a new key - which may be present so value to be appended + Key = lists:concat(["key-", N]), + Hash = hash(Key), + case gb_trees:lookup(Hash, Tree) of + none -> + go_gbtree(gb_trees:insert(Hash, [N], Tree), N - 1, M); + {value, List} -> + go_gbtree(gb_trees:update(Hash, [N|List], Tree), N - 1, M) + end. + + +%% +%% Timings (microseconds): +%% +%% go_arrayofidict(200000) : 1266931 +%% go_arrayofidict(1000000) : 7387219 +%% go_arrayofidict(5000000) : 49511484 + +go_arrayofdict(N) -> + go_arrayofdict(array:new(256, {default, dict:new()}), N, N). + +go_arrayofdict(_, 0, _) -> + % dict:to_list(array:get(0, Array)), + % dict:to_list(array:get(1, Array)), + % dict:to_list(array:get(2, Array)), + % dict:to_list(array:get(3, Array)), + % dict:to_list(array:get(4, Array)), + % dict:to_list(array:get(5, Array)), + % dict:to_list(array:get(6, Array)), + % dict:to_list(array:get(7, Array)), + % dict:to_list(array:get(8, Array)), + % dict:to_list(array:get(9, Array)), + {erlang:memory(), statistics(garbage_collection)}; +go_arrayofdict(Array, N, M) -> + % Lookup a random key - which may not be present + LookupKey = lists:concat(["key-", random:uniform(M)]), + LookupHash = hash(LookupKey), + LookupIndex = hash_to_index(LookupHash), + dict:find(LookupHash, array:get(LookupIndex, Array)), + + % Add a new key - which may be present so value to be appended + Key = lists:concat(["key-", N]), + Hash = hash(Key), + Index = hash_to_index(Hash), + D = array:get(Index, Array), + case dict:find(Hash, D) of + error -> + go_arrayofdict(array:set(Index, + dict:store(Hash, [N], D), Array), N-1, M); + {ok, List} -> + go_arrayofdict(array:set(Index, + dict:store(Hash, [N|List], D), Array), N-1, M) + end. + +%% +%% Timings (microseconds): +%% +%% go_arrayofgbtree(200000) : 1176224 +%% go_arrayofgbtree(1000000) : 7480653 +%% go_arrayofgbtree(5000000) : 41266701 + +go_arrayofgbtree(N) -> + go_arrayofgbtree(array:new(256, {default, gb_trees:empty()}), N, N). + +go_arrayofgbtree(_, 0, _) -> + % gb_trees:to_list(array:get(0, Array)), + % gb_trees:to_list(array:get(1, Array)), + % gb_trees:to_list(array:get(2, Array)), + % gb_trees:to_list(array:get(3, Array)), + % gb_trees:to_list(array:get(4, Array)), + % gb_trees:to_list(array:get(5, Array)), + % gb_trees:to_list(array:get(6, Array)), + % gb_trees:to_list(array:get(7, Array)), + % gb_trees:to_list(array:get(8, Array)), + % gb_trees:to_list(array:get(9, Array)), + {erlang:memory(), statistics(garbage_collection)}; +go_arrayofgbtree(Array, N, M) -> + % Lookup a random key - which may not be present + LookupKey = lists:concat(["key-", random:uniform(M)]), + LookupHash = hash(LookupKey), + LookupIndex = hash_to_index(LookupHash), + gb_trees:lookup(LookupHash, array:get(LookupIndex, Array)), + + % Add a new key - which may be present so value to be appended + Key = lists:concat(["key-", N]), + Hash = hash(Key), + Index = hash_to_index(Hash), + Tree = array:get(Index, Array), + case gb_trees:lookup(Hash, Tree) of + none -> + go_arrayofgbtree(array:set(Index, + gb_trees:insert(Hash, [N], Tree), Array), N - 1, M); + {value, List} -> + go_arrayofgbtree(array:set(Index, + gb_trees:update(Hash, [N|List], Tree), Array), N - 1, M) + end. + + +%% +%% Timings (microseconds): +%% +%% go_arrayofdict_withcache(200000) : 1432951 +%% go_arrayofdict_withcache(1000000) : 9140169 +%% go_arrayofdict_withcache(5000000) : 59435511 + +go_arrayofdict_withcache(N) -> + go_arrayofdict_withcache({array:new(256, {default, dict:new()}), + array:new(256, {default, dict:new()})}, N, N). + +go_arrayofdict_withcache(_, 0, _) -> + {erlang:memory(), statistics(garbage_collection)}; +go_arrayofdict_withcache({MArray, CArray}, N, M) -> + % Lookup a random key - which may not be present + LookupKey = lists:concat(["key-", random:uniform(M)]), + LookupHash = hash(LookupKey), + LookupIndex = hash_to_index(LookupHash), + dict:find(LookupHash, array:get(LookupIndex, CArray)), + dict:find(LookupHash, array:get(LookupIndex, MArray)), + + % Add a new key - which may be present so value to be appended + Key = lists:concat(["key-", N]), + Hash = hash(Key), + Index = hash_to_index(Hash), + Cache = array:get(Index, CArray), + case dict:find(Hash, Cache) of + error -> + UpdCache = dict:store(Hash, [N], Cache); + {ok, _} -> + UpdCache = dict:append(Hash, N, Cache) + end, + case dict:size(UpdCache) of + ?CACHE_SIZE -> + UpdCArray = array:set(Index, dict:new(), CArray), + UpdMArray = array:set(Index, dict:merge(fun merge_values/3, UpdCache, array:get(Index, MArray)), MArray), + go_arrayofdict_withcache({UpdMArray, UpdCArray}, N - 1, M); + _ -> + UpdCArray = array:set(Index, UpdCache, CArray), + go_arrayofdict_withcache({MArray, UpdCArray}, N - 1, M) + end. + + + +merge_values(_, Value1, Value2) -> + lists:append(Value1, Value2). + + + From b09246ef0482cfb9034f79d29345e6786b75ba00 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 25 May 2015 23:45:35 +0100 Subject: [PATCH 002/167] Removing test files and binaries --- src/activefile_test.cdb | Bin 2048 -> 0 bytes src/from_dict_test.cdb | Bin 2108 -> 0 bytes src/full.cdb | Bin 582764 -> 0 bytes src/hashtable1_test.cdb | Bin 2336 -> 0 bytes src/leveled_bst.beam | Bin 4012 -> 0 bytes src/leveled_cdb.beam | Bin 21212 -> 0 bytes src/leveled_internal.beam | Bin 3068 -> 0 bytes src/simple.cdb | Bin 2124 -> 0 bytes src/test.cdb | Bin 32 -> 0 bytes src/test_inconsole.cdb | 0 src/test_mem.cdb | Bin 2115 -> 0 bytes 11 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/activefile_test.cdb delete mode 100644 src/from_dict_test.cdb delete mode 100644 src/full.cdb delete mode 100644 src/hashtable1_test.cdb delete mode 100644 src/leveled_bst.beam delete mode 100644 src/leveled_cdb.beam delete mode 100644 src/leveled_internal.beam delete mode 100644 src/simple.cdb delete mode 100644 src/test.cdb delete mode 100644 src/test_inconsole.cdb delete mode 100644 src/test_mem.cdb diff --git a/src/activefile_test.cdb b/src/activefile_test.cdb deleted file mode 100644 index e9784eb4c8849062d374dd2058af8814b024f3bd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2048 ccmZQz7zLvtFd71*Aut*OqaiRF0wXO100;m80RR91 diff --git a/src/from_dict_test.cdb b/src/from_dict_test.cdb deleted file mode 100644 index 5cbf317443081e11d62055b2b9d52fdf90e452b2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2108 zcmb2)U;qP|QF=5SfFUrN4o33>Cwr2{4)LkC(vfM^ik2Ff3$A?akOh65we q`K&;kJmap{nG_f=v2fq}pGgp(F)$q2%EZ72jc<_HF(97-$Oiz@yfu>m diff --git a/src/full.cdb b/src/full.cdb deleted file mode 100644 index ffa584bafa2352c088b28bd9feec3aa6ab14563a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 582764 zcma%^b+}eV_xBH72L&ai6$vHw^bBDT2BCmTC?cJQE~NydK~OplpoG)`L6H!oloA9b zR8m?bq*LPEKW5GQ=d;%JT#vH{zTeq1vp@ID>@{=WB;8Dx&^93>>3N$=AOnS5F*t51`%rBA37|9zh)d^##e`h-{hue@A7y^tq;!fSE9 zR6d_36-b{@BhK$E`Ks+}G!@^1A%arwuEl zPk1~2`+G0=H2LN933cOqjVeC9_e%PN`f+~k>ptyMBYi@nIG^;UPaD?p<@b5Vr^)rx zCp3=#KKVUJ8~OVjZUXmhkv^em{P(3=`?Nwk->yFGeY&=zZ&&^zDpf|0fOb>Dod5KKF+Dw8BXL=c7jZ^zi4t9}*|{`j(mu{W{gBDapRy z56|#v;+N?Ys>SzjINPTwbNzi2=le8iVfutF@!#KD?9)-pe81mY?$e}I=@XLTzu&pW zr=`~Wbkru$ZBCz1C;t0U+kBdThi~VoUwnFRH~jq``1}3-x$OMIr+p5lPv{o=^B?u; z+T*^RiKl(q@T~vyd*^+cdDvFGUmy6iPeO)AeB@8#)3xa` zJbE6bGWs+*bB2UY|9Ah#e43m+!z2C@bNaMTo(zxpUz;z(qvw=ZD8r-vN`Bg>rHVm0 zB{DqX`rdOspH$kXhs$P2s1=vrko)wo^!4atpkC!NJZk^iiau@lvQO_-@oDmFP`?`f zekpJIKdrorAo8=gQaEv&I>prksWLorCsW^l9=XpI*4)xoCLZr#o-@c9y#9(^2<*dLbdB?}xNL zP08TX#4H&f@!97wpI*qG@e$7rbNF;;u8fcRJ29_MYvjxL==mfU@acs@{?AK2?bD>9 z86Q2*QN?|l@?6G8JRC0V(?pi>5%&$HPg6|BN1ula<$PM|g^Z8*PI<|vr7CB9^m*G^ z)u&0X`9Hr<&8JB<{hue+@@Yz)jF0%QQ8(kG&(+}uK23Vxr}rBBb~kM9)3vR`*ij9x!2vNeLnR4a-kRWQ(x$(k9}HUpl{cx!9G1a%-^TNDBr&+W1!#1 z`7~*wPj^o7X{qVHoaC84O`HYon&ZDpiXeH#Ai(|da|J{k`$?9ceZt$w zKF58UHy)cZ$N!Ib)Vy8Sq^?QLoBiK^I`)kJ+w$3>&#+nDliuyoB&mA~_Eap(^1r0f z$6KG6=e1G3SeW=f#nIYaWj+LTls}ec{onFv;p!+=&{GWT$hw3{!N3T2;8j-EE*%V< zU=U(ylQqF02nM-$-}OAfAPEL_aPLo}gFz7ty7IvZG8i<$VC0dxvx31840cY_RU#NH z!Ql3`7&bQ;91aHCk-gx*peH;w7JWM_)k{H70E4lg=Ca>|fpQ1_eoEt{U?70O#B+;k z-3L`{1%*Ab`Q$ z@wd(u4+a7l;@LtUZvs7`Q$;jje6#AlU4M`uAizv`HBFBVqa{(z9$$6U?}c3%xxVE1Td6%W9x(m!9V~* z$zNwDydMk%FqE1%C)yDVlwQ&=EN;>{7zkh}^ZLTte+L5r3}p}GtkEMF2w*6;h0EQMV&TXj}30k=YoL%h7o^43y_!&p%VMC;Q}j zvvx2Lz;G%{_K&v)0|5-D$L1;1Hy8+DICHvC!SlgD0K?htU!>_33tNQ2w-rNq2%J+!9W0m z+es6b!WxNj0vOyK`@BD_!WbujAu6PO1LEO~(5WKoUpLbYXn`PrA=s_ju30b;z!1Dc z{niJ9fdGaOc|WNCW-w6NAcvO!1?FZZ2w(_xxx#&z{h1(uA#@v+vLRH70ERFHd2-!g zAb=t4=uZd08i@%47{c8du?$vWOc213vl<_Qh(06qk&Jfdp2?vFgfJX}{)1_Bsb=crVFa4=B1)ikTx z`pIA*fT8WRyWIx{0|5-}2HmfkBNzx^nDT8#z74@3jgB z0vLv8mWz)B0|5*p`ikk*gMk2sktgeAJR1xIFpL^s%k~Qf0vJYTZP4bYU?6~DOs_@_ z>I4JjIoNZ@tNa)Y1Tc)7ex}kLrh!-clZ%d#YdSFalTt81C?I^FodAFe!6Uuhqvth88#i7>;FK_uD_g zKmfz>@hkhk6$}J0oH)63>E>V{fZ=4{eb2TG1_BsPW!{_faxf6UaC+RqDzGcT903ex zP95n6(Q4)hU^v^YX&cxnWsU%bb7@*MfE_gE@GvR=e0$fCzR4U7lk#YE;Tm^AjYGks zXhFqO9fO`?U`NaCyAupJMLgoKiounG0jG#ZB9h(P7#mXug4qi2nKw3;p|TFK1rY_ zJT?|BEw;Z*&=bI5Y-^FztAc@Y2i~Z}^3=gV0E3B=We1H91_BsNuB@Kj1OovKrZ&_l z0>@`KBY?qlgQ}hKg*yEA(5 zcnUn4mSxDrpeKOA*@eUvy@P=O2It+={k<_5C{+@vQ~yyf7zki+ncT9{^tlZf0|5-~PW%l+(L*y=5;3d!# zI#omiv#m@CdIA`N-N-$(Trd#85Im{i@BM>;0EQ6x@}9{R43sv=!F{`a7Yqb2gu2?R z@-x9e07K|@U7wf|3MIlShulb|Q` zk&J$+U8G&m6Tpz{`?}fh2Lk~N$)EY)rB{Q20EQHA*YEUOFc83y^5+Vty95KJ(dvit z`%eV}0Su{&@i$EHU`V(8(A}5WrAsPT^B? zgMrdZx?;)i3IziJ3}vd789Y1~2w*6C@bk%ef`I^ra*HR<9~}$?Fm$WR{{)Z-Ezr@! z*Z}1d!lYcYVU4@O%W#}Q4;gXIv$uH~URC1^!5~_8cmiIAxh8<2Du0vKA2 z$+{9=hPftyp>?jbXW&(pYf878CTV_$mtn36U}$?~?GAWV<(dG7c0<0$UnhbArzno9em%AtY=*}f^pS~3{W+l* zY=*}ff4MIy3a6;0nZ81Q*PMDDj|^u1ufhZn()Rj64S z^n}O8qGhMn=M8!S7>sQ>Ic7{SQ0~A#ICoS90|5*sN?iPRb}$gYU~u`-=Ab`Pnx2>Pv3mG56u2?1|KoF?z-^xOQA{xFt|y(_@j@4fdB@#5doz!1Fs;k7G+fdGaOPaPUnBp4`dkb~Ce zg}GS}PLm3uuB|Ewvwxhy!4SIbvJaPpDiOdCrof8VY%mbO5cbnd@4y;K5KfZ{;qE@x z0#;#h1_wiaR(JI*s1aehmC?>|)f)yq0Sw93jPLz(Fc83y{ORe>bqWRo7*f15^{F$# zKmbF^9S_D-4F*c1)!IAj_XYz245^>J{BN6JAb=rFom)pA1_J>MX*a$0V~1cMOeNBN z_vW?R!9V~*{at&w1U;e2D%wA+S6JE-zH~1Xn>D7|$Z!J!7>bu4l8*%g0SqN-40wKC zFc82{a?j~?X@h|PhEj7+j2;^dlwQ&m&i_>+7zkh}Q~koFb-_RYL)k-bT`e071Td6K zd3)39U?6~@`_=yw!m5JM0v$b=TkM zBl@j;{n=n3fMMjZy@!Vc0|5-9ChfbBEf@%382#9h9~K4!0Sse4I{0aYV4yq)`%jC{ z76$_X4C7`r{i1v@5Wuigj@=jzdP4a&YFsRBt)M5YF)YhnL~aQN0vMK$F43Y-Fc847 z;!fFj&IJPj3@ckz-|%`c5Wui1TaC+q1p@&LtA|(pvPCcuz_8{)rO&Sg0|5+cJKP`D zG#Drk&gH$k_IfZ7z;OTmy}1zR@i3`y(MJ_k_Mj(#;n-v4(hUs;0vL`@5^q-w1_BsP z9Fxr#1p@&LC;Qd9k}((vU^tbj-o_chKmfz(&l-JJG#ChAIP-UdFTM!|0vOKrJoCdd z!9W1RxrF1NeisaQm{fka@ON1BNc0s(Mt}A?4+Zm|O`IXzLCDqHVPi-}I7Kp&3y(B}Esr=uxPv-!FbOtQBq2;HBVA>0 zMF>g68NwZm*q60m=p~#Y8QIzGvO!oS&Jgb4_9YdDu!@ZE;YG5q`yPTpTM`}{i;U~t`S z1S~Kl;p54pnM3Y=04*SV>0WYncEHcugMk1B=beX5>lzG{Dv9)C7M%_T0vKFQKL5fi z!9W->xVm)Vv0sCM00!4RPJhrg7zki+lj=mrd%-{egWJ#F?*dylk`N}9+?{#15^UYX z8GI@cFL(VO7Lt1R0{Vn ze$wF%1Tf@Zxy!<0TN3(6M%xd+e*ns-EJ`F>cc}fF!9V~*^5T0c+z18&7*f>QpV%xI z2w+ILYuS_y!9Z!WTD@XH-C!VqA$8&Pw;?1UB>@a+>aE%aAqgo7U`V?)!+r>>NJ*GV zq+6PB8p0}462MUNvlW7uDoJRvihkXiG#UDc5GIw1&D+rMv0xy8q4*2ivn~$?0vJlX zzN_NX!9V~*$v+m(9}o-#FqHaY@#NgWKG8W!`gP=O#M3;C=br%|88ODU?7Yb?#r*<`V;hcm{hu`PnT!L2R#7{$Fg*PGIcNz zz;JwQpGvEOfdGaRr+arP6AT0}oa|lb?@xn)0ESbEdH%>33zT2v3W0Qjr+9oVKRTQOkxeC#< zID-z5Dq@GaZ-O{%oFN!^r9R`}ZCeq-q$(1s_iTvU#2LaJy7UmpwvFa&Q?dqcWl zAb=r6!Mc~n1p}oGvOjMDbF(6xCRIY+Qt!a*A7}8VAasWcY4e9F5x@{8Z+ST)7zkhp zJA6<{SR*Mym{bY(;FG+t3X3y17;;Nu!7xe^MpYSYD`@_O77)UuO0xBNGkh2f1TZ8o zmaXolU?6}YMeW=z-wp->7*g);dhJ#)P#UdPb^EbNFc83yx=7!#hk}6shBS41&8is; z1TduCvU&aCU?7Yb(kEK*rWL_J07I!SJ6(w8TLfdGcqPgcL&AQ&j!YMR&BusawC zU}$^&+L1=VKmbF#Pp)6w77PS1%m?Y-4C`k?g-mo(G%pS16Hb#V!^X?E*8~Fr48yZl zOqVJc2w)h|tDKq?3E{uc}cFs#b4aa8+YAb?@@r`wMF5ex({thu}U;>*E6 z0K?k0oi4+95=HpZy|S)A&ke7HI}pHdtA1!2-iPrpsd7=@rLWJ1@(EwMSB_;~`H>0+ z0vL{uTUTOCFc83S;?(Lqd4qufhLgRHeENMb5WsLM>%kvN1p@&Lr^oNTFfkYiU^sJf z-{FK{Ab{a)*LE8}4+g>~+PSnzS26?x9wyZj=d2v-077PS1m|R+E_4!~R zfWg$(s>AyR0|5-C8`b!8YcLSNV5VgCv$cbP00y(Guiw}Z3l^T@s!82yT4wGCFTB8A31F*7^+yhigKZR9jbw^@D(WoWa3x<33~WLYwh0sdmxF zH*ehydcsuVSjH<`lY)T&hU1^#p7?t(5WsNa%)_r<3I+lgPJTE|oD2p67*0K&oThs) z5WsMH;)FW8gMqNdaOU{frVWCD0EV-jOMeU-Xqxcbq}sXk&%N0w+yM`hn&Y#F!V9-S zDKgQudaKs^!U@xeiK4G>78?}wI7KF6e_qM;L@?kKnTWr9`_=iufKy~5p&#~qAsBFq zOeE(|I{};FaR&V?n@IhYyazVJ;|#%|Uzza9vQQTf_f*KK~TN&+`da{4e6Tpyc?ex95 zf`I^r9}fls7*aod=auh*fdGay z?>y-7OfV3@kaqLS?Y;>H!c-#NH?O=`G#ChAs81@^h(J$hvWoT$*t0R{31BETcgTr) z!9V~*@d{(Uy%-DxFqEi1?33QXKmbF@{TFim84Lt4l$v$E*c-t>=_OtMM9=HNKmbFT z8mC`v8Vm$5ls)|JyVrt&0ETjl-f!C?7zkkKr(}Ts0jv2x0+Vl za(o#K1TeI{)iYE0Koi2GM!WuN!L!EO-fF@!LwG3v7;RUl*-XYe(K(V6z;>=CL& z0K=Gmd!IcP43y_!|4p*6A!Z0+Qe)hdb{S!7HO}B`410f66g~`u5y?ia%BGBh77)Uu z#tG;&VP)e=ttSKnVZ^X1ch&l7f`I^r)uU@< zS{)1oFs!*#-IWan0vOh|xn2~`lNdsn)L2*WS`Ij|5@&EQ+=!70VOYh(q{c=4!^w{tG}spm1TdU>tWldv!9W1R z=}GnM?O-5);momG89M|60Ssq5UCQ||7zm$e=Q5mqwoNeLVN%;U|5W(!Sd=0gb=y6s zlP{bwTG%L>v#<$#aN-O)1lx%1U0fSJIB|wx;ML|Ng!&V{bZ;Y3adreB{y0OpgPgr} zE(~Cn5GJ*e+P7f>3}A7Fa0gv;`%f?kTf&#_ZDh*rx(=!a#s;Dcicr%5eiKd)#EADlRYH<|Ib)-(96@!+xAa55N8R$O=_9@sr^|vau;WCF!aQXR_txpF7KJQqr&fL7UJQxUIaQ*SauO15qeBQC# zWV}6baxf6U;CAw>BQpg9KJQrWF1@s7W-t)I5Er-H2;>nuRYYTk@?KCr;WVig>~{au z7lMHRhTu(xzg0IF2w(`2Yjm>>!9Zz)9Cq$zvtS^AA=JaTmx6%+hR`ifO?opJ2w(`4 z{p7p@!9V~**Z~c{e=QgYUzDQNNelQTg(9=^=RRukv1v=&Lqw}!Ab?@iv;k?Z1OovKqf-y7Q#}|6 zU>MVVOw%L5KzR=Kbo6nJU?6~D+*tGG!C)YOVSD8$72fp-<=ZH!>-l9+J|*yBS-x(E ziU$J$49f@h9Wgx^2w+%oqu2LYf`I^rmF)_aogWMYFyc8jZ=n~0fdGcpgR{LcC>RJ} zSaUUZzbAr$0EV?q=QsN#7$^_U<^1}soWVc^H=?eZ63C z)>%deJwETai0$cEK2I><6uF33Z`pQMFyIuqNK|O@t_%j8A{WV7>2^;F2Am=nsr{)R zrV0j}A{XfzS(dB`2Am=nnevGih< z<#d?+;|v;Baa{MSxYLCy5y0RkQ@MLz2Lk~NZl?@B4r?SwI8Exf`*+}PunLPaX#CuX z>qmYX2{j^gs)$BctzR?f31A3zr&8-f!9V~*@W$0$lVBi#Aw=#PnQjFGr44d;iDa0Y z9pN;o6Y4?P6qxEhA`QR{PJ%w5Wo<&|BPQ@jpPWYNu6-FrW}J+ zSe(JZkZo7jE&(+njH)u)n0D8QpeKMK*@`p|@&^L}49Ux8{`#w6Ab=r7{j4*iU?6}Y z<(4*?#{~nW(P~+Ta_NGB0EW~hTD4mr3T!Y@UGfdGaUZOh;PCKw1{XnFmE z*4cxB0ESkd)UQ7@7zki!{Z#Et6@!7&t)_ilwMQ0@Xl+&KmfzYi&Yxb z3I+lgMoq2UW=k*-z%V*ZHQOf`2w)h~t!Bn^!9aNq_SDs!y@P=OhH>L=JbN)12w>P| zj~5704?_7iYQJp25GbDzCUurQwPIPeU?6~D`6ug-Eer+%7*Ms|p(*{Z zgMk2sQ)xF;c`X`-T#{X^kY|&?U`92koJyP?hVp*2|C5_%b_p+}$kFd8L z9!2YFeOAPmjGrTo<802Uap0j5zcJ?0-jz1(8&8z-pV_!DI5n$}s8lT?;IiYm^desenkCHPS0VZCk zblLAva)u+o^2TI2gV8)e|KY|=bfY}9~m8=7D z90BHzPRjce$Z-Vt_ooW`L6zgZ&ZAK+ug>;m6Gj=%{%f*HfgDGG^QNuGjR85KG9qWk z*?B>ZBf#azEc$zp;|OqdFEM>7kmCq&-6CDRi6F-j;O2?ctr9?vBf#yzA9{TbavTBf zZhT)Y1ITd%#4jha`H7h0-6*07eSbXR%NF=3AlQjs*SdimM?mo2U1#kAIgWr3iQUH5 z2RWhca%{E?|A8DwK&aEXO?!~z2ngM$V9P&1jw2vUmb`Ue204y^u+zS({wv6F1cbZr zW$#x&jw2x78ami7-Zz z0!qAFM!^84a0Hb6soEC+yTTDr>f4&1g^nTgjDD`lM*pV86^?*1?^nL!-{iQ$5m5G* zdl%~ZXR2@nl>7d_!y7=3BcOj6ehR9N*IY+uGn;w7Y>gwJ+03jN7|3x1G*6$l?nsd1 z2x!qcP0Iox#}Ux-Qmbq0K#n7z)#S!MmH;`9fYzznj2#PdLPwe&9cHBkIgWs~XMS7n zV+CB}2x#~D?_+$-iEA7I^YiF^fFoWN6J5!k{*14=!4WWQNG{z86_@%~ z9|v+A0V_M7IvRl-N5HD|C;$BlWWw(a58y(K~C8M`D3R9z|61N+A|hTp zd79tbi?6Yl2N8*4(-!%iz4#iJ%T*6wcpNH&(jp?Y`}Si~K#tNPB3<{&2hV^UrA0)h z$jy%5fgGhpM0VNhU5bJnrA0(;%Nv!x0XaHg2v+XqOa2v4;17vKUk%?;&zFq{5(Q(w zjs9;V$O)zM*M=_iyLbW*Bnl>8>_64-;lhU+AGB^TE{eJSrKS7Qo!1SBv zIy412jsP?EVx#LI#}Qz5ex1rKK#n88++Xi>y#{g|0Y3X-cK_Ng@DCo3KAkgpye}IM zBnr;%&YqtdA}GE_5#Vyf_J)2g6?n8naCLuIlAo#L zYrJ&bd~3xop{Y0m+&sA zX#DyG`+dy`^M+t2S52u5avTA{doA1MPyYx4k3wW!al0dwPN=&ax2L_IONGEk0ijOq zf8Wp4Lf{Ap-S=?fF{liVfH0X4Re-rV{>O~sJ;KEQu}>(3o!Y#JU&F-LxZGWAmff$A zguoGy)smO{@EW0CWOV4ki%HkZ#t~Lsx(t zM?h7aHpque39PGV)9J_F@@30XdF-jL5?F}-0;Wubq6_)fUW%8ZNFVb zSZCO%XZ216ec5;*QCOBx97n+F871~V4{{smRs4jxVia+DSs@wfMWx*p^xEiw|% z?E9f4$WdBkB!5_1Vg$%hT4bbtUYRF9$WdBkq~BWi`d1)FX_1jBy85FC#+de=MU!w@H?kj)q=Ieq#Bo@uya%2yvaReCKxA9^XkP}MhHMgzx z8x;}{BuXaA?H=W~LE>w)UznV;D6bFpONJxB)ZUbmek(!ZClV#o)#iTaU-IK?yfUWZ ztk?WYzGOH8%q}kQjt@{sh9kh-!KYgI0EJ{Y0{p_dq5cRMe*8!BXxRDD+5I!c1BsHe zhjHm3#}VMX<>^ZmL5`0CTx37-*CLSP2yi)ONVSY0#}VM__JCe9K#q?BTsIl^d{L0& z2yl~YOulbGjw8VBkSM+1tdMvhQF3?1XunwzU!$Xdn6`SQKh}pI>5?M){POx*zUFu! zQ3`hE)|f3I#}N?x;|E9kfE-6ah>UmsJqL0e0U;+&@BKQ+aRh`qKDGK^Ajc69`onSY zC$C9T_ZCv%2nh4|_~)*H97jOdna>-)St0QSz7+0ksSJLzBECiukm-&*?H3vNF+C}x zqsvN8@--(c9VA<@B5x{?;|Roq%=*{XfE-6aidR>CSQ_Lw0#Y70d~r0$aRj7Va_C4N zkmCqQZT5`vFZmKbktn5kY5!XPk{@5A^Mq#KT5m0i&*2B9& zjw7Jd$_*D9fE-6a=`!1Y@GtoiKanVvX|U^4|B@eHqX;Owz4K>2Kq2uHiBh?BJ-+Y( ziuf8uK%d`R`48VxeEibUg`w4Rf*M~AYBsHZ?@vIEBcOTe;m!*SoM?mXzr~dT;e;NM{C9*{nj~QJ~9`ynL_!^hn6AgbX4wb=| zgW64a@7gku;|Q4KeO+JnYP>2Yx{=)LZBXL~7&dTPwM`($5imU8*nA&?97n*2qzTXe z3vwI*Bd=arR}7zW(@>6j(}CEJB-Z%avTAxr?vWV3CM8-thvzmnguzIfVF-8`gZ}y36IHT`Sa+D zAjc7Ky^|XG@R~$VBuW>xD^%~GFB=aeO2_i&X;lN{I0B9j&aQ8Q97n*3tGUuQ204y^ zlSy61{Rna#0jKhHpZyNVaRi(m*yrjckmCqAbEEgBk3fzi;B1SZ{yGbC90BK^`0-Ld zkfVV_H8;=v%8$efTQG{6>X}P+C-E zp6&gl-v)`V@zU*gKRq@Po)Sunirl6jUq}EsI$$W4l=8RoYWyLwXja;lKG?7D6N!qk z{b`2!t%Ud*Z7Ab4GXL&3Dij__R7{l5dd6>q#MgM~a&DWhKG?7DK%!!5Plw8WDlnGg$EK9GZh*??Z@f(8ZX_Z{Pm(=-zz+jsF*wS=i`1Y7+<5K0KYMJ#TbtO zI!r4b4KI1B7^v}4fU^f>_ALcDjsWMaiVw~VazbT9w!+^fgB(YI%h6Q|`?*x%ClVD` zcPeM|Gj)88J|wOiS9|$Ms0=;|aFe@c=fNPy5#V;{)g-@$QFtIxad-JfL%%|buTccV zz*$4``nurVD5B5!{#KyI5fJQeb@~d(aRdbKS7CW|kmCplk*WNkBOoW#T~7KSyPr!H ze)3KUb*z37KU2roC;~!%RJ&6hs0@yPFpt%J=_iom2najlpZEP5M&W@(CEP#%w)ZQf z_!>n(=HK$9->}BN?5Sk*SJ%p4_?qK^L?zk$Ze2CVaRlP+hrUmb0y&O=6tDDp;wg~h z2uS&7!J(@`j;{rz`a18*vLMG1h~oy?&P)I~j({{T=l(qn$Z-Uu{cZk^u^`9S0@8i; z^?zwWjw7JjZG5JbSK|#;(U#U-j)NK>1r%G>u~JWv;|M5TqUDpnfE-6ai4U4Q^B%}? z1eDyEZpCeo;|M6VBK43CASd*UE}P|#eIUmXP^Nz3-<3d)BcSZIZ}08}IgWsG>zDsr z734Sq`s%hWu-3$DuA_@pi#+Yi#=qLEG@Du}`*M)u2xy+B`b&?297jNlZZ$eh204y^ zmgh>GE)Q}X0j;K&-M<**I09OyEjB1O$O#>3x)xbJ0OU9V+Mb-TAREYW1hgAJWy%nc z;|Q3+yDP!_16~yqU2l}FnJ*jvYO^xzlLm!vfE-7_@TY2ZeiP(40!Fm2_woUd;|Lge zP3-FhavTAp2FX(wK#n6|bb*TB)de|@fH7^$4c-89LciO)gD2MpIgWsFpAMYA734Sq zHfN^~;C&V^+eTftB&PVX@e_&4va}m3lmj`AfaTM-weJsd904oN?S4NO$Z-U$?6zpz zcOb_RuqsW;?PoxaBVhH^xeKO%97n*Ki?gOY4ssj;Yx@=$lJpI08;RwQ_q?kmCqA{mHt?e}WuGz?ti-=f44R906yWr~l5Mb5Qt+MCII*sRsK~ z6!A6oql&sh_B(eTRYhYj4?2p53~KDP=u$8iuWn%PgPI^`yBcL|2Xd4a9r1eg8vF)w zlolO{!nNAG3UZVd9m#d_p5Mleudx?G9jWbNr{BhnuW`AqRc`t=s0>Prj!f~2iyMI) zrA0?}<-n!8L5_~=I&vEZ_iq4lbimN;)d9qSzALuGITn0cXC zav_l82r#>3%7D=z#}Q!ez>H;iK#n88j}ALs!>jRL=h4WH?|uww90AVmwQlO{er%Rr7JAaw8CyRv{BM?jdwY!9Y`97jOd zAY19C#o=+ZUYd2i_!@mFG)qoo|Is6WSH(m(^DOaUA&mzTwP6Da&G12| z_!@1f;dyfH_F*9%pH4&$c4{L!W`F2|PVqG^msh&C^4y$%267w$%V&<>djRA(0#=+Ie)3I_;|N&U<6@5sAjc7~D%H7HdVw5A z!0O2-b8i4Sj(|0nP8F{UavT9`KYp*A4-RWQkf^Q8*f6sXJ;&GRDBxayw}Ri})@UG6 zyQuZFPy7~9e2sqa94nOkt>1vs4oAT8Arnqa_Wj~;1e~}scF$uV#}RO{#j207>oc>*rN1cLC0w(^~R;JbY8Pq5(CStpjn>GYFN{fkj-3j6h$WdBM zB#Mkp(+T7#EhdudZ%*{vxbZbcx{1`bD_{9--1r)o>)N+(J%q}jw3x^gd$_d?$WdBM zWLLa&@HWWNIm1M5FshcH0 zjw8VA*YBNQ4RRa-=KgGWs4U2F1o+l*`?7j9-s?OXu{e4S)Hnj1-Cvk+6y!JpoHw6c zyE(`Sl@U+QX>t|hI09ULy5YK?OAUS^(QtKl>rZ~Bj;~P!xNf#+4) zfZ%;sHGUT4I08asUSI25kQ3@I#~<44=Td{8NHjv7Jbc5?)bTZnfY816^F!4VK9 z>z?roL5?FJ?DY1-{TjyLClZZt=i09JE2Q`uML?Fh+}ZlN;1i^b4&R!z3)DCQk}bMC zuRh3e1SIEoZk_-+j(`+ZA8hFcavT9E4^4gR50Dd@s#2ymdl}?70#d8-{6CQ62uM?D zT=nW6D4c6no#r;a1ace! zZI3mYP>2Yx^?g;SUeKmAr0$)g*OQX;+_$Xj>!PQM*Ld64##+WwiYQx}b z2>ovFq^j{EJS7|f<3^|NvjF5c0=D=o6`%BKylfkFs`Te3P~)S3Wf`iTeH-NXC}8=F z8ms>WIX((l@lW;PAA%eo1+45@Hvb`z^i+`J2so8j6i5U)j)2qs%hgx`avT9?ZdL44 z800tt&bAuZ#Gi98cp%X@mu+xue~Kc$#sZ1<(1Lvr9#zF+wAg4#x&{72if<*@DEfA5 zPXDopud$aZ8?m1@JnKIe@ii{z?{DwsKhhTeYO{^RbGxeek92&EmoAqt{4KN;rNu_- z$Hm9}a1>wTrR#U+EcHXF#lPBYBU5^Ie?OGQ*Ldmnhy2t1u`7%JMxu?}&-oU=g%*Tl}ldmI<-HyZ=bX z*Ldl2@v>unII?&k(K2;##jk!iim&m~b>;N~{7`D~uQpp|(UD2=c2((RlKDgM}% z#lPBYncJI?><@Lt*XSs~U&^u#=tYNV%cB8TdUp4C;lGh+IlFoD)srB{5#YSz!(0tO zPNk<)+Yt6TgETM}XTg z&pmt<`$Eo6Z zK#n6IM5>dyM}wSDcR6$Dz|tVc5fJKZ|8LfS97jOt&cjcp0y&O=FzH9{odj|m0bwVy zUFkuNBOu&y{opf@;|RzFt=q%wj`xd<{#f+uHD5MGJrFdi30DnLyzD5UADSzFP;~Z3m&{Q>VW6?e!#}SbFg>BupfE-6an%8$%tp#!% z0cj6(s=pEBI0Diw>Djs-$Z-VJ(T?xKiVbh5igqnM{*5mi|BXbe*y@$P6$LqtfZ~PM z^`8N9d=yZk-s+|PfR4rg)n=>Y_9M?Og3@sWlv;N%XGM?`dPW!DTV*K7aRij9wXa)t zkmCp_yP;j1{vgK@P;O;XgIplT5zw!;OoUZ9UUMCtp8ic|Up9Us(P}n!>cGE2jw7IX zmT`OE2RV*_7JbH_+yQcY6wva-FgeW&;y zkmCqwd;XPjl|happk4CInfHSnN5Di)D!`i+UKJDF8}doy%f;*ZFfE6cl@2U!N904o)bbap($Z-U$%F?Y}Cy?U^SUt9Hxt~FfBVf(xUYQ$$97n*~ z?wfP}0&>D*a;Y~Je-Gq10`89k*{gXq8c4J*YMO4zI#A;XIF>W@j1nNn5paA|mfd4P zjw9g2e~Ay%f*eP{$ri2K&I37)fKyL&e3yY7N5JWkEwhXSIgWrc_nMS10CF4wXOoT= z8wGM40q61^$n_M+(Lkb0S#ruxkSHxKT3K+ikFDSbQe6~%lkVdUkE#)0W9xMnvCXO9 ztP66K78mh`SqfeNIZBI*#Pf+|dVw6J#YOVF)@u%c9HqrYYE#FNZ-N}9#YOtPmIrTu z9HqrYrc{%2%|MRQ;v&2D=&h?DM`>}9+i_rPbCBZ*unl?V`*<0C$k?%HQNcF`__Fap zqGRlE-hR13jw8T$)of)KgB(YI37)%9d644>FqzVI#AJ};2rzZ1+xL%w97llZN`22S z2RV)aGpg62r$LS*!0g=3TfPN3jsSCeHr;#{K?tu?_z5Gj)884tQL* zskN#nR0cZ+ps-=`%N1PN(_P=P%? zLnsmkAz)BSceg=zcQ;CRNeWVuqM($tv~;(`fB)Wn&v%{A^ZwU*-sN(wv$^)np1IEK z*>lf1&wX6{-$-=Aow_?ZS|Nq6;~fHWX}hmhMQw?Hc9g-PCO>VZv*TY2NcKa^fuGXp z@ehIU*Mu4!yGWHYDA~UKLn&byqw>s)8l^-NVjBVqS|zN{6j$PXx}_nbawn5Rl(l2gZj|f z@ecvT)^u2xo=%T{2q>Pr^X2F(G)~1o1eB=U>1gx?9T)#bqEm8b&I*&MG4T%pr8Z{n zmXc2YAI#`H=?X5T)8iik%2djjBM+S({}51i^Ss2X>Gb%AfO4xAi`;a2{6j$3>{4uE zbawnL*TK13hd-dR;~xT=jjDR_4>~>mA)tB6N^6_a>G2N%ExJ@5@Ptl}e-_a4EYDVp zPLF>Ggx@n(@9d(};~xTArzzT{9i9F^h%_CGls`|W$3FzLz1+KEdpbS-A)wuqUbWBA z>G2N%lfPq{)X~}TH^l@`OI><_&W?Ww7&frvkr8xy{6oO-^kw?LO{d2{1dOO(e$CHx zdi+Dc$jA8$;FIVT8bo&3`-QJm2N73o= z4*}zPkFPX?PLF>G*tOfzPKnNrziu0JJbrH{ogM!Wuq@57Z8hoi_=kYyqxX$IN2kX> z1gtoFaAsRNJ^mqJWtU~_PdYvRAz)R?RS7?&)8iikR*%|P@hP1i{}8a|+?v|W==Au9 zfVJ&k=<=9O{~z8ampDQB=5%`eL%{9Xk}FkocDx&j&IR>$Y#l&n$3Fxd%dqKgPC7mQ zA>jCsU9)D<>G2N%Cm!z^Bk1(_hk%py=O-9Wr^i18oJzkaNJgi}KLnf}IJ3@rIz9d& z;LOt*mENM$;~xUfw#Zg)9i1Nk5OD6*tep$c=`Kl_DI#U~fBkm#+4K(0tN-WUS<>)d zQqjK#-FCbj1rRJq7XDjm|L?*2j0K}_pRpi#A^g`g|3`xPYlDPQRY6?FPHadp&NGJP zc&XKuqNasaa%4g*trPV!WCEF7bl`6EwUdxZWa{YttkPE}&hS@7fZgvC8Vn~ZIp#SVEEryQ@b zuy%Amvw)#7Cf-_5F`61ehQ^p&v3FbaU9FIzF{Xamc`y26R>;s8)73Z6?BLsi#+WIv zee_w+&=|8ne>WhSk;6LC7<0dNT^mjAAwy&Q)65km7&pnT?1*BmkfAYd$9{fvneRFpllm-Bjt#>AUByKNzaJrF^U(e?fhp)tYk zkNV7u7HCZHRudEXVJpy>5SgcNKM9C9*OPr8%{k_qg2sfp^=w>u&(N6AjUQ}(;29bd zCg;ujbv#33!Var&<)&w7Ot^<19c$2eTp)tk2-*tDNXJ{<^371W?q7STC zpwO5SpX?a(%$GxBO8zk;K@HE)m{N0Q2D?3r(@y>FqB?CnLu1N(JipREo}n>ikGxXu zYtPV_a*JQ?e9AL4roZSg$PahCnhu_v9oLUaX*8zUpmTH5d4|R`fBD+|1)iZXEgD?j zUd%HzrscmwKbz_q8q;dfK(ZFe*p)tz`w>@+ zCw)CbV^$AJoU^EBXv~@?Nea#P42@aad>t?1S)9JRtm_jk@(hi+ZNH8^PRPM3&IO%1 z4f@C;rzjYWjTu54K_D@{&XI5Dsl zu7MYHs;;RF#scAL?|A2?-+Klo!}U<<4n@V2Q2dj=+1AQxtw8hvua!h_g% zHGHxhOSd@sm?&gmk_Ea%=I4ohonVp$M&;Z)#xpR<0z2#beMvk6lPqxi=A0Sn8H_!~ z_N6RD%TR_rXTkE?&Ay>hacnZSt_iOo*nQEjWN4^-PzKfp)uxmuiwAlGc?A_ zChkg$dW>`!4a2d_DVP<@o z$K>?d$0#6+b0^4PZ>DLSu7Uq!5SorIYiT2j=42>yKA?fp7o}n=%|E$xrsb^?Rso4#y zKk+P1JN0|5QhneV8dK(@#zp?{42>y!_^&+gdxpl8Tk=Q7zdS=@x?IbjXtRLf)pYPQ z_tT|R3S$AL*}y!%=kW}UX`a5&7oG47jTzBJOeyOb8Z+{2mE;#aLt{pbu4ub@hQ^FeQ?1EX&(N4L z9cxzm)U!DKvzJeo`o%LeX55qu?Z5C0joAf<#-$>JrEE}d`XBFlgmlqre ze8|FOFMEc@ta!YjM{m#2n3eVSX2|Oq8nY_>&Nr5MhQ_QOxVgjsbpef8^K^UZDZU&U zv$n-|RsHQ{27p2hsGRBv*FK6o}n?vN3ZVw znP+IsiL*q|H@6z(3sPsj+UYTkU2Ew%()ZoDb!&OjXB%C zZWG#^U=EErm$*SS3d5O$Sb&e%)oxJK5+3^u<-yQgC{6o>Fq-S7~2O|7h??KPNBoE}`in+e_3{3Js9jTJ`lxJX)2fBF8_p5sbCV5~? zwKl(b1}1r6=UixB!!t0+1NX=2uXcL|V~?{(c{?{Jggs}$(mV$XdW6Op+mZYH8qeaC z<2CcGNaPtBV6Kbt*^ zQ>RFj=&vfCp)oEeH!N}2Gc?B4l{#(fdWK_+>$Z(wKkOMA<0fgV^q+W!#<(5x$BTb^ zhQ_!%^Vho{dxpkDru^B96T%*dpl90E3q3+(g5As1udrumOz`G84t4hojR}!8%Y}@d z#krpB)1}=?&(N4qw>y@6-7_>MbhCCZPw)(l36rh;YcF|*#)KWb`@N~2p)uk9{q3dX zo}n>0pnU(cgs_z|_`Oo@W*(t2$$qMw?x|;JO!C)j75~sPG$zI8Ra^h*85)!F*P`d! zdKRZG>gOT{&v}N%q<(|)b5Q z;hksT=R5IUZa&ocz{eaZ22;+NVrC0T0TtKk5(33qcN?9rCCiY60Xsh z))|vrpp^v=bMVD{4}fb^C-EU#k%R}x?5*{?X=TAR8q==tx_@a!!ZjK*X-ggPLlbK> z!G+FkY4yr67T|`B=v0DMz2O10#PH;8)6?pe8#HD_w+^qg5KF8U0euzi6q+4H`4%+syyas+XHM{j(S6Jf_tvH)zbbY2W`!t6pxoDI3%|*ODS6Zt)q58xLgE+;aO*PLLljws2e5*5 z51!oh(Edq6TOh?+S2@jCz_b0#I!q*97PXtC!%KEQoU{VBj z_8ZBXdItBM2;85q>nEPU*c0qj(eecdVb58x?A*pI9-%SDwx1m~%(FPe_?aCLb`W}1t_0~ljmx10a*4PPf3<0i?X@4oX4jd45r<@z*d3jAC^ zaCi2V&uOj=58$p8FYP5%k)3VvIc;&UzsY;wXWboUlvR`?G#w1%ey3?ORPqdsX`N~N;47ZR>4T}gd3_hp(3rM2yZ&(1Gc=}MzwbtL^bC!e6tyl=&=vj_ zWy}gynBbocnWs@Hj0J>Y!`J3Z;Tak;JjK%Qe)J5D8PRq1$8UOu#*93*|9D@|(3nx< z4qQ#^85%SC#S=fz_Y930^WD)QMLmntKYOXc@I{`XG2^Dz{l182Xv|JHb#DM6EMVWCXK2itXC+45@eGYw+w$q)x}L@9yUX%;{aw${n7jVZww#0@77#A@u9!;a5gK#s z#llJZd4|RuA16L9<{27u;*_kvz%w-FWY>zfl6!{6oJvt;^Hk5!nA5{+4u8WlH0I1d z)xKZu85(o;n+rd`?im_$F2U&`D?Ni)Kz=*_5Y3D-_8BUJ!yT{CK*7(?We_Y{I{rjd zTX+zAC(D2xT|Kv~XJAqWyyS*Q6r+X*SWd{byC}Am`1b-bkn>M`MKNP|faTPQqs=MS zlo$)hK$qHIjMiJ>0hTl3K$@<;onTT1c1E+bw7v@uu$(*4JQuC+B>ugCWJ|gnquWPf z&snfy!Mw?IHT;f{WbBuP6HryQ?7YbV9162CugCWbXHG($V@ZJb*FA zb^C!diAfv}Jebz^aV@$Uen&`hcCp8=yF5cJ-U_EjaHP8sl>Om3Kez z496H(H?F?;hi7Pv>#xt(YU&vp<0j#mR!=-bW899Y(U#)W@K5LAUEuCQ^%4}PhWRG6z+NG#g{{4^6yNA z=m8|Ll``0Myv7kK6=#l{MLk1fnq~Qf7DsX3 zA07OA`Mocx6vhHlv%%M1`rR`$rui%9YJTGx8q=cg*_Nk0Lt|P#>Gy95&(N4wgSziN z;29dz`sD!=DQ=Ybxq#HB!O-~>H--l=S802Xt#0D$L}S|Z)cu}&hQ`bXIlBH$2x~OK znOiTu>JhFk3>$O*-NBxrF~d_oZ>2p$V@7m(SnGSw(3p|uC;yt*Gc;z@$VrdKc!tJ| zPBn4?#f=g_7m&tu9y5vJ#_#})8GH4uzWIEeXw0}t1y=dh7=A7w?V%s)eL}^slnoj* zdHJA6Xw0&gTfSD(Gc;!TphoQ;d4|TUcv8D$E6>oFm35OG`pq*mX4NZ+FI4vojafZ7 z_dadWOcF9+R%`51yeh zXU=3?WjsS;&bFPmV76yyEc_^7@ua{rhy~P3i!RW}RI#fL6>P3^ltLh!^;HloY!H*oda@@Bqu%>GL|$AW|3$sK6auT!wDR@BoalRShpwV5qR? zELhp(m;Xa(jIk{(?|9MpcLs{_FB?s#7&SbARxJiv11?KG8>dxplCUAMgK1kYgXDdv7#+36+E&=@b&YMt)_ z_KpWrOAT*Kr7#vyoLwsU{WH(dH0Q0$+$!Z+oI1q|Vs%O1j%KG{)`lX_;xx48Ki{b-I7X=cBnc{HseCW1?!~ zL4H7B4@A(f)(`pVY8VSB!ERO^vC%U$CU}!do058l#)QaG`Q|9k;#^O5=M891QTVxl z66%5ag65*|0D32M%c4oL`8v^IhhQ^fmD0P)rJVRqj9%{U0m1k&7sUKS1&gEI0 zc52&j#zfE1m@*aWj7sep8dLV~lS8L?hQ^eedt^f@&(N68m#CYc8u4m6c%1Kx=2Qw} z0j1fHLJb~!hQ>6{kVh8x42@|~FL#ndo}n=4T|XZqr`R(3rM&@0_UV85+~B*WGJ7Jws!rR?-UoL4!4#;H;=$fJ))#0?M$_^7D0` zp)teL6ib@WGc;yI$HHoyXK2jGb2Y!0o}n?LMpYX=-7_?1bjpgGGkb={jOkM4)=tpGgIjop#;hK)^Tc1Cp)qS7@4fcEXK2jYrmb(% zUYx@32r28bf3xWWUk;7AH9yz&ixP+hlnc5n{dfkI!tV$v$5O8TPI-pL93QnI-!RY6 zm=ovLX3pXn8gsJai6K9EhQ^#qbM)tTJVRqnkKTWEtY>Jagj7tw?UqvF)k;yx}485G{)7{#z!}JhQ_$=P-jpQ&(Ih*i5jl+_ZH?6(%g?&(N4?SNbr8I`~atErLE-a<-;Y7z=2@Zs#a;!80@_c(Y7hzw!)? z36U-BC%<}z#)RzI{#X;w(3nv7+Fg0>85$G1dB>khdxpk@$=YS$e$Q}>2|H-ZxDuYB zG2xzUn{&W3G$ya*F8>c9Y^4l-tvX<}M-ZWF$$qZ2Hn(SJO!7A>U!3F_8k6F)N{7>U zhQ_4)U6o(#S)8`0pLiGR85)!N^&;=|^bCzj^Lf#189YN{(r)dQu#abGOu8R?v-F;! zF*Pwy7Yd8`%A95Po*#x(2C70)xWl9XiV{9<&PZo42>yKrp%>so}n=%4;1Wt z!!tCd)QtS)zVi%?DP1^M_Fp_hW6G4zTi^@N(3r9(#s^zHLu1O#pO)ZL&(N4oT<9GN z`f*QP>)>I_E3~o*58(Nv*{~+ZXhovqJTWxSSbHF?EOeZ?QHwf_exenLjx!5r`8;e3 zwG#hcKx@@MNlsdkga_Cboi5EMw6f4~`(PTT>`E&Vjh_o>ZSSwFM=Oi)0PA!;mVZVo z5{-W^piTRH*=P$TPK_ow-Qg&$UNy!7+OV;0FVm_wJbZCTIcW8&@$Ut+Q6q8|q*ZTtfaP@ZjNNGUs_}CHZA`ax6=>BP9$-0pWAUf7 zde!*%0@}Fo^Xk&7H#~s#m_0Wx;cxWDqGua4ytbVpB#p6vwk+M{`xIG)2hbAB`=6aN z&R2v@L8tgO>L(R9!7uCyv+KQ28(V^$9vT8|l ztUsX$Nn(y`>Xb~N`4jX6Hz;n+i- zp)n^eJfB_MGc@Mpx0A$K&(N4tFHKI|!80`G^w=?M853*L1b3>e*%%eUJJJNflKXjjdjuv;zz*Na__AkU z(gghdhab-M3{09p=;zlv6dft)+(4DHs02eISC1nTd}`)Ic&JV2&D7}IN+uMTF}Ig3spA6OD0`c;=t8 zJVRsLj>`HQ&6x&27cktN%l3rk+VBAGO3}3DrKD6P_CN%^k9GLaBQz%1o#Und^bCy& z-t=IGW}cxjA+qm#PKnrp)YFedL7?mU~S#JNKTyC|8fip}bKriy21O!1<_e!S)x z8dIWd|6ZLuLt{!FyqfW_XJ|~R=~wc6;#r(_>LO>px$7AkQ>NVc59@k{#*{r?z4{%` z(3o-yYBX)&85+~C%IpIP;nj5TXkm{;9-%SKhAvn(+A}n!d8X;73VMddw5UDnk2Rj5 zF)g2ME}hjgG^W*n?Hz`AhQ_o`yY~&_S)4wYMmsb7;29dz_Q5wP{9wUYz-ZUKwez3gUQnUaM4`3cMI>mukzV>yZF=M*!fAf@Qar$SkH@6gO8;k{vaTA&)r;Us7 z0Om3Kba3F`JRF%eXjEw7V7eN{0>-km`6j>W85*;EKu!rRTF!L#;hJ%F2!2U(3mxk%DO_Hp)qTl+!=8hFB=3i=hvb4G(3pCLu|nIFRDcEX?TE)mzkNsUkkq@WCKxbMnGdI zJiu~t#*SGuW-Z17Hc$sPjiE6c9$-0Le%DsIM=X9v$OfkHp1X99ga=s8&Tn&x_Kz)o zN5}^5#Meh?-#I)0V=VE`(HF=V_M8PjFRMuJsm0F)EMvc}s!i`{cmOS7{IiXW#*oGD z2w5iHSd);(P^ymBT(LG}EJ3^M3*AH%^dn7!- za`wljhiU)V;@=Bc=C-!DNc+y=0T|;GH*fO@?kvlLv6B-QA({BOfaUD;B(cadG{*V2 zBkN}JEKZ%`r7@rO@C=P{IpWri*F1x})pB*={(}{sp)szzJ)ixeXK=S#ZjwJ7JKi%i z#_jkI2c_@~?pDj)jpFO3d4|Tswe9!3XzYOqhV|nesT6)LUmx_B$>9or{&C z+TMG#PZ}P;T_WX<1+{1TI^(oOEnQfVdxplO&Oa+*f6vgEG&QHQ9G;;uY4>D*XP9Sb zOuDtNX3OFk8dFJTRip=r#g?ex$cZses1$xKU=>?@bXHT((3s+8|J^@4!!f2r@dH~w z@C=P9d2(r`^PZtGrRJ@!)6TOv?bN}BV3%iTOqmbYCaC2Z8dLVKWZCz4hQ^fpK4F2X zo}n>4Wnse7gz#!QxP5c`dmiDlQnNk}?$7fKjcK0k*_`yAp)oC*JsQ`?Gc=~N$}w^$d+^opn^bl%B=ugK0kcv&o*JF>Rl`{Ye_n(3p0E-sw8YGc;x{FJSu# zVT~rZa^*o+kIDnVOZT0p2g{(Js)%{=NTF^ZltMj)H5_@J7##tKkl)V4Vt&RvW!Z_ zS*ux=wf(WYo}n?zdv+N(#WOT!#l4O{rSc4oS=lT{p}C%+F{`p=$@#8lXw2$9X+P=h z85*f))qHu&4;yazQYwajKyn!QJWt zwy#x@%$|Ws7x1zTn@;x(Ou9f6ty5ik1}0q~rzhPz&NDFS0(CIa^Msy(Nf+pHsTQyE z3{1Mf6iGd`fM;OR1$M!Yz1DjMCSBl;ulVsT&(Iiqb^ok6gs|rn1s>?+UD7?;DvBAQde->8pW!-c;=|BswPb5VGJOm{8zoAY&|F>X>6ezL?f zG{)_OKBsBU41ZT1>vY$9{z-Ff_#^VM(UTK*Pi!4TRbmfBFtl{l@*bfv!5)=pe9SX6 zCV1_#u8wDDOo&Y7Qatc1&h_Mge3NNT34hWZdLY!ZLJMgw3V**HnQoNl>9@X4G$u^i z+`nJ<42=ofed-@HXNEt+j&-^R6Hd`w8~)}x7?VxcR?0_JVk>2^Imw=Z9-%SGRwaIx z%`-G6d7+d`W_yOlq^O!^THqNPlX82Ll%qV0(-yU?W#Ocrp)slRHEOoeGc+bmt=eDa z_Y93myZ7%8)_8`-q+4_N>w=!4F_nEooehMrB`P?a^Z9O%(3oONGViV785&burQ`5$tp)q9-ZT;qv zXJ|~hAAb3;m1k&7_g)<&A%s`c!5zLZxkqSBv)*d*RL{_u=Glw>{)T60OpB&Po-X$c zjcIweR^xP@p)sv`Rju03Gc>04tCdm|^DIsuOpD5HfoEt;+ozY_DC`*;({AAP3=2I& zW9H7%1S1JyjV8F9f6B)mp)tdz6kPPTXK2juBzd1S@C=O^(Jt3-cRWL5MqVpbt)gdW z%&3Van{4+CjTxP|jP2|h8Z)MS`Q(>9i_<@Q?)EF4JVRr~jk@>dHP6tPt@l!Pzfi+c zHfXV|M_($1v4FGe)m6*VdWOa<@3ryNe9zFB6?fPCQPeXuW@XdkrKfp@#;nSItV0UV z(3sV|_r39>XK2itI|nno=@}ZcwqDZ|D?E$ScbB1sd(AU6<}xHNFoqDs0^#o_x1Yas zhetTZ980{YdnM1%nBx;?{&~eSH0H#$8K=8=hQ^$1x1;?@&(N4tNj8=G$TKwN^pst% z-1ZEOIdgf>oAo_IW6pMLWAAx}#+*y@b@FuqO4i8y;aPwyVnbs|m&aqFMNp0~KrDinbNP*D=rV@COI- zR@)qHDul0r$hE~jw^Unf?xXO>2KE%LznDnuUqXjJHBd#U3`Ao8E+YJ)fm||aPa?73 zk_vxjVCkUD>50UCW)l9$z>dNB`VfhYtT=o9TAS#-4HNH{xEXC9a0YIe{60^E@2EC#!_;57zo55=fg7ej$(Q5{Dg$nq zafReBM8FNR^M>dDlnA(C?&P>ETZw=h{?oZ!(U|AZ8y*a9czZ@v0!JHX|JK=}h=3c; z>oy)Wj0n~yUTHNW3lVU`<)BphCnDg6t0$?Gy+Z`taNQtjm9a#?4L2_*YLtKoxZ$?v z&mBh)0XN*;`>D(eM8J*s{cPH3uHevS5sc~b%bBQzfO|@?Gac`=CjxE+@6>Mk9wOjI zh}7*zRwaVhkt5T-@GlW?Bh>j!rUemjBXs8+4gVqnZiGpdrSkhkz>TnzW|#eg2)GgM z>JObhAOddWXZ`v_b1a9J$>3m*7USMMWA!4;?PSjpX5f(W5GS6+5d)QBoy| z9&lWO8!1mz8WBAfxx^Nz`IUc&9;RG^8>vODo6#efOK>Aisj8cz2RE1CM%n}aoX8Xz zkl;qT8K(d0Tz>Q)na(zxylma)37s#KKZcqhol&D@% z(ePB@M#-&ZzNa@wfg7cMEI-^g4cn&QDz!O!*m4DKl&MkjR`iJG3fw6B`;)7cBLfQD zDEHI9$2SoHH~QxR=jbwU6&+kmY34*FG`P`hTACLa5pbh;L)$K*1lh=3a>?oaBtj|jMNvhnj9ZHRyyr*c00`yV3U#_7Jd zhSnegZk)Mwf5UDf;KtcHAJ3>x1l%~6@sm-%5s6(%i*lXPeEpw)7cqC52+H4T9X+Jr z`5h+a-#GGd#_0SY)|d#`&N07sClYHcOv+zr@}wy64qwOI>0wI#Jd+kgVR!f%h+Omh z-Iu7gSYu%Z{=E-hoIoVjm#h?3rtCd7zjfIK$mwnv!4I;6|!ZiHb zKPj=CNbG15tnmHf(PLUbKUpw)!0swh30N%)#{L}o?`9%chJVy=eiUL0SS<@C-s?Uw z3b4c1v2K|B;oM7~P;KCbsY7SqJ4^)JFkRtt%eq9s4Kwyy&AUXv4YPAUE!ltwxMA+^ zFWTK90&e(>=joz_ihy^T2Sa9#A03r|)w1C1@r=2Nh=3c;>&<@@Z3+rldkZcyEZP-q z35Ks@-EcW@*H_UT5a!qiqMq()9?ctJhJ7Hq{*GclPz%5fH|aN}F+{)(x4pWhjpnp4 zw?5S7?tYsqng_$o`my(vIPfrcv{Vz&W)Y0uIPYL|6};O7JG*8=Nh07z@Q%xNM%#Y^ zO$(7`)x%a)2CpMW?Q0Ru0bw@%*ftAw?qH2*-UxH)1JPZMr#?lsfg52`9xF;ShX6Oi zPOP6hn$yB8`cRv@Rxe#N4~9APW8KIylUGEGRcx6Ijy-$t<>>mbS{9NmeDqRpBH%`H zb+hKXL|_z3QQ|?%xkSK?l*dQ?n~Dgyk!r!{-BXBw8>#t}iFt{D8)-^UoWG0+xRLhY zo2zpX0XNc3f34ptBH%_9o76j6uH$>7f-UD?{46Q~tK~3J{>n4&ZX*J26o2PRt8PTV zjS^p8t#yeAxKVOj|6j`x0XIr5ANKe-5pbjQ+kF=_Bm!=f`KrgHn?%5kvcJ9Cw+<0- zque^T>H!gOqi2phm^K1%6&+mKSz|Z>cur|Hb#IGgM8Hk>_pRHDt|tO+v}nCK^;<;1 zjh5GEO&CA~+-Nmn`n;S(z>U@k7Cf9q1Y2#sS-4XW0XN$Ilm6gzBH%{5;Tg_LBH+gC zA9KEM1fVVx+}z*%PXge^uzm-=`j7~?F+9hKG*5|u8zY(=E!K<(xH0m;H+<=!3bRYt5Y{|DCMnNmiP&W9c zZ0j6R30N%)%Mz3;-k%7#v3x@5bessdvEq7(+%t)Q8!KDqS@8l9a1(x~EqCt`M8J*J zQ}Z2sn+UkE=2D^aKNA5r)_ynqVYEjiEV!}m#c?~MeI;SRjgtj-MqxXAJ{hL8Z&~sE zTG3TtwJaRVQl-rjn; zb4`hW8)t6H15b#68)s_|+7;~)2?uVR%QW~=w67$>q&4AhN2Cn@uivh|+?R8F^rV(C zPcIoHT6*Eke=Y&fs4!doPbaR9A`pA0g^BBbKYBbVkyvA4ruxtK57|g0)>xRQ{`CVt z=O+?tEX-2>^U{0+iNqQUbJYL3I&(H6vBts-^`C9{cs7w(V=^#rto<$^5^F5XPQM~q z{b;2WzK(gahq>v0N%(oRQVL(kx?vqwA8H>}2K{8gjO{1(5db%g9oT%W6cH@L%kNwt zMP*@r`q=9;QF!m*D3S}a(*uz+7i5W|r7$-=5Ve0{{wTrV|&NmM84$NgnjSGBjOe09MOk zGWzFX86x0@^M>bd6e9v|xJY;A?*&A_4VT0EmPt+o+;H`xCsZ_#*fw!0XI^7xaQjeM8J)dM~+_`N(9_UwfNYH%tXLVn96(K z;AjOCW|fcar!>V6u8&qQVNQ7<_K&7NL@T8*qdX8dr^Se9r4;6qk9DKk9(_5Q)M0O1 zs$j?b%!i_u!D=~7CBJl0zT!l{jpF%de)pINxKX0!jE|cW0XIr++Hrg@5pbi_>P=Ux z5dk+!7u@x8w1Nrq$H(?lnQD86L@Ssudpr=ktIhCer6gf*St_^T>+hqLQusR7jlQzK zJlR0M$0RghovS0ZnT|?^=XO-<##Eax^Ldt7Yj}w#<#n z5dk-j_erPk69G3)+|HP+HW6^+Wb?M8ejx&GoXXl^#ur4ujnh3l-@ZWv+&FWu)0Xdu zfE#BUZ2kKp5pd(&%fH;{N+fo*tY&4NTk=2uu3}~(6=YrT*No_&W5-OG&wX9azEQLk zzK*${!))%qWnL9USmA3R{EKw|L{V9o*Byv>E#qHNBo}6N2O`(EEfGaaVNQ1-YIldW zQG^v{bO)k8?fiNam4*4-ftWWtrH>-HFq=CNyK?J`W2vLD#=`9FTYh;r0g>3zq*(KX z|ELH+KUpw6$?7OtQn0tI7(1A_Uld`5uVb&zc)65^qNps);0{C-Npm5Jw`=lg-wTZO3R!$}Y zZnzv;DpxcIgn8Oy+sxIYl4+xPBh1ncMAt6!etN16+;Eere49Q*VBB!q?{@QOP7Cw1 zLv8Nny|1EqFwD*#>qhjP-Zx8B8MIjh!}-BR1i+18|ETk~h=3cxyB1whmI$~JB1Mtj zCy3y6M+Z_A~()H;OH5RpJ{W;70L$4b%Tl1l%Z5tIq3R z5&<_#Zce)DArWw+)T%^%TN1&x=|ZXgIzR;6C{s1{KP8BO8)bL?_;^1NaHHJD6~C4y z0&eu}oo#9QhO6k{TIt-cMJ3>y7-3rVi6zplAOdbQPh7V6i$uVU7VXQm9!~_^Xn8r` z`65KXjaE|%9b7~N+-RL7Pwz}bu+^qr?iD?VfE#VkPMw#Q2)NO1^n?k0iGUl^XKx95 zfI(d*xLY%Ay{H6y6C+HZ-m6-!dqlvE;jdO~Q-KJ$F``A4_m2<(H%8tO2Rae~H%9fA z=dKa~H%4bKwz4u2aAQocOu}%ipzUzWF!J^tZctv=Sm{r#;U{%AHGfm+*mzv z*1QQsz>PK6rcZc@2)MDfOZMK;9#NPZJ$5d(F6FB$l2aLQ{?U;R?A^h^z28|bRq(79PhpVtIb5fjT3hcG_OJg+&I}{>7%QO{uHIFb2)J>& z*M{+jiGUkt?yjBt2@!DPZ2e>_qdg)8d&|nX^a=Y!`%2;Kn7w6PG~J^||EWvIJiWp+ z=Y4zEj{Z4z9uEI~Z8iIp01~#RX7Xl4VvU8V&#P3a_9u~8V`1X+Tos#qNF>%+nCX0j zd=l+5gs)@X0XUU>}b;L!yYT{M*#ZCg6~VDSWf`lF!onz_ZAT>!#^q4csLPo!-OkaH5n0b z!{nSozeoF18dl4isXy{PjrOO)*RgJx{y6u-0aP2fVdmXDlXDUQH_R@c&|@eOaKqe@ zsmn4G0XO_)|MTS{0KMVCpjOqp5db%wJ!#zZ5)p92d4oDdJ|lv)iI*Fu+C~K2a5*^9 z)P_W0+;H`8(#1E4fE%vsrhZz}syjslTc8%zO7S%ja3i&>T=h31 z;6|E~l^Rzk0&b+;f4#$QBH%{4nU_k}AOdbw@vE(GL;$X@f~^IATSEZGjbcCMe^QVL zxKaGAT#H5%0XIt2$U7ww5pbjAFC~-zKm^<| z_Sbu*1`+``%Kd!1eKsQCMjtt!`-=#`RdjG8@wDjq9cCVnoe4D?pJYk&%n$R91K|nN zJd2*+Vb*aV;_H-qqGx`Xa~z1g*z(Kh`K>j$(P~9ZWfN3Io%!{LSUzlS& zwl2eZd{&; z%q$K>=j@a!TK9!{#etZ{or_GM?qGlH!(Z}5>%K6jIFxb2wq}ghieX0aSU2|F8mXgA z0JLd?Hr?Bz#Ib-ja3OR zf6$Q#xUqWt*-V>=fE#OWoXcC82)MDf+n0r-cp=Ol9_z-s8|^cMuVZ==W&+=NYj(8H5WWVYD?NPhoN9|T7N!Kx z^L$4WBC*E8gy5@+AALw9)>xPheDjBwTM~&KO@`H+*k^SFpr0)Gp?keS1i%erhx&b% zod}lU6^167Lj>Hg@YfPKBLZ%ioPBLhv_BPQ0FO1m)Ss8fMf+1>{%;`qqqEyLP;KCb znfK1!&qoB@FuUZ-D{F~>8|Dswb*vB(aKm?uI*=v;&>J2MToimn0Nil)bbf-9M8FN_ z^=DM7PXucd>1WotO$6LhX@P(Yz66`Uaxw?HSXBY6CaiWY{(9 z3K4L_ZSSwwL~~l0;~Q#ociQxi=D{$-ckDeSmX1C*DXI+GEP_$15~d&kZUj5Gj7=i~ zZUpbLruLgez>N?oH&*AV3^B0){QK9vyF`^gOO7Tlau zl?b?zoIkpMh6uQkqV%)v?TLUJDUVJ3>@OnN0<~~Tz4wWL8>!W3{x1=5BTb1>iCPc= zH`4BVC+8C);6}PxZx?Dx1l*`<58huK0l2;jwq2b)i~zV%Z26V3S%`oe#os=&V+Ij$ zqr_L|A1ET=M#(Kb>W(D>Zj@Tt_tOMKux` zLiPm_aHG}K1qq|Ys)5zA34f1gdd2Uk47S>|o>lt>5pbjJsaLynB?4}=8<)NOMIzwF ztbUZMX9S=w6FfM&l_qI?x*OL0#9f-AV6|)v&vIZEEeK$>Y>a5Re+)eiVYM7)+`hRw z0lhh}S`PDV_gfnHrop%|I>*{NG!ejR*%;GgLnXQm4ED!9N?7ha>L|D|ZfLU3^N4^O zoA-la=_3FoY|y&I;VlGU+=RcjQ~KiPL}1)lKDFH1>qKDOSaGTBfNzPwxUuq^LfMWH zfpKG1f_(3kCjxG)o{+oC10pbPtht`2d>ta-#@enEDn@%mVSeq{x!AfCQ)@^2N?~^G z*tytMd_PT}sIu6-<675ok8MA@7U$ZXyEHVS%Tej>-q6yRh=qC&2AAciy$HH9Mv2`(a;#h|0 z9SbvM1L39im5xS6m?s;E5C=O%BO}a`4MZ+lb}AZyVUBDd>gcLJq7fKo$OfWIZtM|_ z^e{g*5F^(ti$;2w9UF+9`NG0zmom(a4aDtFFge=U3^QZLjvHP))lPZ~pf^0|aqF87 zkxlsCa+nnR{{0Wn5&<`yw|btj8WF5bWPX@uFA;FVW#7p^HYWmZxVkl|*JC2!hU>;7 z_Z24sZn()g=FA}?;D*~_Z$1Bz2)Nod~!Q{OfahGZO(fLL@w!X($oAj-1x7X8|JMMyQM3m#-rNZiH?#;A}!7;3oXb zzM=ca5dk;Cj%9n25dk;Coz~BW69G4JUgH)ttw76U@YjMr?nEVEZ#hhK{r$pI4Tyl7 z@ay8UmPNa5VV3LIaVy0q(|bfabYYHbAo7px87@<8*a9_Y^BbLsfE%gb-PvI~5pW~T z$9qdxBm!=vJ<_`BW+LE5y2al#u0jOdsFST~(8LPYSHYg8r`tA2HYsM;pi*Hh+x}v-uT(7 zJ}6w02)NO1^7|VQA_dk=32)Hq# z!LWS;iGUj;|Gn~Iej?z;sKHmi*+>M&jnS{1&y$o0xG|>gnT(@|V1MlG8mSTy0XN3= zsa|9>5pZL_KbiY@1fYZsI^~%5oB%w-ElZtcNfRRA#_};~pY10CZmc+yX-{b);Ks_% z?Y_J~1l(Aas(rK8M8J*JBfAv-l?b@8=6uJLUl9Q})^^yI>31U7QSrsg98ZX6$+YVSxQ;Kqr6Q$J5a1l%~;pmEbVM8J(x zFSn}Bh=3cX2Q^GJhzPiG=1H9**@=J~XPckQGnfdtaW3nTjIR=jT`jwXi_b-Od#o`R ztj;k$T9Cn>dzfT;dD3p1{!>r*I_7y3rkLKAs6u5TvBts#(_f{^ag|7{u`s>#+o=n7 zBob>ZOfJ2$@wy{KVvU8VrMI*iRDnpWu`sdpmkp2JBNA&YOe_6Poy+xz#2O2eO0Pfp z;5Lz1V_{0^-A8uRCjxHRrYv)#B`JLB;8?IA$0t3a60ll!j2+L?H4_nV!+7bmg%%M3 zH%#zMIg1bhH%u;UH*h==aKqHG_CLKy1l%xPqRW*PM8FL*s^hWOh=3brXKma5BN1@J z+`cXM-y{NVc(t^TqD47;?&5gRU1Tp2m4MZ<&cM?Zefz`4T?0TVWV~Bto!N18` zBSeDSUDgu8>&U64FGh1fm>D{@%|cx&aX6Yc!o1KxbnCJMG}Q)fgn6Od+V6>g8)3)Y znHkM#VMb`E&7HbCI+_Q=e9*CO2L462-8^zXiSeKp%xKTWJ=gZM-^z>QKHGj~f#1ly+bq${|T2)I$EQpOy4h=3bqH_uDFnh3a2 zZq;Iun+UkkHM7y-D74$jp&`~d-QquHpc7ylpvZZuC>X>C&?FmANyQhC4=BH%{L zvpicZBH$+c;6c5!iwL;UI!)0o?TBEjO~)eT&l3SR+FtHmu{{xRqurEVwa*X%Hzt3_ zG^ry1b(!F4sY@>q05^sWEO}%E5pZL8`ZE3BCIW7Zs9%1~&qTnDk&p8i$Uy|$7&W9| z_Wne`jnNtMmg7XgjWP9db)HEC`(y7+tD}g38{>M9uQY=QxUp-urJWK1C}D$+$M5YV z0B$Txb8K5pBH+gI(fdZ9BLZ%$ID2qrTO#1b$}Y>;pG3foRVh~`{E!H^v3k_Ticg7v z8*9$3sojhSxUshV3tb))!JfLr3CcGo0&d)%ExA%fAa=DJrf06VW9t9{;Ks2Go9^Z$ z0&W~1vTN2XBH+e}$9u*IBH+f!`tuVECjxGqO1~&bMg-hAJ#c27^+dppGf!t!dW#6S zakfRaa_fkI8|Pll+PMG`mz;89v})kqeEqF_YrP2Tm=C2JBM%9Lin*W=#6B`nY^PK%R=cQIxikci&&;OeX zv9wOq*^rC>=5o=2yV3Sr$mM@?b#(vMXcI2v>c6=zd322Q?bH9wjXW`Hy65Je zO*9EO^bwrK`3buDzG~}gh3dDf zC7=?bgpvj*p;C&3BB69B@tgPA^L*dmv)}u}Kf4^CnVmVV+1WW~uj@YRCbwDOXg5}$ zY+09V{05G8WBt{}v&r^v;Al5CPp_HLjoXKIV_R~+Y?7hGe^7f&f>3sNf>)4qrRh>|1U-+LfZ?ge-h(NVF?up~-?L zrtm%P^vopHfQz$tbMw%y)F-cut-~DcO4IuM`t!`uuC&EZoNLM)?MgTNv7;xLqh0AQ zJiNOlbF`~g^fFB4;YpYXfjdpeX-lj8KnWNqC%iq;AlkKTE(r8!9M>mYV!sXGfR^Lz0^c-`v ztM%*|(KpO9Xur94e$!6O(XO_S%&GSWbF{17&YNq$%pC1%zu<Yt9mSe4N4swEce82|#?mfo*5tEm8KYen-`r}*ROV>c ziMKlJzLPoHb@JafM?YqccAaXGw|73~XxHf*ay>eLIofq*Xx8GDnWJ52FJ-GRhdJ7H zuKfy8iFpQo{H>p7na>>U`gLEA*+baFGuuZ!dJKJ-G1~QUo}MdyVUBja7~Sp23(V23 zm-{>KI>j9AdeyVQWA&J$U9WTJe|-aUwCl}?LgjlgN4wr0Dqid;bF}MSxA(GkXO4Eg z&oWycWA4*&T%2?DZE-^5bfghSGxHXw#b^v>Lu@j)$J2(&VjLG%*L!-leUCYq59`Ys z-74L}9L&d&6rFRw#T?AXk(yUvO0qE=hm-Wk2v5=7`4=SH!-0eOI5JfWUCqk%gZVhJ zy7-pS%)xvdxtZ^6&Bh$e$C2MUd;bXLFs|cxYpx0u3dCq9jutm*_X?HHz>(Aew4hDh+~Z7;#hx~YijahB5<@Dn`g3auFCbJ z-Pqof?aFrMXg7ABuQ*VHIogf=H=l3Y#vJX6T3LHjAcj$FETZvs+$U5Th0IjF&2`Mtu254-T%tKI#yBn(`p?qe(wrGiU~CKXQu)b!xqh@O z>@{WQ7i5lhg&R9&A;G) zd~@b#SBee|Tkm9!cBL%*e8b0>qg|;6R{DqooN z788@|Ge^794STyU&H3SnGw|t2e{skndS-;5p~JkZrZw43L3jpto{F{osNFYv!DxO9J@MxKlhYF%+YT6 z+fG@wJkK2MYWZ09tDBgkU9Em-+O{oow5#>3mQP<|omzQ9k+(XOf8;WOHRit*|ux?J+|LMn}MTx{YYrM@f09PJu$V}(hR zn4?`IUnny#7jv|0)IYWThs@Eg(Qntf_CDrl*O;5CHtf$lgZ|m(HCo@u9PJu+`k!uZ zFh{$__q$xAAd{l);u`F-wF!5 ztN;I7q1f;FVK!F}aa`=9?#q9Bov!ZDu8;Gs{NYdLXxEETOZz;*9PN5}VBx}b%+an_ zy|&-kk~!M-I@h+FPcTQj-i+K;oi+htk9NI1xVI~Xv9U+H-gRx(hBm-rk9NJ!+M*F{ z;lv)|IPvbLE<=;niL`kU4J%RmAA(>uBuo9U%E69|GvM(r6}Mkx4(3Cu)Q4pMYRtiW zNSGSF;+P`AX(%)xv}q`I>~zJ1KWd`PEyf8z(9W)9{!e}Rs7M9vko-x{u;|(PbEn}WRdC|DcCs~-I z-B^~b&~FrTv>U6Xwenl$XgAiYYnP;5Z-H@~h)ts!FWt)J(Qa(ZSH1sj=4dx|D=$A+ zggM%c{ic6j9Ksy!iYDdD{YV&lFQSRL2OMXNc13(F&ypU@(XJ5Pvj4G$c?R{%ELnbP zz#Q!gHK}FQGtAMh&_|ngZpIwPt}vZj-~Kamv@2}(jyFEa9PJ7>`uppCWsY`*-~ZFS zk1$8OvS8Vw_Y=n6$>@!IOXo30yD~moa9~B|Xjh8%#eVF=9PLVZYvJDuFwfxns(-I8 zpE5_gQlIKx?RMs9SDJQRZWzxT?Mhpu>uuLFN4wI!_04@#n4?|k|NZv*oXpX#8eC`4 z0m9gN6@6E)WIM)aSH+*z&wrUY+EwEA=kI@rIoeh7$%Zfez#Q!=^>yV#otbCQKK(_d z9S51CU1jbN{TeezyUISHKiS9}?JD=xkPjO%N4v^@^7i;G%+apCQgc~1!q_?;ZLgW{ zF2-nA$J6VS_?$V~)uLkcPC1yPT`lWWd*B`BXjiLkrMFAwXjkhQB@WJHo&E5sb_YCnI>2g8`7UDK-CpDZ7*ZlbG!Q)!Ijgoy_==|Zcy zaFX5u5k};1a~rM3!U=NOvcnCuniB@?8g(JpKw6CngLaJ`o_8s&=7d4J#uUi*JFUiq z$)JC>Y1SWUH75+(HSXl+-_U9-b_**qo3Hv#9{O0jjehUhnIZ{+ zah$O6yFIE>Boa>G>Q?0Jd?Q5?BD|bSea}Hy`Fgk8DG~`M$klK8mrx`jEZVhtOo0Iu zi3p2!t+~GVPqfq*7VTR5YN0X)c3xoNJLn)>-zk=mM0!8 z?V_d!J5UrX9L@|bE^weRMbW~cT_=VgxxN;SeTR0Pym0)!z0A?BQ!NM2>CGJNI-P&m zgu~3yt}}xMenU~PaA?=rtD*l~9_>2U@xI?F3Kkg03FiyBA1R6sCop#X`AcVb+J`tU zB;S4Y>hrXN6HZ`k`S|(^9cc$hc(m)qn3MPBqSkw~>*a6fuA9Uh?Rxe4h>7a~I(m<>sMKe#%qA>#~q{Og64o0)_8ki_@FlYe(*4(3BD-}v0G z`%={ose0_4jNaV3cOql7E8{aej^|~LcBOc6&*w{-qg^R)+5J{Y z<{4aH^;=b#=Dd)j71AbU&J3A~k(;)QUR}WTqg`o>E_v87N4wI!eeF{;=ZD*;|h}+BK%&`nQfU&!B&{$(ql5F-N<`o#_4kG3IF3_<^sz+nqVuwbwp>n1XDH zwcF^Am4&8KX^i8fjo(>b=33@x*NSTwzVjq~FqU>vlTz90 zGRAq=#RW^M^~}+(6T`~1=*b-II(f0eQ%9JiU8h>rTKx!fwCi-f+9!Txj&_|HT;u%~ z%+ap1SE{~ynmO8auEXWGnlaD7k1zbs=VzFsUH{h~>xvVGI4-1xeyxhm&lv6c__~VO z2Qo*yUW}DbR$-2Iz1*i>n9CgPdeyt`$(+p5uGiN#STlt=+V$q0#_!z09PN7hN2B)^ zGe^7Lz4H4Pw=+k(-e>yl?N6CQ9H(BL^CQjbD!rai(a+tF(!j&-?}YTy^B0cWo75am z(yNt_WO~=q4{9+7^C8uAwUrlX|00}Vd8w9fqWzDMeHyu%v-errKL|Ofk?Xy?+SC3+ z$V83YRNq#G*0LcVHF7Jr=k3ky2lF8X_4Ic6Xe}L1u)N>iz67nMLsn|A8-LVmH{G@h z?ZnY1b3dF!SI6&FsyO~?-uPV1Gbk^fp1G0sFG99zXk9GJO}jw*A0cNoa`o~07is?> zWUfZ8*RFYv_8&t2YUJj*jd?!g_MzR_-nFIjeaz8r>^^&`B(0@GMr*9!fA>m$T1$t# z)?indu0v?jRoHtGO&##h^K^CmUS-I$cDV1?o0y|rAvz73*oAop^~)T?=N@8?c7+;u z^xg-Vqg|o@K6c&r%+aneFCTioEpxOhZ07wPFEK~E!oB-kXWGLEf1CmA3jh1lRcQ|+ z{HZgrDydK=bF?dE@lg|IFwfxn zs_%*GuVIdMr9OB5zIU0UU1>UA>2No5v@30)i;cfvj&`LR^k6mG!w6Zm!LIaIgFh~h zcGa&1E6@X1Ve3`2Y0q;zsdNVOvWi#ke(^EpXjh5STPvSsj&_x-yCY9?=4e-`&5Opb zW}ZR&^ztPi)@P1(l_|0605!KjxnG#+5Ur&{ zc5WEE`h$GMX}O}XbvpWfL;G>mdW_>jy6)MlpS_Mb+STITO?f|Ij&`+tWOJ336>-Xl5E66;9_8UKA6Rl-KHgB+_?W5B#(pom;^hWM}DROWQ*N=9!|M1rB zm6@Ym6BmAz7MmG-wkG=b$bHXHX^i8PiQhVY{ddgKt`RpMZ2Ssyv}Br`yH@16+R-payH@tN@cets(XLg8CVici zIoh>)#KeC_Ge^7O72Jrqw1=SZeVnqk=je&FhY(J{u64)m8c>GoN4wTfEVqPLyZAm% zxgXze{wNj4(k^Pz=7t@N(XNYc=x|##=4jW6p{=@HWR7;7yws#xN9JhPsb<-J{FXV| zb^7M4zdy|!?K<;TuBH2! zD|58#&FK6C-e-<>z1?46iDiyukxDC9*)ZffLjK$Bv~j*i@l02kI)8zMmurzX`8S97o*)c{<6bq{&4X3bs=^7Gp(l4{zW)}*2nUmCL?J7 zBVzSinQK91s?jH7DL{qB2)0#?S9H&M6XSMgPFvm|vh?i=ftj;`x`sFos z)@)^tc7+;W_DP!aLRNOLBlKV8U!XZNg=o$X zzv#jG{U77X&@&_aE>KMm`H1MkmU-K$+w5vp^eD%L% zj&_x-SFq)i%+an=Te_S+&pd1AENYRVk#DpRu8h~3Q5uCn#JPpiWm?JBo^-Kssz z(XR4~z8qeYIoj2Wa+l|c4O^$9AF}88h)QSh)YI{-EXFfOyIS0ryVaY_(XN&c=V@>= zbF{0~kFCF4!W`{t{eH(&C75T>e&bqBpTHdLYWrBzk$IS-UG0ATW!PlqXjl6WcCO6L z9POGiSzhO99j|Vpf66@7o=RgJ7gEu`U7^K4%+an9HbF^#ygunL;W{!5s0ggpa07o?K*wanzvqLj&_}S zd*j}pn4?{1|Jic<0p@7exwbEzpuJ^{->cNl7ky>*gIpf%`ek1<;{_eWaoR_{7Ctha zO5^t`wU2WxeN8h*yIzc3S!OtMwCm-;<%J3}N4s8i-~0Aw%+apbd3Swr4|BBZ&8Th1 z#xO^_-X7S#ClhnD>s^<2Yu;s!qc0>zYJc(?<`Bo3F*BBqNT^A#Crost_F!7f;+$)u zXo0HzF_pzQ&P4G}+4D~3VBSRH{)*+MG6(Y}lD10GYng+26R8gy4WgaHaFSjvnMnWA zcp2>+h7;uGp$2~r;QGP5iELDNXFld&-bC)bzt{F<4(3hde>!}oAafYkCZ1etIfWAV zO8^r`ODps`MWyk5oQdPrW$L`Z9PP%UQOP2|Fh{$wEMMxLhnb_@SS_r&{3vs@8|w`< z2KQo)c4O1H_Rkxbqutn+t#!B_bF>?~WW{!4azxnjK`pnU;$X4|*1ycAs3L~OP zS<1&$8sj)4;-lG%&1Q~vh3K5;v7*e;u8`Stzdn>X+7)VI$0KE!qg|nowcfRoIocJb zThpQ0n4?``v$R~n_ci{#N+aBeowKrXd9*A1pmC`F*533xcEy{ z8sj)4<5R^d{LUQhO3|)h?`N5#T`7y?d-QAOXjiH?y6$en9PLVdw#(70%+ani?YniwLTIoeg~J6&f1^9?Un9%gE`t&_Q}dc zZeoshm0SB}=KjpluJRxEi*IC(cJ+i(y=V&se*tB5w5{5*b5t7RIHThk)dw|Uj&`-E zQfKEb=4e;Tnl=Bd%^dA&wY_}LznP<5t*4i*{Tg$$t4+lcMZaQ>cD1cjy4+LD(XMuT z$3<(Iqh0OiOwIH-bF^!+R=9^Yknnw#G0}w%M`<+{PT-kr;^A#}(`wFS@KiLSK$9V~ z8Z#Np>qa(h^%XX*Fgt>Yr_yt2eFY z48D&u#-01LIjzRR3D)oXE`EYma|VAOXKdFpMQHmjgW7HMTen>lNf?adjE%>1K0=X5 zIDxBMaea>!6iFETeVnoK)t*Bs5(y_*Uj1IE7)25We;;S8ez$meibTQ*mNz*IyiSpX z!S`{-+Sl_xMv+K3!Se3!1&>oCVet2H#`N68;J)lJoEu`FmEGiuD)_Fb1-is^})pbv~w6v(&NrX z`qxQYY3DGUAU6+=esdAm59VzY{!H)3rI~|y8@cz(Em*=F?t2^gAMct{f;o(98~3X_ zs4QW$6Guyq7hJ~}?Z)w%Bc-2UoNyyrT@fGI(DfbWXjh0XoA11fc?R{%Y@2TWoH^PRYQp?JH0N1-A7_O=K4&4# znc)QNhHs5d|4nlJXjj;*Gk%!K9PJ7>^44!@&bRnJ&I*69$R&DagcFzW9qh0-GmD&+uY@Lp__T9RM zG1}Gf%mMowFh{#uR385Eapq`O%UXlp?7wJ*wYMW}kL-A4PCF095F?b>+k!ah5gqg^YmTmC~w z=4jW-*H->^kvZD6>W^JrX{l~8j%}`2Fa5?G z?Rxn~m2Y2Rj&{9ztx==x%+apb*EMcal{wn=W^4m@fjQdsc3<6`9hjqC?_T=*=Bvzc z^m%{H;XB(fhd9o4Dl&n_hD)1wQP(XqUP_AMxOGu9V@^{V7vUs*SGp+PHoqQ?i*SNm z)SQutuZ`cUbdjtwJ)&_LPO!Y1zF{Wa7Y^e%7wPS*N7H=~PO!YGvuQ2emkz&I=^|Tk z^BKA?!wHsmb2|M=?_3;yuhK<+@5?*solQ6ayK&Zyqkbp5XeW-oSX7tBg~Ru8E{?xm z(uBrEIDyv1;)zu;jZ25$t8}rvV_9Yzm*E7E5UxX7ZZ=T$G zmhMZ3->Y=7y?w_zx-Y{CmUkbw{h8jmIQ)H_i~ZUc57RrFZ~}J4gf;6J!(Ht}G-guP zN+ciO$2k%IHc`%Jj&_B3bwsm*%rmH8UO)QrzRc0CQ171Ha2s=&HJs4DpF95vbF?eW z>sM!8#~fx2Cv46OW5zK@yTXloXy~=fVb*ZM|9$_8sm#%?Jih)ce>m7X84VvOx>IR< zALnFzp-+}$%+anCO$R?wpE=rA)AP*H zu694=D7u+B+SUHO%;g#~N4sY7yv#KScsl{7~`73#Qo2o`;a->HKNFs**7vr zyGFLVIJQ4?v}@Ga$yrJJ>Q!#+O_edV;c@LN4r*JKQ!iX=4jW-&iiMrWsY{O zIyOMIVvcsLp4d0*N#X!eQXzVwClRx zbPq3yu(XTXcR9Lm*O1A~(XNwcyMLCOIofrqU9k!uFh{#i z7b#r)Ugl`mng016?Z+JLI(w>M?;Dt-UFVv8(ELs28Tj!xfAqx7%+apTdt(t^kU|{i zeAKP=t`k%m<2dKzEFF)uWR7;dnAmc~9_DD*%VSN4)?|)$z3QC3*q_YNuGiVKlz*K$ z+Vy5q?$_5bN4wr0&GXn3%+aoQJwAT?OXg_T`#hgCdy+ZCaX!bbb0}5<^FDfU`{+0+ zie`Nj&1{`}7-N_58yO^Wh_?B%mXA?#{ zarAkYidz_?-8lZHYpzDjGbk?}@Ab?9=4dyTcXn^rjXBzl)rw+Qeq@ezWBqO6&G$1$ zyRmsH-_(DYqutovR`8?t%+YS_7Jsn#GIO*W`*k0^(T+LV6>si3Qj8 z?TUC`qec^$qg^3htDCa~^9<^j*EMiUn4?{x-jVZZ&hz*_&I|pAnoM(MI7zRiy)eD2 zeCxS>v@7hj6)%0n9PJ7>zW;AD=X-n~=Y{|4jUVWl5l+%kvX^J}t{qC%WA9`%tVY8+ zjM1))FIH{6n>pH*qDd{^lsVd!vS96N&oj^9`f6~QNi^qqd>`kfzEWWx&6(i@>`K$B z)a6&XezYrXzLMYl#T@NQ*JsN2H0OJKALpe%KYkxQGr|ejRc)8oD?`=eou{HT*)|Vh zjCNJLB@zvi#gg=vSHq-5p%Sw)cQ8LMl#Q!eR@%cirJZ?U1iF&YPX6x z+Ew=XCeM^*j&_yX^6P`knB&Z#{IZ{4F3%k8>Y^)~t|W}D)6vhxuYSWA?dtfWLR+3= zj&`*$`KSEB9PMiPK!F9Fn4?{-e(hXoD|57~^#|Q@S7)9<`%T>Q=_}09uC@>MXxo}O z+STsIwXa-cj&`+w|Eq^OGDo|n-_j@>VZ6GDPK$Xt8KYej_tTT6Fh{#a6s`Q-9n8_L zk!>qoUd$Zr8g=IR*7=#EU8CP@*l-|ov}?>Q^{%bLJcIt(7wh}E%+aoKm;bz@B6GBB z{E)wHn#UaN+S3a&jUbG*+vrHy$&WBbyEdL&e*Uk_(XJKQN?&Th9PL`!rNpzSinYqHjKJ(;6jYrEFTd4zce{dWgX-Q0sY+O>Y<**lLjN4u{1 z^+kCxjHO-l;-bC-s5HiL-o>{pS(J}C+I8a1Rr}^JN4rj*S@wNp=4jWcwtH$!WsY{8 zF1owhwan43GyS&S@iB9>>+IwI@{{%4>L!* z-b~(f^C{+N*V`kT?|gwd+V!q`CwG=P+Vwu~%Q>4f_c=pOkpEdXbZp`8y#1~+qEgQ) z`~Cw(DkFi2o;>nEa$OOnRvIC7+{(J|luXLOo47!uWqGFl%@|9^n;N{glyExxct|T( zxADnj%PJhhm1XHhYmyDCkYEm|T3+QzM>VJa{$@xhx4GgA3y7xws!K>G*FfrHD=jIFPrz9I@9WB1`ZWgjOB9%KK@*uraxf=BV$!4k=RCZM+>daLEB=}Ac( z%_9D{>6e-)cod>p>yg8WV$JgAj?)Vh1&=}v&22s-3Lb^NlqbhMM8TslEwVQlLlis; zdqb91nTUc%;okV7`@2NJqwr@xt9cDk@F>4KkT03f1oTHnqkDa|KPf3;Jj!@~_tRa8 zf=4NObeXo9D0q}IPuCF*iQ;wDhLn4;L(`OZAOeBiaoL~cbJxqD0npP_a9az zYs!$tEqxCe|L%{&ll6Sa;g(((*{_Ffrw<6~wb9A^Iet&BYQdw82NW=;YshYKpn!&hC|}DdXpw5n>yh*H4BBb zYo#X6O}2ExF;KPa>b=)f&1u6S%i5L;*NrEdHY_7k|K#(x6HOZq>DEe~>-Z_rwBe9! zZP6p0?;x5s98#^Vf3)giqUn(;;ek3LazYjyFC- z6gV*1tm$pTQqs1ecIGscZdNiqA#a+$DcK$m$7zp34cYW;GDn3xYC!en&Fz!9r@% zKB^f!N}FqUWt!o_ao7XO^MCt5Av@ZH7fL2`aySO+k2lYs%;zCbTH2$kIq8!msK6Gh zX!n)-Zb;e$k&aaHyo=ZOBMKgs&?g$-OB6=Bl2y-l_<$&QRBF%2e{&NBk4n!S_043W z;87Vdc|vKT;8EEc6Xq-;3Lcf)ap%$!M8Tu-({3BMgeZ8_RVMaJ!Y}-6b@b(->z+tT zLZlOtp?$jl-gQL5qZaoZ?f5!T@Tlc8$DaR_D0tLr-Jq{)5(ST1FCPBS9-`n;o4W_h zZAlb7YWr;8i6@AHNA147cR*93;8FV(e#v>F;L*$&u_Ips;i@M3bK`UG5QOKoiKlFN zF$YobXv8(^D}PQDJR14Znml(A1&>DkHFNx6qTtc!@zXvmP82*ElWFdSnMAQi_LX@X zB~kEb+#fgYm_`&l8vo8shZIrpXt#|%G$4UcuZ>P@YySg5@Mzio(NHWCGoR`*#xt}#*YXiedTn+_5MkJffrda*N6?613$^WXuZ z;L-ZQ*KhAe6g;|WcU?#}KX7Js(JQrHDwdRlNJqLjQ|f=6fntZ?WHqTtcF*WS61?0HHD9-Y5# z?8aojR66kJRk@AHHW7UHA*3zqQ1^l7ldD3cBYj-BL8nbb!J`*_8kguo6g+x)zEQqI zM8Tt1ZL0KtkSKWcx>&_ezb6VFy%{JEwj~N4y*;6}Um^+~y=yXbQ?lnNJ$UrK;9D1y z{Zi@Ek&fD0d|a}!QK=`nin1*HegA(h3D4?~k?ga*$3_xN-|-}74WG9+79I}x; zx$W&$MAL>tF0$LVe^Hib+HlB3_Qk?7Lx`pghdg9oFD+DrXxebdLiWVUM`jUC8&;9M zWBF?l(X`=^f$WnUFC?qdaGY8Zh5Tb*WqvYQm4@TA$GF?lAG;64S-MTJthnW^B-#%d$bhODa|PV{EHTdpKDgsu(=RZhq0Hl2vKQI|eoTUAMGIR;3~9SlXjFwsK(d$^dq{ zm52r%9hN^a1(8ljICeFZB?=yeXnE-GDn!AfkooujI+rMT6l(Z@nmLJrN1-qD?LLJl zcoe4Tpu6uN3Lb?mF#OiVM8TtQ1EL&B)E+X7r9BFN(wZb{5BbH?9_7U4OOux?@J>L< z=-m^m>Lyo(NGGHg`~Cdz^+ds=6t7?Tr6*DFC}qxze;pwT9;F&HxyK_!!K2i_O{n!N zQSd0ut0Tj(Y1+P93njs$wAYWi>oigDDBaY%Z4$MITw-aD(jUI(nj~sh5a}qDeeZ3_ zv4bD)3+I=)pD1|LqU?;<{viq;wQM~7k@iHvqgJao z?AbyTJZimk^|3}o!J{_iH+_+;4nvNxv}Hvt(ytLbH>CI`}BB|D0nm?%iy~_QSfMFw_&$@L=-$4 zb@a&aJVe2x(UXp^nm`mh8k7CtUrF>FvVo;d8Qb~5FG=(q1};#y|JkofQ_V0QjUWBY z=|x1rqn)(9%L567dTn%eQuikbf=3&_F|p>CM8TsKw~n~=Rifb0%J!r0{);GhwCdE! zm34@MN2~jv8@8J$c(kU-)nA(u1&`LYyYTmUqS#;e??b0r5(SUezjgnYCy0Vam#1Ba z?-K|mUDP>S%ZUWRql>d=t)G`DcywY?u4|SO1&>Z1&1*{%1&>a3YcuT)qTtc#EFDJN zL=-$aGqKfI3y6Y8XOA^G?TCU$=X(D7*M~&WhR^-;FZU4zkG@Cy#z{~C`4&TBuXe>7 z>`F>Pq@#RXq)@BcM8TsM{qvi1M8Tt%rwZg~LKHlD)xPt{uZV(2uW#)({VAg0(VI7V zp8A_8c=Yyck1t;%3Ld>{vG&))M8Tu?H+=PXZ=&f)N6#$uLAC!dotimybnD!orY9&p z7DLXi6~zZ6k#jgs%`+im*SCe1BvE%b1}dJ)|3?zBhumF2%b)H4#(*>#iDkuACrhZWa$DbE9Lz? ziS|Q|E}&{=n=VP@95QqP)muAMOQPe=4Y>}A9U?v zh=RuunD@HvEu!F2$p^dNa0^lJsMOEJ1}-NG9+m#6@X`uI!DCoX=KFm#QShkj0|kG~ zN)$XQ_rnL@j35dgm7n#|zuAa_N8N7C?e`=Q+N7iPtvmll5XPg97j>-q3Q_Q=MVXd2 zen%8MYWaNA+n*r{9<^GNeaQu);8E)(Sq5|%*D0y2_6283_QfE!5n*S4`;L(VzweG);D0noo zYwefD5e1J%9Vv6D5>fDI^yCUV<`V^v#$+qiuOLzEk?m6QlfFd3qj3kOe3*|YcrP!Aq$GTMEhNc$vr&n&M8TsKx76+Q7*X(O<%e_TDH;L-ZG-}qnyQSj() ze(8ByZb3;Gby=ThUQ!a?>4bDx+16C9NEAFeF?r*QeTafbCy#7-t^iT+=v3Fa8$Trq z9-YoQ@51dw!J{)1W_~!HD0p=C__XoY69tdX^(xvg+4BtfuhP$%@E7)O`6MTm1&_Yj zD;;TJn*R1$NPX3IZ z9#c90#f$%`*QB20A&u35eoYdTe#Qj0ypg+15NW)*an5!`(}qKOs|F1k{XjHrI3%|! zQMb)QMAL>tHmjBDQnKe2j#DEh9Ds@Hx_5In~5Pc{5qM6s-Rw07%v zh=Rvh`dSTh5CxC1nqA?$WDnLrq+??JeVNP29&9*HdyLH^CFc#Mn!#gi?=3Z{I8pEz zy9ML>4kHR4W509CqC!N$qxfaeq1p+A-il~w$ERN>2p&azsdd{wiGoKVS~RWn1W~M6 z-q14lI-=lFsJF6AX-O2uqtO3mUvPpbcoe2to~t#9f=6L*&b?(1QSc~S|4+8oAqpOa zKlSnc-9*8o{BZb=T?vH#$Y?~@cJ~qlk1{^g<(UtNf=4NOcF%PKQSc~b?p_u95yk7O z(S<)MO%yyzy}#JhMMS})G(8G#&P^0NN}DI&mB~cGqjZx#I*^Abc$EI=2iqnP1&_)e zz5lfYLW@ z{IxF=1&_+8`VGG&3LcfMRr@{ERidtq^mhWGO*&dz z{@Y~)VLa;icQt5ha+_(6YWc?npNC6dv3T#Q%;30<;P}wp6)nxr1GDrbcC%d&x z*5Dz36i_{&^V7-tJ!FppYKr&Bovgt_?kJ$PbBAt*UD#yS#niB<&PD~xPZ6{Ii=;YzS2Oc8| z9-Vsm_{+zLf=8z_A9=7lQSj)@xB~@O69tdX{(Z1?eWKvex!0emn8bu3LsZ(M^Esc* zmBf=FKUCVIf8^=P$yOcwz5I|Is`bP-lRb)XoUYf$#V387>~ENm?)SgHxox}{Fna&E zq{SXQdUw?d^)zIA+IVtS zvgZ|!ftq?3&R?aP(}qLZr&3opv>}={9FjgQxqsIMqG`h+<vOJ|t#rlUM zW0O7DkmCucdHBHkl~gl$jO~2~&y^tx9%J{>Ge?&b1&^`+`PtnSh=ND4VdVC_354E? zXvqBNVS?aM#Fyt}`h_TX6yk;H^NCk474kWyO$q(ahPBE3 z7P2`3HO)7V?nO0&M`3T;H1jA?@F-lrmzO1Ta>(NZHT%<@1|{=($l{c~ujRr~2PY;K zLw{s6a!KZE34%u%A6yhqB?=y;=(ViLokYQ7_)XBNx*rq8>#9+^*ClgQ$lR3fkJJbD zoK5Dpkhck_>Aqt|WvUrGN}G4sE9<)w1&>PYp76v^M6p$R-sI*F z5CxCQ=uzTdqTo^4sw1rV$fo{1>7+1xWc>t{s4 zqqffslJ^h=kJ^1>%YQ)>JZk@WRP1h|;L)tUlJBMj!c|T5*P64n34%uxk6*uT7g6wN zM5ZmHnh*t#M!vFf#yO(k(WpP?#ZM6hk48_Kn>h(8EJQlim}{oheT~XukL*h`oBT}_ zJQ}y}mR`Myf=A=W7OiubD0s9>FP3;Cfl#lF&hJ`F4hAq+_ipynQAu zs36j@R(9AnnwAL=>4dCGCzfWS4;LbxkW*>k!ibv;T)GHN=>?R81(dkTO?x{l*JUTPJWUuo?VLUqfSE)KpiGoMxdXKN0 z?0JS9O6liJh;F7dN%l)a2Bq|K#@BryZ~vs?^gEr9KB-NkKi?z>9(`P_@y?rxf=4d~ zHW>5~QSj*HiMq=?QSj(h2UTtYQSj(>VOcZ}QSj(ZpNh4Y5CxCkp0Cog1X1wlU8^^m zCVQS1A|2~}zW()+{nBupigeuW54T_WPrWW>*hLGnf0zs?h`K{&q>ne;oP6rxIQ7Ye zyhv+T-s;TWiRZd2D}v^n_pTF8lX*XHWUXbZnUR3na?|@HcUQ=Z1k`;|WOA~zAF?6=^)E1)PQK$iv zK5kDGJPLhs;+y{v1&_kC9BhvHa}WiO(*I_zyh9W`st;SgNK+uRSVce0 z{r+@P65i>AWJd4J+t-39cnm+Snz<<1aSoY`()Wmxk522G>{f?7MnKi~>u)+jHDjyv z>@|1vBnlptxp!l?^+ds=vX5-3QI{xqRBq=>4c8C_kIFB2rF8?M;8Fk5@i}@}K$~>5 zdEsx1lalZ|oshO@`O+WmAPOF}D6z876r$i!%LdCACcD)R{`Q*FYSZ33=TcelsP)QS zH&-EwZ8oL1RUb$cJZf8ad)NF#!J~Go+qLOK6g+Cbw0)xjM8TtZsAV2{tin}IbZGM8 zPDx34r{hdKV!|7L5CxA$0Iildv$eyDPd_xq*qfz@$yxf{7cr<$S`3GtB0g;X~ zCeM|E)u}A@$o9BcdMi=zXxz~UD^?>49*v*$K&~A`!K00uR;C3A)N7+l1Ky01k`U=Q z8xQUK@hqa?(TW=e?Jq(UJX+ae_|_pr!J}3G9(|}RQSfN>TgP5mMHI%PH8&qBm7OSf zw6@v)0walHf8D9)a%Uk59x- z;L*!}^IXkF6g+y>qIKKZM8TuiH*|bDCJG+C8QL=UP@>?`+e=L=6(tHDy=(tVskexN zNAGXlS>P6;=}5=VTW~O$iPDCBw6xf`WHAiy#X=&X#o1q9{hvC+aq4*+(g>}~@>qSM zX~Q9j(6hOV9V41H98w6~ou_*6xhC||@&wn|8ifGz!NE`Ic&J8aR1&{IS!XG4I1H3`>aWuEsqkWT- z5b5|h-cz`DL89O>7B%u!m`@Zu#!?h4UWq7pjMcm@L&gyWkFnm}^|R}Ug2&iY?RE4M zqTn&My8G_ih=Rx1&0M$sW1`?O_FKO^cPCNsC>rIvm;_kxuGNdEk1Sd>DG8B|7x8&j zZaY!%C`5-UJv$Huk3tr%So;D|@F>*4#ygWaD&z!8-v{B3Xf^sXncqT2AfTpA-DR&( z&EQekVhslUMie{>H|+4}WKIsbfS_i7@$Z?*d>%3Zr6NWz%NN{~grXk$Bct(U?#Y^5 z6(Sukd(pi7IOap zH80f~Y^Y}NDD5@1m%m38JW4nA^o(Rq4q1PoX20*usAN75Ie*d~)xs{%E=eke7OUvT zHlM8}2=k+g-|sNwaiZWc{E3NHyAKluk4iq)E0Rz$(0az8H?=ZJzw^_k*C!J`*%Z<;xiD0uYppUtBsQSj*13v)8PLliuE zedGKn2T}0o&5#*QKPL(vy}dlW-d#k&qjxVBsl9?Ic=Z03TYHux>a&M$>;2EX-szPV zo#JWT+CTjt5a$U0^*`wY|MTC`>l*C4p66{skUph;s%)%v}*6nX$)-&x*5%uB|{)o>r&8fYcVJpJpuavmKT4Mzt~?? zYf3>JeLOAm{mfGG;&|=M_(z86lH$2}O_IALj^dQOSl%_aZgPVKn35N(PqwT}-em-s zk{9c*Hl9shWdxX#7n`To%;?4qO390Dx%HzCFigpd-50M7PUfLdZAxD3zwW&}nZg20 z$rFzkdhAK6HPuTIjT`=W5(CANNXZlN--DVZ3;O_5@`QM~uS}x(0MpgV%mcC}A!>jr zc|uLyKPFjM1elU1^x;FZl80r0DS5(lI(9x;AqSX}Cv1+R85zS$I6llsqX5O%^;; zgulD?UsHut11`?q&Fx9alltV9v2_@x+r&97%ynO4rkqlGvlqj)#R9S{8c}h0eGh-FQ zlsu(2EsKjVOgBicT$OnU!<0N_N-wD!F-*x*w(g=Pvlyo2DYyE%UY21>p7Kktt@A#^ zlsx@y&t3d^rJAFopEj?pPUUn;o{rz!bY?5Vl)Ug~HdfC}_UGbAr{rn*=!Vf(xMWJ6 zR^Lz0^c=&KJgsNXh`wQ%Zmqd@e$!42Q}VQZWKO+57^dWDxAW%OFEdQZ(|*AXJ@+w8 z$um!N8_IV=>WU`1bYSd2DrZviOg!}9?EDN<@{G9Q__?_ZQ}T>#ab$fJhADYQ{X6W5 zDGXEcjDBly^PCJ*@{GB;k66qw-4)wxV3s==rsNrS$`${VVM?Cy{qL)AJHwPbdt2uf zJjbLewb7vqv({5No04ba5m(35Wtfs@MecJOjxbEgv$E&O^F0}+rsP>Y zdc@<8Figp_CeOsIrx~W?S=(chY{4+yPj~e8@@E*Pl&!)XhADZ@wO=7BF-%MFw|<^wKEsqezwYZXdx-t#&BFgJl6};p z$IyqFrR4cIPtO&#^0tha#PqWB*FNtIF% zB$PA=19oy}2!k*P1%oh1=`K-9P#OWH8zn?KM8Y7HG(agyr6rXviU0gQd++z}z0dJ@ z&a=GE%+9`ccJ|IZ&wH+jl~)*36|K&J|37h!)px*#*jR zvqG$E`o=+oyrg`yP3AWcGI>Hh*1YsD2$?*gzsj~C`Eo%BHuVs~e4lwr^4&s!OrEfB zq~BZ~t7Y;;{CI|2+YvH(!hQSA;YtXZJmG)(cH1_DOrETgz9-Be*aS<+Xk0b71j@0g zhmhj$%6C18kjay>ONBaP5i)sFrK#BbZiGyp)RPO}fU!rgsfUo}M6sV?)Cvc*TT0um z)3KjLLnduUo zQL$%Qi7PdHBZN$zl5J`?+l`RPQz~D*+Akqw@|5mh<`)=ygkq1aGH1))gi%WG|LMBhSp+kDZ7#semjNTLFuMJuRqjT6>gN}K6YW=-18{|)kj<)2P zBH@f|>LIjPo_)b|giM~6Me^M$h>*$Cs%F8>0}wKKT5s(7#$yPXJZ)C>X!#vNCQsX< zol4({kjc|=jgFZ=LCEB3x8~LFzEjCv{K?M(=oJfnYW(4qxGCeN79o4kGlA#bgHs%@5M5i)ru ze!f|mUlB5S#_jsO=rahJJmVMqn)`QzOrEXU=4)8N5bTULx>@ArLMX?k9>R)2MSm%Z zkjb+$XQ_!35i)sJy#pMnnY`GHtnPXuWb$I0txof`2${S%?pn9b%Lw^(x|0Vh z{(z9li~XcyonAx8 z-hB`OVS}^@`Nf; zs?SJ-OrFros^o|fGI_$Ru37|^G9{aONMY+#YWDz^Wb#B@vi#FS5Hfkft-SeGeuPY( z@SCo;9)ys|ll4nJu@e}(N*PVa((ep1CQpjfS(kQ0$mB`cIpbez5b|nOx^%zSLdfJv zJ+VpoiwK!KX-+lh&=4W(dD3=h_RubbOrDH0w$1q>LMBhTQNP~(CqgDq`a{1z`2s>F zPvt35;Avp&LR9oY_GNRCF?mW{%G19zLMBhiRt0zVLdfJP^+3L3xe@Yn>b^ZXeua?9 zQ|5fv3J)P<@|11a@!oL=nLHKe@ATk32$?+PKKkjY$q1P|<^TQpp1Tk-d3s>A0f&LH zw(97Y8bw+nWAe24x@L}>2$?)BAFB8Ca|oF{tzNC&?l*)?p4LB>J=y^wZ;<(}%#I@n znLKSDmVN3XWb$cR88943jvO_OrF(OGxvuuMOr4$nqk?NL4YGIlV@%2496f$k(OUd z+#vl<2ymoj@~k`e?N1PKq9RT@Xfx_5v8S+c2zjKwG2 z%0p-!4yd14-qj%|1jsTR{P)XT8LRg@KLnw5IABm8%&`;#WEnGgu^FAaAB5I2X7XZt zPodu-l$9})7sv1B{R={C8T0FO$7f!L09nRNUhJoQ`XhwaGG_8zuYcF#V8F_`sKJpo zFg25oJ+@BdK3o?jb<#0;P7XVHPZj91j>&WC>Y1nZBV_WNZZdFoPlQaKGdYHiKaP;e zb9O-gpI~Yx9h2wc+u=H~B$MY{+oz7f)J(Fehjc!#+X<7pa6p4(f91wS9A@d%L;C35 z+x1{kARN#x;>A5z+rn~!^h}Bm z0!ecev8tjmWqN!Ago;y9^u_I=wUKcs{=B+!GeT-rMe^Bm|8zn~t*R*e=;O~r2&vUD zD_JnSGAt;C19JH)GP_1~g$1Q>z@U9*;$y9`TG}O56x#_8{ELuURgwGr;X7I&q|c>_ z{BIB08whEaRN_F{YEJ-T3PiN%$f|tEm^>lYA09RgAulQGo&3WfWb%Y6e&+9K2$?*g zmk#U(OPPwT=PF^=46QO6OEP)F*6x4*;|Q5N5tr)q=+_9DJmFS76+ecM$rFB~yYoAQ zOrCttyMy+r_d!Do%XTR_dR?FncIK#a6-b2XbNjLJ|w_%J`>{|~d{o(swh0!@2 z(8pHYEgw${wXzFQ(T7|2Pe8`xDRFVfnQRD|JSAK2`*s;ZCQqsR_kL6aAwQ??vnn5q zJ&Jwnp_I9>yabF|;eg6hw#A}%=VP@@o{9@BecmBt@{}8r=`|Q*75mmhDS!R$#xOdE z11e8{Tyw=#sFe*dI{JA;l{b+wd0KokvfGacnLI5Yob+frgiM}RuZ_R|7(ymb>z{57 ztAvm@$b5Tk)i#7op0~k_nLMMn9$lFMA(Lm! zj6*|*Bjl~MWl#NC93hiu;wq>AS&5LzGj8uI=Sv}E@{FJR>h~)UGI_RV?OiY}QS6L1 zx;3N3A5e}>J(Lv#r{;SeA(Lli_Bqd=N66$^)pYK=ZzE*#tiHDK%m#!^o;5?a?5}~4 z$+I@k`j1W_57oai!;?b4ii4bDvhJRR_2`o z<=E6i#p2@?#WNvf@?x23;d@^qWb$IwbJ+`zAY}4lePG+Zeh8Vo*o@hJIy*upFSd8@ z|86!yCNGZP+cTsrLVlg@MB|U=A!PDmKe^$jWe_rXZrp)O1A(z}E~;NNLv>_q%yS}7 z5xpKEljr2n;*Gl_Wb&N4R_e8r2$?*mn^sx<0zxLwne0{1{)v#ub9P{*&l)3S@?3nY z{3jO>GI`Fmx%p8;gj|Bpcm3Oo2$?+p_h0J@0i#n7<)in?nHeX@xnLMu!=(pz}Wb(Z3S^eBy2$?)@GSyl$86lJB?Z=i|TXe7YDRljmKR zW8Xc5kje8t&A}mGA*53e{qF3YFpz5U8rIRSuBV{$u-_NxD4Mr$?Ea*}a6m!^9f>{5 zzNmtbTGf%Pu<{yAmBRrmsq~6XFv-^JZ#{IRXYYR#rmf+Cl{EYJw1Nq*W>XIx*^1lB zK^PYfSjkvz&(;%LOReh2O>3DQLdS5xO8V`s3Pb3q+249-v7pCZc(^oEAfhF6W=(`M zvL9S&A%2)UE;B-2QocT8BTSVw`zewZ>WQgWVUisVXj_H;a(!!-FG_><3pNmEZB+KW;+E$O10=GI`Q{@>U0!E{8vTruRhpW3QKo>2lZsipo>Zbo?4# z2%24piuz6ZsUVz@{oqO~ac=y-0}wKKN;VrYTOwrglqxiG{B(r;oVxefd+tQY zXw$y8c0)NnAZoF4Z|j#3GI?4S+gkP#LMBhE>N~PFLdfK4y?N2N)d+cm%!;M6Y9eIv zv@N{qDui)bGkH3$wR|IlaauEZ+HJUV2ZWAVv!18@!Zb%Abkv&3Ghbvc1TS~Znq#70 zH?$fHZDmsrZN$vgZ{CfN$usiFP1%+pWb%x9VRN|$5i)s3|33SR-UyjIV?LcXHV;DH zTI;88f-p`SCeOq#PP+zSoHk6Jaew4LG8?O9@{FJL!1l5TnLI1_sccg9?giM}Q4G*_Dh>*#%`bPhM%Ohm+tQp+vr|k%tJZtYA zI3A|1A;~Enxoq5c=xms}YWA&%w(gQx)*P#4@~r>B_P>pg$%~&Y*z-?d?2g3Ip>ucN zkBo&Xu^4sv$&U~+d9loTyRAjYsh#?9~D zfsDy>;@&n7RzS$)IXSp#$7=|gJg08duh14Dljn593_E{D$mBV5U;1OOBV_WN{V4OY z0|;5qbMe2~`oDva$#bsd;_-hW^$@3yh z=bY&fGI?GO?_Pd6LMG3vqun}`M9AcM-L24HLl81~-ek@D`vVA>Ja0$k==T{yCeO!* zaxaY$GI`#0m^J5fgiN0I8Rt)k5YnlKxo6%n=&^>xFeX~lU=K{a*vrC1(cBK_A+in! zb9EK{+<{FjAbT@aZjtOuLlrDxa%}kXNhjthQz=Le}%79#`U37<)p> zd#XU1ze~OiqgKdx&!Fv6%)EuwGI=siQ#j*K2$?+TKAw^n#@O(4cUGRE2!#Bcx|eJW zV^7F)PcKB7E9NyAwL+SE25pEY~1_+rvZHx36z84{rr{kJkr&dGAEnLOqijc`O z=Ciiv3nS#MwXVsu@d%kb6Tj48L{@}Mo^gNvF?13_CeQdUcCXBWkjb;f)4h))DLbQ$ zt`~o;6_jIBkC4cINU6rx5i)sJ<|?Y6M#$t@)u>2@od}sct8Z5Nq%%S$&zeEyKRb+& z$+I?Rm9upa@=J-|uDW^)LMG3;ix>9SMabk?|KY_m8xb;jalMQ$;X8^|8b^oK+fP6_ z_N_VB#bGI_BXQRm~S2${T)*rfWJ zya<`RIPOvF+)#x4I^CJ$rwbuu@?t;!pM3)nGI?&x*DvFnn3Z$U+jHJM1m)Ndu0qQC zoO7S=gpkQ|a?p%?n-Ma3PTibZqBcS%&*?_%zx@{>ljlsXH6OJ`$mBUYWaIwd5i)r$ zzP{zmGYFYH=UTKo3+sm=Sv`GWIA5U4>SwVeljoOz*AQbII`s&N>U%7FVH%WUKe!5c z>N78U&md&-yd1Hz_%MV_o>xa!14^1RQ`>fD_O>D0rHp1y2&f-HFr+vsG~fe^>B@h&8%pRcQa z38mQ7!$xAa>i!5qYBi*$f4cM&QxHzE@{Q2Dk~sD zWa{=K*;2YxI~rXH4rj+!mYS?X*EJ7Px#Fj*40GFkHb8G{Xp5u zXkxmO0?M(ehn3<~#)2~uGI>&V$of(NgiM}P8MC}U7$K7<^@O%3iz8(6q&eMe&q{<$ zp0u4C49e>t9T@`3QM~%-6DqL&)T5`%sz3KS0Rj z>G;*M`Ewy;^0Zs~VcNb3nLO>k>?3j_Wb(}TqCH@S&i(+vnrK^v<(Huxn|fFyrdJ$L zA0d-xWVvd)_aJ2QjH+DuL{)@Lp3&P&cK-(?+r$mAJYx^RIX5Hfitu2$@c z*AOy!#_b;)twqS>89#eUnwJqWc{Y8i$6+0UeN_yS^LL3vNNZy-Pp>Eszy?{*&yp>@b6&!9e*w;%+_A(=db=95AtA+!#; zB+Jwguo949Zsq%IuLGdelo@XRq-E0gEc?Ql+nOrFyXdZn9+kjZl< zcYmo7GI`Dp8`=n_W|sY}hjsDQfv>=%E*!Adx@Pu8n3{#`@eKMxq9;u1Lh^Vj&yV<6 zybkT5Q;(1>{{72Wt|MbTFZ_AOxeculGI?Hpa&`1hgiM}S$8LZAG(slN>vt!p!w8u? zZ|<3xzB585&)dS9Sc^@tBn z>kq+EOsvMyg<8v3C56}{8?wbOxLmXkGHNv>i{EuF_q_9w)Rzr69FD4v<1*LF6t}rBr|8wG2SWpTF4BBT$eYgm#rB*|lcrpCTVhE|#kR|@J zC+06jNUeq>@jD-#To@tklCkJheLx9dOo51&p2@Qg8Ivc(nv=y|LC8zW+K1mcfsn}) zs??EZ-$%&g3B9P-Hdx9G`QfP_Vb=FQ3`?0IJv@W9-q6Kov05fi#Ki}G&;%iqC){#7 zYm65wXzFQQJ=k?pF_svDRE(6rQZ-Tc}lj}k*g&_CQqpX zTOYoOke^e3xO5SWJs}f3y%1$CF6s@VR!9WTpl!YU*A7@Mlc(bQR~a+;H2!ceWK5nG-%i?=8zGaYO0jVWd4tS1XY<^Hkjc~b!E4Wcg^Awnk4xP7m`egPqqXZ)PES~Nz; z?n&RH}PA(Llio~Z{)B4qNcsz2k`gl$48S*J>s~vdLlp6 z$|{Yc1IrdxK*r?7V$8x`I}kE?vAlc5Z*37Wd9iwL<-uzRnY>v4wWkvVq#@Hg9e-l; z@&1Ys>W4({4BAZF?|TQUW%A;<=e9= z2yZ6qu`X&_YVJpHMmF_`oycB%;{6DjJSPVh`Q=lDOrBG>ir%ycnLMZKmv1%-A(Q7! zo=UaTBV_WN9a=Th3WQ9ai?3Djr4TZC&NaXIFsvVjbnf(?IA8EWE?9XC+1#l-KWK0o zd`#%nBcyWgsVDscXJp@c#9m}l^WH$nXw+J3=PU zo4e~aFOQJP^L9)vcNHO%=i>v_?`ngP$@8w=Klj~6$mDr{=kZ6HBcxLg*FOJvxY3SS zby24+)7vG5Sg#8y+o#WN05>@tkXscpwr`tP18#CSU{F?`o(9jyesC4?wU?U~LB|RQ ztfZc{VFq+&$EF@3Tl@Caqo6Z~16I;j+q4!Qppdzp!MOD1i|_!2#O(~a+3iojVtB~g z&Y<7_&Td#nckBmOPNd&B@)*ct3PkkXqUvyyL(+CysSrOdtq(UjHm59n<)sHJauOMCA7{iL`B2;%dSw4ed`gDvtR9%?leLsPss)YU#W?Z z$x|x#&_=5f@^k6|CoeZb$mA(=JG2%dlc#KxBV%4d$mFRw$Kjc~5i)to^?vi~=Mge_ z%3pqC&>skyJl%0>eHbVmYpae{ub=)Mlw&`*3K`j#tr__kLMBhkk{j2-f^EpiP9FrV z>TJ0LOT8f>JA-<|ochzTTHYYDaBg*pkjc}w#Ei595Hfi>t~*r}M9Ac6x4FRM!w@og z+ON1je?Ejvp2;wyI=q4$YmSL_?;mvo%CT=fLK^n@duFsi$mAIrZ@c&_LMG3sr?;o^dAdr97V|F88d5HgN_J!Yi+bL+JunFGx2jP($qu9* z#wv}YQ>U)+PdNQizEQIn-w~%Q6%h zeV#))^{s8oO0ciPK%IQ z4Vl!dW|{vDLTWW6QZJKr@)HQD)sRPh&X*s4i;!9kY1H>E`SMYOOrE%Z+l-mOm;w=f z+p+W(WK5nAKXuAn2O%#hU+(e7VT4ScP>*zN*%={|C-gT3Z|y|LLU8Ri1)NP&>8Ivc) zfjV`@BV_WVe6RXlg%R><)!nt+QiM#N)E}#PF!qGB=_EP~iPHbllVH>e2MpSt<$m^9 zEt4nXOr>uuK*;1tH?HqN7-K`qbXKkZ`-9(LbPgHQ$-K?0i~H9OhFaN$sAy=V+SQOT zc}iR>-)t{JCQr%wReS@4OrBDCs%E-^ke^czEItv&o{%e@UWhWcO3j5)E2K(i&^9f4 z^Ifc#$y0ImBES5NkjYc7*W_Pej176xS+)MkxC1achcxL_o^G+CMscW>wN*!JGHf1% zjLFktY5H6F5i)sNmdd=~bA(KuR<*NDi4Zb*TCZ=Oc?3e5K@OJZ+0NZMg~| zlc(c)_1`Fgkjc|-%b(9KN66%9zkJs_B@r@trohSuD}k}*m}pm_+dm;=@{CxJcgtG{ znLHzHj>&%^Wb%x9CindI2$?*i|Ljm^D?%pEm@hhKsfdub){5@0-$KabnfTdmEt(-@ z@{HTLw#zkyOrG(d{qS5{giN08vnP) zR<$T|b1_0D&+3cyn&m*qEfOnLzslNXCgCFlK#kjaZ>hGI7wBV_Vo)v@r;7Z5Ue zu|89=PIZJ#UTnr!XucjHlNa0cm0fp)OkNy!s&>~&g#0?)k@NR;L&)UCe#E6m&LCv+ zT%&smU(M9Ad1_`;4{k0507oNLq~(-MSSg3s03KZuaY^SSPNViYhs z^$5w%JI!9W0U7IgUZkJbs|G?Q&&%=Ce>;Vc$@A*Ww1YhmGI?Hi+|cO{giM|{8CF+( z9wC$G?W9fjokz&z`S|4KN8U!r0gvd(}aOWOJ^N@*@eAH4V7a;P-Eg{24VV&^< zK;*|~A-~7M!Fh86B3}-L>>e9FD&7|mnbk@0(3$qh&6M5zt;@7I5 zTn5BS$rmcEPS!P~pqhkwvi#X(RYMA@N$5|DHhvE(q?&~Jy~t~Dy9CuF?2E-SyauJH zCK3Bm`UgN%lW?;>F7YxTs!8}i#^hTIh-#8wA1RzX5RzV%j6P~|ep*t7^+GBBZScJT zL^Vm-u-S-VfLMjaGN?$}bQO*2@x!5{jG_;n5{J58=mdyrQnFje zsha^&O-g0$G`uz-b}l_U`}mf^qTT^Bx) z0Z6L~CPVU$l$vVNdVh^il6R}rtQlr@&Cil|veZz5YgQDSteBGRP63E&vht1$H3tKt znyhM{zDWTJG`+AA5Y=SO*!n*d2Shblo3{Dz;ec4<<9FIj%>amMvhLV#tCCnM zq}Cw)-TF^<4ohOxkXVC63-OObw}Ww>$$+S) z*bn$+WqCkUlY3okf-XqQxaggM_m@b@II78sv_l_V1&C^La%}%@839pEPW{uX$_PMI zlhf^w{1E}7nw+`g@ZX;UqMDqYa&lFEKva{9kDnPf2oTle+`BIie*h4>WPi_VQ-=Ve zn*0;?8a$39&=1Km+75rLSyIMRO+NJ1G$ndnyteNB!6f5V04sVbqv7Az2RypZ_}` zgj^U!C(KD!1HxyD^zz$J-UAg9dm;73maBJ<14Qhp$ksep?;$|MUdVw_ExBK=mm@D^MOCrO3KjSTnX{p(0|teVx{Es z{bwgLe?^0HCDc>B#wYXs@R?{5`m-bVya*LiO~ULv{M0T$RFkkToov$(5Y;4Ncc$(| zKva`(U%XtQF(9f*_&;CkcmWXAB&XfZk%T{r-h3GinLc)8QicZSN{Z{#zDNg%YLc?i z>}$!orlQflk}B7{P06Ze_)IiOJ!sRL$ygDRTToEVXymF{JCZZ8=TV8n z%g0p!L^UbdbEU6~_0-bzN8ki>!_+hbqW z15hE=q~gqb%fc9=s3zsczg;9514H@>TA@GFC`U4KhU67Qlde2*NfH*b_UUNvt*7oy z8bO0|rN!K9_w)foHEC(i)_oEX?VehdztZLlKva{~`$qhm1rXJw&76@xO#(zUX)7m< zF9wKe(s8BnvljuPnzY;T$g;wKs3z^FKG=ULAgalfo6skjBCva9qVJF1{Yp}X2InC| z#aD-(TnC71GV<|LZQln(H5v8B>3SysQB6j#8}MUgKva`4i-%p`2Z(Ai_OX6*ngF7j zO#Ei=31;tW7iL>I^`v@o|^A8xS(`&(CXso?6y|F0il{=(SQ4!&jF&EVp(v1wwr*c zrdT!KQ?4Z-swvi2R(`P&5Y-f$UMt4d1w=K)Hs8WcM*vYxaolFvwGM#Tb-7!29XSk$ zYKr~9d$xB5L^Zhzk6ult(QG_(QI{(13MOS}aITz4Q}yWqfT$)X$5qN90Z~m({awDu zbU;*-)9s2bxf2l8%7g z$81bi!<3_%ynbS1vZO%2whg%?+Ejn0UUDWHoGUN#)oQ;95Y^;mueyah0-~C{x>6_m zQ9x9a*UiiIeHIYaXP4|Mp!Px7S+>jZNPA`oAwg9i>}%?9l&CL*H^CO~lvxPmcgXp2d(c z;+H-9G6Ev@LW+o2w+&eZh}a7WA|BfQT?s(MUPuq|-NNF701lQ_)Ij3&dYXoO6sE* zETU=a_iqJ4H3_kO&6$dTSSeX;DiV7E(bF zP4e{0{>c(9U7FM~8gOc8j>H8G&O;W6+o2R7s!7TwNB=1Yh-#85$Du#x0HT_t9@ek& zU4W=2X|DF}IvEhvByEEMk39^CYLap8VGk?@L^Vm*FS;X{AcnLLM3eOAVw+45L(&JL zNljR>ELqN`%d}cWpPXG)JvkE%&O^S3V^@Z)2Sha~`Tngxx&xw`l)CHM-zNc4O-hfR z)a?a8RFg6X$5;6i5Y?pYyCcHgZ_ui9W38wr72h-R(F=g6Cgr9)7AF(Lkl=x6QvUek zcP10Wkn4eH(i!(Zm<(og$y)2^k3}WMBxhp7mKL*?=1U8RY6_ij)$`u~qMEdNZuz@U z0HT_--o5Y4P(W0ZHuLxH&kKlZ3Y+C`{U`~HLOKU>!8(3=$G1sf6p}emJ9t00HT_VS+@Fg9Y9o*u_ZTsmjp&3aRa$v6W7^1BngZ{+6Ic; zruH8vL71j1=i2y{?|hmBVc|2;WKV6Y@O#nbLVtxMH%4&e9rr7koJajK0swuYlZ~xf<5Y-gNEwBD_1rWO~_wRG( zn*gGkV*k<8-=76UHMy)U+x(gkTE;~kGBlY0glcjkWBQue08vd&PRxAgGC)+5Q>U`U zMF3GvPIqoT^#edulQZes49^9KYI1f$(;wyoqMBTMy8Zm!aXW8|~ zQ-G)@->p^MBrK-cXF|4xmW68VNy^aRTzio}Z_}!Ps3tG_=CGFmQB7W*&wWRIKva|0 ztvZbO0T9*X%>$jMy#|PC^7e!7=l=mjHTn2bx9{HrL^XNWcByzrSK`C5*@ zlF4C6s6df=F!%4tlrp4Kph$n)p?or73&|8HGCy_hkWBMJDg}z{%iSMJCWj%B0!8s7 z-Et;V%8*8ZBKOtWyGO$n6MG?T!uLNsnFbK)WJa`_`AL$ZTy>A(`fdqzM$+ zDoyWA#*vURfugu<{Rfk0$I#&12sii7r;>MB_)JUry}Ry7#_y0hf%GPMX~xoFi6(kg zG8$OoNKqiPH%W1;|A9Y2$2XL>+XlXky-@zZcXRFn3fFZeeDAgalAL|@O;&ZP+HNc$s>$k;#gCQ&L^WA6snm{nfT$*G zGZgKU2M}v~+_A`#-hikk>kdzzl^qb(Wc|o-t=71l%YRs z37HE%tW)?BAgU>r_g8QK5+JH6R;_D2vl|fA6zdCWdsjeIQ*8R^Bc}mTO|dOd?yH)B zsHQk>QM&JHKPqMDrUG-u;ifT$*C($Bs65Fo0_+3_=GjRQnAx%kY~ zarXeCnw;xVpii={6_OH=A>8@Q_b<5%N>NR|VVAZLhmoJPgnR@o_Ln=8l%c_S$VO0L zPquD=s3tG_YIXB+tOxrhE4GW+DyK-*J=D2q4f7RLlz5qjdfqs4JC;v`{ z?*LZRaW{c5BsSN*t0f>}FXSAkRjbZ#fQY@2aiDPZ=Fb5l_Ckt*mHI}q6cRrF_r`|= z1Dn*($x=x8Op&c#deTOykk|{E28xxNR~Hbm7xD}&`(WV~K%|p}ECZ|i_NoJjbTTWR z>%HW1Lg)pH=+p9X&3;&C=l9 z3iE66o5{jj_)Ihj`$Cbq1EE5yNyJYUomdDE)g;{falMBEqMC%?J$X@HKva|bW5Cg> z387adqrq)oe;)|dB*l$pElvQUnxt&ppv)_PScU4|CRx@2qMD@sDBa{HfM{=$=HHC- z&jO;Fq-~h>c4a_RlZ@}nvSlA2s!6)OOSV=6L^Vl&{>wvq0Z~nA*03FW5<;7Yvy|aQB6u^=~225Aa*W2D&K-)fT$*A4i%iT2oTkzY_~j{ zvjC!+RGc;Ytx15WCgmnBIGhy_)ujBXFSd;bL^bL7)YI=Jgtkvd`^`-l&1i5QvIESP zzqExyRFjsf%)}jls3xr{mYrJx5Y?php4$EZAl3{suU@8i08vfa>YBBG21GUKxI&F) zuLGi*wA=P~=br#kP1;XCQRyu}RFir7bi02NLK|VCwIzRE4utk5Bfczgqa+}z$;d|w z&l?GdYBK7rVw2JVqMD5Up~79C0iv3WSyIu(fLIId6V;jz0Yo*K`1Q(l9soo&8TaF* zih}@AO~!wBzEgfcRFmC(w8(1-p=YwuKk26=!9dvZpNySWjLonh2_eFU{}joz*={Al zK-lh|BK1z@%}EFmHv6YYA8+$U5)4>NHCZ#IdCMe(2pj#=QudDerITPFZ1Yc1+`eh% zB!mc?{8Qu(?#hz{17VAQiu~w5izOjM*x;W81o8YsIethq(dy#pa^Cq#Xc;#5C)LHG zccIBiKpM98r%2|_y(I}P!^Zv;skS+8Cjn{L)}JDMu5*hdv<#d2Q)K#ecs&V7!1f*drf1=4f`Bv6sje@Cl zQTtwtFC}N9!FkxY|Bn72Gy+65IXPwMw%vfJCZ~=MJp2+Ms>$hh&b)IP5Y^;N+LOkJ zbTikmV}G*=A12e@@R?NSMWKmbCX;3BsU|P`jXE?oX`iQ>ygE00>)n8;Ca>FGer_Hh zs>z$Y=em>uL^XNa`)bi-+G{=4!~L1nmnB)nf8Va`ssD~9M6?WCd-xv z=W&(Xo04}{{9k)Lk>Tmv0V4LoCj6@|k4~0C!sq{9QP_fiQ{A->&*#;2lWU;6_zVEVx&Wj%(5jNQ;E~L4>VQn(Pge~?dvW+&6>H!r}O)}24X~ro) zRFial-dUcEfnjrfTA{zten2vEhOPCaQ~1rXJw zWRK5^yLyQDtaJ{|45GG;Rns!5AEXJ^(1 zL^WwCuU$R_h-%WR(yjHK08vd^?;Zcj?|@h{%-l(ho&iKPX=_Hxe*sZVIxasVU28y8 zlXhDlFLVPC)ujE5$4a#TL^YW@SDu-l5ZVY6tvmhsFd$Tu5sOca&IgEUGV-xQ8>RuG znv8n$=oJHqYBKu!-VH|sqMD5Ps^80L0I?R@$A_i=8W7cF;x`7U#{p4I#{CqR{0EU08vd=rr9#GJ|L>esxBL+Uj{@q zS$$%zcnuKMWXcC4 zx<9)t4csgmoQI9>`!9^JfwVWpw&02eFksN&JdWe$D{H_*60_@a*V0yf3a*%Hiv7?# zy3YbcHMwHXmdlwCTE<1~%J2Ff2<=Ty+*#@PtAJ>4a&mIj6@LSwy~(K)RR+Ebi1sF@ zyOhem7ZB}D&ZH^+cr`#&le6QB^tb|u_9hqqU9?&QKva`+J;zl~*0sXM_GAcmKGWp- z$!b{G)}9REzWOuS`X=>}mGiKtee*geJ_JHFc~P+L?)v~yO!Rf)t7rf zRFl_j^b_L&QBB_DQw6dDqME$zRl4d@Kva{DuaxUv7!cLuUDFR5B?_kn3!)IE`zCL|n@)(5e z>M4q!nAR(K48msh6uIy6PfC_t!&dba`5zyc_evs({^s0?-RZ_bE9eD_X!gE`lUo=z zsVCJ5v43x_M4>Hw^mBJOW2^EBBge8PP$9jo}MB-Z_$CIYlh9~DKdMO{+e{n zur)nJw!*63$zu>Url%;@%NHe&LD-g_A~*fcxyh1i*p!|kzb(zgWT7@}Nl$u{d^*cU zxFhtcWYqgym(Gb=`kV8x8~x?W&m9ItHA&g_cJ4ZWScNL@)uLMfQB6|!oA_lbKva`7 z=O%o39T3$dZL{H9p9VxV$+*y{LpuRcP0|f}^!9Urs3z&JKC$IDKvYv$W4;Q#lQvyN zV^2P-lQQ%-=V9OZf6lx!6%f^=)6ws9e!Y;C zp)2QMhxt$E9%u}RY6`#npRp)eE)3htlP;~*i&J|ii;Q7&d5ZL}>vNri3RyGE%ry^p z2Shb#`{c&X>j655+bZ@4L^YYX`u0va08vfGt!~-87a*$1_+_o?P$iqJYqP@xLLucP<28e31X4I8u;T27Tb7yVV zTX`x%Dc1P7+qGg_0Z~oXoqD!(1wd4j^%I}TyaN!`6h{rp!dseF7e_bxeHbNWXmIXg zF}U}ap97+rVtMa?L-_$wO|fb`Z0jIER8y?~J@s4(KvYv~K04iH6(HK1Vte1wq8R~E zO>x}tQ0@_c*mb${Z)Hgbh-!*`-`C5G1VlBtPyZ;gFCnyyi@Ft@avKN@2%N~8Z$WcF zRFjjVvftVUh-z}`P@c_|08vd&cklSdF+fz4Gg&&dYzK&Ha&~x+(mw*Cnp}LeYvwlr zQBBTuUYF+=K<@}n$U zh6AFSy!tom?F@jZCa)VeYcUfL)#S~+ZC@9Fs3va*H_0*>5Y^=48x6`707Nx;*Xoa= z9|5A8ynkSK?)w3e;M~uhe-ID>ycZ_vn`kVR!e%8Q)+1zpBFL|Guxn-tOzt zy;Ktru@`oBe=|$L(}0M*u&?`LSxa^WMC^rK-M?z~&2B)%Uf9$9`?iB$0z~YE9o^q( zvgZ;YVlV9H{&<6vjQ|mQVK?`0|G07<5V05ba{p=fhPMGxO=5MvFOt}oe$nDZG^gN; zy^}IDIQK&A%hxjxAgW2oO4&=z14K0mCG!+21Bhx8dTz%-V*yc3!tCwz_1%D|CSl9> zIJE>2)g)rmb?<|Ks3zfNtXuyjAgW3Dt>0gM1Q69E>tw%{%me6`PF_a6RDtqI85*2> zDX!=zwgaM?q-;~JdmBJhlT`UiSG@{|YLdEt-QCGp5jJWkJvZ#vQRhT5!h~(wDYDJ0 zFYf{sQcW^0SZlyRKva`-LywP2#=x*aJFU=P`)5Wna)#~M|2v2As$`x`$(+d3rmJXN z@yF9AXQIKmSK{wd`9}ewnw0EPv|1@ZRFhI^iu70kh@DGMu5>&ZE5erSr0L3>D8DNi zVZw&&6xnuF23n|)Y6?5FR9*2YAgW2ZF&CyMV_?{domS`%TpXEZ*l^nnv5*c{baIS=;>C9-l&@0_9Tmp;WN37MsFQQsx4M0?r)rV#NdVr{=@Lh{}d=nt5$=YmXdvpZE z8XtEpQ|%}qs>!;OeX4f?L^WAI>BIVm08veGiLTkQCWKZOM>i{;xDyD~6pKL>c7Fnh zYKmpf$^#w)L^Z|g?P|-v14K2&`g(~c3Id{|iEMk<)dfT~IXQCc$RmKL zCZ`VXnBD;p)#P-KMdCL=RFgBAm!^FV5Y^=Dh*i~Z0-~B+d}Mk3mVl@x=Q`cl<2oRA z$v%CWYOMfKO@8zDg|j4t1m|I2^hO)j4g^9qd68@N#X^9nCNGC$oOXQ%lX5Y^;O&Uw)tfT$*K2TgDAEg-7N$2X_dcoY!T|ydFiADg1NR|DfpG+$EDGV-ZFF zud;0ah0*L4QQD-=urqoJ7W-GG8G~4)U|Dfljij-mZkd9WTG$}D%z#x2*7LSsOqL4+ z)+yNR*|s)WG7Q+HU|V6&D2+|CDLB^qXG}#Lr(id|;qs&-gvwnC_S+f{NV-tKo+}kU zd6BZaEel)muy*= zEK~=aqEwh4HeN~=s{>9^D(vfPrgz4sr6?8g6YEDFMx3HlxbNN@m<&Oo@)V`Q|JZXy zGI#}?qEx<|_oY{%?v$&R(b!=xCsPIy$rPnh{4=0o5-JCrqEyOvdaGpm8gOdmDs8{? z$+Rut6s1y6IW#&62?9=0D$VhupC_-vfK!x8+y3;GBs>i`MX8MMIJF@O%K}bOD&6Q; z_bfsyDN3b3_{zy*h*OlRM+&|FJ@bjsB0OC{^O}h*$7_s1&72wjG}ip8%Dj zRH?j^B#t@ZcR6g(QM%u?nR~HiDN2<&cWX>F#3@RZZFXh-6~rk@Rb1%ohpV>AWl)Lzv*@o>rQF6jyCVOlo8766s20M*t%{6;uNJ?7T!Cu1mYB> zTGiS&eHG#qrCM)VF7hKzZJJrRD(xV|DN401wzPVLI7O+Bt1qhmIpP$h+O593M~pZ{ zsrJhxL40e*0V%pQj{9? z;)YSTuyl%2qko;2<}Ji2N{yL0J^Bf8YRm1D^BS~AoTAjkFU+p-7vdD9#_hhZ>N|*2 zlo~((-tGqwrzo|rbsmh*Ps*8XbmQ=t{!rGYD79knk(oIVrzo}Z-ZPiyAWl(gRpXQE z%OOrtYW2TEUzv~S zsZtLiPEi^^*x?%-u~MpzqoY?pUk_#D6s55ketUFv#3@Q+ndS0^lZaE4#;W_dE8P*N zD2?@@Q4Rk@oT4-~qlUly0^$^UFzQ50$)#I7MmfCq7)T zA>tIJZt*XlR|c9=)9M$>c+lW(?dUdG7p7V%PlzQDg_e(Vp zrzrI%OOE$9AWl*0?eM%Mdmv6x>f@t@3Z6ooqSU+2pQi7MI7O-V>1L|ai2s+E>pv4K zao6exMFItz1&C-ywn7j$3a&HE8s>C+-9pKfiB*Vz_jPaaDPnFr#DbMNm$@G?w=E(i zJ7oC?F}E!uJvaB{WT{t#1J)73t8j0Q`N?u`z}&WoZ27#m(_`h_wuoX=Xv-+X+_s3^ zj8C^_K+J86$Zwr_XgFftfrZ$bxfDd|B1MXb7T0gt1h*Ojb|I@eIwjoYYDyyXL33H>A>7ZNSb2(48Q)oA zULM3LN~Ie!dLfMALZ&E{{=g@_;0+*Tic(cJ!;3H*O__oV74@o_=@Oh-rHl(oT&dw3 zAx=@MWSiQ}b|X$vs#LyuwO>MeHp8EqSI$ICILx zN@%e>`-16+QDQQL4?V9xcB^oT60QqMb_L zi8w{6j%#$x{0ZU|rP{6eIZqnIDN40p`ctvdh*OlB9ZgRy1)9<@6Yb18za!F=UTVZ= zStcJvoT4=BoRw}%J;W(Wje05L?M;YNlp6h8gBC3irzkb%^Cqv~K%Clg`&8R3&mvAy zYU1acmH8EMic;ft{a*AL#3@RRU+`=0-w~%MwN=}E4Qo9j<;*s^S>)zID4Q~|5>^Z< z`b$y7DN3!(S!&`$#3@Rxdb{}C%!pHzT7A8$pM^L@sWn5YWO@p5ic)KHm9O0waq5M| zjVd+Ei8w{6br-I8{s3``QtSKNtdt9Jiqg1D^Fgq>CQ_=7qa&q%J_lt}CRQRABg)-q zia14SEHmqQ`w^!oja3gdsS4s0rLjI->#pO7QmjAx=^1=Ik7k1!ziH7d4vt>yt=R9(*Tq%{cG{;uNJ$ z4w<{?Uc@O%ow`1!cOS$lN}YatORi#wQy6|94#}N?m+&eWgiQ zIz_2-t>3GS&#p_=;_vU-4EtgDBag6R`JeAlgx~kuOstzvtb~uct~mHUoY|)+^&;EK z-%cP-QR?N$WxZZOoTAjL!wVO#L!6@2>mJ)5X@WRKsW+Lo-FFsoic)V!?5PNIaN$#w z`uNEHPB5PoK1Hc_of3W|g%p`<7%B`pmSN|(etU-z8f^>U8a=bvY<`pAl2r zA-nI)QtQl>PH*B3gr6frL?>J?p- z3Ncd}%VH(_j6}?o#%f8G>=rRo8tYY63&G~GV1bp0O`S^Z^I&PFG`1zmm-`qoQyRM! zx9a9a%#_A{+~TS1wbf z`n?unrc|hjP0L?G%#;d!x?#sgh}kF=repI*b|Gd;g-zS;{udE5rNWK+_1-@aGo`{G z{{5*J5HqDRXR&Qz~Vi+{beu=I2&@ zdvy8+F;gn_g{~DILClm&)2h>b;}A2Y(&p{_a8|@jsdOK0dU`Tqrd0ZWe$JW!F;l7r zRvT~xC~LQheyLHY6;h^D#oyMEs>CDp%RPsfDOK{-+UNVOT(g1BPYOw)esEOBklqsB0Pf!vaberqt-6S(d=^OBklqm>g-2 z!vabe-Z9%S^-frR3B!~ccmBIguz(6_j#t>SJR;;d>L9q--| zmidAOR>H=^x|N3@Aso;%TalsT{jkgzAdb)tn;;k!jwyBaPUtt5W=frF_w;cHMgdi|y zjnlv?WKw?jPJP%i2?ul_`8eyfcCg_hJX7k$=<`oygm!zT)XPIx?wyF3DfR08;S-)l z%#?baVbq*ih?!DvhE2QyTPA`9R>Irk6MljXmvF#3=H7nvci1u!EU*&Zr+nlDY`BDj z4B_(o|DIUM{^HJ%+DJK$|G2hdGh%8xWLiO zOl^nE%%4A+t|el6U5D(N3`=hg2DjrJ2&$)b9V#lZctp*p}?|__v6e z(%7wh+CG7pDUJOGm;QUiOsRNJ=N$u-^;$&ZH=fyplqnVQ+0FZ^B4$d3=)C^pD~NgJ zGWDwO8z5#%g_<E&Yn#-m_-K^az`guf^H<2=>DqcRa+fRs@QY9Xq z^mu#3OsSHujeqDkVy0B7O}B?uLd=_{zq`J28)BwZna9rk)dDe7s_e^G58Oe_lq&bb z^FOvl%#ryb>d?xAu`pArMd^NdV#G|T zmQ{N{F%vOUs@2wGE7BllO0}MGc*tYs!_H>94wJNA(_InAt*v@~L- z)QvlMc_2_$)TyV&805RSxBuz%#=DYq-c}wh?!C+ua|u76k?{-sb*DHy?~f0 zbvj$sbAKRaN}U;4>9Zz?nNnwOmmhW!F;nVX+gl$sLd>=J+&8|vgqSJyf1Oxc04NQt zLKf`z%IfS$nNlC$TPkgT#7wCdW8|x45i_M;9#n75LClnT)wBBf42YRhukWd~dNN|B z)SFN0eex(`rqtULbw2$HF;nW@JIB9&1TjRgsj=~7L47WR2vS+lo2v#?^*Ii6~xqb$evwc#dX*?2nVdRRLeHPwnIp#&9Iui z|4j(qLrQIi_5MArA@mPvwHY=Qx0QuOXGpEhu$9}h^u(4^+aZ(ov{u<*!5a=(X}`U7 zL0IsH)Y?>O{AG{5aA7JYC5{%&nKcp4%w7dlas1=laTyWw(&F_Q8(`xgq}rwpi)Hbt z*I?Trq}yg#eZ8&?1n(i`Hp6<&>bem6hqT)ao4Ok^&%&lLrLlc{OPQw;Go`WnwtXR3 z@P-uJtaAU$JK15u8`5x7rNVR`1QV%Zy%y1wemCmFnc1s=Ar;%n-al;+$zMpX?oe>Sv3(erOFgsc?}kwA^kU%qHL|D8(`5HQh+n;)~DYA z3*L|hoMFEp#W7g$hE(8mlyi+^Ty^x%v)~!^o_9S45`AY6m4Ieb{!U-Aze7b z?m*t7v$1lfRQp+ZwwFQ7l$tpAi?A@{U#5xvIra1#P?iN&%ETX?&H4*srqqZBkJft! zF;i+}qa$q(A!bUAy4nAq@`#yIqX+lev>h>1YRr8D$3p-gQeskP#x@x;8v^){TAX2b zIbPBND`!fL|G@OWgP19`&*ty>2T)eEjSio`_aUTg5oP01SDyL^F;i+q<~!{SVy4u} zZrAF6ikK<2>e$4eQX^(ctsXw%#wf&0sWq8~&w&75u_so_+U}z!KmZ>Os8Z|BJl?M; zR?d`KKcVT-W7#7wCd!@HMXikK<&@>sWyB@i>E zUUe&Q;$y^2sn?lv{hkLgQ|isA?EOAN%#?b2ILBg(m?`zH9};~*T+cE|F`2E$?7A*4!YSbdcyJp}I|T{^>hZE6dle@L0muz53M)5*yg3u)6C zwohcKnE^3V8oT9RRT+nv4&abA{pWAGWkt-CijwV?W4~ChMKrnMC(WTO3#_z=|E%!o zZN%&g5u$zN^A!>E%4Pa$tG6O%N`)F%>{Xb{LTYs?Md-guyah91NUzSYd8df3i8mDrNqf=SCpr=T^N$6PU|F8g|+(sjuqSU?vQy*cmo$%cRMR zl{2N%<}R%UA!bTLzN9|IU=9yy*;(cO_J_IQ4iHkaQ>AKM=KOg5uy(6xLw@@goS6ky zAxrzp-09y%%#G*yo#7NO)u^IV?)GDsWOFn4Bv~G zDOI*+*QwPIGo{L{Tf1@}Vy0C2g+C0fjF>6a3p16#8H+VcN57>__a&6&cP|}(p2~Q{ zOsN)6XKMB#Vy0Bf=QG!O5HVA#)z0QWEJnxt`2Zl^S%#>>X#qJfE5HqEwXsY*do@Qq@(T$?7wT7}RunO7VKQ7ti24bevhzAO* za)_ByBO4b=vlB5>YSgVt!@3}5N{t>={<9;9nNnl!uX3&qV%{tc%{7^X_3N%U%TxS>^Aa`+R4_OsNxtX5`+Cm??Gg*3@FP5i_Mu zHD34KKZu!9ryp4TQ5(cesWTsM*#A3XrqtORTh2a%m??FxW&3lmPa9Is)7z8t`QBOe zES6?U{nGCn;R=ifRw4U*j|DGGgR<;Zz>tAH+=!V{ zue$F4_*=wGsn=QdeE%e3rqr8}+s=$e%#?b2Wc$7ph?!FFI<;Co3^7yceVW$i(<7#V zl^H#K$?yakvKTPYsj35E)yd}3kga~cs`@pQWPz25;@z_QV~DBkkhQ*Csp3-*Q`;eP zy)B#X9>mmk$XGwC&H&h!3kPHng{<`l>MezBxp2U+d9K!<{jhRsJ7ltts_)K*nA#56 z>_7c`O>e~1cF1V|`^if=5z_%|;z?DO!8)J4hHm0$Ny#1;pe%c0W#V{M(Q0oYW=dmG zr%>Jlh?&w@mMHw>^N5+!SS={O>@;GgG}h}Y4eWuKDUD6Ns=L-BW=dmQtjft6h?&yZ zExUAi6=J3|_M0!Rt%;Z^6=}+!gy@F7x^F}@F;$5e%Cf-9i1>8c{4)_Vr9yPf{8B!| zOsSA*Grd0;F;gnkgm$NjB4$d3KGS^93dBsQFkKoBPJ@^!6*g7V<#^$;{|ab?8@~JV z)L5D+75>C_03(NJ(OjEm67p<{3VYgW=f@Km9yuYh?!C; z^JaVTC&WytR3CKS+X69DD)r?~r|%$UN~LMt_1j8_nNn%<^cb`aF&m}Q4gO(FdBjYq z^f%Ye+>V$jRc8xTI{}onTSY(B9=HT4-BB?r{=WXQLWr4CB_6GLaspzeRLNIr?8<_e zDOKthU2Q&M-ZcHK=;07ErOG@~`pFLvGo{MDS|;xUh?!F5)_j<sPTYPuJTRzK578)uvRzd_N*)O0}(4r1)!ynNscckB!zKW=gf6Jtf7A8QZyNUf3VwRom!Lmp!n|H!QF+HXhya6a)$3fS%cktlgHwGT*S@SQ#td?LHWS zgmA!0tK+%y!!qBn-&h%|hZQIRK|(lSrA>w$@53_RuqRf=+V``+1VKVLV5QyP^IwK# zzG1(yGS-is)d+%wa6s3k?&z45Kf_pHUx$mDo?QpQs9}MXadGxjS0I!P2ee^M3^+1# z4A#vqd?)XOb0cO-ood)C)l|gn!go4Hf1waFrOpf;(innK!+v9BoV_;i6$oX+0c*Ky zZeE07)Ud$HIA0*{388E_pi2FSPvY01Wi+rdK6?Mk)f-6Jh414G=hwGJ%#?aD?Aqv^ zh?!C^kKg&c9Ac)_t9K{JBZ!$&ud_}}-32jI>doj;FK5HV9K^3~Zr0x0XXh$gM7m<}mZD&kY?JAZ(;<1q*%tg$UD*5t=uSy|iN|pNgYLRh>dDHarb2+mjW=fTL`1-ToAZALHeeL$! zk054Bm0SBvtFI8V3t#!Ko_*s{#7wFFux!-`C~KIGw)Wn-8Yxq%;~D)9*FwycYEfqB z*JlwkrCL@Q@L@N^OsQ5o&g9sIm?_nI>gmESBIYeOr4PSz2{BWuZPjDXHA2jkYPaw8 z*DoSwO0}O;w`CK=OsRQDwqh;F1LukTRt<9f)XMi(9J-E}DYfdv zp3bnCwk)u+R)4a;B7`sDfX=1X+_U|`x3O}j)Y_ig9y^Gb_uKu|+QFuXWr3Boeq5^z zuniRs=v?YOIR4Go{X6ui{H0W=frFap_Umr?o7wvd-tf_yFwk zh6Ae94;q{TFE1KcSs(ROlYW6SvnN*8$M?v2Zy;t$z4)Zm%|nQpQZG-G{rMflOsQAz z)v2=`F;nXGz4cm@N6eIZGp3fihL|b!_F(l4Z4oo2-nIYx!8?fANb^4Z$;VnCrh%2~ zkas)`1xIYVsPmTT?URCRgt{o2KD!|dgK$7DJQu~==GA~<5Dpj?m8Yk`^Ria~T_np+ zi(m+b16Ep1TR#Jazhi-wi}dzYqhR=l16JBp+qecUJ;z=JbdfE!`4U`u;eeHPvpf6= zk0~5`70^X~|J%FaAxAi%O5@ZUMji*HOiCPmzpy$C1IM0Nxj6o5aRV3z;ea+Q7O$*~ zVF)_*Dxizyqf1l55DW*bwEA}6NErT(1y(NBzwDh3!#^Cb(&p8zm*LWL>{UP)+edb+ zg-b6Su+r}9mb>6Fg=4?5a_wRguFJd|;IAJqf z8$A{=Q!3op=LX+{n9d1K_`l07pMsbvm1oyo#_xl5OGZQci>^?XJ+X2!zSb+%8N^Jf z6b%QyQWG&#DrJr#jaMP&=T-wwU1^M%DV6$8XgOl0RGOwo$Gn7?DU~++k(s*@Go{k? ze)HSs5i_OIUwLED0mMwH>NK?hOu~+}TScqZP5&OsvR46}ikGY&`2=F7REZKB*20## zV?Q%=O4iwO88+O*0loN?T0f`3bgY~=O)r>RT_9#kl_@qOkqvRzw>SGo{)-w=6|{#7wDnzo*N$ z88K6;{ii95*G9~gnn`n0Rszb-Y@!S2);*1s-SJJ__v)2dh?!C&^4^|#KVqiT$X3_K z^hL~+8g+S6s=|nwQlmc@FBTwXN{z`gqH#vVykoZY$X6yJW=f5_`NWG^5HqF54}P-e z1jJ0KJvArZ29#B8qtmCa_C(5*+IZrb^(PTCrBRh8Q8h?nGYw-_$`O1TcnNpwS{=B$$q=A+5QJ3a> z&OuogSUDf3YImwBVy4uK2~C&pL(G(Vd8Xmu%7~d#uR5m9|0iOm)a$gVO1zJlDfMPz zruWw(W=g$1o%y9#5HqFTb^H3|9}qL8-e+Fa=vBltu=45h%!UYt+V)YK?W5wPAk*oi zXh!o)Ly*!r!AJ4dcBOM6rnY?~sx)mm6*0B#BUz^5>k2Wo?ISfc?Upf!scj$W9jWf5 zL`-e_$W+ZVe>q}m+efx^=E=noQ`r`qBQl>PHH+9Zf2Qe=#UheV65yVVsEFbIIstaPKG*-*=-`D2+O||N7uJxC4X(603T7Y5$tRP&ezAjD}RIT@5Ky zD&y_NzN}H|FFMlCsN~P;H`B#|3J$qv1rN27vAlw1M0adD6E~`-#>SkA$idLuDJP0XM zs^Z0|Z|6nKlqyj&3iU5kt(5c8(#g>6fvMa+~cQ?yyDm57;A zW$QP1qZneQRJkpGJi8PzyYQ7?y6f!{h?!EIZ$-luKv~0dw5!0KO-Pwi9ec#Yf@Ks-4QdT)^@Iz;S^%tZ+G;mgS=hTDlx2aHckx4u7iL4ulsfU@%7e2JGo?;mTKa1l#7wDEE%#NLf|x0F zI^W(d_aJ6Uo$0gn(XSCRrOsa5@xWta@P|0mMwH*J)N&d>%1V z>dmB$4_-jblzMw=^J8xzW=g&5+QD5$%#?bcT1tAvokYDM)=3MHlJv#J22rJ1Mvjg*y*H`e-e2~d)fIAph~ zUhjOeHxxe8v&tfMRwp|{AR(fkaH@#qz20=aM}j5mR}w`m}J9_n=xTFV?>odJVqLn97UI zi$&AC1|_My*!q&{M}Smb>}GvZ>}7ydUhEHy$-M?3l_$PES}?hk1ns4WK5BYlT2hXU zS`q)#@COZ$$`hhd^ASSY=1l_&h=Z!4zXrU|lUsnGZB@~pGOs0fJ)x*P70)Z(B)MY>Mdc|G z{*t?G_(Y~PB`fGO$(>p#Do?5XHHIa3d7)T?^z53SB`XA>s61t4{d39cLMSRv*^0GS zB`Xu5s66GipV*%>@uR3b<)%n`v^hHZvB+C+uhCSVju#bt6(&nf zm@V{c1`7JwWji?@MUT$PwTI%eS(c;tu>EVT%D{Vgr@Sety|%I zvIY^F%G2(bn`de!el(S*{kQ+@TLqBHGv5z93a6xJG||b7c4kt}P3zag%$v*JZo}woVE=hm1k|cE+Z-dWc_q^ zf&_q6p7jHMSy3J!mFHfMH^Pvm)kUkQ-Pb7E}&ZfOBhc~1V_ ztI7y~RGw2Ejvj~rQh83NJM!1(0I57@rkq-t7a*1A?8&o32LYt=oO}1>;duZu2|nv< zQ$Ges<@w^~I$Y8ulnLpe+6{lAc~Z_(c|OiHD$i|zRGt^TCswWqkjnG&%7m_40aAHh zHNW$B2Y^(b*9ETqaRMNf=S{!!L+S#g^1MBNWyL0dRGxPYUzqkfKq}As950Ue86b&F zWYun2-u@TKyRT;%RXg84xz*BTO2`4Va`=rLNvVifl~KH5)K9$t600HqQ;mrelHHT= z`R?f@WPd6=VNSAl5 zl@!nSpPlTJNE&*`SU%lre6mXtJ`;Jd`s`@d7ol1zFV;JcJiQAbl^2_rPPJ_Wkjjg# zJ6rD(Kq@bGU%XtQ2|y|@_J6$A=^{WXPfWX$Jz1(rIz&YD@$|7HlX5ilkRrY@?Tb_Z zsXQSX&%U1QE=wBGNFg7Xw=vmU4xfoUp$2VyGnsorIwuPCt<9~ISt}%SqR_mxzU*hv zASzGT`&VVL0I59TK72o0GRKBAPPAHo>D_|K>>QFfk;_(Yzg8$&%1YW18I4>yYe#ZM zc1_Fp$kK5Y08)8UbX~Y1*+-V_JCQQW;%n`oBs-@Xv9(Py_k_evq$5%v-BCB0wL;n^ z3QdoFnGZs>R9;A_vbPM(TawC?Zv0z?k~uacY@*frvyHPSvvWw-MC7T;6Bi{*eb!bL z?Y;fP$ zRGu`AeE=wj>nc11W4s6KlS1Mivd!3y6l8LNhrj=OC9}i?A}+B zay0Y^IhnpW{M1^2RGt=3o^JO(Kq^nmH_p`m6CjnR)!G3+RR&1qY5mpE8~XrKdD=YD zZ%$KyRGzkP_MUJKAeE=x&rkJh2$0Itez{+K6(E&orVrndEn)PGCi-(j-A{ngyRC^Q zZ)uYbAeCoC`gLW#14!i=*?x8A#{p7#M*THo+(3X-p3&o`&ME+q$}=X#oNF@xvaZ;7 z=5CMxsXXIO+`nTgKq}ApPaZg?08)8&+o)sx5=JYv(YbA{e*;G4*|`7qH=hGY?zv{AeCp;)fHcC07&Io-D}y{dH|_BYjQ8xcoZO&XKmXh*E<4a{dBi8 z96bV%%Cml8*6m#YQhBbz>o-; zcw@4EC_RQty6xkY0EyL*66ulc-xmW&tcHY0-!CXS2q3W<(jon{Bv)R5#A--}^va4C zJ_ksws>nXN?7aveu^JK~ElT%RvUCie?_R@%G)OaVneM=bldXd6^t=n*m;9ltts9Y88iiuZ3H=njy|lQP5gzfJ+9@}wF)soM(x zsXVC0%8SZqvkMm8a;c-G?FDFIS>VbHF8emOuYPs!(&zFQn1m8aD1 zeP@RNr1F%WzjuEwfK*;c__Fn*WEmF{7!d`^mfP`NvWyGqizsxzw)`wvI)>y$6#AKM zh9yhKkh+MBJl%26eaS6|K7>&^T0cA2&ZMn0^axpt7R)PJ4j`4MMX~Ac-2h1CX<2XD z3#|cCd0MSnzi$gbDo^VrtIpH`Nabl$V&nJ8GA^VlA_}yvv-#s>85a^2QRp^y_#|06 zD*DtzX}{v_Pm`r%_)O%P)7vWip0tOa(L`tZS9uT^yWRMqQh8QgIKQGAKq}AbzE_6q1xV#tllRUa4FOVl*0#F#_f>$bpYEUME;I#5N^~ zO#J{LmFIM-w!;A`Ivj8#)KGUuPPXna#e7Dy1 zlBGV)O$(E8In3~J@S%g2>1dmuD;tv1%seov;{@mp^li4{W0wVI{2UGjyPHLqckAkl zo{UfEW9I=>N`55P2%^y8Dw2F8Yd&R?#1w zDtwxpk%k^2+s_xBI~jmfUI>YMJp2(rDo@F0yWaN@Kq^nEUHSVj14!j5{blYYB>_@- zVbz!I_$Yu>p0dy6+?g66m8aZqUu+rYePwU00`n3hfT5C#X`h7b< zDo@+mnNO4lNabm_;p-dQ08)9{uUzy~C4f|(xv-%l+-c|;O?0+Wp@)-l^o6UC!DoE= z?27T}&+0zv=ox@io;CT(ep3@5m1k|sQhiqeWc_qE z`i`vukjk_E;}5=A50J`p54Nult0P*@MV;1Vo|}}TPd!3bo;0h=lmbZQIWcKNn_d8^ zJSR_Wsha~JmFHCFIUBwKNaZ=5dhWGH08)9*jGr-U96&11*|Sr}Wd%s(IoBg!pJe|q zq~Req4CgaGv?v3Vr1E^DciO?~k-TsfGVrwAU-oQLj)oo~|4zO=S-JtF^1SG??akEy zsXQ+)Zf{)+AeHA;n+4Y|0i^Q0erUAp$! z4>f)2p}oI9W#44yF?_!JsfQ_({rdHPD>ZlDhC@1?etjAwgxo9wT2{y10>)sxxn71= z0EyL*Wv5oHI==xVRzr@Rg4J6*2as3|Np)7Jo5_Y!_py)>-nwf-L~akPLZsR`u;w2Ot@eCVsB>qALlb3gYO~^7niP zjLM7S-z)jY0kV?f#j4Fe0Z8S=(pRaS4j`2mtC=N#NwzW#4LwY(e=T||*~$!`iM-gn zP-yNzsFuo$?Nfy(763@)#cuw%-a`OVd9mL;d0{SqRGv66;8@j!(O!yZaJ$#v2S(+I z_-6B#e*&cPglN*R^eX^awer5EnbrcN@`U;*)#RoC>BtlMpS1JO0i^PTX_Wa+Wq?$k zun%V1vJW7YCtTk}TdM)2@`S(e_2In$sXRGr=#D)JqaBga@XoEC0!HP@_*kbmz5qz& zNzuJ)#`^$Lc~WNTQK}C>c1|@a_m@QgQh8Dz&Oc=#Kq^m~ZaFt+0!Zaan>pLB%2R4jZGR9TYmlB-|DLx2QhCa#nzerhNaZP8p+@u90aAI&ZTqXsCV*6) z^3(sUR2LwXr^}sb|98S@b9A((#Lr8C(UGU)uZ!I*0g%em;_-s>MgpYrw5(fXQYwH{ zo>o6r$nY6JDo^W06`cjhT5F0|Ywbkjm5Ur^^)w0i^P@|NcVfya1^@ zv-?<~*Ahn0XrjMUPf3<3A;}GypG`bA&6mjnC#1NcP^8RqJ6WcL1UD4Qw=-@|7C0fj z4Tb7t+c%PBiZN84(NkKqN)|XFwGAz4(lscREK@>a8wzcQW*L(OPDpD*p*yrIXR=HQ zNo^?fqyH$9EO0_f8?scf^ABhLF_A|rwb7Ma^OFT~NM}PTwQ=tPlar-&NM=K!$dzMD zvOo^0Y$%lNvfoLT)*+D%h3b5lmdOG+q_Lq;_v`q2va}9KY$!AZx@Afh$RULdg|>P3 z(&M0;te@`MkA;&3a!6l8OZuT}awJRZki3S-b5GUHob2#0=`QNf>#NJj8ENPdQq`pE z|3PDbRGt%4hHTpnkjiuNQhRd4t)9yB^8E0v_X4EyylQvlxp@GoJg;+| zf2TA+D$kqV*9s?_0>)E$-d?$rBiT+cp33vCX}L_vra(wZL%ZU$KUX^0P6*j)?hZX{ zl^h$Bd#1hHs*Q?G`liso4W*ZC$VIbs!plDaBUVEunk^F>y$O(54S8s4j*`a#600E# zO`+kb+XEz4L)w{@S4JlrO5yX}Jq$@_Hk|)F*-#3fDKs^%UA+U<600HmOyN80TL2_h zL++Wy<@Q_yNUVm;GpnCF)fONbk~Xe4zVDKRQ3Y}IS+B-}fl+yJytDr+`2ez#;-w*} zW&))0;*jB8!~m(hSbctWX0nwTlFbl5vHtDUm}Dz6q?)17Jbz@}3aFOKi|x}#uM`DH z<;CvHH%>1DNae+T*PDAw0;KZ9`Vrf+B#ib_M1$r<&jX|KM0{&@iURA8~{KLe1;Q$~*z{{W=&lr29ZRU3d*o^o5C zEN~Mbm8blSCrY*iNag7|SD%@mFxng)tv&PkP+(M^j=wrRIyXQnPm3oGub&2x%G2`A zV^=jmDo?8)dN&#kkjm5gn|?2+0LWTvo*bI`TYyxawr>oOPXeU!wA*A$d=HSy)Bd|C z{}TYIJhSR{wg(bM&uF5*R$s0PjLI|dxOHpy0HpGaNU>#P1AtVXk?(Anegz}#{l}^42kd8bjCRbhd7eG4locy!Oz;^-Ck>}JqCG+kD zNJpO2DT+Q>4Iq{0%(y~5t^%YZ&)L5US8E86%5$#gxa!INVMygdZWtkX%j5>h&SOaA zLT(to`ZHPjCbg2M9wCcMi#mUP2#m_}asGO{9|TC{dC|YtfG+`3d0w8YzSIMx^1N!R zijN0K<$0Z3=F1F_%JZgIsj7W*;<6_75!$J*@ZH`qoHx9>eFm zLl3ui*7n=~R_gAy>Z18+XC;G&2D~9b%h&54Oupps`R=bOq-R;P>apZY4xcF$bvJfS z29~2QT!qvuk8iG+46N{(mQ;&o|Ay@$Rzq5rALkuP26OmKOPbfGFGwz+kf4P^TYOrt zhlCn@JDsHWm46Klng+eKJbV&wQNXSB=nz!&^GMGa;77F#A#lI$lIV5AD&{SC2 zJGp>DDi#W@TDmZ~fI=b`3f=VdbCb>RkcNdqzb(bYWE(vsVId<=l*_aMzAxHK5%oU* zPM1VF{jEpHzjEcub4LJDc|x?ilcNqmR;|o+t?(9rRGv`%CVt%-AeATd`3WE107&Ht z(|q{Wasa73VGE2pyb~amC*07-?>q;P$`k%t@h!grr1HY^fNL{y&;_nh3eOJ51fK(S%dV<)sJ=uNaZQ>)P^qW z08)9%zObcIb%0c!a=Y8tUJa1SQ-1zC&1(Uq^7Mgrbz$D8&C${31&6*$%F*9?gzPEH zmi+c8Kq^m*f-8DW21w;;S!>yXWUJTF|8V8B+PMF*IZ%?y(|X082g?Fvtu;lqRqPLt z%G0*`_RiS>QhC~~YSp3_Kq^oBC9Uh^07&JT=bC1Q`yxG~iH=SBszXwaKJ{=W9zOnq z69B0^BQlNHRu>?ZXJq%0M>YYZBhRSA=iY7(kjgW9)YWHU(Lh5FXH4eXIV(a*))m|B zdXcRFsXXIOKU=B-Kq}ApiO*!*0g%eGQNuE@ZlIOg=w`nUqof=SJ)DgP_x}2GfK;9p z_YF9l7a)~qWs{*>2LYt=torBlbHxBsc~*aP=AD%Q>BzI@!DEHf0;KY+ZFD%t2!O1g z?n2#6sQ^-W*7tqA^hkhIp8NDbp?wLXH0w9&=#Hehyw*jQ` zoIIR!b0vUOo>SdBy>T2MmFIM(&aK)5r1G2@-lNn{0I57@k9E!XCO|6Bxh`vS{sNFm z@Tq<%@&-UE&;PzVd*y_Y&?DqIX_R*U3Sd;8j~`4mxhOy?&x?;TZ5a-b%JcG{%y-fN zr1HFK(!AwNfK;B>_qBUH21w<3Gq`D{!2qc|Z*Mj%oev2mY^WFE$kj-Rms+Vd4BvwN%lQ%Qv zKLe0h4Vg@y$Xuc;Kw>rIG5MzX^4$Q5)sV&HhjxQs0!XZe942ox-E$cru^KX%JlXJ6 zV}QhJ$Y1i^fvXn)600G5$)?@w-vUVG#jA3Ek*wzF<1`;fbMn8~J1Iv)4RGv`%>+MeFo{*Y^jP~%~i|YKD%vvEa35BLb^`-AXwN#$4`D+b0 z1dz%TZpg_|$s8L}lF(}X^}lB%vvWvDa(CS3Wr>^{lW5S>j>u?S(I-YdqNJpgpv;3}P)(R;|C^YS>3^Y(J zl_zcbs>?nFNaaa4=Hm2Zjt$92Xtn;}rIE?(98!-Ed1^tYHy0R=w*OZUid!&&Gw!INaZQ{QiIby0aAHN{g$TZet@h&`t#H;Rsl%mDf4v30_Ooz zdCESYrDQXJRGxCXz7kgeQhCaM`Aw<@0I58^zH{q>38T%?(Uy*b`vRl#biB07^7{c& zd0G_eek$3p_4J?XdM#^q+mmedhR@`S)@nn6mnK5BRG!u=bG@GtAZx8DoV~;XfK;Bg zHFD%H43NsxZuP9xO8`=N+Ap3j3jw6^OuZgu$0v-Q(L_hCp)w>x%7Ky4o>-RGx9C z`c&@>kjgWD(uWNW1Elh7v94J%CyZ8VqgxgKOb?98v+>q3qX4NqCy(ry-Vq>`=Twh{ z@oxaBypR-jamwcaQhCmdSXuoRKq}AKqe~mK0!ZaK*ExNU8vvOEpE^ag)&QwIzxjuP znG!}qkB|wZ@%l9bfl+xreqhz50syHzFFxKlV+KGf&&wN|M@fKGo>yHmmG{52ADvq?X|K9> zkEeEN{rbP`Ujv!`KN7va{=VcUog_{F-|J{tDvG|#Q6kybkE7^+rLx>bquI-%lu5NA z4+!V+{uL?4Adfj06_?aVnjESZoJ+Z&Vbb5gCFg41_Djh&ZQzP?y=U8+WT!T8&AF+t zXOzPB8P2WRKVvF#%ekB0XlXJ)Lj8_&zpcrDWON1oZvvaU?8ck2=l=_;=XT=g>!~Tr zA?9}Dc+HG>C-VO&FY3;1kX%i16mz?=e0)yzVnk z;M{I(USB=E3$~BjjcxIDBaa~Gc4PPbdjpfXE7Z^J#{Q?C%aVyNaBf$;oa?1mp?cnX z5se-CauWK)k>GYk{P%!H$?86EZdZu6d&?wf5Bxvsmnr+DPU6zQxm}^A93Gu4Ljvb^ zg+6)g^W;t&IJYZIhcj1`1$5xtuCVD&uTR#9fpfdUjed2{Le#_U3V-O8Q$>(-yYjID z@BaXp_fAH=M>hUH%~DGwBmMEylm^*5y-h+B?|5xSqwS1t7NTx(^n$rc9q(=G|r3sKia2PtV}rw zIk&4!k;T;`U-yH^xn1*Gm%(^x@w1!g=8-Y|p|s(4 zO+5JM%QTdAegQeRYfa_}sV^eucCGC;Q8q#T zAN_ZyA1QGOIk#*5#7FZtLeA~FuYUQwGGJcXMU5JKdk`XjkaN3U9`3m3 z0&;HGtL`~os)3x_^*U4b_tzumcD)&%t3(gv+^)CB3gkbHoZI!T%crTkBIkC!Pc>7X zLH=(H{6B{ntHl{sJscAr@ zd{|#r>Qee4>HnkK^K4|B!J?@ zq~2T}>*sc3`(&Ei+mUm-vHNcMkxIz9-PmvXZre8G+^(pSx+er;Jc^A)G_IOk1f_W# z7mN6>${8L;&g}~EPK7#Sk#oC3rl{EBUgX@aP?HPZggGzfaa=6)pGAI#IWruvu`Nvd z5|esk{oJmw>5I+FiJaRNZp`QfFz3gD+ZF!cuwHP_hy}MR%cOY`!eG7!5X-1n&3i7x z*(IMjWPG)TZ;YJVm7;C!=DU$|yHe(^U;8EG+^$spOaB6MUM%^{AoaO2w_whUCATY0 z3q5xg*3a!qn_o<IVa^Xfc4Ob3^w$S1gnLH#KOFztal32YRZ|=6g&_Pt zt~?cO$vH*9*?Alnt9WU)FQ+5tc9kfU`*wch+^&)}^KTx2oZD4uL)SN+K+f$dy|PEE z?~!x6$`tNgDm`*;SJ@h!G7dw|?JBqW=bR~!bGynf-c)2Xa&A}eXm)xrVBR_%?aVyC z6JkDgb^KYT$;Xg$yWv0orP@*-Ik&6jOKIt9_?N=Sxm_dfFFA1{a&Fhiw~Efqh@9Is>PA&R3puxI^v6~1c^WylYs>@XYxhO| zAN{k9D>c6#Ik#)v#T#8dK+f$N-{)4P2at2Swsea@u>BYF+HG{S)X(RkG>_wA8;>Y^ zvl()3*NTj4-hSlVu9ZFHq$aHhpZr7SDby}=J&h1*;wO*Z< zk^e{k-Kj$re?-phT0iM{=hu*PyKc_TF_{4K(k^N|_1C8m^O?cL56n3D1#)iJiI3+l zybn3I>*S3&z55{NcAa`_%L7G_bGuI8zv0n^$hlo-2CeS$f8FDDoxQcL(j+X;?K;=y zz1nzbyZ`XxAL`j02WR*PZ1k??_x&~#t7mas?4z#B4!sX&_uQ_Jv#j{-PvqRL7bBPS zdIdSR>*bLJ3)Uj%cD?Gc{jsLVxm~X_ZhP?+jsV?73a< zIyY(o8(^{LcD+yCqz-K1#Gb`*V%Ww`gOkg-zS;{uhyRyTXn7 z_1-^_bGyPH{{5*JkaN2-XR&!L`efs;-JB}jfc9nTl^r?rO+g0`zy=Vh+ZdbV<2hFO3oZD4?(Z}PqAm?`V z?aE8L0OqaJ(e}#O9!JdW>UdhUg5M$McC{!~u|qoK+^&|@Dm?QEa&A|vZAG?Aal*#Kx|N4WBpfh%R%GaSKSUBDJe(u%IS4D?@A3#lBH@5@btwB{ zh$MvNcC8+rqaQ>f!g9OTWG(PJEcJ!ucCCFk*9nM3g#C~HyOT3-KqMh7w`={BPk(|) zL|AUu_4;QGPCUG{iy9tn3sJCed}eTQjwAITiWZLBbzYiFO{kDS|e zs_DSlJ&|*}PG=u7{v>j4*O>wRH$fCE9JlN2ozQ!gBROXU7Rgyaa?T)GP-Fp>Bv}O|Nd^g$b4J2_&*`@g z`^~d>Jm>rFy~7_fy}#`+zkZ+Us-9WO!f~ACdKujVJc_0T7)^Nh9^KZ>K8)im&-nOh zeZGT33s760nRve)-vP2bwCh>pZ@eDENAICs&;I>x!c@)Ct|vYnJ>?C}(XJ<>#w}i` zIokEq=Tjf@9UKeCahB)&F=Z#;0ip$be{R>@zwsR$3&(Mm=LNrVj_&}`!au$f|Br3x z&*SVrc9qWRADta&-Zmx_OZt`}n6(XK!P5~nHBN<_C+@G?r8$_l4XfOZYn?R*^R_`> zzdCVBb1+Y%ye~&r;5&!35NO6W`OYCNAXjfrEzm}9AL`mR95voaMLc1?5Z>5(HfPi1|p{>9@`bF^#P z`L0}@qdD3&#i}7a_^!8&p99@7LC9j&@CV{TogJ z&C#yuZFeKT*BtFypU5l|7~{LQjLADLAJ!P{TIQ8qM=NQLb}g&(w$JWrp33&^&<)== z&>ZbrV%pq%+~(Q%d7N#@i*qt^n@I~$yO!#-=u^#5x4m^ zejaCA-szX#=W7NnK+9k!SOtT5^Z4#;qu-umQ#3}qwt0L1m3Yn3u5GnB`dzZ-XxFwg z9~qug^HlaP`mfK#ZJv#v$Jw^LwK_kynX~|QZPjwc$IJBg(XQ=eUG3(e84aH`!q{#0|cE8I~)(>>u1+UlH+pm9{izkE}3Eua0)5RczqPrsil@+Lihg z_(F5EE5)9(>%ufgyHd_OHDa{psT{v5e(6L$&C#xODqg;}PII&?-6QYaETTEumEMx~ zH?7ee?W$fi5Arb2#=R@!$-Ml>d2JlW*~%O;JJVa5qg`dCS^U;b&C#y1n=SeHBhAsS z5)Zat*`_(#RdU4cV^uXryGmu)I{cF6sk}d@!N%{BG)KG2z20lqWzEs9@&+%tLA9ewMOJ?(o=J^>)H>Bd~i{7wCjpy6*s)CIofq) zno8GCXpVMWHKgpUCYqyN*LhOv^IMvuUDs{1jA0yS zd&VavWLk~Uu4g6`4IiXA+V!jn_WLC?N4uVVQhc;nbF}M;UR7>HX^wV18C_%Jbj{JO zr$*Hqm0NSP>pACY&HPGpwClOu{`meC&C#ys1^xcnH=4sZPJF!R0JrKQus$vxDz{2>kmvVTeV? z>hS#oTEO+?vBRzT{)51AoG?_Gy(PFWOAENZWABUarH>!X3&Wk$Dh>CgX#v;w_O;Hz zeQAMzA1BN&la6p~1==xVam-L>+<~vTEO*H-R*G;_3@!y(|K)o@i#O_yQaIgeNOI6 z3;g>yVR}DzOUr#}T7cTMROg}G=nAYY%a}3nVSU~^ey@^d)jZ#4=MK%$u4Q!?Jf)N7 zschel7`gbY=4jUv6ED49PIIhXOJ2L2@QdbX*HT^2)^Djf+O?eEQ|%sUj&?2g^ST}R z9tQol5^(k`?~mH0_#OuRk9T0#ezVhBzGMk}ceXKb+K<_I@A$n+VVgH5|2bH5v};?< z$1JimN4vJ2eeC49ny0dV(dT+%q~>VXmUr*H@ww(`*H-PGv@M`H+O?gG59)odIoh?| z!R1QxJq&^4IAMEF$)8>y?TQl_itq(j;L!_X$I-fnc*tKHP2F+7BKDlPq!s?o%U7>QUzt4SHA<(XHYOLPQeOV#Uu5h+q+8S(j4td@!O&WeKbeAQqEjDF@xr*9KZ7B?%=+xkZ4yr@636?eOV#Vu5^!Q zIkQM_AMHwS;miAqYmRo6ndu$wY^HK>l^)&(s|4I^yiqouQheT}O_d@^GByXqP^M8@-tCAqf0DPB>1_aZ~sn0xf`j$G!a8 zzZ&=~b=_89O+Kf2 zDt^2y-z@5&Io7T>YyNu&8N)bEct+2TuZ-0g?b1hZU0w{)9PN76=$@rkYmRn3`)rSn zg*8XJp6HSN+-I7jT~Eek{Ox7U(XOY)r5!j+bF}L@r_!%-G)KFh+i~II`I@6$&kJ8R z#n2qaaWZk~A6)q|u(u|SjSUa;5Erj#(lC~EyvYL*S_n*)G|VGiHt{f!7LZ$|dXC}F zpu};UG;G;p4tK?A0oNCcvn#$JOB}~ZLmtf-!QCNR!1Yzxv^BUY!`r{C0yhh^0Cvq)O)v5d0*Q7^Rh*(%Yv}?JeW@O|xpZ=l;x9^>sn2)a+^glO1?bN4qrtTmSsr=1UyMN!xqUFB4xgXaVest#R3Ov@m(X*q+U~%6rFg zoD}BzOpzaJj&_BWCr$NDnxkD|SIyA$ea+FX5W73wx~F+6$0t{J{-L4fXjiD5Nu!Tw zj&_Apz5DFSnxkFeZr!r}sODI^!dtOvWChL9u3QmYST}5Vbkg`WJmO1UJC&=RH0OsX zPjj>@tv6zu_0t^fO8c$28ZT*%cBMGbeA6n;(XN!U+TF~dc`C=R+@^CTYmRoM^KQd2 zahjuD>7F<~Vw&b?S9%K$t&7zh?W%kspXz2E_pXeG`95gPYvVXhD)X}cxuJYiXX!`*S|3v|+dAXji$n zZyl?rIoefTzuQ-~YmRoE`r+^Diz;s2F;3eb72>t=^Em04W5xU5YL0dtE51Z{u;ysj zvAY+Q6EsJ=jyO|qrqCSiI&w^{QL{BiyN-&fvN5CPXxDL)YTOv1c`EPUy>k9?cFobQ z<4wMHbcp6?*WJCgn(k8I+OF}@;*U@9+W5Un>6$Mtd8@PLXxFud&dan*bF}N)k7wtv zsX5wpMdPjC{i!+Hb!EDZ!`o<%c3t(^_G7ic#IokEyPOUb6t~pkp=Y_St z5ve(h+mfxR(h$m8`^^PNLlK(5NyxG+#}AIvMmF{&I&qdAyY zhCB1x=02K(d1ZLNoxhzybEs=&POG?vhZ6Xg0LnCyizMCTwej;fWttoERsKkGv};ZcW-mY6)s%wsRt*}z(c_4*4{QoMI<&Hiy zKUA-eb}jGh@$us{N4s{XvIlt>i+_cpY-2#C?Cp7N9LFi!yqT@YADW|G+iI1eR~^mK zu5D*Y^UhAq(XK7}c0STVbF^#AJDn~))g0~Gs&)6ZWi>~;w)1k*(7l>t?b`0JO%qCK zj&^PD(UvdvX^wWqm7JB&F~*}8#?G2Uk~M}qJW80~*I$!UbF?e0+||!d(H!jx`@O0M z<26USLi{W%FVj4gcQeDdT?IOWYfy1-K?Fb7YGn+?!LWTosj-zQIuLjj2@Hb!9hfww8NyDwUQ= ziKldYeDwJDaZ1U-VcEGiM+LEOBti?YcgITX@hy)e6#jjj za_oAA=W)t$K27^Bk3?tz z*LSZitHvVDa z(_dpS?-=I68|h!r9Lzh0_2&KZ3p5Awj$x~(-Coxm%sYlyFy$2AIi!Wax^oP9V(K2g zb4UxwRk?BfR_N`6dB>ptruXZ-nuB@AaAy@-wn}rzy<>O>UYnjnbEs>_>|bSYe#U6W zG*(^7utj6EYnmG`=6z4|RMxj@p6+@xm-gm1&%w{*97|qVw4B>a zS^&HBt|b1d&nmPdTepap0d?3VQ|g!AU{ z-PuO}BVEdCjCO7F*3q)RYL0eotL6T5tu#lwww-lP?#G&^vVYNU)e3I&9Q-`avE}U* zeYnk}1+Z(YHmiT>sJD-HZ71{klV>zXyS6(t=rOnX4t^fz*xuvFpZS_W3t(5=F4}WG zZywt?FyCaLAh@?ah&6{u?uD4&EwV`<7D#k(i)>($DFXd_kPXM zu45&v`L&(qXxFhnS@-(`&C#wS&K>T|U3CY?agHNL9V^2FC0c-X*HO{?Uh1m1k9HlW z*WOo8YM#pbcdxc~`4)|X<2c9hCbf#<8)UQq?XLTHxS>xzR=#UAE3#xb?;Xc+j%%jL zH#M{7XxFud@fdzG$XmzZO4UD5)sFXn8U$GLEE9OoKx--dCV3tGVS zRplL#O(o+~Hih_`OQkbY9uNg>y*@xW4;!%Y*!xi;I6B=bGN;Hs|>>8(ILn z*5r*_G=@~WmN9;6Xfc+LpU1hD`TG=mspe?cvOXT&D1+vyY~M~CSFMlcXx9>--`JK* zb7(bOOa5{9-b&5UuBASGIzK^kXf<5RiMl_2qULDVawnD_7Ogq78m{GCEBWmV&C#xX zW$PV%a`4^R#>hcdcU~Jmk8^GFe(#XWnxkFYYB=P*>YAfn+fF~C@dnLP*}oWk@or9uyyYDyf%KX(iLX%#<2x7N4vr*ynPGb^LO!YM_pmp+I@#VGoS@XrV!f}H<+up zpUUyczGq7 zZ#%C!+I6h(v*W92j&>cpyRK-Jt;hn+(XK1AWXk@!=4jVd1Jbrv z>$;5=H14N)Dt^3|zI^W`&C#wG|6&&1A%$_A>ls~|AHL3O<2cUs%#d~$n`(}BJ!?wS zZ;xt@c0K!Y!(kOPN4uWr7@qBd=4jWG;UR@T)g0}5YHIAKTQouQ@cN4uUEx3baun!`BGi+FhvkCni@XSCTj&h%GBvz}qhYaTm7V`w!z!`#!ZSVqmk zyk}Syo3@;-IhgkhyLiLeLUS$3fGn`^^ z(+g=1<~_q*{B^(YGzasZ;T>K1^=q1=T{H9Ed0#L_JErknr=q(xM!Tl@W9OJ!ny0e9 zRW0em)0(4Q(|)ykt1gn5_OZ>wc=Pz~Y-2>(nw2$1ySDkDRP!U6qg~r- zP|<6sIoh@D43(nqX`agd#gKecxy|$N^El6zPl_zzHj@^>uC1EodHk{7KH9aNG&z61 zsyW)V-QLrG;WppH&*MDXyEo}1Uo&U{?249as^;U(WAcQtF>KdRjnS?!SA{;wqB+_X zR*{%5=WC93g{?iJv@5f&Yq*Xv9-TA}W`Fvl#%NcXUuN80S97#0EtPiqInB|o zwBJm>tb^ugSBeuIi|x@I?Mk_zOKcgU)+ z(wp@|`F5J4UDZF?2xE+USH>-CNtDKDSDF3g)aja|U1eo0{&Q~4(XO&v7JK}a=4e-m z+x45L)g0|A*{^2JL7Jmor7~BIE}?lU@6Tyd-CL|V+Ewo33%QGGj&_wd^lG{#nxkFk z*7BgyjB)FZaWVh2w>3t)jybLH(i57aUB?Q``>2WLXxFhj<@o8A=4jUuSIX3?qB+`i z)N+h|599YwCjqNN6XI89PPR?>ya+e znxkD;_1}~GYt7NF>)hI(?p4jvuIn~#8NE{TRQ!19+IYD%N4s9Ss6yiy!#K|KjLwUe zZ_^lS*E2(x_O7Zq+V!l-bAP?0IokE?D|3EN(j4u2qSLm{$2CX0o($Vi<}J<9uBWE$ zc^2~4J(>p4yagD;(eF59Qgll(u8umihi_=Y2g3rLet6Buw^=?fqyGVlgZ@} z;eD9~egl=Jl3PA3WHUJev0FrLOz8rYsXYnqp?2CZF(@!J{dE%lQGHUK2c;>YaRH zAMm>1(R92b;s>VS(R3G%%3qBscr?A^6EbaP3LdSsXL9&Cv*6uY#_*;$=lE-4HCyJN z4L3=q;L)-gH6JsQDQ?+*sok7ROu?fihQ+G2Ou?fiAH_u!WC|WF)g-*ec&6ada$X2& z7Q_@hT5jL(yMN9UJX+qJwG|?nf=B!3(`o$n%!2o08{?9GIOVTtLp|E&sqVKrGX;;f z)uYqwT};8FZO3&UU6U#9SBy>*`6pBGXv?!1oHk6sqpf;oYx)~g@JRoAcc$uZG6j#e zJ7s>wUzmbN+q*ogM>(e8QM@;3fPdYv;E08BxcCSDCDRh%QJ71MHSsUFmH>~!qJRG7 znqCFAfWj^>yD63ckHRTabA#W> zu>^P&?!I%!GWezhcog28lUMpM1&=a!?{EC;vjxW_jUV!U#1|vj#FF4qI(19m@VhgX z1dr1F`O)R-z9|VFrMLFaqZ^olNA>-XGrU{at1`~VIA8c{D)6Yx8S#-OQ}C#)h_LFz zn1V-TcL;5ol__{s;##v?>zIN^B_}rcAsxh`T*EJ!J|W9oeVH%@$0-qoXo*oU@lHcyye0UB;AUitpEbN;a5+ zN5>od^SV+@!J}K-+`**`Yr00)A(`|0Yr5dkHG@aIww@_?bghYldW16tkFI^Kcf~PG z!J{iWoH=eV1&^+bIDK_KQ}F1j85h@QVG15y=lqqCLz#j{*ZsKK=$DzI4KMM7*`F~5 zk6t15fI}ZRcA|OM+Kn#I++WiJkDi%v+{;gxf=AElGqpk)rr^=D?@sByhbegUMDwTD zIxq!~p3Hv##5tzm(NhC&jHt^LJbKQJyX$^r3LZVT;oEa+GX;;Hm;RkGKQRp)3E7o; z#CQECvjY-u8o=KL$3o` zA)0nA&y>afEggCdRIGmbdLnN*&@j!iw)=jQ|c?Gft6~TMemmMyEQhrFpc>`eyriI2_qfbG=ClO=SHTu zuJzWSMgHv@8%8>|X}{5XvVZf2UI%(K#jG=l@9>tvqbU!Ze&Zlh@Mx-cFSc#O6g--a zd!^oOrr^;lk`FK}rzkQ=w)PSmwww0L0 zM+F`&=fw^24pZ=Gx&1y(hzyMHU(#xrEw@vH8)<8^VrV$7a4ej7zIrUkwqTb|ip*Kglw zzBHgJ>1f6G*0Wc=QK}Rphr<* z>Pml5fk!NiBTwFV!9OM#>Da}{;8D2yUro-z6g&!VcCJCIn1V-HVoHC1_=V?I z8k^20yyve8BORIy?VD4tZ(#}^rB(1!yHA;dM`?d}x&8&F;8BV#gLhV73Ld5WYUIPC zOu?g61qLo|$`m|Gr%sHhTkz=ll0qx8P@R^4L?9@X5@`_uRk_Nt5v+v|>E z2-j<6PT$=of+=`ZR^-;=-!TP`%5J|g?lq?1QHiVbCJkW<9+jLldtr8_;8Cfd#rNki z#dqX%Te96|3Lcev?#2DHnSw{Xc^+ z9vv&&vG~VK!J}ihI9#F?Q}F1Bd+Qc#X9^x2*?Y~zdQ8EiqcSbuafT^)bey)y4>~f% z_v=21I&+#ScyzoWiTk=R1&?m&*Y5i_Kd@zWjcyg&XY<#Dk&f+}L6u4lW(po%Yf{;? z7E|!(+E+{EoXZqEx}tram61%rqbnnG_WztIcy!hDeESP91&^+Cp~%_qnSw{x{bbaA z|DLDqf=Aa&n6TZyUuwJH(G!KX`?rbUyAL#NS=%aa*7x@cBOTi_Gu7y@gDH6Qtlsr< zbYcn~J^Nm*G-sKDM^CgUF`yh%@aV~GMZfulDR}hMAp1;9rr^hTzdO_iemVhAFOVRo?!cKVqjj$O4baw2ST@?vM6q1~Q;x?&3`T$eHFJ z11k3}$?uQ4Y4$Oos=~aF{obK4!K3MvnEjUDI}|2(G~K0HKk$2{H18N}**l!MiQg-w zS;qoBT9?-i@;@?wJKe%E245PH);9$s9hz|LDXq&CJX%)Mv)4*61&@}U_SA{ROu?fi zMh>hH#S}bR@_wK0)0u)tOEnx^AU9L+XgTRezWfza@MyUMjR=3#PBV-JdbGS7j`BzC zG{0D&M|;YeWdFkz_)I|9#^=}9SMm1>BORJr?2mgRw=xBfw)N?g<2{*zN865iaP=Zn z@Mw$i(|Wwk6g=AU_sJDcFa?jc`gjbLO~ck*ng|;_+D_uw*KRQdkG4Cbfa8zaX)duq zkG6NdV5C24r)k9kJ&N#Sx%_4oK0Oh_IKCqP1b?sCdJ1#ls!YL5!6Oy(`nSGi3Lb@B ze)Y$Nn1V+k4jsKRf+=_ua@mn%8JU7dno?=caKCp*GlvDb5KhVc-}${mnl}un`%BAN zey@~f4Fl?Z(dKi%S4wk+1$vYn55M4Fyx?;tA&qT|G9K`c7)ClYUD)!a`ARYckJ8FN z_mhWA!K1Y6&3U^uQ}8IohHXc8GX;-QCU3Z0iz#@Ns_>5Q{oWzX5fEHZ=g1su^@}P<@F@)=kGN<(J(T^#3 zR9484*F2`+QQ2KaWd4#VcvRxj#gTDL!K0E>udJWU6g(;we&(t_dZyXH0!=BW;$R_u;J-Ou?fwbzi49eF)oij5|}izt0dnI%eM~6*e&ikB;^7 z=$Aic3LYK1^|%68nSw`0+`O@_GE?yA$N_gp9AOF`9hK$jiH1zUqvN!?f9)Pqe829W z9?uj!y4KW~$YiGA(X}td zJ2{zxM^|)dF}p8Q@aW2rwxiQA1&^+p((H$2Ou?h;TyAj7WeOf$x95qg3z?z~FZST^ zH<*G)uSe^8{-6TpTcnA-T4k?s*k2PyI>IxvWNcQ6DR}g(0cq7;rr^=DZ>EoEz!W@s zqIJhHKQIN4o_x8>oDZ0SM^E+bdGi`m@aQ>rdTjcHDR}hUCYw*3X9^xY?}Z<(^drvtVNo$>)Y%D{gE@h4rpg+#;%_-uJT9S^ctx3LE3Zv zh@Ix{0&3?<|C>MBr@<+}zZx>McW0#KpsGDZ(0;;O@e8nHJ)0|yEomYFj z=#Tbk#x9`lH=7g2vy4E)G(*>>A6^e)8d#~)Y`x^1^dYz~joD$z{>WLvXFAd}_lFMh zN8R)~@R&@iQp^E=#7?tx0kw<8|KX4JX^t+SVqS|*{>YhT=mIMDv@PwAx@mqcpsHfC z7yKrSX6FLx6mO8r52l2Xjx^mRC*JTc$MhQ3^^P1&^e-|rNmpP!T6gBH8tHq4cWW6# z@}J4W5bDt~pA_D=oGEy;tY&!!#4yDz+i7xqGnFZLw8V%qIs7(?=Hd!`KbCw@I*s4H z(M()GRf7s|zQ|h!kCv06a)$v-p&l)F(9PC3&2Ifh*hBF0^!Ymln;8y-G6j#q{dK{QqnUz7;m!Z@&oHLoQMTInN`sr^Kz;8BT-`OX$&3Lcf5R%HKDrr=Sjusr=U zFvWM|bjrE14^!}{+|$z+reO*ml{a?Mq=8JqqcdQ4DZZG)b{*q(y)=#eHR0Q9G)Y#! zS~>181&@xExk`t3nSw{hZd2pULrlS=BW~IIx-$ijj_fbaTxJR$9hJ4jH`SSfN5^Sd zbif9t_d^)(X}t`uA81IcyvYQ#oNDO3Lae-Nb`T!K3HgUbEmG zrr^PgU10MxmV~DM( z>FrarWrh%2J%<_F! z-t{54FpZg|qQ7GZ9!>MNvfgV=vfr(GtT$rZ;5@^=QdI!_w;L%n+GVF?F z3Lb4IF3po^Ou?h=PW|$998>UUdzTjMoy-(G3g=SEPkabRER19FF}JHQ(xHi{7Fj>H z<5j_N1;UZnm;iGkHRTkwRvr( z;8D1HuXg#7DR>m#+zVyvG6j#a&-}Xbl&!ZH0RB}d(R(^kp=9GeURYZfLexHYClmhB>Xcpu5muNmI zpziMnGx&WTnoSC*H~vIkzrRFtNd@+MoMord{@{Cr?K;NYjLZB!IL#v!xLwEWlYP41 zf2UcbfLa;T@AmuPG=~&WyItC+e*c|jkOC@hbZP1L!D;>|pmJcx+J66?W{(1@viFGf z``|Qp6i}ym&tjAK?c)1&@Bfg;?}O94QLwHza&vmW|4y?;1$uN}uN&vzfWkB28XbCn zb;sW;jC5#@sE9#*8#4uut~F!C-a|~mqidfZa{67S;L#ObuXMf46g;{z_+q*4Ou?h8 zCZ5i)fhl-&ooi?ER%Z$xUH8)ui~3_injtFCqw7W0iSfsiG(S|JNAK<0#r<1#@c-r0 z>iU;Fnr6ixU0Kfk%HJZs>%Qxp9o_Q0cO-x$3ofhlCV^T#rP@x0fre=csNGW=)nOWFm?nU# zKF+g>D?|ISdfOkr>J#D`+-@oTYuYszn_wPOBEe9H= zX`k{u-PVF>pkbQyX;sO?_n8J7rYWB`mcQ7RX<(%~X1&P+l6?p+Ok-B>#={tbN7Fno z=)J5=ab4@(5g}hN1&^lf4zWz8;L#NGuYBR(gQZ!X0!^9n*NYSUd$2Ue6HxWm>8uSGErg@xT%igUHgZ=iLW^oG4wY_}onJNB;;r-agm{q~i48fyq zo>^heUD`|CuSDAuG`kSEjRla75`xRr4Z1LMDnzV9!25n9bA9IaKyqmdT+um zhTu_{i?4rClPP!32*kzw=?aUNB3UOred%rQoqmoOeHGY#Rcoa&Gwf)xBod>3j%#RmRngcPcRikIJ01b<1I<;89sYyT>+Q3Lcf+ zZTsB2Ou?fP7nYbGFa?iFPG21C4=NlO={QP7&aUzauZ!=-&Cq2iE_ znL<6fqFa$HN0>rAx-uwV!OBd*qpK$6OuEMu>d|$s=BeC}DR^|3`&958L!Hl@dNw~2Y#kQ(K%HCDR}g(K{W<{$rL<# z_Vp^OJ*ME%6KzGI$xOkcCo|buZX~IzTWndf7a=Bz?q|Ykv4C5 z)j#X>8mLuwM`ypQooeIGwvGzSt;N334qr;BDl0_x6Y7dV7PW`kxBW>_>t1XqAlJ z&ZiIFt!4DN(XESr2H@{>XyT*0cgvq<3LY)1-P827nBtc0jQ8{GW(poHF>vbFt(k&H zOWv5$?;%t0XsPC-_mpG`9xW&PxKjt1f=A08`P$R+Ou?h&-7mEJSEk^RKIOR2g%0n> zHYQ#yC;Ts&3jDU(?wx+wtTgIc3Uu?|X zlPP!<>h${anp?>VO6QCYEL_SR(z9+lm5?CBqwLOm*R>U!7aOu?g);L(x8FLztd6zb7YFP+U3&J;X4 zPNP%l$1uhB>)xyz8^RPkI^KZV#l|uPkM7LlIgk1f)^v>?*=9Ut2>l+{jLY<83#Q=F zwZ^4+vX?1%bnR0ac9mrc9$nG1(}#aB1&=gyTIW{nnSw`GjZP}MlPP#~owMCz>M#Y5 zuG?ixhM$?D4KHL<-Vd3ANAI^oX)E|JaHK;M3^fX0wvHiq^vst+rsrb{9zAP#?C#M_ z!J}vY8TT}dDR}fmljbeIU434Im*>4tx9lU@g0Z)qB#Eg|n# zXBudjCK0L=o9!~wK*KbJP=UC@-I)d&rU`_;Y5wgYrh$fO`k+nihP}%)&@fFN^kLJ( zcbEnmrm2GpHoVxFX`o@6IOx0M_ii!`G)&V5{dj2GM@+$^xgpa6f7k$@AbO^;INLjY z{54^u1EIRZ4}M^6Zn2?d7;!n zzkQ>*e}Jm?6^AI^GI+F|$VzKwG6j#eJK@$`zfGoDe_+er$=hT7_MGPY3G^tIcdE0> z-!L4pFb=d>yO|-hkHVbQc4#%G;F11|iDpO6GX;;rez(D;UQEHG5Wj}?I>r=_O3n{` zry^7EDAXG<*>5lfkHUE?zDP5s;8C~-zq0N!1&_k}@|%zbOu?hv*12^KAHp$7V|T}4 z0~mrwX|C?_?TbvoqqK7Nyy)L?_TYb+_M~0C$6^0&HN6fz^HOZj{_a%XGI*47ea25? znBp<3JZTFrX9^yrQ#F0IJWRo(bT=*xO=b!nrMGICos%heRP~ZdO!gt{RT*dMA1ucZ zJSuZc&GWx71&{QNwW@1cGKG3nc2e~rkC=i-B~Dve>N5q8^dX`wxPvKpR4TrBQYWVP zj-2krDxYNv9+i8sf0fQm!K3n~^=oj7DR^}9caM+rA#B$%9+$Zg$q+m`=FrlIK4%IZ z9qYvkg9|VPkBdqrr^<$pB2tJm??O4RJy#CET-VmaT@37IhQHE zU-#CGhLS0GbiDo(tIlBx9^Ext(oFLqtmzuvkKWnN5InkO{E;p7n1V;w8oOug8K&UT zwNLM#+mR`FbVbq%^H-+e(Umc)g3B`nkFFZCzRF{!;L&x?tZvYXDR^|<&XGwEnW7CZ zG^lcGrr^=rwJAre4+BR!GzCxNZJUQM1dpDXZo}>DOu?gPeYRuXJf`5$vmfpnXEOzl zp7>}{&?u(h(UUJOH6oaTM^6o%+weQ4;L&p)&#C$vQ}F1yZL(DQmMM7jyv#56EX35K zFYEo!M^tjIs0Lv{LD9*%CbpvI=RMjbt!;KQ^m#NWs8iC~3y+D16rQnh0c|I3a;^=% zQ*5tW(%KIX5PvB-SMd+&=L?6FE~i3h@}C-seQ$*a1@%Z;d+c}OnUZtmC_sGakkU_H zBfjWfSlwB)+fS3$4*G$3*5q8vyAWS7r1Yxn#FyO*3m;CrU((v>-NbVy=Q>$|cyh~6 zyT2J66!guI(rwex@4cFwtIn(R^Yk?mmz}2ltr=1}Pc;2})xEF}8q)e--3wdVm7a$u ztu6jGZRh(TrH?(;sO$1QqZn6$Q2 zLMr=dSm(ll7s$iXq_uZ45r3mbV)`lMO+T8{xiEejt*;-ahm<~0iI&UMNW6H6ey>HH z3tR7};OB;v9`PnEzf~i#*K*=*X!}`c`~T^qN{vLdo7V3_#~=AO*6&WgSMYEA-hX-= z&-wje%F~7VVL`!^r%(POPr;O@N`H~3V9Hb2lhB}G%F~mDv^~nxtQ2_)raYCTJOxvp zKBPPaQ=SS^o`NY)A5oryDNpw)Pr;O@V9HZ4RpkT_=V9HZ4<*5hdDVXx~6XhwG@)SpT z3Z^^_q&x*vo_bQAf+Bqs8r)8<+DVXv!k@6Hwd20CoLY{&tPx~lO!IY=4+w@F%s+mfjf+Gt$zc?qUGeMfl;rab*bc?zaHEl824V9L`1%2P1qX)5I@nDTV{ zzsOTC<>`IOQ!wQzEb8CqB$)EF{r|%}J^zgVPZC1)lu#1eu@K5bxqn<={*N9ZR7dNm zjzZ`>m!R_;LiK;~AJ>zBM+lwg_5Wx$|I0^6`Q%*3NB#Hf|ChTB3ZcAo|Ht*`Z$3h( zeJVfy-ydfPwI9991_y;uUT#ueLWwQny_`xZj&)0|<{&7iy-o=nTsPG+ZeGKAW-$JN;{GQs!5Ndb( z_NMJo`*t&meolFLxQmXD^0J7oKOxjkzCrC|2(@pEsC^5e_V`I{`h9A5>ruNKLhY9G z3LP)CZ)zuTYTtfzXqnpGtJLm>Q2V&49Ia37*9mIBLa2Rwh1$muYIkQ-yBk97X)?8^ zA=I8GcB5x%zyF*`&(!{A`$vD~zkGyJUJ9j>mr%+}zW>9#gi>B+P+meQFS#f$p_G@X zGtbVuKKxM1%i;e;UP37^kNhI4TA%WgCV_rVdAV7Mo+&T= zC@-Oum-CdDP|C}3>VJe%UUtr;XUa>76nP1w->*~g`R5^we*a*~-zVQA=z1MS=ezKU zf8RcaQNAjrk}qBFVRU|u)AONjDPqMM)ff8?O@t}jl^}-{)Ex_ zu1DuPjLy$Hbbi8$CFlBRd{|Hzcz7Dneg+eG>~ou3FgKVeh{JE&h5Mm}=Rre`|e z@6h=Uqw_R^`fXw4s{^$!`VmIwyB_r?!rIokFd{EKQ(gUegP!TUm3uFra*>HMe76-a4& zRA0BKzQX9duZerc!>GNAq4p|_>Z;2>#()3IM;O)DTVrZ+6caN~M_riYt>G{7EMmZ}(ISadRFYJ?K{Jk*B)dso_J?{}l?QtZv z$6=Y1a|uPyl)t&J(X*(LSZY4`r1p3SwZ~zUJMoY4cEBT?@>Ak3@)J(^xtAh8;gp{x zl%H_QPny4xpK!`gtG~!kIOS(^D)|Yg{8Xa+gj0T&r^rt@gj0Sx zQ+~oJKP6M-C!F%rlkyWz`B_5w38(y=r2K?aei~4I!YMz^f23#1PX)?PIOS&|r;MSqWpwYeg;r}!YMzwDL>(qpShHuaLUhn zspKbu@{{c^@)JS%nVcd&5tN^5l%EL7&&|Kl--w|64E>AzL{NSfrIMcr%1=wmPXy(s zQHuOTP=1D24yOIn{W{7|1m!0eGXeir^kej+J9qf*IFB;{u?6AKarH5S13P`l%Fb;pGeBjhVk@s%Fp|hpGeBjM9NPjUIKarH5A5!EelJe7*@)JqrmLIT0EZMfVvW9(*>wjG}rST{So;ipF~{(ReS4?lX?~G6kpeS)a~l z;3JCae?HZJ6x|QKdy}5&y#7w-HHxm2t7TYF6xH|L&Cj-@*N>ulZ1n51delGH&rx(f z(=EmQ{fpD}KK>`Y^z1o``d{g({}n}f-cEUrqU-%PtDf=y?W-udA9}X{ZJ+WdKMo3t zqVrd+U3gFw^(W7dqy16dj#1wJ?jwrMXE8dTQFQ+|g6{uDQGa6RJlY=Be+1Qk6rIoS zsXdIM`@ic-5~uqfxl`Qth@$!oYDkY+?3^$<<S;|qNyHkw+Ro5rh3>x^$<<v=zkrh3Rh^$<<< zaJeG+qk33E^$<<fv{4ucE0Q`cOSYQ$3VP zQ4i5n54ZoK9`td=(D}3I{Ke4qq&r*R#oVJ&U2byGwN! zL)WWM>3S7I^?c&uv+c#u`LpQrjTrKjCxxdN>TgV@{(21gnoPc8s6Vlj`V%o!XYHun ziJ|^R`aHxbH;O*5iJ|jVl=2%x^_3y{+0n*OZgx^`V(9$sEJpjG^LCTYTMX6n!Zr5)A$dCxJ_|7l6%?-;7bJT%_@Zyzz#@7P5Bj^{mM zsa~p3y~I+zjHP;srFsdcx{RfIxfu5R{l-$g%%*yYrFvQaH|iyp>cwsK{CbI{dTIX` z^%6_Lr%yr4-dmEY-_us+U-*m*1#fVyRvxQ@zAey*x}&FR@fF>8M^} zsa~Q|)JrVYOAo4-SgM!RR4=hqFXyRVVyRwUr+SH{dXZEwu~aV^sa|5KUUE>q#8SPy zm7-o^sb0=gy~I+zoTqw;rFw~@&;4SlUVd9l%TzDbQq)T<)k~dJ>Lr%y<=}r&FR@fF zm8f1~sa_)g7xfZL^|Bz9dWkEVoa@odu%I~FerswU|Iay>apWxzd5fbuoJn=~yhj|>)8Q83L2-1R-cE7-h@<}VUg|&VM;!IPU#HKP z;;2ruw4&`(|M>&DuO3JBRE_E>j{4v4WTj=Qzv+4Cna=BtZ)lEVs;{S1Uvbp$&z9ra z@&0SOR9}7Q`Wi?5{sYwSkE6OeoT9GcsGi=T`~7j$-a;l3X&lv4 zN2;ed>hBfoitjUy>M3IR^YikrSuL|`iZCd$wBoKPxbSd+PQeD zpY~Kg@l-#8>L;G+r#012Jk`&4R6p@lKe?!%5>NGWhUzDt>L)Cf`iZCdnM(B&PxW)1 z>L;G+=erd36HoQin(8N>>L-cnC!Xr33e`_M)lXKcpLnXD_o;s3seUS^Qa|xjKe_)# z{lruK)LrZkc%)K5IsPsjhFe&VTq%A`_12{e9MNcX=J z==@Jg<$9E`G-+*t%*4shBJz_!b?`FPK?0rsMt^bs1M541#+StwV-Df>$WOHteiEoI zYE!$CKy|b@MIGt&5~v>DOQqkMK=rYM>LY>1mrr`r`c#kEnmx-;0@X)Fs*eO3Uv~e; z_wWK92~>Y0sQ&aLf$E|=)y3a^Bv8FgPf>4ry#&hFY|2*xjn7wAr2SKU%%4ZkRF5xE zJtk1TYE!-vsQ#8w{UuQT7E%86BZ2NqoU0rZltA@Yis~(l+4?(}_v1ggVQDeCZFzeo3PEc(7w0_E>86!n!z^;L}OE0OA}dn)ynNcGh}mHJAg`dUWyl}PnT5%jXZ4au z_0^Q>E0OAJQY!V8NcGh|MSUeweSP>J)K? z>dT<|N~HSgO!bvW^)-s>E0OAJdy4u>r21M#^_58V)qv_Nk?QL<-JjEsM5?b1De5bc z>TCHdtgl3>uTfNAiBw+=sJ;@Zz8a)bUx`#-m8iZFslM)0eI-(T4Ws%>r23lqH|k3t zSDK3(R~BCR{Bd%cs~cBl+wd3s=EjxjCp_b6ZWoSubNsXSo#yVwl@W7-gVIdc*!mEC z|1VARtNqVB4GT&$abxQiAJFo{jVnj=qvsPpM~~e@&yNbnq)ke({*#R>dyP!7{h1qE zx5-Vv7gQwX(u@?p*Qxl!zC*)<(#+l1`m^H1!-~ZG-Z%yCR{Y`Qp2QbyY~A5jP*9qv zA~Dy!w3Qt71APbIvh>&S#}hi+zi4c4b5duh zXZyr~z9olLw*Mr5$_&P191R0~eeY6V_9u0TPqTh(oWmVFS8%-Mq|R@R_5BZBN6tWB z<)@bWAIdil^cmK$f9I3B1M}E_Mx4V9TiE|gS%ZnUQ(wgceRp+}e|X!(du|dqcr|OV ze2fw|Acwf7__%mvSupa)5#UCg9*^EC<__IUs#|eKn z;g1vk81csme~zj6cd7W}gg*^DFL%P9XYj`ff1L5h z34a>##|eLOhT)GB{@kkJj}!iQ;*S&loW~z0{E5aNC;V~7A1D0T(?dS~tQGum!k-CO z;Exmjd?Wbdgg={1yy1@?f1L1Vj^K|I{v5?0C;X`w{Bh>GyM^m^#-H=ADg2w4voro2 z|ErDU47WGHtuxoTgzJ=(GyYW5{y4*J9NaqN&#wf3obe|If1Kgl2;a`|`w#ea#+R9! z_#S_n@y8jy+u_?8et&>pXFRH)9dgE>V*GK2+vKTi7fjsTW-B>!yrn& z@_w9sw)ZbN#`WUG3KcJ$1Ge|iIYa)wZ4Kvc8z%AOguSX$NAZw)E)0q_T$_+&?lYeeDQ5G z^M#y=BU6YY=E)h~F5#Oq?L=y!X&E^#Byoh~f^V|~-(2wRpoVWQ_-0Fc<$`ZVf?49* z2EjKMd`rYP7kul+Hy3<6^|(@wF8J1tZ!Y*2gl{hRwgTT=@NM!szQ?x=d~?CK8TjUc zZ+ab{@hukLT<~oyzPaF=_Y}6nw`P2E!M9?3bHTS+_~wFd^YP6E-;9E9F8Ha`in+v`@pyHbgzOBSJ7kt~Q;+qS; zy{6%t3%-4fZ!Y*YTf;XOd^@P(n+v|h;hPJ-{iNcX3%)g|_~wFdw)p0PZ&SvyJ-(g2 z0^eNlZN1=|3%1+W$0rNym8_I8Plseq5s*SNBZTpk7Zt@n8$_KnH*0Rru2p57yEC>hO2mO3UNv zhzG0bzv{l_KJXcq@E9cUs3RUMVLROg?gL+936GBOC?_59;1cn`TG9~@28ahbINYkz z57NQk8u-&u506t1a?-)y(FbfL9sIq0Bl+<6$6@fNBksLQ+|$9Iv%sGY{(9k0N8I~o zCi(DpX^O(l&~b@(ub-799r57sU*X|}Z4-;^SRWo|b3djd-hJxKXZYIyf96R?+}nL0 z+r!;@fjb@X)RuUvBmUj`CHchl{E4@}74Ey`e0ixOE;o%+IFrA(wsfVuXH(v;T<;>T*A))S;Lw%xxR3L2 zrMxBD8CR}%3)kz)@fQonccr}NP~NVTvm52?3V-(S=gN7caUQOex1$T+<7F#ey24)! z{JBz|N9Wl~u9Wwyv)G<;m_|9c;^!#*bfr8qC{I_)dnx7Z%JnC5{jPBM2^_lOsm~wpXHzLJJP*Qi zIl1EZQT%o#-jow>T=8(D;GrviJ3OMSFK_ohy>ul$Y$rZkZE__(oW0rl_O8SSKcRhc zB|eQOKDpxOEBNV3JQ%AnKI2M!IG_?A+^Ekv8ujT$eNNM;PdDoGu)w7o=ku5lFWjil zz0{{0=ld<^>vrVbYa(iFB{z7koX=d}z;8FaeO2(* z4ga>|pBsMPNj!DKS8pM1xZ&TyyUE9I7y6HG_-iAyt8%@$;dd^6yWwq*O1tWY-%Dq* zec-oeMjd2-c$<#5a&p7(Yw_C+9#(4b;D*1?;IEq(<6rH{_Cx!DhkBvCkiVCgZt#!; z4{pS_Xq9%>?Q!~NKAaz17*)7%dxG)H(c}}~I*D&?@G(u`!wvtVw{w2Pw{qf}8}an4 z(0;fPKMxD-g?wJ-OE==%9F=xfUeAsAc8vJu=0g19{cJblrWZp6*iLVrr$pBwRT67leI$sKt*%S$xKyHG)6x__Imy#~pw6 z3I4d_PYC|FMnmjIjz44Z#~puyRQz$rpH2AVjz78h&IXZYieKLz;Xjz2D& zSmMuE{Bg&hohtsg}K;zZb2ycbD_dJl})s-?`XU^1zQXJa_Zp{5EiY9*ifB z=e*9*ie8A7aVOuV*secd3*Y8C;$%Fgl_ouQRTt5NV z9@Kw3_3uIXI^9Y>;|kNZa$I=66P`WjzbvBv;=#DV%MbCH{?2NlzvDswWlbdcjHB9* zW54usmaJt-`EDjYd2oMyw4QwUPF3OCgZt~jLd)?ixm-L<&pRe5`5wfNI_A53a6ekl z{iuiOc}Fk%XWajrhReq}dB5_~1AnJ~!1jzk&s(RoUmiS9xOhpqACuQNUwY8Lo;N}{ zt_SzKQ}xPzEY^qTaCr7$d}H7kC*d3;WOtK z1>Fu7^)g`1}YyJ>hdZe0suXqY9s%@ad&8{xej*rsoL)pPuj;1fQPpnF61l zrsoOp=?S0PuYylc_}t3-;-2t1QQ*@PK0SF}e7WQapX~ylp76Oj$~vE(@L33-p76O( zgHKQR92EHUgwJ9XK0V>nS%Xh6>eq|wIO z+>>=r>n#tFRpJj*XKq3PN9Ci;C*Ac5_jeOc~QUfsb4SZx291! zz8CG?AnlzOybqqbd_Bqglb2rfTYfL}`-i@#{*Lk9trz{4-_URIqFvukyY2=5zkz=* z>iY}o+Y29;VSW8~; z?g8&!c=ThNQcqs^cv$e!i}*2z_#r1ReB6tVa`Ga6I5VE$MSsxHCwvd@27z}k`h#J^ zxIgy7pA-A3AKLdY#udCwzXUikEu;M36YiAT1?BQN66Z;3uimEdXcb<);dMK_dc$iyyn4gylVwUd z%je+@uWR7d8(sqhUcKSf{ivPf4X^G(e3rMn+|nCfr>OAi4X@FWBP4HlT?DV*@VXLS zz2S8cyn4gyMtJpx*V*vu4Xsi`6Z+P_*c=d+Y27y;^cr^}#S8sTY zKS2G!Ycaffo1XK)t2ex^R^infUXOaSKD?F+y!voo)FV9qu~t$(Y`>lD<>W*CKQGk3 z54<0|-d^(I{%#J>w|poc1LfmG{RQ(J$p_w@;oXP(yJ^qzJ>?T7l#dU*>xs|Sk`KI( z<@t+`X*3v6@`3l=@a{wTJj!z- zA9z{~Pd?QDPU4ld=Tzh4sk9?FNO^YAp>dC* zwf*>Te;sp9qW-o`eAkw8!S4h3?ZbVh!za`$_t$MYK69V{yL&ia{N9P*KHO)1&wZvZ z{LUN(zrOIhQsCDYe%}`O^@ZO-`1OU~3GnL+zh3a`3%_~r>kGeU4dlb`BKY-%U!%aU zFZ`Bk@aqe|M)>tLje83B>AvuL6a4zZ?=txHHH~{twUvC~_lUr+FZ?dj;MW&^PrkGf*;nx>_ zjVk>5!fzt{`oix76@Go;_dN}Mec|^|JjaLMSt|Vc!mk_8y@#88;djNquAEkGfGd+j7Y#;5)sVk7y%*-6?t zIr+g$5WM&?KJ{gmrQd$=k|Er0_;LQbc<z*^;gR@QPy22y`4Jx{5Fh>F{YiNDhxd`f`z8MHZVT`J@a~muxqp9n zZx?v?hj+(+vOc^!z`H-Z?}K-Lcz+1q{o(y0KKjFZBE0*bI zEb++SG~TVkyFa{p@qVGdX}lZW{o(x`c=v~QJ9zhp_i}jmhxage_cx8V!n;4bFM)S| zc%K6A{_s8p-u>ZyyuiCZyjR1!KfKp#@a_-qGczG`8hkk5Rz=Pr(r0+@I4VU=b52QZ)Z<`2kc-o>s$B|jjR->FOGJeW_L z`Z@W`FRlGV$Y=h8&slpZfZr2%Y=?4u^YfKI2lU;u>FK@7dI8J_9R0L%opAKCF5wrAeR zu1MB<i|m=CxyfbW@4bRXkp0nBIlb&ZhEe58ACpxl`kxN9`)F`wnL6d|Aa zxe*RRJun~d%@84S zTr`0BH+{nIN|>*2?Q?+B8$s3gD&=jyUI6orLRvKPneVtR(t3U%e#GEMAb!}_S*{m| z9}f$D1med+!H+=v2o?MY#E%8|5r`jWRs0CVkJSMx2*i(Nf**nSku;I>z>kF*egxvj%I6h+ z0`cSX1xh&v;>Rrf2*i(RDt-jwhb?{t;>Uda2*i(re^hEY5I@G^M<9OeRPiGaKh82= zKM+6G;YT2T>=gV6#E*UW5r`kl@gopFmJ5Ca;>R-l2*i(d_z{R7>jggo@xv(i5r`iH zA;S6L$3nr6K>QewAA$I>4?hC&<1o(`%u^tKypA7%_)&x(f%viVD*OnnL6mP9UGKy&&StuZb@~%%^;13+09X{rE4ZAmYp4g!mGK|H=3tM0|9-OYu60_;M@p zB?$jFuT{1m+7Ep1hwmW#e-i(L@K}e(LCi1x?oCMwBEFouo$c}eSP)D6y@|zd+r)O!})0y&V%sx5dH@Fy%Dth zx2#WlFq-y2PQmbRJYXXQ!+(tm|H1HoqXz%M@IMazgW>-Y{0GCoP9+Wo!~d_~KN$W` z69>&xF#Jc{W&W-5IT-%C;XfGu$Eoli4F9*me=z)KY49Hm|2hr+gWvfeIvD=Hh5umq-vs}`@E;5R!SKHW{)6HFrAcfL z|2yG782*o{@E;8SBLmqE{^L~m4~G9(_z#AEJ^Tm5{|xvKhW`Ti4~GBIv|qvSf303Q zf0>+M_}>oy!SH`C{0GDT*TQv%!2Kx|?&a-6;D4(M{~^@hE|vO|w=-Xc!2bz@<@tqB zZbuB3aY5cbg!-FK{e@6|y(;AwLj8U3I6?}c9BYXqAzm2+DIYr z`z!bj!NVjx3?Y6@{Xl8|L*UotNAmITg=+Gtzq!<(oI>FDU+^2UmEV^tAs-+8h5kqg z{5}i6A$a(oV0kNL83MmQ!EXrub?Aiei9f4|KXM9z-z1@bIP^Wg7w~+xaz66+=F1TH zoq3D0UI_6g{)n=Fi}mqmFoN}YaA*~r;L9%Bix7Ut;Q^iHenRo* znsUqUS zvBaMl_!Ejh=ZQn1_!ERbq4;BqKcV=u`a{K^Q2hCx_#cWt2k|Eqesp@h232oHhIj#h+)qDR2B)YHQt}Q2Z&zpHTcs7=}Ng_;X6bpHTeyM#G;_{F$fW zPbmJx;!i04e2PDz_)~{Jq4+b1KcV>Z0{(>J&n<#KVO;Op0!weg;G~c^8&=kG!;ZD~ zQW%{4^JcmBBq0V!XTyIO{x87)Fv@4X&>szh|4-mQjJUE^ zXqUs_^A`9Fga4=CKa99?NhPj?!T(3_FQ+j4&%*z(6Y~muvRJ}@3jBw`Q8XNd;s0jZ z&#<^Rf-;O;H*sbSaV8A@e|(qk@xSmn)`Q=3!uY|^{_#Hq|IJev{GXq|_W0k4|6#X%AsipP@gW=^9->~t@!=>wgyX~U zKd~M@98mEg93PA-K7`}LpKoP*e8|CvaD4dsDtrjXht(=RgyTc;xS#JsI6myu@F5%@ zW~lfOjt?K=LpVNsj}PJauoNG{@nJbWgyX|2_z;c{M^$_X$A{DS5RMPi@WEP&pno!L z@s-Ds2+pU3^NC>Gc9hDvZ3OL;JMB{hqWrx4aY}F5!~Nie3RqDgXF+x z?hl`Lx0NE``8IftV0=-D*Qyk&qWvV z;peM|Si;X<4SpiwXFku3BjG1ZrGFL)KQF;gB>YstPbB;-7x;;UpV=DxM8eM|6@DV& z=N$1c5`MfGABlvYqr{U)_*n=)k?=DIej?%L=%eJrPd)rZ!p}PRiG-h<;3pD(*27OE z{B*-lB>cP$KaucL4L_0a;|xEM@bmr{^5JLFCoJJ-g1}EC{P+s|M8c0V{6xaf4ETwJ zAKNS7ClY=>(cmW%epdZBe17!At8v`-=&64v>R(SeO{JXVq$ghOAYSPypFNb19uBIA zBYHR}CywZ8Cnjq2`{n(}OFie;VPoAFJ>@e|xc|_@$3*zh!%+(y>1i*^epKRy9&Uau zaHFTayyipB6FzQ*4?P@C?y`P7`Mk`RdiYx*@TZ51KDf}+4&1hi^PoMq2r9}!{1iommWU$_1H*y+V=?VqxJZegW@Vu?0T#@O3@+kC#h&`0Et-)6)-nQ@C%`)1L3AJ(rW7_|ks`@g)kr%Hb;t zzM|nP3clI}zM|mkFnmS9*D3glg0DpQih{2w_=MP(Y3cma#>*W;%U)2I%QSkK; zd_}?6T=Z){iHboB1*dzP1Q_MZwpz@D&AL zPr_FedB66a`;70$)+^wLsu23cfZ7d_}?6zpj8U z^W#U;f1Uoa62C0+>A!w>!Fs;=`ts*!>bbw&UW(@W@8kNT>F$HAG|2roJ-W-T8G4M7Y z-eTY_4Blek?Qd;1QVhJk0B^G+t1)F2Hpk*-eTbGehuDY;H?baV&Ls0yv4xVGAa>jmCox!-)9@yA%o<4=@FEZ1>JXg_5RVyWLk>Nl2h`IK^r zrQCN??pK;(Dc_qfTHh{~`aMVe$|;ueJwf@#!o_1M_ldF8dxA>6$8!De(|*P>{<7d+ zwukrcTP!({k7K@!h3^QV{fwo2N6~)DDVFh!_l5E8Sjsnu@{OhbU!eYD;X4h!WAR~U zi!weRi!ZD2B^JK>Rrro&9OWY8D6#lbi7&B?w?v;|3Gb!w9?Lk@KX^VJ%Q#lstt=UD zSSjJgargD6IHsW(E-0z3`Sn9t_qaBT9ykcf5`-ksiDtr%JFXKh6XW1V_=0wlYdUfP5V=PBtsPy#;f$EO5*I)qON z_;hgw`S|n{J|*DOczjC0rvq@8fKNN{DFL7M3qB>_Q=8yZ0zOT^rv!W&{3FM~r^oOq z0iWF8<1;>m;Zp)Wotw}8@M)FcQvyCs!lwj$+Ko>M_*5hKlz>lqd`iHlsrZzDPyZ2o zO2DVw8`&P8W(z(g;8Sl7=Ydbf_>_Q8BV*VPJ}nk}O28){d`iHldcmgzeEMJ*J|*B& z-!I@(0zP$W_>_Q8kKj`RJ|(O8lz>km{{^2C@F{2*J|*DOs9lsRK22BgDFL7C_n5Uno8$;8U-PPYL*xj86&pG*iW=1bhm?rv!W|Rq-hSpH|^h0zO5n_#}Iq z$nmcej-SYUq1ESw8}DBBP12Ood1WBZr3O?=3WC0;&?mvTzP$8()*kB>JAJ`UyMlxl0UQ_@y_i9@`;~~^heB7B0hdQp8XT=uFn>@Bp%pZK|Dw# z-o?HuY>&So_XwI z#zsoQpJWw(lJIA);7=0%jGn3RnuI?(!Jj1jxsLWM34gW<{v_eg3j9gJpFaFa!k-C( zKS}tLg+EF7vj=~Y@W)QYpCtU*gFi|5(}_Px__I#%CkcP9?_qoVDOK?&34d1MPZIv@ z(C{Y-e-^0tlY~FJ@Fxj>rfT?;gg;YF=ZQZz;!hI(9Gj%@mxMp#H2g`zpT+o-gg=j| z_>+V`H|-lCCE-sS{v_d#ZaM4YPa*y!;ZGy}B;n6#{7J%}KEaV+N%(VY-0=ISN%-R=_>+V`dBgB034a!= z_>)ZeoumAc;aB%N8z~w7^58ES{_lkUWXda<@=Au^Ti`bt{+^^imrQ+KM|~yZ!;Scm zOu3B{$}JiG3xxI~nflpvP`U18_`jF-BN_f5;{GU^`U%<1{;03U-F$}sCsN3#JQvfS zOU8pA@F1D_s{7yPGyFerMv{`5FYIE=deqMm>L(c=I`APG{#{4%J>35t?vtsXTIxqm z$?!j)``2Xnj}rJ#rhabDU_bDm@HzSL8O3p|rDW!tc3w|;;K2{I9EbY3Pw3AjGvBl} z-Lig@;V&fC@_TE?x!UiOnQy(Q-}-Tq;cv$k@OQcMNM?TVUtU+@p1giC{67x=mrKd; z+^NBHGVy2}@hBOdrwcqM3u*mCAmSiAQ$)zJ4<8!A6z-Tnawz$EOs0ipQrEeCoxg6nuIFpHlFt5uZ}< zsS=-3@M$JKrQlN}KBeH(B7920r$WJ}6nyFxd`iKm&jp`S@M#@BrQp*fd`iKmU*S^< zKHdFyKI78_d`iKmsrZzFPaZe&JwBborxbiz59cZPWW=Wwe0mF?Qt&AUpHlGYxIN$F z(+PY^!KYsfKBeH(v$wH5K8+T9O2H>xI{U#VTYO5vr&m8VuNCEU3O;4wQwlyE#-|i~ zx>4{c1)nCX_>_WAcK-#RQt+w#HS0d5;M1yK(5Do9TCU<#3Oi1u4*m`J6C(IyfbUP?+W`05 zXh#hAV=MF*4DkI1d>i2XadT5d-mZ1@Y2wE5GwCJRdR;Poz=Sj(X+a z8}PRee+}!i1{b%p#E%e-{_o|EV8D<4_+c#>@S{(-Uo;T^w-Nshv=`~L7Y6*;q0#>} z(B33B+DfT#I~{JVrBrwgg4fHXROIc}j)1LLr`{;>TctQjV$c=E(h8Djxd_9?Sbl zg|~6=mWs#k;<21kso!zbZz}aWmHJJEx9^4fwxRXltqx z{QD69%u^cuuPHAp`_XHZ`{>MH|Y?uVBQ zc$}1@@RdP)e4F@~LH(Vh{xYc7e(F_D8N|nw8?DDrc|B{(4BF9d?kBG_Wl+E8g!+}Y z&!BxiBlHI|@SzAFz2Qlml?Esg)0C5E(893>nTs_Cy)Bc zz>ANYRrZhHKHPs=OBwj>$o*#q^?EDyDyIzkH{-bf%rK3|<97x;ZBdCw8K&`gelITr z4_+5M$e?|`b0p`7-<$C}6JCz#Y@|$h*~I@<&xDr~@RA8H@4-tZyu5$ABxSNRTlgIqs033BY&VDDW@#X zFFBZec#PpWLKgk|&xHHaEY5F*&|b;g$;&Lx@1k&iS@2xHMk%i>cs?VvS6QaH<4^wF`<&;JFJ}Z=O7C!94hb;Qfw+Q3q^7_MFW|_vjdEO$YEPVP`{|G4y zUrr0YWKn+;X+N^yvgc0ayyWfVWfuMO1^;4u)3~a*xG4+&lea5zAq(CmcsEbk z@O0E*BW1%=mI_bV@boS`Wy8~Cc*=&S-SCtRPkR0jlbo{QNe55a@N|jxFdLqB3Or@Q z(^d_hvf=5Fz*9CnIlQ2pZ#FzR^8SLHvQ6VtDm-PwQ@_AdHatBd@RSWtTLqr7;b|>A zWy4d&&)_NBG~O%llnqZC;3*rPM(tDTEgPQZ!BaLoE!5yC8=hX<&-SKqU4f@;cq$cm z%7&-Y+)ro2(>Zv`hNt>$mhf~hJY~bvMtI7Gr(57D8=k7+DI1>FTmetn@T3!X%7&)^ zfu|fkZ{`0lm2daJmqj5A8{Qy@YO5ul|%VoPx~mR9QevAWdEk|!{=ha*JAj}VSMV-Tb1j~!HfUkg`9Gz4;}5b zoO1BuA6L--%`uHnF5o!ulBm*u&7nS~3iTl$$9$Os|5@;#!?@2!6O{6h*UN!_=^Xj^ zc62SrgPXB%^Gl~3(|8;2>*U}=2tMR6KD3GFBIYTF@udLf%jV#Rjo?Qvd>w(WT=-h5 z!dEVQ&4jO9_!?BIWIZo!dHZi@EN|61-^3OYsD4tm1`RRfv;Tns!`!97rq*Y!B;MPwZc~}d~H_Y zD;K_w5nprRYuSq&559{22l&c0jnBbXE_~&|S1x>gpu$%!eA&TQE_}tqSDx;TphxQM zq&(*HF0vmX*-{4Sk8OMS}? z>t?fL{@P2|NK)RM&XeoY*-y+HLAUTePTsK#cJ?316j(0vPO#lu%J+G-%s<|v#K}Cj z+h#5ZR^pJd=Fj^pkN5jKuCw%4-fP}V{Jw`f=U>eHiN~z3C$Ini`ZDht<~@E%J@LMJ z^?9XyFL&N~%)9*NPs&<(%)gxdsZt+#%=doc5h0&>sWXl%`_E%u_$O18eDlBZHe9g# zT^HQg#6DfNiSq7b|6dCG-*mz5(OPAzyk7Xy!58z5zC6d0`SPd7vfY*ocIlbyXB@wK zdly_YU-j3LQg3<8AF}r+|9O7@BaVFLpH1QS-}0EZdiN*nhxh&0r}261yn;>emdE_d zmHvFsy!jtbuw=f~gc!bOUQ<9f$78rMBebV;er^|7 zF&`iI<6}NPZpFuZeDpZU5+D2UF&`gu@i8AC&+z*d`S`dLAM^3?^B6nHNcn6L%Ez2f zBR)K<;e!z$Dg+-awx@iGDIX(T7OQY+qE5NrY8om|a+swDD z9aWMF@NFEv72w-VD!vuq+k1j<1^Cv@e0n(*;M?hYl+W__=F0+n^U(0E0N?rq-wN=p z58n##Elb6>0(_e+_*Q^#r}3=--&PkmkRLh=0`X` zd|QTZ1^8BuZw2^vFTNGv+bVo3z_&(xE5Nrey=g0Y zTLHdB<68l~)e629;9HsCTLHf9(eSMR-!^LaR)BAF*ID{j$b8!I8||e+c=Cd$Le9gM z^C+Z#e@p+gkomM3Pmm8!?a6%J#P7ylOFs4cAL_S|^D5`O3OWAq^EOi97W%UipDC{m z+mv>rkmDcUKtA3rSY#&^QopMnWqam3pPsA46-y%iyWbSTZ;a4CGCz7D+`j*N^KX^U zg>d@JoyuqPzY5{CtXq-_;dL-Xr9R-csm^>W<#QqY{s_P3sgSs_M2H)OaGDRNh4^;M zjh5wG2(KrF_sa^23)z2FSr6Y1;hUTai5Ip)yeK4II}oo6@$R|?Wj%R)d0B{Wb2WS` z#J4?DEVnNtUJnwl3-Rq@iAp~4TJH)+_!jL8S7C1iZH{0`90{8Xx5Vr6d)c0LBcFDo zka)h)#3^y}UgBmUeow&fLgMw{cP#Pl7~T~UFKvjIh4>wW--X0$J@LAb_T}Vu_Cx%< zf%qw>Li{cl{4OMZKFWPx5q_28R}p@t;a3rU`QTR(es$tk5q`DeR}p??;a3rU9l@_6 z{L(SMqX@s^@v8{GeDJ#nzxLo)5q>Qd{3^n)e#Qlh@askVD#EW%7qLBlT|3XR+~o8A zZ!e4R>*6ZwM=ipykt+Q#d5a?anyleh5q{0UuOj^VG0f7hBK!)`@T&;FVzcC$W z7q66UF@8Nr`&GQ_g53vREb;3NmG^Cn@ylzL;$1O*?fhKnmlWeyG3{3|ewAzZRg7Qd z8h#by*FqJ)it%d;?N>2=Wzb(VPsR8(9>0q5YXW{1T|4t|wTJ`t2p2_Aj9)bjjGIG=T#PYLHYiSsj0CHS=w zze?EeboOhWN+_R0lurp>oy05iR6_ZbQa*AjA-?EmDcqG%KI0y^{QX!7<+FnFDWQDc zqkPO$3FUL0a6eW8kNpCV=Ifh(E+PI*CH|C9FR|2%c`70PNW`BK>g71~QUb5%w%bW1 z@OTU!OQ;XM(4Vl_p87aSeOOB+#Je!!op~xD9&{5AN{DxdRqn@1@atOqDj_c1q7oNM z;IVHW=SRHjc%1VlK8zOPLkWKUK>JlfT=1aXl~W1v;R4V7O7QAdyec6s%poq6z}sSY zlT!)t;1S|M34TRs_*H^mFX2}Sy!XL-32}cvaleGPa4C-M@yi>(O5lAmyq6FcMtSo+ zabhEJq6EL@;a3T9WgT&)1l}+DSm&mMcH{)@h@49B=54%@QwibF=T1@{_l_+_IGM@dvI| z&fk3fQpU&a86PiY9Kx1yh*Ivm|MY;eRw?5Tn|Lp?l=eT5_P><-@2W|XR7$;i)+_6o z|5Zx;Zc~Z(=IhCyOR3LysZTkT!r$_TDF>cUOwzHWUVlTpms2S`?1P6=#(R(QdrNXE zC0^|zUX{Ybe0V6uw*>xga4GR*CGn(`@!w_9d=HP)1RhI?C)@j!{mA>1m!xFnyO8A3{_j3C0rHl)#KA_~6;@49CZ)_>!0qwj; zTS`35R*9!W=RrLEd_CpQ_|D2m#rsm8x19aEvR`@q;Vw(@xm9Q{%i!?{JeI-Z8RB6X zJU$1HW$;)6k7e+fcNILA!Q(9&JeI*@06dn#K7gU23tEQ7~c z@K^?qCp7vWW$<|R3V1Ao$0r3I%iuAO_Xf(~(b0t^JgydaEQ805@K^?qy96G~;PEAR zEQ7~Y@K^?qliubtJodq389esGV;MZQ3OtsN5mcgSN?RyzK_6af_^gB&nQwy(H9YIlf-P*K+FP0pdeB^*VuiEr*YF@L`_Hsb4?p zw;W#GcwQ!_a{TOW~~o-h60ZD&Xx9yj8%P$41H% z-hQXSTLru|UIA~FlxGIzDW^)#_W|-a+|R!p8*qf0gv7&z16-a=EpS?cpVe=TDXJy(66O=}(9Fv7}s_Za3eX_0N^u zPY!ZFX)RUa+che_RpP~QPs#z`UiGxxx)Pqsh4@to*R=xImE2!_!TnVwT-oy6zYksFephlo;5CB%!jm4JD)FS8 zxLk=((`cV7xgXfa{Xiu=9sP+Vp7?#B9A7?OC4TI~k4o+*9Y(TWyt@nU%u^-41>##J zetYqJPfnHiF+sx*dA;E-tKhL!;IRrGH^O5TJievEV--C1Y4BJDj|<_k3Lf`s@K^

D|=tKhK!9;@Kdcm+IG!Q&wf9;@K-!&uAG zs)EP49+U$-9#!G73LfJH9;@InLEy0p9{0dw6+HUDV--9e5qPYE$36`ntKe}iJXXQu zv+!62k1oX9DtPpP$0~S~;IRrG9pSMG9&driDtK&#$0~R{4Ubjuco80};89ZHu?il~ zz+)9WM!;hgJeI2PSOt&u@K^<+&O&)sGwu@j-3Y0g`}XC+ zbJU^r8FzV>@#bpEqnq-m<~rQDj%v>LFy~u+?1J6Ryk}ZXdAJept0~V!%Cnm5`s=HF zPkpcC{p;!z7wm4>A@B9_%WC|YCiqiLJ#QE4`SN@CKi81X~JcoL=ma3`e zqe4I5e7$PwdkOVjjc32YvueiMP6+b=)x?KT-sh{vzvu9;nsJ+(dH+&Q)x?MAh!55HZyUi9 z4{PwSn)u{=p5rs_bAyAt_LVQI@pqQsZ#CnG>v*4|8ow_6#1d}}cw5c*-3Ff5R5N~f zSQtO7W_;naUiYv6T<2Cp^n+7GWa@Ot0@t{Yz6iY)sVHSp>KuQl*$(@ik)XZf!7`IS_7}6 zXisY3HCTn$8hHJa2Cp^n>Hx1b@H!1%Yv9!&@LB_}+u^kaUIXE^243%j*BW>|M|)ZW zuX#U%*BW>oC-7PWuaCfM4ZI$K*BW@Oh1VK*-4Cxd@On|;wFX}I2)xR~)E<8Kn$wJD zn5WuE<^l75XD#y#7Vc#ESk_<;?+@25&Kj&{oX$Me{vm6y*^&IES%c$8D*3g{uWJ^5 z@5^F4=G&&dr+8zYZ~nP@<@Xw< z{EhXUPU^}Z752mY-|ueV`_H%akA2^AKefycFRy3&liT`(URBBG_Y(ednEbD{^*g`G zex_s%F1(-mWWLWV#*xia?X;}H)u#Gp{^F8*Su$U6{S(T5Ynh+D@=wZVi|v`8J@$T% zlfmzh{Kc|fYx%u~`iHG=Z@#}P{alBi@2R{`Ux%O1;Ab6v-hrQW`00nAb@&;p;%6Ox zPQ%YS{G22BS%;tL8h+N{XVVqslzOSd&uaXv!_OT2ti#V&@v{y;`|-05KR0Xm zS%;sG;Ab6vp1(=qR<5@?{9J&ab@;hV@Usp-X_}=v{A|R}I{X}1uWVn3pQG`!4nI5b zvkpIZYxr4*pGyQk>+sVLKkM)_8b9msGehvR4nH5k&pP}J$hXd89e%FH&pP~U#?LzZ zoFMpFho2#ie2<^ARQ#;N&xBgG$Ip=}e%9e<#J|bM&shAd!_P^k`ozy}6+i3nb0>b* z;pYbYti#VGf}eHxxkbazI{Ylg&pP}(t>R}Leop!y@Ux!!J4OA~bA5ZczIx7M8RtFa-5|-58!L@8pJx>$rxgJjc0H<=Q$4?*ptf&58 zr2gyS_a^wQr{3mMZ}s?j=^x}%|9_?a>*4nc__dbmssGc55hH&6A%pC!bfdiX7a zUpdv|=U)8$rBXfd$FEt5FYhU=PpUqP}@o*jSupXcF zDn8c}KezIHsve))ReYBBSC7x-_*_q1JfIR6>xqXM#KU_0ogw%;v_0{#n*K^XK2K2b zxdERXI&7r|d|HQ34fr$*pBnJVh))gp^f5j);M2&d`66`vaL zX#+kr;L~`)rv`lb@)z`}0iP}j?YUf^4fqs{PYw9gtm0DxK280K^TVf636v{7HRDqQ zJ{?r?sR5tXsrb}@Pj~-S6fQfKPrZJ~iOeLVRk#r(zYK8t`e9JN1Z9IT}7S z;L~_~YQU!rDn2#fQ#(F2;8VAXPYw7Ki%$*sG)KiJxtx|c=uB4nBaQf! z$h<5$HNx8#c(axo;q3stnWsj4I*L!$QX@X?pnZ~4BlGDKk6Om5p?rAjhPOt3k6=;- zOXk-H4RHQ=bszV6jqo;&`@2Sdk6=zVOMIH}Jm16H<^etvFP;$kiH-Q=d%tpg`8e{j zk@|8Rt;F|6>UV(pZ6t1V5jPs~Y1A9a_Kn1gD2@JOBk>}gc+p6`E}>oqXvwBk|6Yc-M#@ zNAaVPxVc;CKQEek5x7K|6EyKI_-rNIP?Mf%SalOs?*) zCj8ulpH29gm!zC`6Mni1em3Ffat%M5@bkQipH2ArwuYZg`1z^eXA^$9Y53WMpE>y1 zgrC*;*@T}t_}PS?PvK`1elEe!Cj4BDpH28#g`Z9MX?KRt_&HPXvk5;Z&sO|0FU2PO zY`sC@y9qz{;%5_n=HX`(eufHuHsNQfhM!IN>4Tq5_&E(foAC4Wcwzte=|MZ*gr8o* z{aq7&{supr@bfqL*@T~y@Usa&yYaILKX>D26MnvjpH28#j-O5Vxl6^*Cj4|0{A|L{ zgY-w5@N)%zHsR+c!OtfAtQ>})P5620yd*W@=T7`=!p~EJpH29A3w}1?=YWczP5Ak4 z3ESi6G!;La@N*)5HsR-T{A|L{0s3oA_<2#q&nEmV#?L1F48zYR{G2cN*@U0f8h$q6 z=K&2roAC3}|A3#(T>oz2`kSf$J=DLPnz{Z>T>s@#GuQ3FbvMKPzXa~(?V7p%!BynL z&mZBZnK-c6pu~Y@u3t|-SWeAwA1w4|o4I~3`oYccb2IV18UDX{M&Y@c`l%*x+b z;emN-hMzyePct6J&Qz{bUauMc-xT_@^7rP;X8dhd={L#iH4{H45qcoXexGx771 z5I>vocdsx$(@gwya}vq{e<$E?Gx76zmH64r?=3Z_aJ|IO`NYrW!}(KojAn_~=ehrA z=J%E+g_DoR?SjY6#LwkIf4Z5txrw;hOq^TcI6`VBer_OsHsh_sBKAZ4j3<6tOU=a3 z-_BxtelKYWzmImMshRj`6ym46eKYZ6jY|J_D4+PTOeKEE`7QXfPVlD%e_j^+X~CZZ z!|H7!5;(uwBSz~ z{A(}F)A;7<$woNyW;wcyVI{As}-J^r-dkBf#sE%>vW|8v}eKhv(lpBDV- zQSqk*e->)^(}F)6RQzdW{PwO~8>toEJ`&pdR_>oS+@{1Q*)w_B%Kh`+E_Xlocs}QBy&UB0l9z4pd|2SQ4W6HX=Qemg zL;KVQ&u9K%o#!@qo&?Wr@Z7D!a~nMGgy%MR-k`#B8$8>n@Z1K^6X3ZGo-a-2df|Dx z2G4Eqe4O^V4W11G&u#GhEAGd8G=^ZSXt|p4&{%D~G{z8$6#+wqB2I@LUbgZSXt~p4;Ggp1^Y(Jnx0)Hh5kl z@Z1K^(eT^`&tdS~&iO7>IbZqu+Bx6V!uhtd{S7MPo$Z|O0nWFb^8BgCMrwzv9PT6K z)Xw=1gelk4PXDH*O!23k^Nr>HyPfuJ742I)?dCM@C)@EO5I@>E-(u7G(!Ra*0Qt0| z2Wdy!;j@Hz+irTkq2gaVzJ7qO?WX4&_&40t4o7YRN0-~K9X|gd@Y#+JTkxSBzFXnD z9ba!3?kC&v;Q&6g!<8Pc+UY;}msxt&PCOVS9<VU6F8hmxY*G_@2 z4){9zqGdUBz}E)&>VU6UeC>cQKY_0f_!^B5a_WGu2!XE-_!VPk|-;obrgYeY>UwZiJFpc-C@YMldBl9>P(|G?C@YMld=l`YnA)mLk zWe0q%7x?OcujTO70biN$)d6421-?4q>pXmQz*jkZb-0UEq)yZG2cCCz!vCBsmfSyX=l;<=b;7@4qT*L4_rEiEUe*bJYrbL$|JB_} zzWMgnK6k=brRliT_hIV06aU{7+Lumv3$Z01&rkE8#!RL>}S2=aF|M~3STIy#1GuVGO=k++} zWuCg>XSB{<>V~%^@MfO6;U^4!y7AyIc+gFG9%`|S6W#FBtin$><6GC>&35oJ34Xfa z^B?fp4S#Fkubc6$j~L(ThM#rRleN?hKRe;48y=3qLpMHj2<>S%{G6wK?}pF6(7%vV zH{*4u81L?epV2D(bTeLeP#Ev;hKEx^``8V~3+y=_{P^+y-qogVc*x;>vu-$h49>bu za-Z3(>fz}tEi-g@9|fOypdZ_`wG>w&jo zcw&yw&lP@YVxw3pIG_ zfw$xE)&pz}o?M>w&jZ0&hL=win)d;4M~#w;p&qFYwj_Z@(h`$f*b3 zZnEQe@ODY)Kgs#SUG~76AH4Oz+je;CF^#VZy!F7_GoJY53cU5e z+iZC2fw$!-94KI5|#<1HJs7#75q? z?`8hOiE~PMn{RiepL>}X^VLCRt6t_i^c_|9-^=ea*u~pQz08;BXI@h;^Mz-OQR26} zWiRt3l7;!lz0B9SzsjSWKJ!gR zwUf_$zXuNs`OMGjXMSEU^Gy~-lh1rV|IgSz^Mx0D!S`LACpVvE$vi#XPc~97^UdAg zA)nuisARrDFY}MT-^};SiyT?P_RKf0c#rRyr!sRYpKt0s8O?LuUgkwcgz%a9KxsVx z@6|IO?@pG?54r9;C0_M1pK#Y!zF)xat~IkC=7+5HX34ztCv(_7x8;V@!n|+!I;|~x znJ>6`q;kFTdcDloX?&UEFh6}}BKv2)=ts}Vxt7ab=8M)WRKD+JzRq=fh5BT^_S^hE zK<{gvMYG|_JoPeP`{^6WXMWEPqj0^<*SYz6u8ZFbxr_M(=BbzYI;q|Y7rlSIVE5Y$ z^R?KoZS2!`@jhKI^L@Swu*{dg>1BS;J2%Lm{`<=@_?Hl)_%{as4p3fW@NXpJoWE2W zgMWS>D|;J*f8z!J#^9eH{*A%Ez4$i<|DI6sZw&r<3I2`2Kj-r!q%ruHfq!H0Z!i9h z!N2Qg+DT*Z&r$Gi4F1i>zcKiCvWwXNP|qm$4oE%fLT7{0qlFJN(;s4gT5TU%85ZcKA07|LpLu z82{|>&$>}*FT=m=@NcZ(pB?^nf2{CghkqCF&kp|zk0{^E+p)txFZ{E^zh=QdJN%2m zKRf(uR`Jgc|HcUZ+2P+j760t;&-N|0gMVZ1l7H1Wx5GdEpWvSc{#L-BsicAbRH6Sd zwVbK%HSo6^{xr6fxkKMr(r(p*~#SiYlpuY+Vi(Jtt1V;zKgFK z;>R{2e#qO=;OnVll!x&i>N6_a$JcLnseBLbkHfnLU!(EWFlmTKw!|Y1+<%{LSju|% z`>bxDq+!14n+bfz*N~;`2OiktfrfZAm3X9qdkY6;eGTz<_QMvE2HyV(?;7IL9^#Uz zq#-UXB`#^;{#F(4HN=_C#2F3o)P{JfA%0kGXTR_ubv)ZAEcK2+2dP% zKYX*tx8do^aT^^6zIB`N%^u(O2)^0l+h%;T$G739jpM4(_VG=FZ}#|hM8!9Id|Qog z_V_mRK5~4Uhi~@ywqu=@WRGuM_-2o9)hfQ(^TRiLd^`S_vK{#^dweUwH+y_rf^YWtHUZ!4@omd^wvTW2 z_-2o9bMeg{-%i}ma`={mZ}#~15x&{u+went#m6g?bHqxdX>D>M3P?x!i&J^`L%DB?o-9I%z36P``VrUpYC@ z-g6$ckR0%JC%!sRzYD2f2kIk_`fz~5TZQus2Yg+hN;&mxOFcWl-z}N))+EURU-R(Q zf%h+59zsq%FC?xxz+G|x<@h=iUmd(wopP8&j(4ee=Rm!OalYn2{chV%4u4O;p98+V zcZG5|wRhk%zRtW(>3oDW>x z9`XJ@Gx6Smc;H4naA30SKi;Muh#yk0ar|^39;BIx2M)x8k;DU2$pQa9!aoP%f<#;} zl^i*qTls${eI-ZusaJ{HjvP-i$Kwb;weaIeea`PR-ku};%z__B{7JwcM~=^v<8y?A zMR4E2n7*bxs_jyGMaj(BijDBGd^SBEI=??`;w z(U16KdducZ9EndS7*7}`N8+a!@zas`bd31q2tS?h<48OmOFVTXe$FF)%E^)Vv^+)G zj^hhGRyIfEQjUxGw2t`Xhz~1Od~hT_jU+xf5})P~pB(Wa2Os)Mj>JnH@zRm_G*XC9 zhU+;JpN2oI!~w(dKkKs-@1I#%Jy3FDe|i|-IKkofaOebgK5*v*uTSEi6Z^A?c~nvs^mkpLfaO)(LK%;BD1=l+zBAX$L3TD~Q0w0)=sb6Y=4w5Fea~e`|&7A5O%#`9go^M7+9}cx5U%5%=CF?#aoC_%Mg@ixYAD zh7RhV_*X&vlamwiLE?QCPV@)M=ntIeKQ_@HIuQq765^l}@u4S)?a(j$OuyhnJp5oB z{1N{viT_T-hZV#JC*J>)#QT4o=pSC9e{iBdX{0}KB0l&KADrkHmeDUb(H|7jA2`u( zETZ3VqCa?%{=kWTU=;m;6a7Zl{Z^7QKDyzfGd>>1M`wIIfRE1jxEvpy@o^kJI^*LZ ze00XgXCAYVobj<3AD!{>-}vZ^kJIqc86W?)h~@C{`Yq)6_&h#3<737)%JI<}AD!{B z86TbTadbF2KF-8PXMCKGkIwjbX*TQQZKjE|e}(HS3m%=qYxkF)X786QvoqyIiS zsA%10S97G39so=!}nzf{)Jlct!Bh86TqrAD!`Wir}L&KCUq1 zqcc9{2|haG;{<$k#>aha)IUD%7kqTa$Fcb6jE_F}=!}nF;G;7>ju(7%#>e7Bwu6s{ z@zEI{d+^a2A2;KpGd}9@(HS4>@zEI{J@C;PA3wlHXM9|OkIwiw8y}tVF#;c*@$o@? zbjC*)e00Xg8Tjaoj~DULh4b-OI3IVxgSQ@%qe$Pm3*O$%|ATO0e-^SorjiTehcoxE z9A4$&l?&Ig=kt7(3myhVD89Jh?_OaZfywO=KZX%MT!;$`pR$l#xE^84^*XSfA_7!-eO(T=4xKe0SmeyERkUFL^&)h!01I4=$X4 z&*6HP3)g$jW^g==i+T-HmgjnP=8(SoNf-S6?Pj)1 zJlT5<@x%pR8u7)&`jwk{ep2?|h4FJ9=btWI|45H#|KQ~WyvWG~p6_*(KN;sP@NBWv z_>g71~;>z)~b3Crp z|EJXdpOsu`->tN-oLu2yIXsw3uC)6;+T9h74{_h!6|U=f&cT)X9mxBnOeI%*vsJnN zB`@y^-_zjRFuCH}V3q&BA}{BP4}W{YLUM)gdiXX>uJ|z)KV0EE1HN7HE%R;amG-~j zz;WQqVSI6=Jy+A7uC!-6?db~dtKr=h4raiCEA6?G_H@PP0DN|Z?|Ja;3SV2{tFPpW zAKm!jiXT_-!^Rf`n+=zc~6aVDo#yH`aaQ*4(_w3IwGx5xg{h7@9kQ?o7HQ2bGuWpC>U9w5| zJ;pmeD(6FPwD-ZPfsz~Jo#~tpxxu>x?{2i$e%?1JCpX$>GWWmSsE08^JaePHchcT& z#Ir%A%Ko@9?x|$l<3@Y?(B5wF{ErXitx1v_yjQ}z8|`aF`?}%NI(%}2YrD6F5vVy3xK_D)G#X{&1#neZ!6ZIY79M=0~!)OXg*SJ2J1y|$4sS!@%^ltz7kG1rw`1_;4sVw;l>Kpsw-|VHhqoE<<_>S? z1>W4@Ee77);cWuExx-s7yt%{Mbb&W_c#DBIcX*p2@a7J0NBhB>JG`~Sn>)N|Rd{oU zx0eOp+~Mt6fj4(}TO#o04sXZc%^luW!J9k0jnR?Ao8Am>?(nw!{r>aj4sWMb`cJu? z-Qles-aHu3=9#&ldV~6@G_tdztVYfd}K+iDNik`1dFNd(f`Aw5yyvXy2tm`+C6R;kVc>?er4& zdHS0?;D0^*d(iGaLc8~OeR(+##_a{%5A(ny5BgaT;=zynzd;YqFD`R_;Q{{+@bAI6 z{U^rl9{9BizdSe(O%?8!dJs?kK|Jx`Jh;cqbzu+AgJZZ~^m~&Bek~OI@*uu!vsT*8 zgLu;QmJ(0o-|>XMb!PbUgue~&=Lvu3IG^`~zZ@0*JmGH^{CUFPJoxj3zyHFYC;Ux= zKTr6Z1b?3Jw;KLD;m;cWJmGJU8U8%s?~n?Ap78e~{CUFPR`~OTzmYsY;t7A};Lj8O z_6huX!rvhH^Mt?U@aGACK>~lC@Hb1~&lCR6HnLs#i@65=JmK#E{CUD(#vkC%6aK>B z&lCO<;Lj8OCeR;w!r#~M=LvuF;Lj8OhJHg1e{u zKfs?S{5c8ydBWe=Aqsz<@K>_lm_M0AFV2VO|7|T0g@N% zuYQx9_UokmylDSvD);BTsQ1=CQ14!}Ul8plC$Hl?Z^-j&UU2{0Zuwi|+>7In;`qJb zx=c8Kl$Z44{M}`$GXC(Q{U+0Xa`J-Thv3(X^ZApU&l@JMw(!lKJWt^Tx39yk7uNyi ztyJQL7w7-`r}915Eq>ZWj)!aT(2I5myHRNe!|nF<*$e+y;lE+>!jJY(21;J|@k6{} zDfPwmkq3A_&kO$-;=ifnh5siRUl=AYuBUv-b1q)^AEx5J7uQq%ei!BNoetk##Fr@I zix=@=R0sP-{HY}Vc;QVZaobe#;(FiweJs!Qnv=P#PaGIU9Pq+B8{ztcH+(w5r#F0V zfKPAu)WN4Wd_J^dfaDFIcda9b&m8#lhR>T+`1FR)J%4~tZ}`*;e0sxY6nuKa=X900 zEFZTwd=|r}H+*`i@aYYoiv&Ks;nP~+(;Ge)!lyTUT8=a3(;GgQz^6BS&V^5J_`Fx( z(;Gg2fKPAu+^~@xK11Ns8$J&YHEw@-zx&F);WI>qPjC3#xnrQ@4WHvx;0y5z2Wl`{gpR-z6YP)@VU(l zpWg5}1wOsuGbM-h;qy}Oalz*jfls-fe0X0$_1(tpD*Y;A_ zxPg)n^F7jT?SK9IFrVOZ7wa>xqRHNPJnX}K0*AdyT$1<8F!#xQr(t^U{xCmn%qz<8 z`!IiQ>QU3{`!K(_w4U{Nf5Jy;10)~jp_B;kPw-)WvELHo^?jJ{v0KCOGCyoXuu?xh zyf5M`pM99maWtNs`6+?HZ1?ZoaqHXp9p*AT_`%-fiw6~1TwTshRn z8|?@4t!+w_av$DHaL1jj&-|60SNI*~TOV!Y_-=pD`a$k@%E^cMCvVR-ZpW+T%sZ+1 zS(1F1Kb*w;bf4j;MieFqzr*~K4{zps<`0iJNPc4b`na!^?fEdT&iZlI=lu~jybr}N z`7odE%ah8tKD_T@Zw&3k{Fp5ntn7;%78I_~PeF z`00zErTFQKpRL#6r!Riq{w3?5HEy@xd2AHwk|F;^+96jQ#Y*&qn<8#m{bcrG9+zb6AL_y~@v{Iw zeetv1jGw;v>4TrX_&L0Y96vV+e){5P3V!E{SRX%^s`%-PpUr}wzW8|tKYj7@2!8tF z=VHN6U;I3dpT79n!+lv_{2VO!>5HE>X8iQU&piC}#m@sOe){5P=U?cj7C#T)qwpvD zsm0H7{FIXxKeyoLwI(fo?x6oPm9+R--VZ;u_&Mn_D@lu=tMF6Hyv~C^T1Z;_JogRX zKgI8yR(?mzd`TPLbEd^l4Ss6zb6~m>|Fq1PT<=Lae%`)JS6YJyW&pe+XCoO)yX2wq~em;kvTKrs$pIZD}A^54q&wGdSJ$^2v z|1?Zm{Iq?F_3_h3#ZN7Mh6;Xa@pDKl?TD8%@lwlt&NAVBFIxP}!cQ%JcHyU%dAB|9 zaa?$MlJ}Ts@iWbgpIZFv#7`}LzNO-)7C-a;LO=cR^TH&ho#guR!_RjFKmG9Y$nWse z4?n*#S`00nAqwv!YKO^ze4?iae ze){3(LHzW?&s{H@_R|kPSHrm~{5*o6e)yS$ zpMLmR*t_5OnWW;UAAXJ!`cFUnY!v+T!%qo6{qVC(@Y4@Jr<(E84?q3!(+@vA@Y4@J zZ3I94@Uw^h(+@wVtN7`MpUa+Ref&&Q@zW1K7Ycs*;pbHR^uy0l`00nAn*~4p@Us;^ z{qS=We){2O7k>KT=WhJ;!%rVGe){3(Ui|dK&q5VH{qQsBPxRBD_WOqRlaoLDzk>aj zlRx$UU!M1plRx`^yv9QEzsPu_ot*mr_jyV3r`_J6-Tc}A!R)`B{HgyNsed{7v;Vf) z!e{Egp69&`lRy4n#(#h6KmU|rsqtrj;zySdKjbC+@y7>$`bz%1&!F*sevkN(B*YJY z{4ZV3_q@-bbs;(Na>*n@o5x3`Qy(vUvkKP*3=?34i>FAMd?l96$W=XAS=JH~AAk z<`6#&lRy5968teN@9$@S;)f0WyFc%Du=(EjcT8=^A5R(?Z~7BIpAzDyspb3n-k-QR zfVk;T{A*cBPX9A}vQkg}#LriVpZ>&;9^!{TelNstIr$Smzv*3%_z_3^@aH`h9gcj) z??U|cCk}qVd4xalqmlUGPdvRNTJVYZnacTxocxI!mw67-pZNEM5dY+I!#sff@nC-f zsLy58X8`rNOsLNQ>MM!*3SfVN*dN0b0H;H1l;brlzt+zI)ZeuG_&wTZ)gNe|0P1_h zM;1~5?K58J*W~>PpuSzG?*Q6oHtiEYJQy=xIlchudnffBK>J*#eFA8gy|jyA3ZTA^ zQr`iz%N6=L!xTWg^C#W~&`ym)I|UFAl86TZ#Jk63c(8?d5I{WGOFRf5 z-c2Ol1;D`tI0zt~?hxW>0P$`-@h*V&9yFGmI9V?AZ$|q=yxk|n+W_KM7V#^9`1UjL zEr57ClK#$A3LxHoJxqxk@^S&hDI4Nc0P(Gk_!dCieQ%~*Dp9}hG$e8TH-Px|8SyQE zxa&mR4Ir*n64wHVU(<EO0*G_5!gYoK;#U{(D}ea+ z0P!t=IJsJglL5r9ImE93;+x0vzT-szK2LI1X;0!;56^7|5Z~@riEr}$%6R~B_Jj~; z1YQhp&jJ!zRE!%foM%J=?|OZ}C_Mmz)gByCLa_k6PlR4v$~LV;#IMhF3Z1@Y5APb@+L^C*|;J z1Fxo%4qk(V@v07gAHiQa>EN{zUUj4Sy;s>DUPt4#4u2~If92)nT!+8OjF)u8zisq4 zI(Y4ZSKTd)54nG(!(VsCOFH7;7Gb=q<9$w}c%PFFZ@Xr(KJo9U%6L_Ww~q?rB^~`o z8vTckIJlEIXqa@wzwyLB9sS1&`VSrbiHmT(M@Re{V`jXnBmT`H{^=fn(E4fa@9OaS zX~An9@y|*a?-~C7AN?GNpHBB1w__lFJ}CGZh@Y#j!OuYa?73;66o{YhDt-py=X(4M z#LpZ1;b$OzzSU+W1>)yr!OuYa{F@m+1M%}G{0ziTz2IjcevbW^-^b74JWp;a1>$D} ze90*gKlkHjAby?@{0zj;Q@#6#pKs%5Ab$F*_!)?wv+y$zKc~ZcAb#47W_kSV!OuYa z+=-uo_<7V<_+9+W#Lqzd^uo_T{PYq048+eu{0zj;%k-at_&LgqpMm&UDEJwOpALeb zf%qAPpMm%}6F&p-)1h~N@zckQpMm(f2|okz^9X(h;^!vnFAzT)@iP!VZSgY@KPv@4 z1M$-aKLhdeM;q41&&w))2IA*n!OuYaY{kz&{A^V5GY~&*@iP!VZSXS?Kj#a62I8k= z#?L_fT!o*3_&E_j1MzdB;AbFyp8ONfCxd8*T-qUs>j;^=FH=rIjF+Y|UNV(}xPB5e zZ=e)J`@c+l4ubDPAMu&-MAtF4$Muu*9(?Ay%Lw7ROAyy(rv6BI@BKkL%HjJgd0oOEV*v? z8vDuho_ZVhkMaG^Gwd(d6@F+|>d)}^g5diodZSelVHygJ5{y0q?=^t`&F>hWCYLcn^m674RMm z@3Y`N7~V7X@EP9E!+S8ikKlS?FuY$B-bWh@@5A6dxc7b=ya&Vkuuw}WxcB}V_rHVT zebZ}JQZT$<;<=V!c)w*-|9O>ii_yuSzU z!SFs4-h<)&G|%}2!@G+a-h<)2M}_xbc(?2a@4@iC+YIl)@IFf5Js95i!+S8i@9zii z!SG%R@4>zImsNNVhIdzZ4~BPFcn^m6O=frxhIfB~_h5KmA@Cjy?}Onzg#LXQ*Ed4o zeVM>}2)uj1dkFpe`@(s&+&-p?nB^xuL|#1w+HVD@E!v1 zh43E2xbhzUU$kKgf%g#t?;-FWAn+an?=J8j3hy@X9t!U^0`H;lo(Aus@P15%_t4(^ zK?3ig@P6e3mWTJL@E+QGKM3AK;r(5B4~6%4;XM@IyWl+(-cJ(WL*d;S-b3Mi4!noL z`!<31P6yBe^2Hr#A{iq7>q3~V^@1gKsPCJLf`@Wgf8@%7a^~z9q zA63ft@SX(kq40h{;5`)H7Ye+G!uuGWPcTfO@V*V+L*YFM-b3O27vg&;ymtz`hr;`0 zGrWhw`z96ML*ac*KX?y?cS(i!Pgfh4(XN zcn^j58h8(d_aJx=h4-Z@yobX3Oo8`Mc+U}d4~6&n@Ghq?cpt!hz_8x?AKd>8gZI`K z$T{EbQaRrZgZCbR_b_!{B`}@jVRwcRfMJqq5#;e9f^hr@f~bCMJe?@{m`4(|y9@8R$sc#Cm88P=Pr z&*AWX1H6a#UdK1XdwB2leRvOt_wi_i%XE3A~5H`^g;B{)WSQkP7eN@V?ay@8R&?2=C$W zzCwlfaCo5_kRrIwI*aFCBdCvsD)kXTd$bDe5kY;7qCO(<#T8#7xW4Bq zJa-vEedJLea*DviMZ$HX2mx$i`9l~=u5%@LhaV37q zzZWr^^?C1A1b)1V9}&c#O5#rhUZ&t>1o6Xi7&+H#)9WZF{x}eSBJg|;o=4#GWPI*# zionMdeEhRg1fG}Sc?9twjCc@%*SqmLqW8WC&mlzMlN~-q^xmJ~dGQFm9;M>7ygw0K z*Ip~!PmaL%7wnDsm6w+(9|`}h@E-~Pw+a6*M&6#BN5X%S z8U7>Te+>Lb!vD+g9|`{+D*Q*nf0n?1B>eXb9VkV@|3dhWg#U#C|B>*&Uf@3x{vQ(f zkA(l-@E-~PU++*jkA(k|v)Lc`AHw;KVTy$R>F^&3|Al7wkA(lF@E-~P+u%PE{y&BP zNcjI0{v+Yv1^y%9f1V2ek-hg51pXu8zxy#`{v+Xk2>eIFzb*Vn!v7ZdkA(jQ_>YAD z`YADM);5Hz3%}3 zk?2YPfDBcHh`g>zO z40*ZM&r!2CPUtMLlA>f}YMuim@Xt^V&>K8pF; zi;f$|+o*?m4}&ldF3MtB#oM1N_1)JIMDe~EO%L_M`(hS#8*g7;KZ^IoMBFw&iemnp z_eFBA@Xc?0PtJQ_ydJWYq8xZX*BWx>FaKysJ7{_DdJpBT$)Cd#KsB!Jii{r`!b#!KsoQ(`En0A?{_%#i-i<5iuXJ{Ncj%l zOEQJ^wqCUQnCGjbc)x>NDdl4}PB`gJd20CPnL5^6Xmi8b-u>To(dy1yS&sMXgu9V5 zUwYU)mgD^qpGWeU_f9;JPtLsa_-68W-nTPnfE0}{E#v#&4stz5lK zFHt}E;$g;@Xna|NFVXn26knq88NY!k1`#$;Fpwd|8Dr(fD#6U!w8lEWSkJ%cuAfjW5sQOEkV*!Ix-!v0>a1jW5^v zP>wIYfvkrwBYKYyUl!p@G`_UsOEkXx_#DgOOD(=cCGSrFo!k5kX z5`!-t^e-{^vIk#c@I`|!G5E3>Ut;hj9baN@-Z)`F0-y2av=z(Y%Pf3}!52$>iNTkf zJ|f4LYJ7>omub@}XWsUbJ!}tezQCIpeA!n{IlfH7ml%B6%zUR9do4>r7GKU?&;H?ykBTp`_|p8p$d_1r$$G4BUt;m){CB3uOR^ua_!4%o|GvcHi~AKx zip7^kGrq*)%S3#M#g`C#iN%-k_!5gR=bBg_Un<|=GrsKaBFC2*_!5gRL+D>(@g)sk zV)11-zQp275B*ClzWhM{5{oZg_!5gRd+;R|UykETEWSkHODw*W;7csNXz?W$U+%}3 zSbW)!FR}Qt3}0gLWjwyb;>#3#iN%*K_!5gRH{nYxzWCrvEWXs^ODw*WmQju`&*4if zzFhQYJ$yO*IXS*`;!7;Pl;TS)zPyPqvG|gMFR}PC%$*!x2H{IAzGUG`9G(orlQ_!X zc9o{8a{FIv9)~9xf+umj@4;!lY0lz!AIaD! zOqXBl@BdZH$KlJNe)tlHFKvgF`tECg;_zkpS=td_4t;OB9^&xj0=~rI%SZSUhcAcl zB@SP<<4YXA=$!bBCokej9G+z0NgTYluOBGI;mZkpiNlwRhbhOGdGs%F__7pV;_&4H zzQo~6Fuug$%N2Zy z`grp#-o)Wc9=^ok+hh0h8IQ8(kmJv(W#srW>FEJdJia`LFY)+t8eih^6?}=umxX40iN_b4{|9`D$Cpk0@FgB!PXDw2zQp6ptLJD( zd^vB%mw0^X!k2h_>BN_KeA$98@%WO3FY)+t(3#Kp;({;n_|k|k@%XYAU*hp)H@?K< z%MpBu$Cs)25|1yB;7dHdjF`ap_>zY&@%S=o80Gj9hA;8>vJ7A1@#Rf?iN}|t_!5sV zrT7w$FE=fx9AA#(OFX_5;!8ZfB*A+;z9ivGJife$FY)+t@*%#*7au*}<4Y{Q#N&%& z7~kVdE55|zOKk7&;mh!m94Ed6;7dHd491sueA$67@%XX{U*hrQS$s)gd(~_&f%bT~ z%XE7ru)XDMFM)cs=w`hMyvKPlpZVSL-tFMsfpL7MUSp}(zET49yPf)#Qv&|&5d2G^ zexIR!6X2;Bo)YkJlHg+k^?Minx2cpseb!K)eWe6=8qRnn0U!4WJ|p&-`3GTnMw(G){JL;r9}4sV4;PS z2=^breIhV^0sZ%@t>i7%^(FNyd+ z9sd*IxC@Tult_FTM0`oa|5E%;_`3#w6XE{wKoyr*NML->LAOh`%oQYnT#=|J!{i$J?cNYnYO_-nyw+X~!gf&q;_ENnCer z`^<1H<#Q7Cx|w=S;=1gzt;+ZEUrE%b7xkG0FH2Q;NuqySLjPteCDHGmXf|Cgub0I6 zygTRfhAD~jxN@G$P2zn1d!C~;m6AA5%!*galN`BT|A}#ZCehxjg?OI?&yUhyCUHHr z-~u`A?L~Vh(Jm)xmn3+88=jN6o_f0_pJ|Vyv_}%xwU?h{zwmm{H{^^DuRL!lB@urb z7gLUBb$FIUdq>dTNsJFS*iw$4{}ud9V*HT7_(4udj7KsVkC;kHj7M%}JYtxVxW2G? zhH*V7aXrV2=X_12B=~KI-z2VI&!`+ICBsvZz*90jMF~75!&8_Uo|56I6rPgdDM5v& zWOz!0r(}5YQ{gEYo@SciDH)!u7{~RMlHqBw3Qx)Kv<04$;c4)#3NMD+ONJ+3fv046 zD&23)Q!+fw5qL_5r!nx93{PS3lnhU4@RSTsAHq{IJRO9mWO$khPs#A~8$2b$lRrEq z!_!1~N`|N7@RSTs$?%j6PlwLJeg01Sq`>h; z6^>J=pDPazlv0TM>xla)@UleUMc!fxaef--6NV{;`thQEQfQy|&9qMn?en}!`^ei% z!M6yZUrd1)H+V^*|Edn>_~2*;=dCGtHv{kFlmb7e;U@*(G6dgJ7=OI5TwY3&Qs`f2 zjAA=@Hv#Wb7=QdUM<}O%o#D!M@FRwK1%@ewe${#g<#?yTJHwPhzq*5dHHGn4IpePs z;?D&k{unNA>T@c5jfSsO_*$dFS1NoRfUi{eYK5;<_<97sQsJw!AAF_4m&4b_?I`b0 zDttx3S1NohhObokS_faL@b$GyyiA2JH~310uahc#rNUR`w1HA8eC5JdDttYx!dEJM zy{N)hDts-5uT=PIHp5pcd}++^l?q>L1in(?>jn5qg|BZSI6n9q1z)M~wOHUQ6~3&v z9+C=QJ_28<@HJZCD;2&*z*j1K&4I5}_?q~%T&t3l3SU`nYzMxMer(KFDtw(~+?EPo zQ7U|;!q*Y_N`&279LUQV=4tTxh0q_RQEz(Udm7{0U5szj z;PX|XKT4zi7U5?aoOY!c|6Uq=%&-|KrNNgk@hgq`*>F&)pEURzaB+Z?#&~ui_o>q0 z?DqGRy-I_>c=$_$lV%}KD@*);|4M_uAzX(yOlj~jSKuQJ{|4b-8sqbSFg{O%zq{ct z4L&a9v%mN^693W|pFhWS`!x8g`-~h;4u3{D=SNSwb3B|Mc}W&h8eDiWeoW*1ZblWK z;cqJZr7?alXZ)VV`BqVsvR7&NcLM+9lm-_gxew4+O5^-4>~Z7c>uWv3@-*VfJ~Qzo z9e!^89sH!j&lZ87bodE_pLF;+4nOJe<1g@&4nHpNlMX)@RrpDVpM$doO6l-32Y%Aw zXT%|epLF;+N&HHOp9kP49ex%F{G`KACH$nrPg+0tNr#{BRpM7V{7e-1Nr#__@RJTd zjqsBWKZD^X9ezd-U((^HXF8wZXAS(M!_PMONr#^;@RJTdTKGwaA74ugDII>&;3pk^ zq)I-+&yDbt4nJGqCmnuVqLfxlhaam)6@Jp;XJ$Y6Nr#`3KY^cg_&Fo+lL0SUGrVNb zexIp~UotqKT*C7h8Px9}>NkV-3l-WggY!x2OBPZF^|~1^GvK5cPBJ*3PdTn|kU_nU z6ymp1Q~&c<2KC!a{bq38$h_?4H;pneY%w=>}PZh>F9T+TD# zXP>}N2E1PSRM}5?Ia7HCJU>P}G)x)PxAiH0AD-WV=M1iw{^us+IFP~hiX!f3X5if( zyvyKxK7{*^8SuJi2szg)?%;Wu47`iMy9}-~?cqL02D~mI-etgD_uu&p&#%IB2G^$+ z%~Sauc>X6m8>S3=)jeVy$1>oxS%`Or>&c%p@bm~Q-GT~_=JY~XD7d&Oc)7$Wr2~QpZPnq!4gQuoaCOqwir%ZU-pu$roJZ*!g zOn6FA;VBcIdf+J&o}M(rQzkrh2s~xN(~8gf&r>EmjaA_(6P_l+Qzkqef~QP)T4ZJ1 z&YAEO0Z*Cm)Co_S@Kgv-necQHo-*NS7CdFbQz<-U!qX{u%7mwx@RSKppTJWlJUt0d znegSwOg&EES>G8Y8~awJDikCIr9Un7Lqfceo`IlGcR*?G5N34teXNA z53(NWj+^#{Qjb}@=b-Km)?@z3%giIldZasU%x<>N{HQYr$d5d&UBY>IR?T*g_aA1t zn@^3{m%(;No*HqOxRKSc-D5k?+hsAI_Q6D!W1foL8|2J;U4Mf0Tef?AZq0hkr+sK1 z-~TqvI<}ac`L7%5g#BQ?)U>H=hxaDj;7U2~JFuI^`giPDKj{?fGY|Il?~G~B8nwl=S~tRX`*Zbs-cR8cLHqN5hAUsFlQ6@*lUa_u_fREatiCg#Qn5bw7UIzJ7HB z<;-)l-e4SevhisVK4s&R8$M;@(`bCk#-}a#l#Nfz@F^RgcHmPsKF!6aY<$w4R~8jVlc_>?C2l#Nd}3qEDzlMbJ<@yQ3DvhnFLe9FeB>v=9O8=vm~p6~JLJU(UP zQ#U?kPtW30Ha>ldPucinbpz|+(+GUZ#-~br%EqT6!KZ9| zx^)WM!Kcail#Ng8d0sRdpRDmI8=ulte9FeB8TgcqPiF+5vhm4R@F^RgLd^J-jZX*u zLZ7nnsg>uU`bydOAI(_s~# zvhgVdpR)1E@*S1+@u?o4vhisNK4s&RUd5+udrOd&*>mH&UUbAkeKB)hC>fbQs(Ed|s{~YRHPyOfM z+h}~tf#0X#H-~stLA;Vv4sq;CgtDC++W#=^FQ**hSa%BR5wE@@UggmKyJ&yIltcV@ zm-r#49NK-4(C$~iCl1|99Lgb{cE6&s9pcr`36v8jLx_`d%E7mCe9IwTxf8E)h^J1( zQ^S-)oO+Hp)mO?PUS<$4<&;A_>psbG60h=wcy+a$c=q*qrJm&P4fCAwKb>7uZ`{x3 z&=34ZKk$1~4*kLN^anYWycc+?vYj0IqXqOwIrIb0^aHv075sy8{JHoQgI~G$b(a3X zFy-P`?IlYo7r!)uU%B{o2Y%(^msapA7r(x{$8amk=Un`n_MIf<;@1mi{K~~IKQn&i z;@1>2e&yoV68y@=uL8lZT>LsclHHq$e&yoVI{eDTuSfAK7r%z#S1x|V2!7?_*D=AbT>RQB z_?3%a)~T$AU%T)t7r#!Z_?3%ahXucK@vB+IuU!1H#;;ub8p?RZFy-RcaK;0<__b2- zD;K}s!LMBWT7+M@_?4jIS1x{O`r%hDezgmJ<>J=?{L00zbt-=4;@72q_?3%ajSa?r z<>J@=Kj2p`evQJfT>NSj{L00zDfpF(UwZt?qh9K3Eu}p8a_Ue#%%lF!4j3rqG5_=I zB`YZpzLvkk@@&^`3_10&C{t;VJox$=zVg^^^h|QNc`;k@HV?k`!B-x?ZynEe@i(0Q zF%Ld%#`F95nvJh{_&p(zayZ)sXL)dRC(mK!!OKN>$%C&A#P2-3-E+4g4QvlztB#Sw z&G)O7-BeKVM@En%jFTTs?Ef!JmOWBnRs<|JItr`eVyN7 z-g0=e@qXtKujUZ1@|gd;+0~fiJmS<`;#3~QU@e7li&m5*;X z{%R%VRAsAKy;ndp^D`!nb^U z8#9jI$G2_xmXB|9@hu~-}3Qo?OpQGD|tS??Z>x#eA}<$ zTRy&ZneixXao_*RK;`S`X*#kYKXdmZ2M@vT_Jw|soN2jBAX?Hs=4 zkn6ij^CoW|Wm-N4KUYDfwb9&y#F?SB-#HR@2lm0i(>+;Bnmjj5G zdc410@Lo@xyqh>_nDqG3j30WuZ^e5(aq^!k=k0ppG5NM89(&IrwHPcp7(w1`-Yr2Fo-yy#~Uxa z(G#D%h);Un_p$Xqst8{zs4h z8G`?Y>&u_@#K~CVq@MSEeCKIgZ}M`6c>z8)3qBU$;}2(*{Vl*pOZuwY1^D=UKYT2}$Kzuxr2>4sAe;vo?ss3G3-EC>J{I8PY!x31@bMTv7U1I|6(0-m zaX3B};GIfR9U5d@R7n6nree$4q=I zz{ge<9}Dnt>=0wF3h;3@J{I8Pd-zy@kCAsuQUN|jenF0pPWV`WkN?KU0({h}_*j6C zQ_T2SfR7q{EWpQg_*j6CcjIFLK4#!!0Y0YSV*x%M#>WDDd;=c~@bNG{7U1I`&PNOI zap(gI--a9&;Nz|5jeRV@$7%RjfR8Qx@UZ|Nm*QgqKCU<8V-kpPA1j1ocQ`KOdcS+Evj2s>_s3M8b2U6}`Ew!HKNfKPLr#TU z|47(mAr*2Q@f=4X*FQ9ag>tTc=yviw*-dj|zz!Uc`+;#*;s3 z*}vZVTX$0q|4+hyA>*&NRw-N+QvWFzSq}bBzbO0;{O=O@FNA+f_?J^5{BOt;w!?UB zw#s;|kob0h_*TexZ3*MGYfXh*KjW2E5xrC@v9JD67Z$4_dZh)IoHn`HVN$s|9bc@#4o4M<+YW(5dIV3|IbQA z@HtC`&m#EzMHqhbw__XIasv`K@Bk)-SpUncFMesR6;Ijxm4+wk~!RKfdK8xVg zPwnsEvj{$WD%gMc%z)1#_&n33@KXezMJjw2!KXER7QttT3ZF&r83vz4@L3O^MZNcJ z;IjxmcM5zK!Dk437QyF>D$mIl!Dsa!;IkNh>fooC{<)m~xtR7}MEe)>dow?kq~hND zBk)m7|C~YpTul2vN&CyGxcB~uz=!<3oEO8#VfZMfUz`%DjBASF=M$c@DTa@8@KFpe z4+!VS#k7Bh(Ei2n(F`BO^cR+Iu^#OoLHifeFNf1F7sJPnrzoc$H&Bm;sTe*&RQQmW zD~698_$Y?^dxid`7=8+c@mn$DDW3_(+rPRU{9dl&_tkRvzlHPTVmNw+I9Cim8{nsy z{xCzhK5}(=+J80eFQ;Ps9xXiARZROY6579*@#g}rPZqpe@_zs zMl0>3K-eoK1qKbYaS1b#Qe zZwdTb3;dSA?}RcdsRVw9Pox}v7YO{8z^_)|w*-Du;I{;R$H8w2{1(G+3H%nrZwdS! zgWnSPJ*2{K3H+`!!*2=vu7lqa`0epfcqxJ3P=VhP_+5IJX?{!KS7(Ob68N19za{WH z34Tl9_cQn{f!}r&eoNqYuE1{z{9Y3JrxN&e6ZkEG-zo500>3FL{FcCPOh5Q7f!{@D z_$`6oEch*fUzaj|4}J&0ZwdTPhu;$T9r_44{JQ;#{FcD)YXZNe@VQ6me`T)ayc9l< z34E4Pe=G67l=GX{U$l@);d3<4Ym~y8RQqGSe zxnESuc~srJ-g5r$`c%G$PdB0eEv5dt-&Z&(h0hrHETz2%)83`dyjNa0|15>i%`T?n zN-2CESZS`D|I_lZeANBVOW`nI=#NX`ZIZy7;g(9_ZKS}PVY%VwQaHRrxZhez`vnQ@ zXS6&#?XRJp@c4p?$A-(xpG!Hvj@U~(aGo`#p8Cb(4+W1)IluPmqMY-swudPv{(O~7 z&iUA8;l5!h@$Ulhuaxt%dd|;E@mb$NPMrLPI9W>kJ3;&_#p7RIQ{sfYA7${^Y=*}& zc(lA;X~!~noDGj<@Hh<~%i!@6@v;maXTxI|JU$JNW$;)Ek7e+<1Rl%aQ3H=<@K_3u zW$>5;k7e+<03OTWae=^N89WXZd@h5>YIrPz$9v$h3?6?Lcr1g*VlzCJ!Q(kIJeJXa zoZ2JTO8@gRcnlJFEQ3dDGdz~T`3eH~F5 zJZj*v3?6TU$1-><7I-X!#}s%hgGcER%HeT(J~=!V3p|#=<6(F#gU4O)SO$+9;IRxI zFT-OQJX&(UzYHEH3+Dy${*`n6W6g9)DyKd6?)&2CKq`mV6nHJ?I>cUe5KW67I*#sT^Kcsyy#j&hfe0P5>94GOhn0Qdm^`k{RZ&(iBU!NAr;rCmWeqG*vIoChK zu9)7xtIH7&+J*i}{@ySzhyOXeceI?iatCqcuTJIoxK8k~0)C&LCP@|WyKxse{8qzn z1^mu}-wOD3gWn4HJq^DV@Vf_oE8zEre_Kfv@Vf(kE8sT)ek2gTtAXDN_+75TZw34w zhTn?b>y9e?R>1EJ_^p6nTllSj-@Wi#0l(M5Z$Q=fZCV{2mbat$^PU_^p6nH~6i9-%A3&74Un-48IldyYsic`K^H8ef{9K z0)CUKsWaE0%C}hm7INm#e3v6hmCUd4Y$fOY1%og1J@b3U4QvBJeuwuPxHGS!lKC&8J><;$sqA4tm|yoS&oflc+c@E1HowpOmktYZ z=KJ3G3cthr78fg)XMS(&6MSa=i}w*eGhgqMHUS%J_FGc`ro-^HmH}CG&f>R4B(;$$YADx3ir)H%=I#HD107f8I6YPZj;ZNOQ3#kf!X5mj2 z{#?SJD*SmJf2#1O7=Nnp$J&GM@h1U)s_D*jaA z&r1BM!k+~Eslp#U{#4;l0{&FtkFJp%f8N5MD*VaApDO%$5`U`jX950H;m;EMsluNW z{Hem9a>1V}{K>$dD*Tyf#-A$uiNK#K{Mm&+Rrpga_)~>H>s9=z!XLMO_)~>H_ZJy^ zR)s$^RQ##JpPjpmxsu!8R9=NY;b#1)!kDP_)~>HAFKFN zg+FB~{#4=5VZom&{ISBHD*XA`S2@mVxH&UW;lG;skH^;zl&az4AY4>$Bp&-pQZ?=D zNjq1=LksUusD|I_L6pNyW0TTvR#P9P)JHX({0mMDQ#E|N2Op+VHT5x{`lx1pWLF$H zeB{H2oT{mhV(PfcHb?Dy!sGl7e2#$6 zYWOdL|7!f7=)?N(k*~r>HU2+~|Awg={*RR^{8!_@E&l)BRE__&_%Ekw=5Ox4pZz3W z49_v$9`f?0@@nFzt(o{)O}w0VkMZ%z+p8vSj^O!=YUbnKegpMKT)cs}D5q-T<#W04 zO8oR7ej27~;^nv)%JDyRkCjw|4}*xKa;m|HUHDLg51a9!1|QP!p#~pp@u3DEj^INL zK2%syjt_^M6(4Hw;T?Ra!H4xKKGfjDSu;M=;Da+h)ZoKKe5k<(2f>FLdds@P=gPX9$-K5;iv51??VkfEH~pr4L(?#@u3DErr|>k zKFq|28ho%7e5k>P5%^Gp4@*>hsKJND_)vonPjWxJcGAWPN%xYoou(GqioWw&+OJa> zFV(`!Wr3HzxA6OxsD+m@sq6XTXz;$Ii>uNEKA z^8SKa;>6>hkrS_Nh}X4Slb<^!EczHOrs&eI34R#+N#LiNTjTe6jokzSQB%WB5{sFGc?se5u2i zjDGl1hcBBx>c7u*__B7Gg;a+x>&*C4hcE5;Qim_+@TCr4&f!ZPzMLzdp7ABdm(Tbz z8DHw~Y!IwIGId&)I__7IK>hNVO zzSQB%UHDRmFT?3i>+od)zSQB%0eq>$m$&hy4qp!7OC7$9$Co;M8H+D<__7aQ>hR@u z=DpY9%T|1;!>2Pvc7+zF4l{GrmmW zytEEqy0gjgRjY~OJC zdfN9>YnIzhyu8z})b!_i;(?xc@Ox7|{?y`6J-&K>V!WMt{7J(fQ>h+*!tkfBRF6Ma zD&rw}y?XqK!=HNM@;1gX^{4p%5^qr-#LG{2k`o`=hgwMWc;taca;nGM@pxO0KSyho z@8$9acwZv$-T?0r0`Cp*{+$`#8{qvSKATDn@UDaR26%r(h4%({*TQ=Pygv@_4e&l$ z;JpFfpY&&acprez4e*`|?+x&NP9lf*!|>h!@3+Bw1H3PW_Xc=B3GWT?9t7_V@Lmt^ z4e*`@?+x(YP27@G1H8Ws?+x%St)U#==L@_y!23{mZ-DnocyECB)0_`9z1H2biD!(u9hnzRSdnLR#!28)T^0$5G4e%cL2Y7FQ_pR{W0Pi#Q8MkW# zylc(y-T?3Ve(>G^@27{G_M!pa%i+BN-oJ+T26$fs?+x%?3GWT?-ZI2OYJm480`Cp* z{uI17!29reh4)6zM*^*lk4LW2My^jiKgvpKq`#R=f78hL!B%Db&`AFjMgL@&8mX^S zpPJsjynG|q4+|e<`&|F5;koTbcz*`o8@W!Gukw8I)%7_32RVK@HFCa}cefH}8#zB8 z8)y9cjrh5g{-}}bcjsSI${V>px0mO#8|j}*KIHfDQ>Wsm{ClSIM)+SU@ZX4^v+%PK zz8BZBUB-11HdtMRy zhj;JcT_Zeq_Jijp_{uTER}*|qg|8;~ng?G^@O7sOUrq4kq{3Gde6{XZ+D|?{!@LQ; zLg1?jzP7?w6MU_PuO|4~WrnXN_`0%Q*;*5PotvoCdlP&;p~6=ad_4kRP4M-(z*iG| zorbR__}TzpP4JcfJNRmXFRj2=6MQX$uO|4qrH1Xo*J$`^g0B$xYJ#utCs7Vxhv2IT zzN{J7Ho=#Z3SUj|HC*7U3BC>s@w*AWF6k8Co8W6C_xt741YaY0zfTiTeYF*G#SzoW6_wBsYRV>A8q&uhr(j~`}y-wb~< zR+*_MeE1R{es5}qvzc(#41Ya~*bmz8b)jEwhPN&7)(n3Oh;MRg#&-vNms2x7e2Nc- zshRPFFXQKC#v$E|L*&#~)2wu_HldF2ry4Un zwZPNlL#DPONiFa+3!YlwY2yeBsRf=&1)f^qsTiJG;ORAZYJsN>0#7aQG|voAE%0=x zA3U|d(<*psfv0Zae7FUkPT*k+JZa#m1)fgBQwuyTho=^JdIO$X;ORGbYJn#!cxr*C zoomS9$pM~P;OQ1KJhi~n`9Hu@3p_1?rxtj+jd<1qPg4b+THvV~o?77Peu1YJc#08t zYJsOSDm=Bo({gxffu~Odo?75(5g`>Cc#rHJnb;U zQ!6~ZYKEs)csjC)?ZA<{z)>sXy21RvmsWTh1y8N;v@O9(YMr-n!eZ|Cv@*UM(n>iz z`7?fKg|qPjXYzix!c(IV-{tS+yp{3dVaAKC@bn!#wQ~N@!1ZZ4wZhXJ;!`U;y$MgP zoIgaz$fZi&3QxuKkFD_Z8uw=mQ!70AnBl1vo^IlP`0q`v@Knfq=v(1wGx4dF^Od8V zue8F|uXjjNE9W~coZq*?)e7!=w8B*u{bVaVbw0@VobPN7JlU9uPp$CuEIhTr)3;`LYJ;Z~Gd#7y(g4W3>y!&4hPP5A>nwZYRmcxr>E zFX5>To_4@f8$9iSr#5)Xf~PikdJ>-6;K>)B+Tdv)Jhj2oX?SXbrv>oT22Y<}15a)6 zvrmMA%3>w%k}uu4*#9--;N(w@T2`C^~7_n za%zWv4;B8~;k^ys<0G;=+yo$CWj zxli8C^{a_0&rh}E&no zv~xYFj_cuaYUg^@60LH)^7nGy0dK#;TL-+Az*`5rO@_A)c#DF!4tR5cw+?vg=m&2d z@TM1d>wvfRdzE^Wf49GR2fURFymi3a33%&(H)~;h(E)Gc;H?ARtW|jHfVZszZyoS< zM&PXj-kubA>wvdu@YVruU2k&y@U{=$I^gXacWw+?vQE%4R>Z%YK; zI^b;|ymi3aYJs;7c$3wvdC@YVruUMjqGz}q%~w+?uFScSJv=2!3LcRFjgd))byh1ALXxrqS- zrOrUxOCzGmnWvoDqa1H1&o684QaJ8x-R^N0*Nr=wA2;epbVw)V%x76u%W)LgUOIR)$ItWi$FC!2e(0L5lrx{@BX7!?CmC^*rPRsu^<58B z&iwYcQsdw6WM1f*FqY$acK1-$zp#D%S*|N~GSAbWd7hojAB@Zt>Wk;^7e*=D>1?#U zA-x}ww?5E-W$)7u!A37vUS-+F# z?`?UmzLWW{KQRAQPMth|ze<=NDt~X7caHn%?4_?v$G1-Aqj@h->e+DpzCL#{Kh*Db zF`eyzo?VEkH(U+e!*b#ERYMUg-5cMgz6M061mk&TFmh=?q@sE7!N zh=`~xMnptJL}d{Xk!2BC77-B<5s_6y7Fk3WktCDJeJ7Lqn%vjqp2>Zmlk4})>+|Y# zeP?=x0Pgqm{nbC6nd-CZdEQlBJ=MW45B!S6FAw~hk6#}6wQVE$`1Q>QXUPMcPfnO)Cz%LK{I;G;5 z2YyxHmj`~W!Y@z!8r7xv;)!2t@yiphY6Y)kLp<@T={8|~{EEOYPy8|j+H&uSUx6d! zt!oNTDWPaGN)$9k)77Lzv;#cBJl6I)#IJ4@zdV`$HQR~( z;@5Hf^2E36g!hMe;#c`bg%|7kls|jo*J7dl^29G!{PM)FJC|R4dwD4jg*@XHIo?izq!UicNJ;+GeGZFAt47k;h8 zFE9LR6xuH@{92D+Uij63UtajN8o#{otA+N<3%|VZ%L~7D;FlME>EGfreytg<_~nIP z*(!c{;a38FdEwVA{PM!D!{N@JoYV75JsWuV%q74SwZ+s9c8zzgiY@9?WlceS{@` zO~Wq@z8%Lm4W32dnTGkz9dDA4XMuPoCk=jGjb9qP8YXzvpO0T(;+F=$YTT9W`t$KC zk9pl1{Fj~TK`nPBP^OP9d>*>$OuY1PWZm+?wg?p(7{CZ-lljMzGpZ2;) z-uN{Kzr68l4}N*$*LD@Zyz$FCM)AuVzwXB`Z~Xd6#V>FC`eC?yR<=uT{Q4b!dE?h> z_~ngXaSr_Q#;=c6{PM=H6bF8Jll7{zOyZFZ~QudU*7m-z%Ot7T8m%a_!W;|-uN}*kWilZ)gk!h zjbG{b<&9ruvsvQTNc{4~uR#3r#;+p$^2V>xZ;_8*#|6K<@vDyC!Fl7?6cxX`@#_M9 zdE-}tieKLN6^vir__b2SFK_(XgzpMm zobMyEq))t zZ!PbyC@bQ6;q3^#X^CUc634Xo`V78mdEbV|TekJ4C0^|o`rF!n(LbD|?7#o~h*xhC zue7`$Wc|}B`S?A9{;9pBC0>1_60hX-wZyCO#49;zi8oV-H*(SvuijA^*Ov3Gms;XV zK5<1(TH@4rt+N0A?};-zgg7IAFE6#kn*=XqKl1nTQcJv55Cn9&wcPs!ebwNJ16+& zgKrD*%?IDM;hPV>WeUFe;9EAn`QY0meDlG#Dfs4tZ+d=r>w|A&-gl9F@GTPGeDG}= zzWLx=D8BjNTlO%P__i3|eDLj^NcM+s@9?{1AAHl`n-9Jn!#5v%+p6N5559E}pY0_d zeDinUn-9Ltx&+^R@GXbmh0DnY-*&0^=7Vo*gx|UQ;M*2_^T9W%nCr#23;5=PZ*Sw9 z55DcfHy?Z}Qt{0P-@X@o^TD?S72kaD?J0co!MA-XzWLzWNC&?8;M)fdeDlG#M+Dz| z@NKc+n-9K?<9GH0O+NV6E%@exZ-Z2P^TD^34t(>$w;Bh&`QY0y72kaDEg9c@@NGyU z<&SU21>bz|ZT~HBh;Iq_=7Vo@@XZI`y18HVr9RIQKdqB5J`MYVeA;J;_Su(l`Do$2 zYx4TOj34Z~aggL|d2W%c#0y_|Nrx9-%k!ndEaBiC?f9;|FOqr7~HA9HB0<>X8M zYa91BzKkDyvWoSIA8UpFpfBSGiy1$-Sn_2&;E=?A@J!upKa=iZU@h?m8 z&zJk@Pw!OL^M$XmYp4g}*r0njFP^jf`(YuU_%vZF$9o?yc;4bmT)Rr;|LV!dv$yoc z&p^RXU*g%sVB58QxzG6Ztc&DJKmTC|{e0Uk|K}S&c#Bct%@5vch~IwjR#xXE`N7-I zD!lo@+gW(?vy9^ly!pYK2HyNE&s*Tl58j%Lw)N#_dENqVe(?5)z?&buEfaY2gSP|l z<_B-P;mr@;4hX#Y!P`O=-u&Rr7vB8fZ7sa{!P{5x<_B*R1m67M?HIiI!P^RW^Mkit z0&jlsHe~?3`N7+}@a6|^cep6L`N3O<3U7Y!<}dK(2XFoYZ+`GL3*P+TEt7WB58k#4 zy!pYKE4=x^Tg!(m;qCs*;LQ)-as=M|;O&u6TWb8^Edk#A;BEM2@a7K>6L|jSPdP28 zocxJ<$Ax}|KRk@)`P;>kKRj%J2Y=c-9qpYzT(rLFB>BVh7I^lDhYJ3`hCf^$hf9C> zb>cauKRnMQe)_{hBs}=j54l^oUzCsQ4-dZZ;13UX!Gm@3w>%#Zc<_hE8i7ZD;=4Qd z$8z$gU-TLMB7f@lZtB;c{*;FPlt2BQee`$y;dCvW`cuCyvxV(w|AOA<`l#P!#1nt| z7l{t~7yh)L=Y;lCKECzRpZaa_Q@-_w&(i{*{?zZcms7v^H5~N&ozPa)ksnR{Dcbp1i;UR%it#feqOu+egZ7dPXvAf;Ab%7 zZ~^c$_eN(ako$uXpF2r`l=l?MI}o1M!c!pS_6ptw(m&fh(M1ZRpZ_G!4FciGeE>WK z!qXIZvX=tk=`8Umka7zb=EVhC#(U{M2f~vpJO$EUUdZ#-K+5m$HIfua|2T#IaUeW( zyg@$qFU35cvQB~Ulpw?-IX@7dX9?|lAm#B8KFKK%o)6K!Tc<$F_^?XNgPIz9wE@ECs^#Fn{~&1>$GE(C-K&F1Ygi zClK!*ds*2okbd@EPb#179XF8szleQG`N-=9!uJOF4ur2EJf921!^bB(NkQ<`!u@6t zJPm`VAb6S&PeJfB9iD>V>A~Awq#$@&4^Khxv~2)91;NuW2RsGAQw%%>!Bd`(e3p_F zWEtOtryzJLdzd9W-33oU@boM^1;JA-JO#niXyQ{4Jb9|{6a-J_RCo%4CwGCTAb4^Y zcnX52Eh;<(S;j*h@DyYj4;6R{f~PI;6a-J>;3){6E~xMn1W&iZQxH7q2EbDgJWX=I zQxH6L3p@qElZkdF2%bg>JO#niiUIHx1W#vFcnX52eee_nPm|y&2%fgWQ!xCThM!=L z8^>|2Q!xA-8vsAS@KbXK`=MMOpj?6(AJ|gGlKOL{{(|9WJ^Tcx@?LkI0|#^6V^f@^ zVCpZ1`U{4i`NX4O%Ktse-#P`u&rJ9UW_;k;WLGJe@-fj*3TB+ekLORp@N+=mCm26A z;72gyE|2p2Bzq~Ca;T(zlT$GDl}@`Rr(nDa5&B2{-@{8fyj&~=Gfwp3aAALpGfm<> zsB#KsyvXe)iSy_GF?g_Kd}+@vu8aDbA+&G7_*95b_ENBAJWu8KTfy)eS4VkS#`9F3 zAK2TU{CzOI#0c$MF#f$K_!j~{d*CMoetuEmCj@?c2f$AV{3O6n2>h&vpAh&t1V170 z;|4z=@UstoLg1$ZenQ}97W{<3&pG%Bfu9NR69PZ(D*S}Nk30N?z|Vc~69PZJ@Dl<* z$KfXgel`gFguu^Y_z8iZRSx(GfuGTnmHG^Up9X=S5coNy!cPeNXa>Mf2>dJ+_z8iZ zVFEuP@bdt_lMjKPpZL9a2>hIcpAh(2D)18mKNlSE6Ji;kQsE~AeoXKaVi}*h41Pl3 zr$XQ-1bz++{DhA68NZ9)YuQVoZoF^uDJ4IY`P!xRiqD}Fe8z7ZFXV4#z6!sW3oYS2 ze@m3{A$k9yTR-rBH=BIpC-V-^Q}RQZ2Y76zs}#EZ1OL`i@}Fj2WS+AW%J137m{>AD zQrn>T9?JZkyfEcFDW*!vEgZbBEy4e3u?XI!MB`K8O_uurZd~`l<;JvXEIL&SnE_j4T)%9HtP+P%Vh^O$$R?>j@8kEU6#l$Ubkf&CTAeA!9NleJEv%>TW? zQl8BJcXzkVm)8$v-hcXZ`}tw`@x22-!tmox!H+Qfh!Ok^17k-4{N2rP) zVff*WA7S`$PVgfPKYZ{b3_nJck&hn<_z{L5-A0!9aZK@MRPZAVKhpmuY>yxA_z{L5=kOy8KMaB&VfYa$_z{L55`Ki? zM<{-T;m0|_k1+g*6Z{CnkCTEQVfe8HKf>^1J${7Y#{t2QF#OnwA7S`0Q}81UKlXg2 z_z{L5Zuk*~A75R8A7S|MI~6~|@MD&xyz%3G2Y!U%hqnVil#;%*zrx{p1w6|s9DbVw ze&z44bQ#Y4u;nLQrEvH?Lwpa%yEAwf&O9)eR7nbFKG%N&l%s^>-73L5CFlRGzryi< zV!q;)e0JgRTk<2<$^5y)CzSXr=MQul4j<7gDIfedG^iX0U+?;keEeTP`xs8#`-Qj{ zj*r7c%48P&{KhclR#Jzind*OJf!^3cVPQ&ML{5Rl#IC1YM;$ApDp25d(58m@S zo+bVd$NzBR-d)7KaC}^akKxRR8_xJnIDD^z?{MPXzlnF@_*jRJ;drwZZ^H3^EB=QQ z@9rSph2!HHd<=*Cqi`P%-$&s)9B=30Z8&~+;deMbKD&qYi2rkm|KY^D>1)`Yd2ae` ze8%HyezzY^{NGFb4<}wc_XMAbFHaI*!ihsa5QoBv|5_pb548NP*NecHzY4xY;7j^{ z!Iubp`4hfG;7cUFMBvK-e2KspeTpPS;7h*XO9Z}b`~QJ25%@Ah#g_agR;K0N0F3kD&-nUIh@<=Dn&X^{r;9Vr5%c-Tnj1J zNUk^kdGe{B&D2jMzRkwBNXqpU$~BU5Sjzvwi==)gQ$LZ^?`OorNUpz|>yM;--==&c zsh_zwkx%{INBu=|{gGUMB<0(fKtAvH~p4K;>B3vMI`l;Vc9SBQ}=h)3u$+~ z{V}CfBB{qW?od8k{}oBSU8gdCA`+k0;8P^?ymu}icsmn`Z%6504KzjK(4MUJMH0_86VD>?O^0uh_;i^2GdV>P z&n64;EE1o>@#%6?BtCiKQzY}e!=nd&-Isg3i)|N$PYr@kQTTL|1D~SsX@&!zqVTB( zpQ7;TGkl7|rxi?Lms1ozMdDKwK6T+!6h67*QxrbA;Zqbo?ZBrfe7YB(qVQ=xK1Jcv zN_>jKr<*tOJw7eNrzm`ygilfUv<{!5@af<96opU2h*wegG;%ok_;m7d^6}~Sw~>!e z+wdt0pN8O56h4h1?nL3!2z-jdC#RKck54W56opTV@hJ+Qx-I*`r-$$<3ZDkkKZwGo zK*o8Z@M*c=QxrbUbl_7IKFw9}DGHzF<5Ltqx&3+IoJ8T%woC9S3ZFvpDGHxXtN0X! zPh0RQ3ZG8nQxra}6nu)pr=^~XFH!j9j!#kebQYhY@M(+SQxrbU7krAsr=lzHDGHxb z@F@zPivAz)Ne6$RG~{;ZLrT zZmQ4tjq{ZK>G0_cKIwQ5Mcps9+~}z9b=0>GuU6reoOJkc*APk4QQt|_x6b#?4<~GQ zl63er9KUq>ifJGpYcjZJEg=>xd@-w9l8D zbi|X{!tYIW#Gi4(_=&y!>hNhcKIwSh$4@9c6Sw80BOa{~;*pN{9ZY}7UW&$_QsTG06pcU6sQ43& zKf4_G6OBI`@FyC7#^O&j{`BBaH2!=h_!Esk2k<8ve}F2 zZ;_;E{JDTX(fD)64ksxZe_H5|N8?X0{zT)?FmFENPZ9n^<4=s>Pc;6_#Gh#VnT0>m z_;X(HCmMfF<4-jHOv0aN{OPUdGyeDr{zT)C`%d!lM~gqv__MHr&-gR!YChx7D*TDY zA2aQ7H2$nt@h2L8-0>$Ge?0Lg8homf1>f{ zZ^Cn*X#ANq0Dq$KX9E62ec1np5Y{zwn8#Gkw?@FyC7;vM)CjXw`K@FyC7 znuYPTX#ANa_!Esk1rGd)Vf=Tp(OHV&{?Gqbmh`{wY;=`kxSutAH%N-%zP%t>l47|3 z+syB&tWyl!?T5PK1n+k1Baf(aXH1{T_WCDrx@;|M{iTgP0o)Ye$H{w|A--eZc>S#{o50_#%)#V ztN(lYp@W}M>aG8K;^#EtXAJQ-m-rhCf5YG}7XD7cUo8BsYjTxh;V%^aV&U%`{KdlG z0TupY;cpZC#loK!{$kT)CcD_$w0l zi-o_J1^!~;F9rT$;cq7V#lqie_=|-b4m;p47XFG<_=|$BTpizrlYT?blJ2_A3tVZ-e`Rra0>FsBr%tM}K20 z{q;Eb{RDpF@Nh34$|;WW9lhB}ii6*)xsQv(!`XNkM}K50_sMazKL>^WTO9uW(9C+2 zfB18Qq&V822f6=_qyBT>XNe!p_+c-_;h$$2$HV_q_#a1mrl&oN!<%pM=3*(1{>?|+ zAH?Cm9{=NrFYAaea*CsWa~1CkiGzctxttGiU>$KF4qt9wA+JSw<3|L3#NmIq;C~$b znRwxTAP)bx3inNM#Gh5fpE&wC?+W+nam1Is#24!nhyQz3{Ex#w9sb3`=LLb!c=-Gd zKI7r@7<|UV=U9F(_3}n6+YwP^BjC8aQ>$_{{-q~JkJjjDBt^PT%`ocXF1P965!)sJoibUUUF61 z?*#aGLU|il<+PV_O2D5$q5V#vd>*8H65z`;nB!9ae-?f}(0@GmdV}^mfqL!{>N$b((d+D= zr@Z~emI>VdOl{;m@GBg@5~%N1cgl_XE^nT5$SHyQu9tXTl0f}`H;*Ov52ycO+pkNY zehypehx>MEKiW>nbI}-%R*RguhMj zmk56?D*PqF-*$n&MEL8(k3{%uz)yQA5&lBqFA@IM!e1i%Y2Ysr{>C!DR!)iVHz$bm zg1`0fmk57(0)L6{=MH~~@RtC8iSRc;g}+4j+XH`z@HbWX{YoPIHT(wrCBokb_)CPp z)9{xFf06K)2!DG7{u1HuAp9l5UlRN!!rwwU0)L6{cf%$pDG~lA3H&9(U*&6@C;W}Plk0%LX^SWy_)8P`lj~VtCNVysV|*Zq z>l?-OB~jmIp}yty{#(l=>Tj>mJ}1HdBk*r8B{6={#PgdZJTMKQolc_u#tH4Syx%1F zy@mEU2@mUNpOYBhdeuXck|^I{{2n+7e!qv`Bs@H$;$i=O@b}j`)}y=^@q1x8B~ky& zssAMW@KNz2iSes3{9ZN*|EdaE;{O=@Pr{d%1z(aFAA5a=yjR<0660dmzRr2$|03F- zB;t#K_>#o^>{BV0d`FOtw@00MQ4_~ZP661r* z`F&gxaiI4c>k)r)i9bn-g#W{Z_BpA9aq_3xAO5Yv zKkJkXpGEMQ44>2BGZ{W_I;g}`>-B%@&&lw)Uf?qsK8HKtGZ{Y9;4>LMM+LMYnVTh44*ZN6+V;U z^F8=XhR?k!d?v$Z#m5{6K79==;j^3mV={am68KDp&-)$lnGBzE;4>LMr@&`2e1^hj zGJLkeXEJ<#1)s_A*$tn`@HtV1&t&*q37^UEIR!qG;d2`ON$ZphpIhKF89v9-P9?)< z0DLCHXPYp7n+%@~w109+hR^fxnGBy(1U{4DvxV{1WcZvf@R`EAfY!e{Nh!>`JI#2G zbxL90<;1Yd=QM@+%5~|kQc3~u^L~Tld^^iL{6VGuQto5kT%5B1l<#MmCr%k8r7(}d zY*gBF`Mgs85U#0eVn3&6nT<-#OE^Ui#0EdF{9N$)6-Ch52AVC$T@?Q}FBo&gVbO ztG-IvK85+S-{g{iL*>XNIpqJ!djo{=yA-B%DGRI<7Yr9^QZl>y*m%26Mfsc()AiQn~KyBb}vGc)pOQT)*{pm-{)D z__1#=`^T?M_;tA{64-xOwEmiw<5=@#`FZrQ)0M4n7l)4hr$8|9JT3j&G^NBO~!hPN~G71;n3J z;`8r_FV)+^^K%T#<@DflMmq!M>}UuJvab1LyU74P2QcaCyOCH|Zu{-hG0Yl+XP zc()DjQt|5uex(wBj6U``N+tg2i9e~taRYHYl{o#AN}Nt5o)2y&pZNR&@i~=vbV7(n zsl=bT#Gh2+^RTCEIqyGT;^`wiAGJ=Y#GTE=omArUy~JnhluF#0O5BlCD(#_;_V7}Z z9v^l1XfNsU(SVP7e2iYN>|c+M9S)v{+uPs8zSrYp{_mV6Jw8TXfscB8+>DRx!le5fix3a{?aN@TfA3wrJJwE=3k8;xEW0Q)H{rUJ|T!Iu4pZFqUK zaGv;Rz(+kkYVc7%y>8E*LY8=`IVDMYe4LAqdVD;Jk9vIk5g+xf6(f`0;(L5tKz~4w zpYIFpU;lODV-r5=@ldDYVgGvgw-f*L_&5t6t&<)fOMWCDAAkRZZN9v|9v??%sqlc0 zhw)L5k4sd1)Z^oDeAF{fb(}E2Ti)+LmudKT#(|G%_}Gh&Y53SC_?U)|a|}v680hgX zwq6=Oh7NI-((v);EATN5AD81}8a|fbV;VkQFZh^-k2~=(4Ijs-w2x`{xD_AM@Ntof zk7@XL0w2@xaTPwM;p2IHOvA@ud`!bfEk35<<2U%2hL3}3AJg!06#w@z4Id5on1+w1 z@G%V^w+TL`;bW!)AJg!$7a!B`(TI;}_&6UQ)9`VLijQgdsKv)Ld^{ugn1+vg@i7e_ z=iy@-J|0u?F%2Jk@i7e_4+=h};bSm9rs3n)_?U)|uj6AHK7M;rlG5;T4L+vfqZdA= z;bSH~rs3mxd`!c~iTId?k1JGsOvA_P|4cqUmOAh;4IfwGV;VmGLB+>3d~~LLOvA@@ zi|qTDhL4RIwmzoeqbEM5;p1B>KBnPgT{ijn*qdbEi*)wyY*6-}j$h;OE1hyL7VZbq z@oQh9vY&Lyzf$O*q*Gs>p{`Oo^U?FPE>b$>Z9HS&u5`+MkV(l;r@q!?*e{24>T7SF@uc`QzPCR^rc$f~~&hM~3apXV5k#zi8fM4lw zy%(sR|bBSpW-_3YqYbol!0Hp_?3ZQZTOXeU!jkak6#-FzcTP^mEczf ze(CTl1HW|mm4RP-@hbzrg77N?zq$_#<&Iy&Rs71pubudnfnWCver4cSD1K$&*KqvG zz^`NYm4RR5@hbzrV(=>izt-Vb27av*{K~+uJ@}P@U(c%em4RQ!1iv!yYn+N-8Thpt zzcTPET*a>p{8~Q%zcTQvNyV=W{F;eh8TfVnGW^QGuN`5wer4d-m@Dur1HUEl<)%#hnS?)?yhr7&7o4O__*?>?nQ(D6 zTx85A9zMZ*+K*dCfgTf4_wF;e8&wXTneQ)08Xz zY{s8VIKR#T=b8A^dzAGC)9*RO5`KK(CzJQ|yxGI`;LjTT$%OYm!+R$D#H;Yre_q7r ze+cn8llbRM{L93n>*;r85}!X6`ahY(zos1W@#r8PWfGr9|3}`NBxMr+#tY-2nZ)N; zh4`FF{JWn1dnSA@p#8|?y)iDlHzpJAUEn^G_&1mMmr47viuS`gWfK3U6aVa`Oyb`f zmG~#GpGo|CR)~M{_rLWr3qO~-DxAps&%)2-EATT5Kd;8mEd2ZiKeO<2Dt>0+=Y#l} zg`al_erDn4-b?T^3qPM9fS+0T`5*ku!q30qXBK`Y;b#_p8t^jXBK|8<7XCr9(3Sm7JdffXBK{bmdg@9XA6F2 z;in!yv+%RTfuC9Uc?LhT@U!hH*2m9@_?d;Df%ut)pNH`?3qPki@G}cP4fvUbpGo+c zg`fNIGYdaI#?LJL+>M`E__-NBv+&dGz|SoFTqgLLg`Y;=UzdfS)A2J4Km7$iv+#2` z{gW*G+=!o9_}O<>rcaWx@N=->XBK`Q7W~Y@&#CyCg`X4gGYdbBKXRVGABFe}Qvl%}gW8|~td7tpR;%x3mS8+d@&G_-qCh{%M`yN%| zSvL3UnFi%NvKcRVhVhbY{0YXNZ2Z0kzpYa?{$#q7Z+T7>&i>*52Hqbbr)>P)BKVsP z|Mf!uOI}Z2X5;TF{FPHS{O7>`rKW7gCmy>|<$U39(XYzhvZ?=Xg!<2hzc7KnZ0dhM z^`Ff+#c;+ctx!cPwTJf*@<4*c{^B_Dno%5Bdh2Y!~rPY(Pnfu9`s*#kd0@Y7q) zXUp>$fu9`snGHWV@G}v9axBkhgy+RM@Z%@&lLJ39;3o%uPQXtN{LE6}CkKAo;3voO zyp89KIq=g#{FGA;{CpwslLJ4)K2q}K_2gv^{5&e~lLJ42m%&dC{A^L-CkKA+6@GV{ z13x@-Chp-{2hLB;KwN3*T~l=FLU5W?|`2i z_?am1lLJ2o;U@=v&cRQv<#~YHASsvgIL+@faw-4mlz%SypIh>&|A{=GvzKzI|Jg$Q z=fcmw;3pS;o+bX|avj-RM=tfhllqraF8s`eA35bx|3|6+fu>yApGPG25BF`|N}Q0l z&!zo&ocs4&>d!QaB^=%ehq<)tH*RG~`(5s!{qEln?Y4n-JJ<5OK&ZF=eA@51+>hEz zxwQXFi#Q(qeeZz3T-q;B+OJ&vIgUTM^rwys;|002U#Dola_L9yq#u<_J#M5Pf2)*> z|9SW?r(FD*fIqqLvlo7HX}5>-`-fc1^95l%Gne>tg7{;da_Rpi(f`Y}JdYy&$SD{9 ze|6_N=>Hk%|K(Z6zllG2@K+Pe68@frzdZQc41am>H`oDxdGNPN;4csUg5fU@{_cmr zJopQMzdZO`CGeLAf2#!k^59Q<3H;^3-w622gTLRa#N9mj+X{bq@HdM1kq3X~otzi^ zndw*N!JqR9`|(RIuRQn*hQB<^__zvxdGNQ!0e^Y$cV{8TgTLJl_{)R8qwtppe;>kM z9{hbM@RtXF!SI&{f8WAi9{er&4f)H1zjFeAdGL1?{N=&lI{3?jzdHJjd6x0?M1}i2 z_zPCyFAx6C!(X1|`HR3`9{d^LFAx3(->q;dm$$skhmY;>k#89nr9Yd``Ba^AmGa@M z*;}c%e9liddXSXQc^L#R^0~ioI^rVbbAE=Q%5n1Hyv1Ai%=w+5f0xgF#Xe8lIF^qe zTKveTd>;5K=Yt=c@gpCeqKIet__0pKk9_*marCG2@na``)ex!UR*MkQKh5lE*<$1|=^0|NgW&ppp%g0Bj(YD8t&nqAP7xBEx zUdo62sc@f9eEKu-X`m^ec(k8*WG@xK+fjHcu#6MJTLHW!!dn5njUb-MsQ}*2z*_;l z1q-|tz*`yjvjype zc^3M}KB++P;J+sFsF9(da*@K#`XegJO; z@bHpDxHk07#MGP^%>TOx7`A7 z@^P+oX@IjO0%w=IKLfn|pmM(=@6P~lBUE@ZSjKO8{$YT(x__~Rx0~V3aQ8h!s~P_^ zSjKOK-?Pg5la~f~3wOX-l;jPdz-Ux4n@MeIwLU&R&w88LMJjst z1OGp|x=KY(Q@?+wl;sPn+^2H?Q^b1&KIHf3Ma&bMvyAVVhn3GftRm)>Y5&e==I^e# zm*wAh|7r#6F&}H?9pp1#ZE5`=sfhWz8s_g7F(1Lie3>HV3+LZNKJzNx|5}oYc)vjA zljJiW;bz_^Q^dUBgc0O3|6?q_lPMa-yoL()b35<({sZgrK8e7q$Y(xt4f7+4m~Roi zf%TXVR`U2-~_t2<}JK1jWKi?EFKiT(;;H2R%im_rqK}xD`iSlEi}C3d z2R;?!)6hoS?c}p6#-}0pRE$rf@u?V}W|XkRrasTiNW!KY$;x)q;_@u>};it#A} zpNjG6@Ay=VPs8x37@y|iQ!zem#HV6>`U;R7O5N^_tiZK0Sa>#rTwoPsR9@gipozbQGV8@#)vyY>!VvpX4(>HJP2H zVtg96lh63{3qBR&lkrhL<5T!y^6_apJ{99r2|g9$Q?LV{it*`w6`zXn=@dQ{2xg7YOE^p0Zd|HQ3CGaL)=PZ?QKG~d43G>m)j=M@F z@ajSQFM+oPc(YC=@cInAmQZfTDYp`M%Z4{Ol~7(YHA?i0RezT2c&Wk2%vzxA>N{(t}d#rIl*w>RLey;Op4pVEHWOC|7j7T!wm zdwi87mEiB4e__dd$<$GlJN`a|zxGlI{(9rDbt)m=93bA5;O||6za_*+S0O%@;BTyo zza_-I_9EMUNeOYTg*aD2yqQ#>)I$lrhvIt)anYH$XfKrzA14qW?WGdpV-xYwUMj)& z)%adQoE*8>cD)jOFT(c{;^P?Nqjf65_v~lb{x#k=9?z2Yz>oe_3G-yD^^`llZ^QQz z;`)gDSkfN6KzmR^yd3%<*NyLo@V$h%K8d(qLVMs%dr-oB(<#h1Eg_ztC7zcs57zxQ zmc+9i#Iq9GgGQ5!REke7-`Sp*e4eG8$JjLa)1@ycd_tCH_n%{*)4ba*02s#J>dMUny~6 zK5?Oxcy~YX&N`Li*KPP!N?a&AYMsk=#DhTMK`HU>zFjKYl|Dy4ye)#aQuz7?zDkJ`8;BF7_%#o|N{I`5RN_J@ex>18DRE&XaiJ8>C&Rg% zO7ZHCcvVVVI7VD3h4bNXZk3@{co(yHaL@Du3OT4pArNsN8#QRd(lVG7ev00DyWYr<1Jt-v~8Hh(^9DfGw zSs6SX7kDb8KL192mT~+E9N#*XQNO>&lfQ@enrE=&_@_C38P~U(>nr2>_R#*9F>iau z&n!89%mL;2Wz=ucKBa!k;Hg`sziNHl|JLU+xP8w-dsc==ZP##}aOo;lIm7KUcgo+gKECPJDcqF7?OmgleEF|3;>Bwf%6k3Zt)Zw3eLSsA`fd)xN7{l_O>pR3_Gc=xPIdsargoS2=#2#;m+^|@oPPP8S!g1ei^sY{~X2=zaGOcBYwH#mvPSX*Y5gi zkYwcjFC(V%J${wrmy!36q^Iyber>`pBYsW5FC%`1;+LF^__cOF`QgOn7MA$6ZZF$E z&ifYkyGlm<3jJU7%ZOiD4*W9W*Z3C70l%Jc;Fl4X@XLr_qe>*n$a_?# zJ|Ds#CuMs%zk>eZ%k&TJ zr3&t^@8SNsg8S($G3=lF^}z95ALIRh<#**3`2WC3mc+y3#6x?jf_Qj1Z-}&3y-un9rcE*dN_d+n z@Kyw@P?(QsJ!<-cG3SRtaz09q?8OZzJK2#{Jsm@KyB!c>4_AD&b8p@KyTySJ6Ltc%E{!D*RfBUvjFVf8r(dPpa^% zL-4B#-)p{A%AtQdcyog{>r_Si9bmo4u;Qa_>A}s_^gIc^VPQJDW7*Wd};+gtKsvAz-KjlE)w{xhR>8xNveiV z%?#W9R>P+OKC9t#4189@XO*yCHGCe1&uaL*0H4+Hc}n238b0&jvl>1PDtuPM=R@#W z4WFkfm3pa$PxE6+z4l)pd=9%Hf3jUx!)I`VlT;0#E8w#lKKEY&pVjbr4nC{lb3c4m z!{;~^KC9t#D|}YNXBGd)yc#}dz-Kjl4iosShR^K+pVjbLCh%DepSc2`)$n;W?U8k= zhR@k5d{)Efn=exS@aZ4S5KoWRKxyE>9$;0@3)5j z>_VR3)lhHiXW7=9_4@MX8hFl!=Njs(L1?dQxPO^cCrLH%yb_*kxSk_iPYvaBhVrSw zt35({T|<4HRcWtl;QtezJIbks`bwa_A@=hVRS!wz`v-=6!g z7?tOHHSjr0;IoGNsp&l5l2Z-$eV_8Y+g_@{mr1owQVl#u!*dPs;WY7~2A&J}eEz&ui^e^?6s61d~TqBXfM^mTb<56Z?*8Y0p4oiEnxt>)xz7I@Ky_N zx`&new0C^@`&xJlhPPUH8~&ZLT`j!bUL#4h@U|V^YT@lXyw$>66ui~K+nWM!weY4> z;jI?l`~}`>;qAP@TP?hufVWzBn+|Wa@HQLXYT@l8fwx+CTMlov@U{xxYT<37z*{Z6 z4R_E#tcAB-@Ky_NN$^$+Z?oa87Tz`vfVWzBGpg`b3vVj~-fH3P8G*N2cw6Csw_12x zQRgJp!rM7`tA)3{@Ky_N!|DIk!rM-GtA)2D72ay$?Ll~}g}43iR>yttOZ*PZI@Q73 zp(tmmj&Y`2_#Ids_wPqley_l<_Wf@mUS2HK!P$gI6kaa6KI2_WojESNorJeKc+(Ky z>Pi?d<9T`=e65DBI(Tyv+Uq)aTMut_@a8MC2-dNJ-prNfVVn$dkx;|;Ozpu)xlc>{eU`n z`(EI!j`5;huPgDXj`6M>2fy#BgSTz)R>!#4)BLVZPIZh2&f)i~_EH_=fh}J;Npf!Bic&mrEBLZ*r@U}*U zw|aP+?|`>@cv}Q-_3(BW-s<6PDDkZx-ckhK>f!Bsfwy{iJL`bAdU!huZ}srDOW>^@ z-X=QWtsdSce&r<9!&_T5OL!}Rw|aOBhqroo+jI%M)x(?4L3=85*}(jy$bEyP2Ie13 z5$0F*=QBUaZJTX9_U9kHbL}|(zwpIU!|S~7&&B?Jkul`e6uYvhzn|#RUy#Z-2E8ok< zZ(#n}j6m{-+%xoAp2IaTKXma!%Jnrc|JCh#@|lP6^;~892Ih-Qn@D*x|MlEi@|mAe z{W{w-KO=1kpP4T*f&a_Z!2JJ|z4qf%|9SL=X}0jYmImgZmEO*h`64x&*`9f6zcMdP zP7TaIYrNNXe+|rUtN74aYGD3ZLWrx>z&zyUZh0$Z*}(i`r{Ae;&wS#OtJt3Ty+_j6 zp7$8c{aDzJ`NRu1kk35dZXNmebDUI`ny?L@6iZSA?<3f!CBL&`nXmCY?_aS_ z4b0CNQNi`i<~TfOYjCO@*|1#c4>m9#Xl)z$%m*5KihSmS%;Ehsa%y1yN5v29pLswp zj#t_bIbU8jFuys!ROR^0qdfE(+cVE`hL`R34a_G_$r6soe4vQ;ZOcnOj=g0gK9=EQ zBR)Q%;$tH|N)CK%#K+7bw&!nsyhePyV*oxj;^S8Se`q5Q^dd^~`UjriE0;$tH|j>gAEd~D*mT_Zkj5PWRJ$65H;h>se< z$3}b{fsc*&cnBXG@i7S>8}ZTkX+GoQe0*%g$C?57*ocqr_}GY#*H2`LkHg+&dwe`B z_}GY#;VM2h;^UBA&Qc>jR^nqLK929Pu02U=#7F%QTfQ3cF%%yg@zDbx8}aeQPn7K& z@o^G9HsWI;J~raxulU%Ak1O!85g*UuVYl{+@j)RBR)>S$3}d7A0Hd>F-7pP z5g!-gVxPd8B?E2xhq=C4gQu!PTH@Y%%t!m;Vnwa+({0&R$ z!|8VW=g~xbId~P@;c>>hEa7z$yf!hv@!!0UqKW!2d$NE0{Z8;#KEAwc!sBaJvp)5E zu$U$D2bc0biYEA74Zm_~g4ccU+Jx5`D(~lMg4Y;$ZNlr91h1R$_dVLvCVakqJ^P2x z3j&|={^ey8d`beJO~l8Y#K$H)eiV?`-Aof2(CPexC2OJHA#?RCE*^Hml@Us~|rwM*GpUwCgsp4ldes&kJ9)1?#XET0Iz|UsKEd)~tJ&Zj?XFST&JCx!lO3-jB*dr|Sb1)h6d z?Z+E=zb){P;yg%dVSal`r>!3?uZ3x(b|qK{E%+7k#l`n4@85da0xwJ9r3KIa zq0*miftUPj^6@N7@T`UR5xCyWdBTeXFD-cXIG$Oj7I-;Lf3^j$zQwB+;!%RopKXDc zL-5jqSGNgX$@^)6m$5&Qk6-W8pKZan8}Y5>I*zxQ&+sw_UM`ke;KdbQTJY=+?k`)2 zM;nMoEqHd@GUdEm@a>oB%Kln-AHmX>SdaL#p8Lxd;Rg{AnRRZz4Xo;9Fu5-{V#3uUsebXNZr|pHQ~^ zKmTeW{%j)tv=E=&RN`|B@w|oK1-1~M&k~B98{^FR`{skcayEuOUM`U z-s~@1;eP`Bw^C0hiX5y5FYm#NoLb?(2L4;&X&*ebQooYQeM$fN@Nzr6Sf^HangdU* z@bVnIv=U#mE(*u3@N^cQ?4?%tKLG!&a5V|8TH(b7URvQJ8$RUJ3jfpKzZG7h;H4G5 zmJ01#EBq9}Pb<6x-b_BcjQv76-&S~92v4o>GgIJ4-oLzTg_lp^rIon3QHYzZ@YFy% zW-qnE)20FNWN-UR{l1lWo9`grwi3U+h~KTmZ!eYj-AbH2Nt|uPpYJP$`XZhW?tnky ztiKRvTZzvbh|jIGKWk}!T8Xo3g*e+vd=4c(x6=OXqWx(lJ}w|Wwi0i55^r0H&tp~E z^H$>R9O7*&@p&KdSx&9Q$K}LF>(s{ijqgfZSZYW`Q2^XPxMpxJ!czyKMmh)ynko*IhOb_1wYz& zU(KxbEaBVx4}4GkcdFEX8~$x{c9Ghs-zTWwHvB$|-*Re$?~U-?M*Uv*jZ*J&zPxOs ze*aGWw!z0_BTGCl68fqA>*4p1zc@>6)blGU{na+SUWnIi*W5GIZ!!C!zAGPP2_Jq> zQ9i_158`VZK2H;TmiI3&+wglAez);HnoYcqrVYM_(qCXKFj-&muY!gm{S`RyX|;d=vox8cto{AuHTIzOG^Jc+wM(XO`9zPQuAwBgTc{As(5>*RT< zoZ4t#mI>{P{Jp$vqkY*Xv@i1a*2^~9mjv3Ewm$AFk1E$^z24=1ZpY8r%nPti?fCiY z75LeXpVfk&?f7|4@UtC1Kf44!+wt=xezxOhLydC2?f4n1;%7U4p2W{~{QML@+wt>7 z{A|b1#rWBdpDXdR9Y5FbyYzPa9L4>nb!x}YiB2w3JASUi&vyKb5&Ue&&u8$n9Y0%y z_-nHt{5*l5?fChOi6wqYw8Pe^9X~5*f7|hMzKWmi__-cG+wn6KKilzhF!8t@KP&LF z9Y2rYXFGn5aNuVezxPM4nN!RQ-`1J__+l?+wpS_ezxQ1Qo+x5{G92)&vyJgG5|l@@$&+Hw&SPU zf5FcV`ajv{Z8?!ks)O;G9R*4|*ugkQ`v)#k2Y!{|R|n(UWhpG-^GVu|4xTSO`GdlF z2lw+u3C>andp19|_}%MSXld&UfsIv7uWa;xHD2Y$KYR|n%sMZY5-K0k!d4({_K zqnxA;`ggZ|Mn2<7E2oi9eN3Z1I_RILy(65D<@w<4#K0`kB zbHPFVbm0FkVZKEN_2VkkPY3>YzpmVucksMvexp#1^j}j}D*4vsVDEDW<5a`HRJQAY z%eUcDP95~`R;u*x6|nPWYS(pPleIRp7G|KBYuwsS`f42f$}1e4d8SPWap|@YxBUx58&9e1^hj zCwv}<&rbL}3!k0vner+5@VNs%JK=K#e0IX8R^YSK^1KW_JK=K^e0IX8r@&{Y<#`!= zcEV?Z13o+9bG5){Cwx|XrEuN}pJQpC24gwG2CpPlfz2tGUEa}s=ZTArW5XD58V37?(txfnh>;q$g{B&icV&#Lg*37@`~ zz-K3XE_J}Cnfh2ueVE~F9MAd8v}0HQ(Md8>|Bq4saxzmNbEpq9<+Jk;OU`#a=WC{2 zyQWu?%+yC9@xlxb*D#(SCo|`JobxqPA0xQmHN#ole>e{9=1Yttm}&2IRV?8v_H!k^ z$lF^l&GZXa(=RYvo(ECBX8IQ;^e?QFneyF6`Ce%@g950O#7Zs`)`j!+gYB65`V0dnf~-##&gWXpH0LcGjVk;akY!> z*9-Ad-mZ)5NpWyJ@_GYZc2OUp)JGTfv5ETV!k;yQKV8&E8THY{dA)2oFV6R6&bN#5 zD5E^O@TP${+J%QscqpeX%kwA7qYGc=;ESBPD32+WN0;UKlS+AXQ6Cqmk1on%Ddo|H zx1&B+%B>3zO?cRa|9kMi3-9xV{%aRLZaqjoe$T@1F8p^B;<0=l^0MoxN6bsve;08e zpE%Hkmr;1xg+I^YPZ$0-sKnzgeBOo6U6$uZLcEmsD=)k7*&Uy|@MIdEbP<2{5P!Px z@>{&@B97EnE9KL_9r0(^CB(}vd>$dh%PxHUs}S$I@VUc5doCZ>dfA2V+4$at*V9zI z?!wy(c-uuh2qhkLS)L=|v%S=XKe_nRg+IB1KVA4fLh!vC-Wr7dg}lFR>S+h{)D3U$ z0&o5K)Ynq#tJ^X@&+|b!byNOblz%tnf0FXQSn8(y*HZr7@OB8^x~b<8JSXhtzB;at z@8NRlF^&UoSs#ui67m>lNme*lv6kLdprH9ZsNtu#EWk3XJ6*|sCDWlzFinV zeCsCuxDtQtrEcQSYMy_06Hkt)#P@FEPrge0k@we4{JB8<>1MoSj>_}*ZsN~r;!ijC z)!*}+);e_)Kf3(m3!$KkmLo*z=-xd)!7!E=x0`GCN44?G*- zxd)yD2f%X=JTHXj9(eu*o_pYV8a(&Fvr%FR&&H!12cGxrB_E!bO=Jnr$1Z{A9(ax% z0M9*^@oRYQf#(!~=N`-R0eJ3#=b7-_1JC;eo_pXq1D<=}`IHLJJ@9(;JF8$M+-dnz;oLRlqWpTh36i4o~y!h4?G_l z0M9+}Ji!6aJ@7mSo_pYVn+nf8@Z9Zy=N@=I1kXM2{E`aKJ@6a}&pq&*q{4F#JddV5 z?1ASZet+M~IMx$;6@Po#zKrdA;rK8d_cDH!%J@|;<(K-TupZloTDFI`6U<-jh2L!W z?X`@r^LxTxe3*p~y^Q}Q@%*Wm@xkM5x z&ot|GoG! zU+|@uIB~BKCwlS44PSa0KfU*9_Dj6z6yilMz8rVpOE11O;7c!YTT4Hr58kH;y!XMo z>$8KTK6rl*-uvLa4&M9Vy-wi058hA0dmp?nXFfN_rZJi z74Y6?8J~vtK6qcpe49Razn0&7_rbdX-uo=$)9~I0?}y;M58lrUy!XNT$P}&{-nXgn z-UshT;JpvtPYb;F!TX>~;Jpvti{QNv-sh3|97z6HSY3Pf*!(g&dQ1t|RhN`HXT44e@4 z0zeuBKr~3d0+c^WgM(!>JVsc00ObKTAojNWWSgKGW(nsTrM};}Hp~dlHx_!)X&q(^ z=bQBO$jk^cf%8p&ckA$lnZo&INlmp|!pz`&bJ12`r7&|i-=ftjU~`xSoNxH&*$eS7 zLr5xQV3<}M$xr~yyD%^v$Y(&}&j9iRkoYr!`~)QaEFj+ii9Z|2cR=FL0rCZq_;Z1L N1tk7_Ain{~2LQchn0WvI diff --git a/src/leveled_bst.beam b/src/leveled_bst.beam deleted file mode 100644 index 09c8b2bc29096f50d8948b26ceed54fa5e2481c5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4012 zcmaJ^dvF`Y8Q=3t@t)$dS33C>+wnOkCw3Is=lBuqByh1KE97B}gQ3YlWZ619vaORV zof3xxP#%U*pg4sDu9G1qJSGFvP8v$eBz=S_B|Z|G}6sOZT~|C&u}j93(*yqgh1%jS=3JG5~v92`s-j)W14 z8^PhoxMs6hY(mrXVq^IcJz_*c<9IMOOeSIp=)`pU&UnPoNK`XI;gAvHlX_Sij_6w0 z9*W09yRfe96o%qj$k2k@wNO}#&(q@LA$`O?6d!6{UN{ufcW7WNJyw*|$Mo1vJqWr4 zfEU1|U^J45hK!-@uK%AK+@UntjCfMB4~NDRnoW=Cnp2BT7`rlHK|@OzMT4Pmb{~hv z$N9ls;7uZOyH*<4hGOw>Fs54}vtX@NHn!BRB_ZDV(O5VM>WDrZD;m*^U{=4z_buxi z72j}D_bER6j?j2gbA=KK2yzpPA~+lZD`C49kH_NWzOT8LS=i+Jwh_fw;mZj!v#Rwf zV=Y%1Yre|Z@~ezBU1d!1A=DKMYY64P5A{WX*k=L-fJBa(G|#n&a?*-cT!5$=K%yz- zOQ_wCJN#Jc#7OMKRQCGCK2^qcT8lWPP@*b!tAp57Y;;P+HZ?A)ZfvUg5^dLThn8OF z)nJXy8tKiKB-MtUHgmv6pK9;N9sO9H!*1`FJNjjbo>V$z)Z8i4uI?FI;CMg`GzYo@ zr~db+@+oXiB%i*NljBy2ndM9wcGKIwC6)drd@~$fw z6-WwLWpT@H^Bf?0e}mn|1sg&*c>&WK0Tx=R38wGAfK%S*Mb7)Y?1Ti^sX!&}r0Q^4 zPM=d9Y46XQ=hxY|OZ%{S2|S-QXTb|M z6$b)RK!(k9l)8EWwA3VYJ+x&eajtiBq#S}KE{K{eFo_6pmm!W+6e@_-7jat!<{3dI zDc*n#p~}OOEL3H(3_?21Af(uy$l?e!Ny!GJda%=Dl2TS~LM841nkzV4CzjluSnQFFx!Sll!FvH<>`)kkOIzvbf?!O6&sL?g4dM*$$YIcN#!~ZM;4l-3VbdHpQ{UL z+UJGfvmw;LzKN4Ujp$6FozO^*1yDyOsa838k6q~%n`3slnaaSORF2wZBq4c=9g2tB zVu#YH*=n~|SY-y8sD?;J223KcG)S!0CItyw;1U#R1%U4Y1%Vi9R1WX4OT8k1W0_*$ zPKu*;3?}e5n=r{Db~1>ZIU2U?lQA-35>F-p(mtC>YN1$3jY$@>R4g*d5=cQMq(HWg znF8L{*=TLM;DNk*umTqd9!Z&`9x&jL5!jbHwzAelu&@diE>>OGL_KtXBTFo0H7M&c z$_b9tr!lP&yZ{OiH88NOU1$Jv4R~hmUhxVI;&HFwlLM04hrzCo5~_4d zIN(?mko`Ch`vWQl*2~xcV-r}k#-UyTIiF5*q(Ke9t&{0ALL$IkE!b1wwuBz+idSgD z4i}iPf@=aJv|ECDUlhQ8kr|n0&@+sk%rWAzj4X$J1>D7e+w2uuutQ9H+>#4+jzOGf z>{1SxWP!CpxdPY{0BiFKE3l(94Ync|Y?eU~YH?={T!jU94V(jjs{^=puh4-5a6MS* z?8wC9Kdo5@LD*-0JszY zuk#9U<7D<*&BaqKpamAM$bl=f;MTwe0&rf5Bb{EM3v<9kXWGRsN<{0D!*I#vRuZYS zkgIZ#ODyEIaEk%*GH}!n;vPHg#n>x!V?N_?cdo}|p#@oOaVZD57=STCn~nHQ@O#by zq3fd)Ox^?J0D@5j*9FSU$RtbVGcuf8BmlAXML_KTggLn)BQJ!uDoZbYO=kFPE!@zE z^G{kngh6KZs+kUGX-0+y)d4X)_ALggv?PRNXv;G)TUVBmnIBFdhQs1w_L!X4ozz*p z9{{m#rjzNIoY6C%SpPbvV{%4cpV6;^mgR6x&-`Zd+kseK z=k!Z5`Zj3oS$Y;5o8JsHFH2vW(JRpAXX$G)dKL%EorS+KVnB@C32bVxxH1-@zXQ=zL;P9iTZKsrPlbTdmmlgbL9p1iQ8)qUU_K>AA9}Dkh751 zpFZ}{8@q>gqk(et?$EK9f1Y~r%uTm`L|%=2STcR_)v46p(?{kt)`x2gw;%od^%w67 z)_nO@`|Pz_dtMgGiu%JZwEgL=SK_Oe|8#YiGtaR}FBrX$8ocK(r=q7{IkWc2?#s%y z6|LQe3hq6&a^{1rhktYm38f!Q+&p~C3GvAC?%n0Pf=e&Q4vg%cREYoe04HC&t;Tos zomJX>$%d9Qk3WqI?|8;_Hhj44!#<(-^x?lofA)}X;$5NGUiQip6$Da$Hy}WmJYulN3Lald}k1m~AT>R#3-p?PKJ3t5L7@3S*)^G6huO84U2oYn zmR(obwUk{y*)`*4H3P)XaA8I!YYZa}a^3!MrrDksMtDa;w>mmS_iy{&+kgD?idJuj z{U76_!m82HuIu$-n0c=?-^S+W$0pc2Gcjs)G7rNsWN&Ry8d{jnW<;Xe_IZO)%^!vA zp^}goY6&&7k$H(&GCrh@q4vgpcz`7uqwubc=!s-h*W!)g!FViYG{URbdijP28x!%N z#;kX61H4p6!I?GiCWZ!Wgl8=@%)hyBJqRp+h+oV5`E~sD{Ca)^zlpz*{|>)}zm?y{ z5Ae6~V|)2&wrnPkUzx#fPaX8n4jh!;E(W+ I@Q)(&Ux8U#yZ`_I diff --git a/src/leveled_cdb.beam b/src/leveled_cdb.beam deleted file mode 100644 index 301deaa84169f745feaa0f75d77ee0ef31f00efd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21212 zcmb7M34Bw<)=#);J1I>`n-n&ZvWi7ObtX! zUJ%7y0hNbOp5g{B=o8SofQSzS6ct6l1s4?k{%7t@iGc6>z5blc+%sp+HfLteERr*N zY)_qT?1w|LGe%~F0+l*l>}s7(x4mhFZ>F!pS6Wa~T5K$<2~-x8`b$Cv`iThz3M%}; zP;^O+uPEd*l=&-shCsEis!3ovs}2PHA%CE%i9Zls7O1H#3PqJv1cE+8X>Db-AyiZA zi=wtcA?TYCRTiiyt%&y3R1{T}3&p`2y_17#%tWoAs3hc{=_?@ef-g|U(737u42A02 zP;~XI8h^-VEUpXr3WEMQKD|RI_l5Lbh2lV2Or<{<^n+wcZB10Ly22ldE~*B_(ior= zRQf8LQeNeo9h&Z|i>(S&25NjL1>*=jttdDxRO9nS71h)f)v@y80{RL@R|EpnYpbJX z7FE>x;vhjIVsveFDM=Odg$z}JDqpm}D(I^T#e;keR4AA}v%1J%Bb3yXM3og)1bs%7 z3T6eWrq{&>ONy!r0yBLzT*)Y3Wp$`7rqowfR9g`es(iByFmP!+@JpuoN~RZZ-a=7D z1uH44C@BD(75hV6eWAuz83lm57xfPHlGJ|h;HLy#3vA?RQrmjHiM5i*z7f#Ffo*PR z1`nOkLjLx?zDq42P>C_fZwDrs32KlZrOgaA+dNwMv&J zHV!fny(nL8l|R(9GEiCzu)nG-KrIU*zS}sjZ`#Ak9Sro!hEh_@) z(kNd|O`zt#v~oFHI^9;hlPsVJ%q`ZPxBJkrRhz4My6Or3N7zFwWF{=QzF-~PV-8bz<;e~o+{oqsn8 zH?rugpTC44w)z@1uC2aCi)5AeHPH5&;MXjzJ>1%K z^WW=TP9v?sJZ@-xy_I_O=2TqHcqc`LJ$h+GqtvJ<66kUw7|jfn`gFQp_oHY=qCZ8a z(=R}Ni{7l5naB5Rt0WjCi=bFz%sNT1NCqLwnr+s*6jzhPu~xkl@3KmkRLQHe zC{m2ooL(#Rs}-z61f6AwV757GEsG?t*&^8lNk~$1W31M6pU}^zVmN%LO|>MXN=brJ zXt0`U)5i(@#tBxl)eH&-P)N--G_gottDscJSY3_C^+28nEJ;YsP9+4FpsXbXgGEZU zC8Vx^+y+6}YH*p>QDG-DOICfZQaDu57b>~9PB))Ifua!<)qOH&Bw<-u(C=`BT8AMxSE(FEF-XQgEs|bPF2-1MCA~!wY~Yz-z_+VVxi~M% zTrDIBTv~&wPOhLZGl@&4f$l7U4D*K2?+wi`Z-9OhR1wO}3QGiqdP4%}rmld>V8jxM z6ZGbs@X1_j&HBBd1DzU_llVQUFiVrf*yd*a#lB~Rc$Zo77FraRM9P?23jL6-_L8Y( zL^q`>ESbbHr2s1>5-WwYCaDzG933z+-P8goEjUUNI?Eu%wot2Ea01Oy+dPs$bIT%! z$!tpki3G4oi7`tKQ=+hFpeaF*9tyW^iPmPxoGmmFvXkI*-fD1cMfT!0QCMqEJ{jbb zBgrQd`D9LBVQpf}TOFn(0UpMgwdKq#!XiCOFuQa)R>7O(H6*$$dX@-LU@`{#w52== zttn3iC(4^Eti>?oEuc4jr@%s#w}d?@Z>6wyuyspG&swLOTDkinS%yuPM3 zLY!16S&}4gJqCh7%1f0pJql~jyP_?WXd9_STgxJ|%avnwdp%wZ3_f%WwkXNe8syuU zBo|0|;0jV+l9Xv6H7s76!mI|f=>c+oxJ0TT*w>j+{gz9yT1WQjv>k;)ROtgud8*-lWlQ>3z;R6oa1f{LI8vbJiLZA_LI{FLwtPt_Raj@Jc#kAxDXa@hTNEZjnUf0zK@&T+J>(8SNs+rpS1{|U zGIN-^f)Q2O(QLJD(o-WJebzV7@(TUn|9??MYI$nTZ*9U zu}E1E6Mo~sxVT?o-AFQFp~Z~ZQ9vizys;g;u!MzE>c%N`0i`Zs`MOxpOU`*wx_VCn z#6*eOYo&%%41$F=%oLIVy_Ku5?lG>5>TJBiEGeuS>GiH_cU;`KI z!>#T?15eoM9w1|NV0~4uaHGzR)yGAvkA-@zCDWjskfyck~J>ZnqWvs8e-AehFHuVu5hYkfd{8^Vfuk#zewr((G1qlmW3g# zuzspkX+TPEM54J8dJc%UV(mu< z4uYT;+WfF-IB?k$xv7h8ElBc zWT?;+Dr8Eh-O-XMThE}wFakV`EcX5dvL$sHUfFBXQ_UKmFh?QT?%`z3f1z;~2EGL@#AWVDqV z+rs`imRPqG>rs0w7!L6% zAEB_((6cveI5LjB#K1c$H%7-=glFr%n62$PXqAiZO3{`sK1Ym3vr!5gLlj`X(eZ_v z2S`G`!p5pb7z;%ZDr%8tl9Z2O0msNAJ5Wr9l`|j2^Fh32x+x#n`Hk36JD<-J3L6J% z<6Wjap}%0tgWJ1Jxu;EI^emS;oyVSA1<8}YLV}yzgb+jv9Sq(i?y+MOHXgHfLlzBK zl*|nRIgnaf=qB5FXjYTlS)9bpw9HsSXgiG^W!RFdWOBo@N9c@U0l3zhT9rsJSG zecz<839w8*v!0EgYnou*D_FXiCS+LR&E}A4qF~kuwUimHW}RggO4iu^fbltFB9xu% zFx?FEoL1NOn^z#L_wIzqlVZ%Q74#PpbznXrP6zm2z$gC&K5-F1Q%Iai3Y!Ws20)C- z3M-f!WoE4fOg;sehFBX5%|25sdW98^GPAaNHbr4YsHtCQ0ZwrsN^ddSmPOfgRGZ5nVP&Wu0!Jx|)1hn1)ow0E zHy1~CbFtRV)6lRz8WsbUq%TpJp9n~LR_ZX7*kXIdTQSUTZP@QZu4J0Y|E3At0TNE|}w!*HDO^1_73(KX3rYb)kN+wZHU~mZM2I#39MNdr4hpF12#*Xw^EoC99c1xw$2W6rGMO)Q+ zHp64849}T0kc7+{gs_x{6jle-yzu4PI2Qs2iko9QpwBU#&Ee|Igvc`^%{-I35;nZ` zh7;@-uICEn`Fu(&hRO}AqEax34y zSZMnq+1yf|jV9F0B!$g`_SoXgfjn~>110SIY_wm&K7?jnw-&|ChC+27(=0@Xv{cMN zYhrw>!sZWk>)Bj|Er0++fi+K68%q~7*Y?t^p0>DPOO~L?bs@LJd~lv0X^Ht{2{-L! zQY})GObQ@#P4&=fKAI4}1qxe4d~)@yKFzd9h+E4&Cm}UMpxs3&1!b8Y%+Hbq+l(a& zTMX+VZh`TaV4zq%riFr(k5!O&swWc>JaJg7XA2=0ZDW@Ltpj%B3R{N7XSu?b(`49@ zCd1{J@=jW)FV#@_qS~t~z=CXAZY9AD2$xov>lJnz&I`gSlf&N5RbC907e}hR7%CfR zb>l9u60}myg}NM9o;QTA`tmf>Z9;}%bElh@+4><_huE-5wwd$NOw01(atkx^3|TZizzy7LFm%T? zlf+v3R@j}DQS~^tvP_7gdBvb+g2%KX%bd$YTaVY0ZBzXoU4p(}Vcx1$WM$h@1qy#P z90c}+aGS@p44lbTZ&%nlh@14ySizhkRcHc*vR?{+ohb z_jLkUV6BvgfhB(pO(vqUm35|(3Uv$uKX|y0s!G)uE(Q=bsJ09tzbqD(pcJ7zYCP&}LGwNG+42 zAlpOe&P~V=#IY_yo$gQkslkw&9oU7w7C`b$h;`&q+slTrjMe?L;+TD z;~3GjNzXQWOo|2E?p4@hXgUGhaA5JcO56A++4w$%J%Or;po$ZNZI~&l`Hr7Ttc}1u zajU|%17{MNKag(P3OCs*SmPzWKP7Kc*p9hTF4hra+LDSh3w)~sE~-|+99z(i>G4|ELtf3L?Dh_cwsD7*F=v@#(3 z%$CY1%hgUuQU_z|#3%}jJlfDyzp%6ki4t^wP+_~E*A$(eJ;WnTW`-G`0mFOBalG-* z;X2;w4=e0hpiKqZBUjOO6Pm*Qg=|ZCp~9Ziupd>}^S~|u_G5(2Hy3+2R<*)j(6Am? z*o(j_1lAKAD-N7r;#lu;oe1`9>8}Zs9-L^5X=&2XLulDCyO1U zXN4JEK)~d1bOyILs5MOH4b_o-Fx=3?N3&!mG-_tBl*4lm!8&w?_!Ly6q5CwdOz*b?Jf#`VS19YN#BcYw~vlc-5kPfX4 zLOE1EtXX51!rmn-JfpB9utF)U@K5qE%zj>$f;Jz4(#rC^3OlNK*Ru-y7??g_?ry|v z3Cv?0vq52>Xqf*}*nfao2F&LO6U)4pwF2gGj(JjHCp66G74|7G%YnIvFmb!!#r{oU zCpqRtg`LtcUr^X-U`_+(i-g&eFxvq0Gmgp33j16`eMwz6SB>AikG~pT(|-#=7MT8@-0uh&C0BqSs602!x=ppST!5 zg~h<3F17g?_iOx@b=7W)Xz$@=ozUVJI(G;kN3hO-{7U3_JWx)z7MG!e6*hVeyves- zSJ-dRr3$*d0b42T_Z#SPzXTK zj8=C*5gF=cpza{AYoUlnt?rN_#-gqUb?=b7da;%(VpB-cRuSX0r3}EU3s}ZA3z(p; zRlvb&RgTLZ(WKFSR}teuI|$khS7|p>X*c7CKk!HqS9E;A{9m}VRKx_0%X^BL2reOT zdH*UGi^?VGf1|C4$rQS_#yAv{5eU@dA~Iz906`LthLGX3$c8nj99LLkbB+H8i!6%R zfp)IyTOG$h29nj49l``bab)m4}2iqF>EUyg=--vLk?)%U5kt-*XL-SiPA z_>UrY;7RI98|wy?cI51I66U7O4EoPW!=Ss3_DDGTXGEsGx(z)7I!E}5X;4R657b$S z8;$MOir4`v(cEzaO~~Osf&`Qwg`AXstcV?<;7llZEWVnY#RUpBn7SP~vzPV6XDZj| z3mO51VVHWIbq4&;L@rck__A(BcwefB+I3aFBC6L_;*GAWO2I(AuKFCrK39o3OrL`g zRoOAxaAffr>OKn_%)_@4Hf&?~1knpb92_D|q!S{8i(e`wtkf%XAOWi)^ zWBC`mVIIGq&pBEAl-tYPAn622MeL5FrIT3;8*kV;;IxS3>dr}@0_js>kxywN_fSPX ziMo@#j^s_{tz2r=r>OcA+VC*efZ8P7;S1RwZgCH&_at>BI(-qc4FYz@=!xEd_X>Ay zn-!SzMfY$WBA9X-*d7!;bP1=3xEI3R^$GAe5f<$PMB@>Q*as5e%qfX)G9*Q$i=Xw1 zm{u9ZV1aA*lauRphzc~{>;u}kwSiIk(Ro`EjdYaE{F;uJ_%)poTRhUCKZb=f-E3PSC>zCawnX%1|piIPT>N}Q&_hMf{bR>Xn)O7}nTMRk`X4&t}kpYZ89 z4G4qbyRFJ`T`Nl3aQ;LQ2a&J;2dybT4%sL_0aH=_sUo^4KdFfPBJdP?m%dLcqJ#3! z;DwZb4o9W@j3V-j!!Iy0==)1W%)l6$jWP69yo<(AdU%u!!P@IEeb0rXkwn)wu5{CP zu*r7bNiL++$FW|I-lN9wgASrP&8T@z1^(6B+NwEn! z1tb@Nq~LHG1TQWLNlMXh1Cm*sOg8_`xt?UMN-_#v4w8#OGTPyc3F|GQ_HXp|a#};L zr#M=r$S{IHaS13I9nM$?DXT)p-hkvVPG&g&9dSKLuS&8h#xh7Q1<5#v(*%+uRg$I~ zkQ~LyjOM?&*OSavNycNY1IcANz1U23LUVWmk}x`~4yPH1_25ErrD(nZpD~>FSpJ)L zJs(NslYnIge3nD>ME(kanc*THm4yYG7f;V#_6Zr4M>j}D5f{2VefZ%N)n4${ZRg2x67>ONl8`&-0iRWNQ;j2`e z!`ae;unU6k9@u+aW`1$bc_Q0OK9bHUe&*wQgy1 z1L{*b#RC3Ycs=#zDs`;>3820b)Z20D2`csW5!BoNE%hQ!xA=PM2`Y6gA|_B@1?n9* zb(2cHqi{MbddC|OFX04L;-%LUH>t!iD;hz3H9R*}^;{o1#i(-V6d`}78?aEVQ^tvx zU(dp*vgnLm6IiSPi!LgQX&1$tmw9A>U7yEjgy=l2fQ_=2r>BaOkXFtIt zNiMu75pcDGSAw=w1e!v&8~S!}%6OrDGDecFe$^&$3zQGCXfUB{Eb>f|~+r zy#B!XgxkX}eZ=tw@HXdaZv;Kr7J;AUiv2v!{y4=DqHzYJ7yE-SRK;QbMvbFKFAl&# z4%E2^_8I7K4hq|6(Enqf+1K0WtZJXZICcU-1@@5~&WsM4!84#}9oT-ZVhzDL7O*w} zE7Rf3(rZ{*z?uWBQz}+A&c%SW8CW?Er?;nu~IbX(;5b> zxxo5F#lj1iqrln%tPu|9NZbo?M;HmLdB8fVV&QemVPM?{tkDiO)37E0YYDJkR4};!&8-I0nnBK?Ku^#5Xb&Ndl+a%4rlQMoZ5yli-EZunEzBUOBC^DU_Ro~ zi>2vKJaa&D;bICq0lH$#L|C7vh!f^T6`ow^rc*$Q4{qo=2Oda# z*>gDYxWn%BrRuSFrkw!2lv%I7(;4V2(~CYF8Pm3I62TR5GHudQ@eqPa)&>tw@&%h- zEW`aM1|5dp3Nto}Q&i-sf4QvG?neZ$!E0_sEWoHs#RnZ!h~M}u1{%hZUl9wT`=gRxyd};h(M>6iZrxB3;dBx9oW0YX)1g=7qYP(-_U_G=#II__ z=?>J#y7{HC1&{v>UVXMjtE&JRyfP9i6|oqM9*5^ur8_G@vQn_(DHn{4t(KWzXG1nT z`V&hK0`~K^B0%l6MXU!0B&n8x&RV@#!>`#13G6C3rCFdfE0WTztCV2KN}`nR z#N!pD2!26JD*k6LTaxxbEY+Jd&>7N;LG7_j>^A;}M=yrxP}%@Ds^zlrw~ZoyHRP-V z%{rB)!&ygX5IA0e)8W)uoC!tA)UzNa<+EWOa)CNUr1wAK92lAMTNN>YuGo%oHy6)E z6tNl?#`OL=a)<%SRmDRMr}j|8;iN|z>H*9QwYTP>b{>gCA#@E5Ap>p(Ntk(xID?qY z2O;cCba|ngeEdHt42l2AmDTuevI-x4kOU+Hk{&4 zn^qQ)x7EsagR`gybhV2O@>bgESn1 z57ywsFGDL6JX0$ZUpb;2?hC>hrolopo(aP^2pP`jWj%uy^Hu8X0ehYb(R(=C{g;stQ`E;$k7r9?6zl?mER(=tAxmMnT zyi6-UhaCSO8UNdj-%_poPvj+9`5(xOwepk5i?s4~YeIKn%b|$}1MWVj%h7=w*R+L*Jk$=&cY>q_u)NdxOY(yS~L^KJHa-MQ(tI($c z^(FBnSwUY<-}I+9*q2O3PI9B2o@`IFn`rRIsSFU=o9O8@_@h(=oP3w$4dauZM88z6 zZ>0Y)zCl-Cgq-9I^LvXmA=!ECRk&gO5W_wg}_jk21+W z7I`d{HTmx&9Q-#LIgO7n{uT{B0(o2n{9X+%Bj+~P=xo;DImqLwtid;F@J!^?Phq&C z!6oG82>3lHlbi#Q)BGKlbDaivBTtNg->JdLXZRei@m;IIoye1@tikU<8Grh&_(`Uc z246#Ez`Njw{3;AzO=ZAS@za7z8vZH*;=c~aQ>d)LS8DKf$jMj3@Y^*w`E9ER_-z`T zXi!}ke}x7o8QVm_muqmc0ga0={xX#5lWa`9!*VVqApWC1Y)@rP&LtY0e8d_7U#!8& z2T8Xu{vr(ypXTdZ7`{-0Bh%TatkIvZ!gW`WlkGTsgg=Ba7>zbbxwG$mA4y9g!oQlEHa<&NK@eowym&cPk;rr+AbxPH}Juhmt}oH_rMEnAf1D~nP- zO38{Ljfed|8^ zuF#?rPGtSE(sgO_zQF9#smYtyrMG)#_`y5h``qD4Ki>P}KJPC)IdiVP{oYLuRSwctk|FVMUC}H=kpu?wRy+cCCQx!w7&B|n(5%gdp=l_zH{)Sm2I9~dZ6#p zDHXxA(U#xt>hj>eT5I}*o1L!h9d~qXnst;7*n0C9-LslMo)MS-c}mZ^*5|s+{PsYb z6~_)%HFTW5X2*98+r$ryu9u$M_(bUCq_@wlI5uN@!bQ)EQx{G6a>}Pq_d8*^ePFZn z;5;_DV)-54ez@wUIsg1)S>eQ1&wWw6%lJKeEAxT1r$!!FwyM{OF_Q}9XAf@Qf6{%v zPgeK$@40jT>lfqp{y4~%I4b+qlrtHJpUYaf|KN_DYd22r&|%T8Q_H?v`_f~Hor_nU zzufPuE^8VNEF1Jv$^LIff3*6iFKW$~^)F9tI%U?5JsF2nHjH}m{Ow2ncyIoO)=zyh z=do|6H64(6Zs^XqZa;79c~kuF6W^P;V`s{NWp}x1<~n~A_H}!l?HGA%*O-R(6DBN3 zaoqaguC(M2Ud)zvRr6ZLmKbf8^++N@88y?v;n&&s+vZ9Y#Z z&D(L?#Fg(&XxOr+$EKsf$Bm(jb6Vt-KmN!swF`Z*)$LM(j?jI^-V2YtHDy|lW|NMe zxgsXmD%`K0-MV^y#|?c8f9$$z@7G^8teaM!@w9%#<0sPEJ^$2ugPL{PIPR9H+q*tn z)wM46jJ?yBW8QJpwL0VP^zPOH4}Cu2?(+2mmVe)-Z2#uH7Y#?+uAkEV)CN*pBOs!s|#u7;Hr5a&zkbeZBI6AtG-}-_MPKZnU>V0f9#BH7!W<} zk5|~9bwTmVYT@zGpT7AY|NX_g*#mDLI6B9_F@IO#Q?JiD+wIvanYHoWx|11|b@NSM zo9~#MGSK_bYp3hlUVg&xW$vZzLypWF9lLFGn^7;U>9TkHpU>}pW#6Vi-;?+BT=U#% zMfbqt4ZWsJEBWHbxEJ;hDqML-`yJhzf3#unl@+_anQdkeLB2!ZvDLDmrFO-D+ z%iY9xzZw)X{#5hbcYn0y_rs(1PwDbd53^Udn{d~;F`+}_23TBw=*EqjRhtpa zZr5)1$~8MW|N84dd-|yjPtIELVPI>kM}O#d=S>Ti)_&9cg?asr-<0RBu64Sfy(bXJU;gQbr|&+0>am;T*9ZRi+NGR- z{ocX<+l5!XNgLkvw_US;#PN53-nwUa%m-HKF0q`kV)BU;p}~^XPkzjr=ZT+w*e8fp7OLY;P(Ya_2)Ib^q~# zbU5{J%EqyVQv-kRYTwqwe0klmA9j3o@rNgOpBVIv=zhmmO{A+_K=SKc;2`t}NJI^>xGZS04E7!-4tpM*s9flfmzNH)~<*uBQ3t^xLJu z$%9LJ96i|0^ZkF~28{SA@xHD9er@#bM}B$muCzApUqC-=c`EYA z*zExgPVw4StxPdmWL$Qi2B%moGX9E;xgz7N$k-|}o{Eg2?glNa+4^+kcWLw~){cy) zB4en?xG6GLij0pUW1`47C^GhmjCUeqoXEH)GM0&qUm|0cC7R3>r!3aW6q`iGBZw!s z{gLS+yeK|f(^ON#vms;WSd;A zOnOtF5u9`yiWG%ZuayZt56O!}bY^R1g4ZJDAQ3+8EwYhFhUrK{kl=@W4@PCeFG7-# z2&VvPfL5N2yq{Jkoq8h?FOrMO)OI9NPb8vAe5gz~nMmEV@?hj$kO-e>w?!hHjz}gX zf;Y_wg=&zCi!1P}pVp*QC(I01@sBFAZsq8qUtSt<{M7PJR963*6hbM|PiGu%9AV5ijx*k59B;hYIKep4ILSEK zIL$cSSYfO&2934G<;Hc!yNq`m*Bdt&?=fyNZZ>W)-ezGHmXc-VNv__6W0@r3b|@wD-b@eAXZ#_&U6y{SFUZnnqUlkF+?*7i2`w)XaRhrOG>_I~z0_B8tt zdzO8ueS|&Jo@*at&$ExS-()Ybm)d>ya{Dy9-(G32w$HE!?X~vV_B#6<`$GF7`x5&q z`yKYR_B-u&+qc=b+jrQXwm)OvW#40e!TzHC75nSn@vzDXW0Pmm|dljODxE%N25^TqOtOYI%lSBZuUf@+`SdzEz$p&zI}vh4Nx~sk}^H zA>S^qlvm4ZL?ee(VCR`~(>LHQy1VfhjHQTZ|X33;2m zUEU!-DL*AYE$@{7AwMJkQ{F8i+{%_J@4{ diff --git a/src/leveled_internal.beam b/src/leveled_internal.beam deleted file mode 100644 index 793f0132628249ea2c7df951c16c564741f8d93d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3068 zcmb7GeQXrR72mztJMS8=*}GkTf3UD;^KsZ-b4P`pukqUF1q{KlokU41eCOM>edc_7 z?5)9d(g+oTP!fRyt=bLwAW$0hk4nUbrnD&{Nom!VNXRLb(pCyZAVJkqN*bY}B<-8^ zL9P5%R{HIG^P4wselzp-Zo0ShwNlige`)cK%^i+4PEo$K6h(!FjJdzH;n znJED?o`zS_v20)33JhEJc+z3A6PXO-*b}Bdlg>MN|F~(7nEv5pCT|9Irn6bo_Cr`| zv@BeN!z>^ zQ8HpWS##3KXVNK?am>84d|sew?J(`TQ{;E1ozbjSm}QWJ4KjJlcKqft;WusDvhO$fn){a1dfy||CMcRt%P=X*p9WU|kjDj(dMc#_SMY_F zC1^Po7kO32I2y)^gb@>Gd?MB<sW1(SVD4gto^~`&A@sh!Qm< zD3Pn;5abLFUUZf~_jcl(cr4NoRNx=gsI1qI9$vO9I59~=4 zBQLomVxu23WVJbZdcoKkY(%lsiDM$((@&%hC=fGEA8{|NH|P`Yu8oL3>z;u31^3eW zD1FJr;NNs_t`F0!;IdE??+Y{=Y~w?y9gv&PolqitNSp&!0alSUg^Fs5&&T2wxN39pS4x55cSd7`#%mEBNaC2LhJ6RYjlCkxps_q*NDT zYe;MjiLFV^-U}0TG#^&8g0B^kpbFixs2>GZB!g=ORVKT8R2QLD=du;+5zW@SY~^}r zFLBu_Se*!~liT>Eu)3uS3YK3g7EANVmtr5uw#;R#A-;^FS*28^u#REtYKjCEqwolY zku)N}FTe=YvCyrGx}-Cn4yfD_wV;f%J&JtyY}lnBn*`Y!a0t27qXMScMvtrna(RL5 zrP&qpB$U=l@)>)iLLTm1dhth)@QL zQLHCAu_9uhg7qBv${>hIG(`Aj=q|29Apu4<3#{mEW^uia;-U|cjV)fHGjS22xW0?L zrpfCF63G@9%Q}>8Ei2?eW{b<#fv{!ZssVuy(roKIqkz;k0hzC&+2}kI1||lKgy4s@ z-?0b4P#eZ`#2Nq=K)M)!4^RXk++9Doz%K`%22=o~f?ox;`YS#eOZcUPk577YVetKc zO2PqF7JM+&asbIko(4df=TVdbmgEW$+dYvbnx6%bJYm8CVgT7-#R>6{Sclnw^J3=Vp7vgIRZSq&;?G52Kr5(^f_d(du*!#pyxo>XxufO`R@psv=pMP)bodXX%_4M`kc299F?(OHq z;mXvXUV8h^gT}??TghEM^@Z`k#vhh9&IJUyLT>6#bgPHY$Nc+O<|eByso%Nth5l=< zZT}y;xsmCoFGmKXV_y^2uoI*{#C)k5b3KJls(n>WhZ?V)z7hFZ{Ek^KBp1E4{oCnP z{Myqq-|sqkY|qyZon5)E{r8tjXP1=*t9InDG#*l4UP7B7glphdaI3fo nr*dE88o3&-jC+WSbKP7Iw~5oaEnF}67`K%h<1EhR$f5if7ID~j diff --git a/src/simple.cdb b/src/simple.cdb deleted file mode 100644 index 14a53c03f8ebf92bd0651a6e1cc5b1edd057163b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2124 zcmdPlU;qQ1QF_FN0~65aHqiJ-ra|I9Q1MY3oIFOuV>CU06T)bI7|jo$5E$y|fCcCW xE+EcMtu$OzniEx)m{Xc+h$L!cU1R$IENTSuF9w+`5+nqB|ZY4 diff --git a/src/test_inconsole.cdb b/src/test_inconsole.cdb deleted file mode 100644 index e69de29..0000000 diff --git a/src/test_mem.cdb b/src/test_mem.cdb deleted file mode 100644 index f6a008c371bf352c80186bc7bc4a60e05367b701..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2115 zcmb2;U;qPIJTwzfg)UTnl!k=YXgG|911JPW)4^yu7)=MG`CzD)11!MAD+t5}1_p*P wQD5$O=4F;-Cgv!VCFYc-f+VFNl19}jYVpPSxv2^zsTCy(iNy+espSw=0CR*#jsO4v From 647a7f44dc6dd036ef5417dd15f1d6789c6bd368 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Sun, 31 May 2015 23:31:31 +0100 Subject: [PATCH 003/167] Tidy-up initial files and add testing to optimise bst bloom filters --- ...eled_internal.erl => leveled_iterator.erl} | 52 ++-- src/leveled_rice.erl | 283 ++++++++++++++++++ test/lookup_test.beam | Bin 4096 -> 0 bytes test/rice_test.erl | 59 ++++ 4 files changed, 375 insertions(+), 19 deletions(-) rename src/{leveled_internal.erl => leveled_iterator.erl} (73%) create mode 100644 src/leveled_rice.erl delete mode 100644 test/lookup_test.beam create mode 100644 test/rice_test.erl diff --git a/src/leveled_internal.erl b/src/leveled_iterator.erl similarity index 73% rename from src/leveled_internal.erl rename to src/leveled_iterator.erl index 874fe61..f9b97c7 100644 --- a/src/leveled_internal.erl +++ b/src/leveled_iterator.erl @@ -1,19 +1,25 @@ -module(leveled_internal). + -export([termiterator/6]). + -include_lib("eunit/include/eunit.hrl"). %% We will have a sorted list of terms -%% Some terms will be dummy terms which are pointers to more terms which can be found -%% If a pointer is hit need to replenish the term list before proceeding +%% Some terms will be dummy terms which are pointers to more terms which can be +%% found. If a pointer is hit need to replenish the term list before +%% proceeding. %% -%% Helper Functions should have free functions - FolderFun, CompareFun, PointerCheck} -%% FolderFun - function which takes the next item and the accumulator and returns an updated accunulator -%% CompareFun - function which should be able to compare two keys (which are not pointers) +%% Helper Functions should have free functions - +%% {FolderFun, CompareFun, PointerCheck} +%% FolderFun - function which takes the next item and the accumulator and +%% returns an updated accumulator +%% CompareFun - function which should be able to compare two keys (which are +%% not pointers), and return a winning item (or combination of items) %% PointerCheck - function for differentiating between keys and pointer -termiterator(HeadItem, [], Acc, HelperFuns, _StartKey, _EndKey) -> - io:format("Reached empty list with head item of ~w~n", [HeadItem]), +termiterator(HeadItem, [], Acc, HelperFuns, + _StartKey, _EndKey) -> case HeadItem of null -> Acc; @@ -21,7 +27,8 @@ termiterator(HeadItem, [], Acc, HelperFuns, _StartKey, _EndKey) -> {FolderFun, _, _} = HelperFuns, FolderFun(Acc, HeadItem) end; -termiterator(null, [NextItem|TailList], Acc, HelperFuns, StartKey, EndKey) -> +termiterator(null, [NextItem|TailList], Acc, HelperFuns, + StartKey, EndKey) -> %% Check that the NextItem is not a pointer before promoting to HeadItem %% Cannot now promote a HeadItem which is a pointer {_, _, PointerCheck} = HelperFuns, @@ -29,30 +36,37 @@ termiterator(null, [NextItem|TailList], Acc, HelperFuns, StartKey, EndKey) -> {true, Pointer} -> NewSlice = getnextslice(Pointer, EndKey), ExtendedList = lists:merge(NewSlice, TailList), - termiterator(null, ExtendedList, Acc, HelperFuns, StartKey, EndKey); + termiterator(null, ExtendedList, Acc, HelperFuns, + StartKey, EndKey); false -> - termiterator(NextItem, TailList, Acc, HelperFuns, StartKey, EndKey) + termiterator(NextItem, TailList, Acc, HelperFuns, + StartKey, EndKey) end; -termiterator(HeadItem, [NextItem|TailList], Acc, HelperFuns, StartKey, EndKey) -> - io:format("Checking head item of ~w~n", [HeadItem]), +termiterator(HeadItem, [NextItem|TailList], Acc, HelperFuns, + StartKey, EndKey) -> {FolderFun, CompareFun, PointerCheck} = HelperFuns, - %% HeadItem cannot be pointer, but NextItem might be, so check before comparison + %% HeadItem cannot be pointer, but NextItem might be, so check before + %% comparison case PointerCheck(NextItem) of {true, Pointer} -> NewSlice = getnextslice(Pointer, EndKey), ExtendedList = lists:merge(NewSlice, [NextItem|TailList]), - termiterator(null, ExtendedList, Acc, HelperFuns, StartKey, EndKey); + termiterator(null, ExtendedList, Acc, HelperFuns, + StartKey, EndKey); false -> - %% Compare to see if Head and Next match, or if Head is a winner to be added - %% to accumulator + %% Compare to see if Head and Next match, or if Head is a winner + %% to be added to accumulator case CompareFun(HeadItem, NextItem) of {match, StrongItem, _WeakItem} -> - %% Discard WeakItem - termiterator(StrongItem, TailList, Acc, HelperFuns, StartKey, EndKey); + %% Discard WeakItem, Strong Item might be an aggregation of + %% the items + termiterator(StrongItem, TailList, Acc, HelperFuns, + StartKey, EndKey); {winner, HeadItem} -> %% Add next item to accumulator, and proceed with next item AccPlus = FolderFun(Acc, HeadItem), - termiterator(NextItem, TailList, AccPlus, HelperFuns, HeadItem, EndKey) + termiterator(NextItem, TailList, AccPlus, HelperFuns, + HeadItem, EndKey) end end. diff --git a/src/leveled_rice.erl b/src/leveled_rice.erl new file mode 100644 index 0000000..f432944 --- /dev/null +++ b/src/leveled_rice.erl @@ -0,0 +1,283 @@ +%% Used for creating fixed-size self-regulating encoded bloom filters +%% +%% Normally a bloom filter in order to achieve optimium size increases the +%% number of hashes as the desired false positive rate increases. There is +%% a processing overhead for checking this bloom, both because of the number +%% of hash calculations required, and also because of the need to CRC check +%% the bloom to ensure a false negative result is not returned due to +%% corruption. +%% +%% A more space efficient bloom can be achieved through the compression of +%% bloom filters with less hashes (and in an optimal case a single hash). +%% This can be achieved using rice encoding. +%% +%% Rice-encoding and single hash blooms are used here in order to provide an +%% optimally space efficient solution, but also as the processing required to +%% support uncompression can be concurrently performing a checksum role. +%% +%% For this to work, the bloom is divided into 64 parts and a 32-bit hash is +%% required. Each hash is placed into one of 64 blooms based on the six least +%% significant bits of the hash, and the fmost significant 26-bits are used +%% to indicate the bit to be added to the bloom. +%% +%% The bloom is then created by calculating the differences between the ordered +%% elements of the hash list and representing the difference using an exponent +%% and a 13-bit remainder i.e. +%% 8000 -> 0 11111 01000000 +%% 10000 -> 10 00000 00010000 +%% 20000 -> 110 01110 00100000 +%% +%% Each bloom should have approximately 64 differences. +%% +%% Fronting the bloom is a bloom index, formed first by 16 pairs of 3-byte +%% max hash, 2-byte length (bits) - with then each of the encoded bitstrings +%% appended. The max hash is the total of all the differences (which should +%% be the highest hash in the bloom). +%% +%% To check a key against the bloom, hash it, take the four least signifcant +%% bits and read the start pointer, max hash end pointer from the expected +%% positions in the bloom index. Then roll through from the start pointer to +%% the end pointer, accumulating each difference. There is a possible match if +%% either the accumulator hits the expected hash or the max hash doesn't match +%% the final accumulator (to cover if the bloom has been corrupted by a bit +%% flip somwhere). A miss is more than twice as expensive (on average) than a +%% potential match - but still only requires around 64 integer additions +%% and the processing of <100 bytes of data. +%% +%% For 2048 keys, this takes up <4KB. The false positive rate is 0.000122 +%% This compares favourably for the equivalent size optimal bloom which +%% would require 11 hashes and have a false positive rate of 0.000459. +%% Checking with a positive match should take on average about 6 microseconds, +%% and a negative match should take around 11 microseconds. +%% +%% See ../test/rice_test.erl for proving timings and fpr. + + + +-module(leveled_rice). + +-export([create_bloom/1, + check_key/2, + check_keys/2]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(SLOT_COUNT, 64). +-define(MAX_HASH, 16777216). +-define(DIVISOR_BITS, 13). +-define(DIVISOR, 8092). + +%% Create a bitstring representing the bloom filter from a key list + +create_bloom(KeyList) -> + create_bloom(KeyList, ?SLOT_COUNT, ?MAX_HASH). + +create_bloom(KeyList, SlotCount, MaxHash) -> + HashLists = array:new(SlotCount, [{default, []}]), + OrdHashLists = create_hashlist(KeyList, HashLists, SlotCount, MaxHash), + serialise_bloom(OrdHashLists). + + +%% Checking for a key + +check_keys([], _) -> + true; +check_keys([Key|Rest], BitStr) -> + case check_key(Key, BitStr) of + false -> + false; + true -> + check_keys(Rest, BitStr) + end. + +check_key(Key, BitStr) -> + check_key(Key, BitStr, ?SLOT_COUNT, ?MAX_HASH, ?DIVISOR_BITS, ?DIVISOR). + +check_key(Key, BitStr, SlotCount, MaxHash, Factor, Divisor) -> + {Slot, Hash} = get_slothash(Key, MaxHash, SlotCount), + {StartPos, Length, TopHash} = find_position(Slot, BitStr, 0, 40 * SlotCount), + case BitStr of + <<_:StartPos/bitstring, Bloom:Length/bitstring, _/bitstring>> -> + check_hash(Hash, Bloom, Factor, Divisor, 0, TopHash); + _ -> + io:format("Possible corruption of bloom index ~n"), + true + end. + +find_position(Slot, BloomIndex, Counter, StartPosition) -> + <> = BloomIndex, + case Slot of + Counter -> + {StartPosition, Length, TopHash}; + _ -> + find_position(Slot, Rest, Counter + 1, StartPosition + Length) + end. + + +% Checking for a hash within a bloom + +check_hash(_, <<>>, _, _, Acc, MaxHash) -> + case Acc of + MaxHash -> + false; + _ -> + io:format("Failure of CRC check on bloom filter~n"), + true + end; +check_hash(HashToCheck, BitStr, Factor, Divisor, Acc, TopHash) -> + case findexponent(BitStr) of + {ok, Exponent, BitStrTail} -> + case findremainder(BitStrTail, Factor) of + {ok, Remainder, BitStrTail2} -> + NextHash = Acc + Divisor * Exponent + Remainder, + case NextHash of + HashToCheck -> + true; + _ -> + check_hash(HashToCheck, BitStrTail2, Factor, + Divisor, NextHash, TopHash) + end; + error -> + io:format("Failure of CRC check on bloom filter~n"), + true + end; + error -> + io:format("Failure of CRC check on bloom filter~n"), + true + end. + +%% Convert the key list into an array of sorted hash lists + +create_hashlist([], HashLists, _, _) -> + HashLists; +create_hashlist([HeadKey|Rest], HashLists, SlotCount, MaxHash) -> + {Slot, Hash} = get_slothash(HeadKey, MaxHash, SlotCount), + HashList = array:get(Slot, HashLists), + create_hashlist(Rest, + array:set(Slot, lists:usort([Hash|HashList]), HashLists), + SlotCount, MaxHash). + +%% Convert an array of hash lists into an serialsed bloom + +serialise_bloom(HashLists) -> + SlotCount = array:size(HashLists), + serialise_bloom(HashLists, SlotCount, 0, []). + +serialise_bloom(HashLists, SlotCount, Counter, Blooms) -> + case Counter of + SlotCount -> + finalise_bloom(Blooms); + _ -> + Bloom = serialise_singlebloom(array:get(Counter, HashLists)), + serialise_bloom(HashLists, SlotCount, Counter + 1, [Bloom|Blooms]) + end. + +serialise_singlebloom(HashList) -> + serialise_singlebloom(HashList, <<>>, 0, ?DIVISOR, ?DIVISOR_BITS). + +serialise_singlebloom([], BloomStr, TopHash, _, _) -> + % io:format("Single bloom created with bloom of ~w and top hash of ~w~n", [BloomStr, TopHash]), + {BloomStr, TopHash}; +serialise_singlebloom([Hash|Rest], BloomStr, TopHash, Divisor, Factor) -> + HashGap = Hash - TopHash, + Exponent = buildexponent(HashGap div Divisor), + Remainder = HashGap rem Divisor, + NewBloomStr = <>, + serialise_singlebloom(Rest, NewBloomStr, Hash, Divisor, Factor). + + +finalise_bloom(Blooms) -> + finalise_bloom(Blooms, {<<>>, <<>>}). + +finalise_bloom([], BloomAcc) -> + {BloomIndex, BloomStr} = BloomAcc, + <>; +finalise_bloom([Bloom|Rest], BloomAcc) -> + {BloomStr, TopHash} = Bloom, + {BloomIndexAcc, BloomStrAcc} = BloomAcc, + Length = bit_size(BloomStr), + UpdIdx = <>, + % io:format("Adding bloom string of ~w to bloom~n", [BloomStr]), + UpdBloomStr = <>, + finalise_bloom(Rest, {UpdIdx, UpdBloomStr}). + + + + +buildexponent(Exponent) -> + buildexponent(Exponent, <<0:1>>). + +buildexponent(0, OutputBits) -> + OutputBits; +buildexponent(Exponent, OutputBits) -> + buildexponent(Exponent - 1, <<1:1, OutputBits/bitstring>>). + + +findexponent(BitStr) -> + findexponent(BitStr, 0). + +findexponent(<<>>, _) -> + error; +findexponent(<>, Acc) -> + case H of + 1 -> findexponent(T, Acc + 1); + 0 -> {ok, Acc, T} + end. + + +findremainder(BitStr, Factor) -> + case BitStr of + <> -> + {ok, Remainder, BitStrTail}; + _ -> + error + end. + + +get_slothash(Key, MaxHash, SlotCount) -> + Hash = erlang:phash2(Key, MaxHash), + {Hash rem SlotCount, Hash div SlotCount}. + + +%%%%%%%%%%%%%%%% +% T E S T +%%%%%%%%%%%%%%% + +corrupt_bloom(Bloom) -> + Length = bit_size(Bloom), + Random = random:uniform(Length), + <> = Bloom, + case Bit of + 1 -> + <>; + 0 -> + <> + end. + +bloom_test() -> + KeyList = ["key1", "key2", "key3", "key4"], + Bloom = create_bloom(KeyList), + io:format("Bloom of ~w of length ~w ~n", [Bloom, bit_size(Bloom)]), + ?assertMatch(true, check_key("key1", Bloom)), + ?assertMatch(true, check_key("key2", Bloom)), + ?assertMatch(true, check_key("key3", Bloom)), + ?assertMatch(true, check_key("key4", Bloom)), + ?assertMatch(false, check_key("key5", Bloom)). + +bloom_corruption_test() -> + KeyList = ["key1", "key2", "key3", "key4"], + Bloom = create_bloom(KeyList), + Bloom1 = corrupt_bloom(Bloom), + ?assertMatch(true, check_keys(KeyList, Bloom1)), + Bloom2 = corrupt_bloom(Bloom), + ?assertMatch(true, check_keys(KeyList, Bloom2)), + Bloom3 = corrupt_bloom(Bloom), + ?assertMatch(true, check_keys(KeyList, Bloom3)), + Bloom4 = corrupt_bloom(Bloom), + ?assertMatch(true, check_keys(KeyList, Bloom4)), + Bloom5 = corrupt_bloom(Bloom), + ?assertMatch(true, check_keys(KeyList, Bloom5)), + Bloom6 = corrupt_bloom(Bloom), + ?assertMatch(true, check_keys(KeyList, Bloom6)). + + diff --git a/test/lookup_test.beam b/test/lookup_test.beam deleted file mode 100644 index 3c8d76474f8d8ff741da4eecc8fc903230a9a869..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4096 zcmZ`+du$wM5#ROR;~U49?A1zw(>CyZHQI zrMaD*Z)U!k-^}cHd*syFAxYZu{-Me7$HpByCrR>GB}pF~vbYei z3X947yf2l^XZ$P}l6!rrt9Bv853`Ovo6TpetHF7DHj_;|KK@bi*3*HUm9q=iBJ;^Y zDmib>rtQT=EA3?Me0|YLI@zL=O&0?Nc(rrEQa(Fp7jphZh>L-=olhqn-y9s**_ZuA z$1YfMzuzhp>_PzhSWZ!ZyciWQfpW}&Y`$m}9D>Gt$|+cum`}~}z38`cOU^a_mE>Z{ z3Y3;INyqZ#?Yy-as>wnjdCi{Vu!9+EE?HW1{GLpiw;ZKtIqHg1*=@^;fURbq&N>U} zWO~60B$t+~e8yMIK4bZFR$<<1@P22#jfuF! z1geH>)J4wfvbq1fHm*g7UGuu%R2Aw_+|%H6C90UiT0ozaPmk=?G&$iW>LSmo@{+sc zxuOpgO}^pYsEfR;$~PNS?B{cD1|`GvMNOaPH(pcaTkb6^8}jY=>zce-mE7Aw$%LNy zmI78its3ePU7PYvDW;}GR879qU>aecsf?J|U1>D6n0qIvCR9_MRO+@5;Sh&7YW{K3~R^)X&Ca|_w1`ia;amiY8VzWD$vnfH%V>npf}wzX4!EYcLE`i7{Rp zfNOL`%UZx#0!&&N6ZAw3+qVVvI}Dqz3Y*yG>1>Bir6|gj&r{tI)bF~Y{kmVel?L1PtXvmD|%{c9BM{3 zt#LT0@7$;@dSQct_O7#Ob452gLu_+J9H2yj&-N;EeVCmJ_VqDLM300*^#yeid5C8_ z^mn+TpG^@#dpxwWg6BXDo&y`=xpQ*@jWXj6)`+Q1xMFY}nveiZ!)uh7D~5b($(+$# zu?JCT<=zQHjz@3kCLVUruxB_@HQZY^^xE-}6NcxYFpOb2Vqba2Bm?OInRb6uGr zL7B#@XdkMU>F^5FTqZnv!?04O3D0n{YIvlkOj9eiUYU-1Q?eUn>aHzQcV$Y}mT8)& zWNn>}mGPZkg>M%S*&T=1^60Fs({b2@*E%{;c65B5qi$Cmj(tx zadM-1I_VXPkA+hkEuZ3Lue!tDsnqDb@P_$?*Qd)O?uygA4N~Zfhx#i;JyTQEGaDAQ ze>4;al5s_ESi<=hZVbn86lHJ})z0JDb%llmg+5oU*Gx^l&ac$V6?nK|W)Q0qM&ch? zS>iUp$SlJZ(0Ws{7|_K;-Mi-syR`eE+2346za1TMLQ50 zl*Is4)(xSjcCkEzvKYj1k3-899}B9YDIPMEX&iR5{O19qJ`)YkAQcv1&iS3?rRIu6 zIXK_E_^DZH{fYBV1$O1b53D~|J+UImT;Gkg53sE zKt51ig%8y5!3tlG`z8=ytl!{qWqvbw8MFn&{~wtTgCZ54Z8Z@4XMQV)eK0kFSl$L= z|5cvjs`A?_d^_$tL3}Y^)nUE^#QvCPedfDBRiDi70D|{4pCYE#DJ1RV5)KcLY zQ>yTEq?G5qSeI=K5XUhJ@-F4z&;NNM>#)uUD1P;obRv;RB@*`*em$3X*2xiH@h;#`>Lp5R=VXa9^B=fQpE2QkLgUSxUQxZ@OX?|yC(zw$y|MpCX6 z^WH;Vl}caQ_l?ia{Hrgzvy z@4uu`Bel~`>Yz^QqFuC`dMQqQWY7RTN(bm5nRJ*YXp)Z7G##VkG(+dD P-KW3PKj`1|ALQ{rJu=`t diff --git a/test/rice_test.erl b/test/rice_test.erl new file mode 100644 index 0000000..1bbb43f --- /dev/null +++ b/test/rice_test.erl @@ -0,0 +1,59 @@ +%% Test performance and accuracy of rice-encoded bloom filters +%% +%% Calling check_negative(2048, 1000000) should return about 122 false +%% positives in around 11 seconds, with a size below 4KB +%% +%% The equivalent positive check is check_positive(2048, 488) and this +%% should take around 6 seconds. +%% +%% So a blooom with 2048 members should support o(100K) checks per second +%% on a modern CPU, whilst requiring 2 bytes per member. + +-module(rice_test). + +-export([check_positive/2, check_negative/2, calc_hash/2]). + + + +check_positive(KeyCount, LoopCount) -> + KeyList = produce_keylist(KeyCount), + Bloom = leveled_rice:create_bloom(KeyList), + check_positive(KeyList, Bloom, LoopCount). + +check_positive(_, Bloom, 0) -> + {ok, byte_size(Bloom)}; +check_positive(KeyList, Bloom, LoopCount) -> + true = leveled_rice:check_keys(KeyList, Bloom), + check_positive(KeyList, Bloom, LoopCount - 1). + + +produce_keylist(KeyCount) -> + KeyPrefix = lists:concat(["PositiveKey-", random:uniform(KeyCount)]), + produce_keylist(KeyCount, [], KeyPrefix). + +produce_keylist(0, KeyList, _) -> + KeyList; +produce_keylist(KeyCount, KeyList, KeyPrefix) -> + Key = lists:concat([KeyPrefix, KeyCount]), + produce_keylist(KeyCount - 1, [Key|KeyList], KeyPrefix). + + +check_negative(KeyCount, CheckCount) -> + KeyList = produce_keylist(KeyCount), + Bloom = leveled_rice:create_bloom(KeyList), + check_negative(Bloom, CheckCount, 0). + +check_negative(Bloom, 0, FalsePos) -> + {byte_size(Bloom), FalsePos}; +check_negative(Bloom, CheckCount, FalsePos) -> + Key = lists:concat(["NegativeKey-", CheckCount, random:uniform(CheckCount)]), + case leveled_rice:check_key(Key, Bloom) of + true -> check_negative(Bloom, CheckCount - 1, FalsePos + 1); + false -> check_negative(Bloom, CheckCount - 1, FalsePos) + end. + +calc_hash(_, 0) -> + ok; +calc_hash(Key, Count) -> + erlang:phash2(lists:concat([Key, Count, "sometxt"])), + calc_hash(Key, Count -1). From c5f50c613daddd7431ce22d1566ff77d40c22f0c Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 4 Jun 2015 21:15:31 +0100 Subject: [PATCH 004/167] Ongoing improvements - in particular CDB now supports general erlang terms not just lists --- src/leveled_bst.erl | 23 +-- src/leveled_cdb.erl | 377 +++++++++++++++++++++++++++++---------- src/leveled_iterator.erl | 163 ++++++++++++----- test/lookup_test.erl | 8 +- 4 files changed, 412 insertions(+), 159 deletions(-) diff --git a/src/leveled_bst.erl b/src/leveled_bst.erl index 886705a..f8b8687 100644 --- a/src/leveled_bst.erl +++ b/src/leveled_bst.erl @@ -23,16 +23,16 @@ %% There will be up to 4000 blocks in a single bst file %% %% The slots is a series of references -%% - 4 byte bloom-filter length +%% - 4 byte bloom-filter length %% - 4 byte key-helper length -%% - a variable-length compressed bloom filter for all keys in slot (approx 1KB) -%% - 32 ordered variable-length key helpers pointing to first key in each +%% - a variable-length compressed bloom filter for all keys in slot (approx 3KB) +%% - 64 ordered variable-length key helpers pointing to first key in each %% block (in slot) of the form Key Length, Key, Block Position %% - 4 byte CRC for the slot %% %% The slot index in the footer is made up of 128 keys and pointers at the %% the start of each slot -%% - 128 Key Length (4 byte), Key, Position (4 byte) indexes +%% - 64 x Key Length (4 byte), Key, Position (4 byte) indexes %% - 4 bytes CRC for the index %% %% The format of the file is intended to support quick lookups, whilst @@ -54,8 +54,9 @@ -record(metadata, {version = ?CURRENT_VERSION :: tuple(), mutable = false :: true | false, - compressed = true :: tre | false, - slot_list :: list(), + compressed = true :: true | false, + slot_array, + open_slot :: integer(), cache :: tuple(), smallest_key :: tuple(), largest_key :: tuple(), @@ -73,9 +74,9 @@ start_file(Handle) -> {ok, _} = file:position(Handle, bof), file:write(Handle, Header), {Version, {M, C}, _, _} = convert_header(Header), - FileMD = #metadata{version=Version, mutable=M, compressed=C}, - SlotArray = array:new(?SLOT_COUNT), - {Handle, FileMD, SlotArray}. + FileMD = #metadata{version = Version, mutable = M, compressed = C, + slot_array = array:new(?SLOT_COUNT), open_slot = 0}, + {Handle, FileMD}. create_header(initial) -> @@ -119,6 +120,8 @@ convert_header_v01(Header) -> {{0, 1}, {M, C}, {FooterP, SlotLng, HlpLng}, none}. +% add_slot(Handle, FileMD, SlotArray) + %%%%%%%%%%%%%%%% @@ -148,7 +151,7 @@ bad_header_test() -> ?assertMatch(unknown_version, convert_header(NewHdr2)). record_onstartfile_test() -> - {_, FileMD, _} = start_file("onstartfile.bst"), + {_, FileMD} = start_file("onstartfile.bst"), ?assertMatch({0, 1}, FileMD#metadata.version). diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 13c3062..da25d1c 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -54,24 +54,22 @@ put/4, open_active_file/1, get_nextkey/1, - get_nextkey/2]). + get_nextkey/2, + fold/3, + fold_keys/3]). -include_lib("eunit/include/eunit.hrl"). -define(DWORD_SIZE, 8). -define(WORD_SIZE, 4). -define(CRC_CHECK, true). +-define(MAX_FILE_SIZE, 3221225472). +-define(BASE_POSITION, 2048). %% %% from_dict(FileName,ListOfKeyValueTuples) %% Given a filename and a dictionary, create a cdb %% using the key value pairs from the dict. -%% -%% @spec from_dict(filename(),dictionary()) -> ok -%% where -%% filename() = string(), -%% dictionary() = dict() -%% from_dict(FileName,Dict) -> KeyValueList = dict:to_list(Dict), create(FileName, KeyValueList). @@ -82,30 +80,21 @@ from_dict(FileName,Dict) -> %% this function creates a CDB %% create(FileName,KeyValueList) -> - {ok, Handle} = file:open(FileName, [write]), - {ok, _} = file:position(Handle, {bof, 2048}), + {ok, Handle} = file:open(FileName, [binary, raw, read, write]), + {ok, _} = file:position(Handle, {bof, ?BASE_POSITION}), {BasePos, HashTree} = write_key_value_pairs(Handle, KeyValueList), - io:format("KVs has been written to base position ~w~n", [BasePos]), - L2 = write_hash_tables(Handle, HashTree), - io:format("Index list output of ~w~n", [L2]), - write_top_index_table(Handle, BasePos, L2), - file:close(Handle). + close_file(Handle, HashTree, BasePos). %% %% dump(FileName) -> List %% Given a file name, this function returns a list %% of {key,value} tuples from the CDB. %% -%% -%% @spec dump(filename()) -> key_value_list() -%% where -%% filename() = string(), -%% key_value_list() = [{key,value}] dump(FileName) -> dump(FileName, ?CRC_CHECK). dump(FileName, CRCCheck) -> - {ok, Handle} = file:open(FileName, [binary,raw]), + {ok, Handle} = file:open(FileName, [binary, raw, read]), Fn = fun(Index, Acc) -> {ok, _} = file:position(Handle, ?DWORD_SIZE * Index), {_, Count} = read_next_2_integers(Handle), @@ -117,8 +106,9 @@ dump(FileName, CRCCheck) -> {ok, _} = file:position(Handle, {bof, 2048}), Fn1 = fun(_I,Acc) -> {KL,VL} = read_next_2_integers(Handle), - Key = read_next_string(Handle, KL), - case read_next_string(Handle, VL, crc, CRCCheck) of + Key = read_next_term(Handle, KL), + io:format("Key read of ~w~n", [Key]), + case read_next_term(Handle, VL, crc, CRCCheck) of {false, _} -> {ok, CurrLoc} = file:position(Handle, cur), Return = {crc_wonky, get(Handle, Key)}; @@ -169,9 +159,15 @@ put(FileName, Key, Value, {LastPosition, HashTree}) when is_list(FileName) -> [binary, raw, read, write, delayed_write]), put(Handle, Key, Value, {LastPosition, HashTree}); put(Handle, Key, Value, {LastPosition, HashTree}) -> - Bin = key_value_to_record({Key, Value}), % create binary for Key and Value - ok = file:pwrite(Handle, LastPosition, Bin), - {LastPosition + byte_size(Bin), put_hashtree(Key, LastPosition, HashTree)}. + Bin = key_value_to_record({Key, Value}), + PotentialNewSize = LastPosition + byte_size(Bin), + if PotentialNewSize > ?MAX_FILE_SIZE -> + close_file(Handle, HashTree, LastPosition), + roll; + true -> + ok = file:pwrite(Handle, LastPosition, Bin), + {Handle, PotentialNewSize, put_hashtree(Key, LastPosition, HashTree)} + end. %% @@ -182,7 +178,7 @@ get(FileNameOrHandle, Key) -> get(FileNameOrHandle, Key, ?CRC_CHECK). get(FileName, Key, CRCCheck) when is_list(FileName), is_list(Key) -> - {ok,Handle} = file:open(FileName,[binary,raw]), + {ok,Handle} = file:open(FileName,[binary, raw, read]), get(Handle,Key, CRCCheck); get(Handle, Key, CRCCheck) when is_tuple(Handle), is_list(Key) -> @@ -230,7 +226,7 @@ get_nextkey(Handle, {Position, FirstHashPosition}) -> {ok, Position} = file:position(Handle, Position), case read_next_2_integers(Handle) of {KeyLength, ValueLength} -> - NextKey = read_next_string(Handle, KeyLength), + NextKey = read_next_term(Handle, KeyLength), NextPosition = Position + KeyLength + ValueLength + ?DWORD_SIZE, case NextPosition of FirstHashPosition -> @@ -243,10 +239,76 @@ get_nextkey(Handle, {Position, FirstHashPosition}) -> end. +%% Fold over all of the objects in the file, applying FoldFun to each object +%% where FoldFun(K, V, Acc0) -> Acc , or FoldFun(K, Acc0) -> Acc if KeyOnly is +%% set to true + +fold(FileName, FoldFun, Acc0) when is_list(FileName) -> + {ok, Handle} = file:open(FileName, [binary, raw, read]), + fold(Handle, FoldFun, Acc0); +fold(Handle, FoldFun, Acc0) -> + {ok, _} = file:position(Handle, bof), + {FirstHashPosition, _} = read_next_2_integers(Handle), + fold(Handle, FoldFun, Acc0, {256 * ?DWORD_SIZE, FirstHashPosition}, false). + +fold(Handle, FoldFun, Acc0, {Position, FirstHashPosition}, KeyOnly) -> + {ok, Position} = file:position(Handle, Position), + case Position of + FirstHashPosition -> + Acc0; + _ -> + case read_next_2_integers(Handle) of + {KeyLength, ValueLength} -> + NextKey = read_next_term(Handle, KeyLength), + NextPosition = Position + KeyLength + ValueLength + ?DWORD_SIZE, + case KeyOnly of + true -> + fold(Handle, FoldFun, FoldFun(NextKey, Acc0), + {NextPosition, FirstHashPosition}, KeyOnly); + false -> + case read_next_term(Handle, ValueLength, crc, ?CRC_CHECK) of + {false, _} -> + io:format("Skipping value for Key ~w as CRC check failed~n", + [NextKey]), + fold(Handle, FoldFun, Acc0, + {NextPosition, FirstHashPosition}, KeyOnly); + {_, Value} -> + fold(Handle, FoldFun, FoldFun(NextKey, Value, Acc0), + {NextPosition, FirstHashPosition}, KeyOnly) + end + end; + eof -> + Acc0 + end + end. + + +fold_keys(FileName, FoldFun, Acc0) when is_list(FileName) -> + {ok, Handle} = file:open(FileName, [binary, raw, read]), + fold_keys(Handle, FoldFun, Acc0); +fold_keys(Handle, FoldFun, Acc0) -> + {ok, _} = file:position(Handle, bof), + {FirstHashPosition, _} = read_next_2_integers(Handle), + fold(Handle, FoldFun, Acc0, {256 * ?DWORD_SIZE, FirstHashPosition}, true). + + %%%%%%%%%%%%%%%%%%%% %% Internal functions %%%%%%%%%%%%%%%%%%%% +%% Take an active file and write the hash details necessary to close that +%% file and roll a new active file if requested. +%% +%% Base Pos should be at the end of the KV pairs written (the position for) +%% the hash tables +close_file(Handle, HashTree, BasePos) -> + {ok, BasePos} = file:position(Handle, BasePos), + L2 = write_hash_tables(Handle, HashTree), + write_top_index_table(Handle, BasePos, L2), + file:close(Handle). + + + %% Fetch a list of positions by passing a key to the HashTree get_hashtree(Key, HashTree) -> Hash = hash(Key), @@ -282,9 +344,9 @@ extract_kvpair(_, [], _, _) -> extract_kvpair(Handle, [Position|Rest], Key, Check) -> {ok, _} = file:position(Handle, Position), {KeyLength, ValueLength} = read_next_2_integers(Handle), - case read_next_string(Handle, KeyLength) of + case read_next_term(Handle, KeyLength) of Key -> % If same key as passed in, then found! - case read_next_string(Handle, ValueLength, crc, Check) of + case read_next_term(Handle, ValueLength, crc, Check) of {false, _} -> crc_wonky; {_, Value} -> @@ -301,10 +363,10 @@ scan_over_file(Handle, Position) -> scan_over_file(Handle, Position, HashTree). scan_over_file(Handle, Position, HashTree) -> - case read_next_2_integers(Handle) of - {KeyLength, ValueLength} -> - Key = read_next_string(Handle, KeyLength), - {ok, ValueAsBin} = file:read(Handle, ValueLength), + case saferead_keyvalue(Handle) of + false -> + {Position, HashTree}; + {Key, ValueAsBin, KeyLength, ValueLength} -> case crccheck_value(ValueAsBin) of true -> NewPosition = Position + KeyLength + ValueLength + ?DWORD_SIZE, @@ -318,6 +380,34 @@ scan_over_file(Handle, Position, HashTree) -> {Position, HashTree} end. + +%% Read the Key/Value at this point, returning {ok, Key, Value} +%% catch expected exceptiosn associated with file corruption (or end) and +%% return eof +saferead_keyvalue(Handle) -> + case read_next_2_integers(Handle) of + {error, einval} -> + false; + eof -> + false; + {KeyL, ValueL} -> + case read_next_term(Handle, KeyL) of + {error, einval} -> + false; + eof -> + false; + Key -> + case file:read(Handle, ValueL) of + {error, einval} -> + false; + eof -> + false; + {ok, Value} -> + {Key, Value, KeyL, ValueL} + end + end + end. + %% The first four bytes of the value are the crc check crccheck_value(Value) when byte_size(Value) >4 -> << Hash:32/integer, Tail/bitstring>> = Value, @@ -356,26 +446,30 @@ to_dict(FileName) -> KeyValueList = dump(FileName), dict:from_list(KeyValueList). -read_next_string(Handle, Length) -> - {ok, Bin} = file:read(Handle, Length), - binary_to_list(Bin). +read_next_term(Handle, Length) -> + case file:read(Handle, Length) of + {ok, Bin} -> + binary_to_term(Bin); + ReadError -> + ReadError + end. %% Read next string where the string has a CRC prepended - stripping the crc %% and checking if requested -read_next_string(Handle, Length, crc, Check) -> +read_next_term(Handle, Length, crc, Check) -> case Check of true -> {ok, <>} = file:read(Handle, Length), case calc_crc(Bin) of CRC -> - {true, binary_to_list(Bin)}; + {true, binary_to_term(Bin)}; _ -> - {false, binary_to_list(Bin)} + {false, binary_to_term(Bin)} end; _ -> {ok, _} = file:position(Handle, {cur, 4}), {ok, Bin} = file:read(Handle, Length - 4), - {unchecked, binary_to_list(Bin)} + {unchecked, binary_to_term(Bin)} end. @@ -386,9 +480,9 @@ read_next_2_integers(Handle) -> case file:read(Handle,?DWORD_SIZE) of {ok, <>} -> {endian_flip(Int1), endian_flip(Int2)}; - MatchError + ReadError -> - MatchError + ReadError end. %% Seach the hash table for the matching hash and key. Be prepared for @@ -398,7 +492,6 @@ search_hash_table(_Handle, [], _Hash, _Key, _CRCCHeck) -> search_hash_table(Handle, [Entry|RestOfEntries], Hash, Key, CRCCheck) -> {ok, _} = file:position(Handle, Entry), {StoredHash, DataLoc} = read_next_2_integers(Handle), - io:format("looking in data location ~w~n", [DataLoc]), case StoredHash of Hash -> KV = extract_kvpair(Handle, [DataLoc], Key, CRCCheck), @@ -432,7 +525,7 @@ write_key_value_pairs(_, [], Acc) -> Acc; write_key_value_pairs(Handle, [HeadPair|TailList], Acc) -> {Key, Value} = HeadPair, - {NewPosition, HashTree} = put(Handle, Key, Value, Acc), + {Handle, NewPosition, HashTree} = put(Handle, Key, Value, Acc), write_key_value_pairs(Handle, TailList, {NewPosition, HashTree}). %% Write the actual hashtables at the bottom of the file. Each hash table @@ -549,14 +642,16 @@ endian_flip(Int) -> X. hash(Key) -> + BK = term_to_binary(Key), H = 5381, - hash1(H,Key) band 16#FFFFFFFF. + hash1(H, BK) band 16#FFFFFFFF. -hash1(H,[]) ->H; -hash1(H,[B|Rest]) -> +hash1(H, <<>>) -> + H; +hash1(H, <>) -> H1 = H * 33, H2 = H1 bxor B, - hash1(H2,Rest). + hash1(H2, Rest). % Get the least significant 8 bits from the hash. hash_to_index(Hash) -> @@ -567,41 +662,21 @@ hash_to_slot(Hash,L) -> %% Create a binary of the LengthKeyLengthValue, adding a CRC check %% at the front of the value -key_value_to_record({Key,Value}) -> - L1 = endian_flip(length(Key)), - L2 = endian_flip(length(Value) + 4), - LB1 = list_to_binary(Key), - LB2 = list_to_binary(Value), - CRC = calc_crc(LB2), - <>. +key_value_to_record({Key, Value}) -> + BK = term_to_binary(Key), + BV = term_to_binary(Value), + LK = byte_size(BK), + LV = byte_size(BV), + LK_FL = endian_flip(LK), + LV_FL = endian_flip(LV + 4), + CRC = calc_crc(BV), + <>. + %%%%%%%%%%%%%%%% % T E S T %%%%%%%%%%%%%%% - -hash_1_test() -> - Hash = hash("key1"), - ?assertMatch(Hash,2088047427). - -hash_to_index_1_test() -> - Hash = hash("key1"), - Index = hash_to_index(Hash), - ?assertMatch(Index,67). - -hash_to_index_2_test() -> - Hash = 256, - I = hash_to_index(Hash), - ?assertMatch(I,0). - -hash_to_index_3_test() -> - Hash = 268, - I = hash_to_index(Hash), - ?assertMatch(I,12). - -hash_to_index_4_test() -> - Hash = hash("key2"), - Index = hash_to_index(Hash), - ?assertMatch(Index,64). +-ifdef(TEST). write_key_value_pairs_1_test() -> {ok,Handle} = file:open("test.cdb",write), @@ -612,8 +687,11 @@ write_key_value_pairs_1_test() -> Index2 = hash_to_index(Hash2), R0 = array:new(256, {default, gb_trees:empty()}), R1 = array:set(Index1, gb_trees:insert(Hash1, [0], array:get(Index1, R0)), R0), - R2 = array:set(Index2, gb_trees:insert(Hash2, [22], array:get(Index2, R1)), R1), - ?assertMatch(R2, HashTree). + R2 = array:set(Index2, gb_trees:insert(Hash2, [30], array:get(Index2, R1)), R1), + io:format("HashTree is ~w~n", [HashTree]), + io:format("Expected HashTree is ~w~n", [R2]), + ?assertMatch(R2, HashTree), + ok = file:delete("test.cdb"). write_hash_tables_1_test() -> @@ -623,7 +701,8 @@ write_hash_tables_1_test() -> R2 = array:set(67, gb_trees:insert(6383014723, [0], array:get(67, R1)), R1), Result = write_hash_tables(Handle, R2), io:format("write hash tables result of ~w ~n", [Result]), - ?assertMatch(Result,[{67,16,2},{64,0,2}]). + ?assertMatch(Result,[{67,16,2},{64,0,2}]), + ok = file:delete("test.cdb"). find_open_slot_1_test() -> List = [<<1:32,1:32>>,<<0:32,0:32>>,<<1:32,1:32>>,<<1:32,1:32>>], @@ -654,7 +733,8 @@ full_1_test() -> List1 = lists:sort([{"key1","value1"},{"key2","value2"}]), create("simple.cdb",lists:sort([{"key1","value1"},{"key2","value2"}])), List2 = lists:sort(dump("simple.cdb")), - ?assertMatch(List1,List2). + ?assertMatch(List1,List2), + ok = file:delete("simple.cdb"). full_2_test() -> List1 = lists:sort([{lists:flatten(io_lib:format("~s~p",[Prefix,Plug])), @@ -664,7 +744,8 @@ full_2_test() -> "tiep4||","qweq"]]), create("full.cdb",List1), List2 = lists:sort(dump("full.cdb")), - ?assertMatch(List1,List2). + ?assertMatch(List1,List2), + ok = file:delete("full.cdb"). from_dict_test() -> D = dict:new(), @@ -676,7 +757,8 @@ from_dict_test() -> D3 = lists:sort(dict:to_list(D2)), io:format("KVP is ~w~n", [KVP]), io:format("D3 is ~w~n", [D3]), - ?assertMatch(KVP,D3). + ?assertMatch(KVP, D3), + ok = file:delete("from_dict_test.cdb"). to_dict_test() -> D = dict:new(), @@ -686,7 +768,8 @@ to_dict_test() -> Dict = to_dict("from_dict_test.cdb"), D3 = lists:sort(dict:to_list(D2)), D4 = lists:sort(dict:to_list(Dict)), - ?assertMatch(D4,D3). + ?assertMatch(D4,D3), + ok = file:delete("from_dict_test.cdb"). crccheck_emptyvalue_test() -> ?assertMatch(false, crccheck_value(<<>>)). @@ -729,9 +812,10 @@ activewrite_singlewrite_test() -> {LastPosition, KeyDict} = open_active_file("test_mem.cdb"), io:format("File opened as new active file " "with LastPosition=~w ~n", [LastPosition]), - {_, UpdKeyDict} = put("test_mem.cdb", Key, Value, {LastPosition, KeyDict}), + {_, _, UpdKeyDict} = put("test_mem.cdb", Key, Value, {LastPosition, KeyDict}), io:format("New key and value added to active file ~n", []), - ?assertMatch({Key, Value}, get_mem(Key, "test_mem.cdb", UpdKeyDict)). + ?assertMatch({Key, Value}, get_mem(Key, "test_mem.cdb", UpdKeyDict)), + ok = file:delete("test_mem.cdb"). search_hash_table_findinslot_test() -> Key1 = "key1", % this is in slot 3 if count is 8 @@ -766,10 +850,11 @@ search_hash_table_findinslot_test() -> ok = file:pwrite(Handle, FirstHashPosition + (Slot -1) * ?DWORD_SIZE, RBin), ok = file:close(Handle), io:format("Find key following change to hash table~n"), - ?assertMatch(missing, get("hashtable1_test.cdb", Key1)). + ?assertMatch(missing, get("hashtable1_test.cdb", Key1)), + ok = file:delete("hashtable1_test.cdb"). -getnextkey_test() -> - L = [{"K9", "V9"}, {"K2", "V2"}, {"K3", "V3"}, +getnextkey_inclemptyvalue_test() -> + L = [{"K9", "V9"}, {"K2", "V2"}, {"K3", ""}, {"K4", "V4"}, {"K5", "V5"}, {"K6", "V6"}, {"K7", "V7"}, {"K8", "V8"}, {"K1", "V1"}], ok = create("hashtable1_test.cdb", L), @@ -778,7 +863,8 @@ getnextkey_test() -> ?assertMatch("K9", FirstKey), {SecondKey, Handle, P2} = get_nextkey(Handle, P1), ?assertMatch("K2", SecondKey), - {_, Handle, P3} = get_nextkey(Handle, P2), + {ThirdKeyNoValue, Handle, P3} = get_nextkey(Handle, P2), + ?assertMatch("K3", ThirdKeyNoValue), {_, Handle, P4} = get_nextkey(Handle, P3), {_, Handle, P5} = get_nextkey(Handle, P4), {_, Handle, P6} = get_nextkey(Handle, P5), @@ -786,19 +872,114 @@ getnextkey_test() -> {_, Handle, P8} = get_nextkey(Handle, P7), {LastKey, Info} = get_nextkey(Handle, P8), ?assertMatch(nomorekeys, Info), - ?assertMatch("K1", LastKey). + ?assertMatch("K1", LastKey), + ok = file:delete("hashtable1_test.cdb"). newactivefile_test() -> {LastPosition, _} = open_active_file("activefile_test.cdb"), ?assertMatch(256 * ?DWORD_SIZE, LastPosition), Response = get_nextkey("activefile_test.cdb"), - ?assertMatch(nomorekeys, Response). - - - - + ?assertMatch(nomorekeys, Response), + ok = file:delete("activefile_test.cdb"). +emptyvalue_fromdict_test() -> + D = dict:new(), + D1 = dict:store("K1", "V1", D), + D2 = dict:store("K2", "", D1), + D3 = dict:store("K3", "V3", D2), + D4 = dict:store("K4", "", D3), + ok = from_dict("from_dict_test_ev.cdb",D4), + io:format("Store created ~n", []), + KVP = lists:sort(dump("from_dict_test_ev.cdb")), + D_Result = lists:sort(dict:to_list(D4)), + io:format("KVP is ~w~n", [KVP]), + io:format("D_Result is ~w~n", [D_Result]), + ?assertMatch(KVP, D_Result), + ok = file:delete("from_dict_test_ev.cdb"). +fold_test() -> + K1 = {"Key1", 1}, + V1 = 2, + K2 = {"Key1", 2}, + V2 = 4, + K3 = {"Key1", 3}, + V3 = 8, + K4 = {"Key1", 4}, + V4 = 16, + K5 = {"Key1", 5}, + V5 = 32, + D = dict:from_list([{K1, V1}, {K2, V2}, {K3, V3}, {K4, V4}, {K5, V5}]), + ok = from_dict("fold_test.cdb", D), + FromSN = 2, + FoldFun = fun(K, V, Acc) -> + {_Key, Seq} = K, + if Seq > FromSN -> + Acc + V; + true -> + Acc + end + end, + ?assertMatch(56, fold("fold_test.cdb", FoldFun, 0)), + ok = file:delete("fold_test.cdb"). +fold_keys_test() -> + K1 = {"Key1", 1}, + V1 = 2, + K2 = {"Key2", 2}, + V2 = 4, + K3 = {"Key3", 3}, + V3 = 8, + K4 = {"Key4", 4}, + V4 = 16, + K5 = {"Key5", 5}, + V5 = 32, + D = dict:from_list([{K1, V1}, {K2, V2}, {K3, V3}, {K4, V4}, {K5, V5}]), + ok = from_dict("fold_keys_test.cdb", D), + FromSN = 2, + FoldFun = fun(K, Acc) -> + {Key, Seq} = K, + if Seq > FromSN -> + lists:append(Acc, [Key]); + true -> + Acc + end + end, + Result = fold_keys("fold_keys_test.cdb", FoldFun, []), + ?assertMatch(["Key3", "Key4", "Key5"], lists:sort(Result)), + ok = file:delete("fold_keys_test.cdb"). +fold2_test() -> + K1 = {"Key1", 1}, + V1 = 2, + K2 = {"Key1", 2}, + V2 = 4, + K3 = {"Key1", 3}, + V3 = 8, + K4 = {"Key1", 4}, + V4 = 16, + K5 = {"Key1", 5}, + V5 = 32, + K6 = {"Key2", 1}, + V6 = 64, + D = dict:from_list([{K1, V1}, {K2, V2}, {K3, V3}, + {K4, V4}, {K5, V5}, {K6, V6}]), + ok = from_dict("fold2_test.cdb", D), + FoldFun = fun(K, V, Acc) -> + {Key, Seq} = K, + case dict:find(Key, Acc) of + error -> + dict:store(Key, {Seq, V}, Acc); + {ok, {LSN, _V}} when Seq > LSN -> + dict:store(Key, {Seq, V}, Acc); + _ -> + Acc + end + end, + RD = dict:new(), + RD1 = dict:store("Key1", {5, 32}, RD), + RD2 = dict:store("Key2", {1, 64}, RD1), + Result = fold("fold2_test.cdb", FoldFun, dict:new()), + ?assertMatch(RD2, Result), + ok = file:delete("fold2_test.cdb"). +-endif. diff --git a/src/leveled_iterator.erl b/src/leveled_iterator.erl index f9b97c7..e065918 100644 --- a/src/leveled_iterator.erl +++ b/src/leveled_iterator.erl @@ -1,58 +1,69 @@ --module(leveled_internal). +-module(leveled_iterator). --export([termiterator/6]). +-export([termiterator/3]). -include_lib("eunit/include/eunit.hrl"). -%% We will have a sorted list of terms -%% Some terms will be dummy terms which are pointers to more terms which can be -%% found. If a pointer is hit need to replenish the term list before -%% proceeding. +%% Takes a list of terms to iterate - the terms being sorted in Erlang term +%% order %% %% Helper Functions should have free functions - -%% {FolderFun, CompareFun, PointerCheck} +%% {FolderFun, CompareFun, PointerCheck, PointerFetch} %% FolderFun - function which takes the next item and the accumulator and -%% returns an updated accumulator +%% returns an updated accumulator. Note FolderFun can only increase the +%% accumulator by one entry each time %% CompareFun - function which should be able to compare two keys (which are %% not pointers), and return a winning item (or combination of items) %% PointerCheck - function for differentiating between keys and pointer +%% PointerFetch - function that takes a pointer an EndKey (which may be +%% infinite) and returns a ne wslice of ordered results from that pointer +%% +%% Range can be for the form +%% {StartKey, EndKey, MaxKeys} where EndKey or MaxKeys can be infinite (but +%% not both) -termiterator(HeadItem, [], Acc, HelperFuns, - _StartKey, _EndKey) -> + +termiterator(ListToIterate, HelperFuns, Range) -> + case Range of + {_, infinte, infinite} -> + bad_iterator; + _ -> + termiterator(null, ListToIterate, [], HelperFuns, Range) + end. + + +termiterator(HeadItem, [], Acc, HelperFuns, _) -> case HeadItem of null -> Acc; _ -> - {FolderFun, _, _} = HelperFuns, + {FolderFun, _, _, _} = HelperFuns, FolderFun(Acc, HeadItem) end; -termiterator(null, [NextItem|TailList], Acc, HelperFuns, - StartKey, EndKey) -> +termiterator(null, [NextItem|TailList], Acc, HelperFuns, Range) -> %% Check that the NextItem is not a pointer before promoting to HeadItem %% Cannot now promote a HeadItem which is a pointer - {_, _, PointerCheck} = HelperFuns, + {_, _, PointerCheck, PointerFetch} = HelperFuns, case PointerCheck(NextItem) of {true, Pointer} -> - NewSlice = getnextslice(Pointer, EndKey), + {_, EndKey, _} = Range, + NewSlice = PointerFetch(Pointer, EndKey), ExtendedList = lists:merge(NewSlice, TailList), - termiterator(null, ExtendedList, Acc, HelperFuns, - StartKey, EndKey); + termiterator(null, ExtendedList, Acc, HelperFuns, Range); false -> - termiterator(NextItem, TailList, Acc, HelperFuns, - StartKey, EndKey) + termiterator(NextItem, TailList, Acc, HelperFuns, Range) end; -termiterator(HeadItem, [NextItem|TailList], Acc, HelperFuns, - StartKey, EndKey) -> - {FolderFun, CompareFun, PointerCheck} = HelperFuns, +termiterator(HeadItem, [NextItem|TailList], Acc, HelperFuns, Range) -> + {FolderFun, CompareFun, PointerCheck, PointerFetch} = HelperFuns, + {_, EndKey, MaxItems} = Range, %% HeadItem cannot be pointer, but NextItem might be, so check before %% comparison case PointerCheck(NextItem) of {true, Pointer} -> - NewSlice = getnextslice(Pointer, EndKey), - ExtendedList = lists:merge(NewSlice, [NextItem|TailList]), - termiterator(null, ExtendedList, Acc, HelperFuns, - StartKey, EndKey); + NewSlice = PointerFetch(Pointer, EndKey), + ExtendedList = lists:merge(NewSlice, [HeadItem|TailList]), + termiterator(null, ExtendedList, Acc, HelperFuns, Range); false -> %% Compare to see if Head and Next match, or if Head is a winner %% to be added to accumulator @@ -60,39 +71,65 @@ termiterator(HeadItem, [NextItem|TailList], Acc, HelperFuns, {match, StrongItem, _WeakItem} -> %% Discard WeakItem, Strong Item might be an aggregation of %% the items - termiterator(StrongItem, TailList, Acc, HelperFuns, - StartKey, EndKey); + termiterator(StrongItem, TailList, Acc, HelperFuns, Range); {winner, HeadItem} -> %% Add next item to accumulator, and proceed with next item AccPlus = FolderFun(Acc, HeadItem), - termiterator(NextItem, TailList, AccPlus, HelperFuns, - HeadItem, EndKey) + case length(AccPlus) of + MaxItems -> + AccPlus; + _ -> + termiterator(NextItem, TailList, AccPlus, + HelperFuns, + {HeadItem, EndKey, MaxItems}) + end end end. +%% Initial forms of keys supported are Index Keys and Object Keys +%% +%% All keys are of the form {Key, Value, SequenceNumber, State} +%% +%% The Key will be of the form: +%% {o, Bucket, Key} - for an Object Key +%% {i, Bucket, IndexName, IndexTerm, Key} - for an Index Key +%% +%% The value will be of the form: +%% {o, ObjectHash, [vector-clocks]} - for an Object Key +%% null - for an Index Key +%% +%% Sequence number is the sequence number the key was added, and the highest +%% sequence number in the list of keys for an index key. +%% +%% State can be one of the following: +%% live - an active key +%% tomb - a tombstone key +%% {timestamp, TS} - an active key to a certain timestamp +%% {pointer, Pointer} - to be added by iterators to indicate further data +%% available in the range from a particular source + pointercheck_indexkey(IndexKey) -> case IndexKey of - {i, _Bucket, _Index, _Term, _Key, _Sequence, {zpointer, Pointer}} -> + {_Key, _Values, _Sequence, {pointer, Pointer}} -> {true, Pointer}; _ -> false end. folder_indexkey(Acc, IndexKey) -> - io:format("Folding index key of - ~w~n", [IndexKey]), case IndexKey of - {i, _Bucket, _Index, _Term, _Key, _Sequence, tombstone} -> + {_Key, _Value, _Sequence, tomb} -> Acc; - {i, _Bucket, _Index, _Term, Key, _Sequence, null} -> - io:format("Adding key ~s~n", [Key]), - lists:append(Acc, [Key]) + {Key, _Value, _Sequence, live} -> + {i, _, _, _, ObjectKey} = Key, + lists:append(Acc, [ObjectKey]) end. compare_indexkey(IndexKey1, IndexKey2) -> - {i, Bucket1, Index1, Term1, Key1, Sequence1, _Value1} = IndexKey1, - {i, Bucket2, Index2, Term2, Key2, Sequence2, _Value2} = IndexKey2, + {{i, Bucket1, Index1, Term1, Key1}, _Val1, Sequence1, _St1} = IndexKey1, + {{i, Bucket2, Index2, Term2, Key2}, _Val2, Sequence2, _St2} = IndexKey2, case {Bucket1, Index1, Term1, Key1} of {Bucket2, Index2, Term2, Key2} when Sequence1 >= Sequence2 -> {match, IndexKey1, IndexKey2}; @@ -105,6 +142,9 @@ compare_indexkey(IndexKey1, IndexKey2) -> end. + +%% Unit testsß + getnextslice(Pointer, _EndKey) -> case Pointer of {test, NewList} -> @@ -114,18 +154,43 @@ getnextslice(Pointer, _EndKey) -> end. -%% Unit tests - - -iterateoverindexkeyswithnopointer_test_() -> - Key1 = {i, "pdsRecord", "familyName_bin", "1972SMITH", "10001", 1, null}, - Key2 = {i, "pdsRecord", "familyName_bin", "1972SMITH", "10001", 2, tombstone}, - Key3 = {i, "pdsRecord", "familyName_bin", "1971SMITH", "10002", 2, null}, - Key4 = {i, "pdsRecord", "familyName_bin", "1972JONES", "10003", 2, null}, +iterateoverindexkeyswithnopointer_test() -> + Key1 = {{i, "pdsRecord", "familyName_bin", "1972SMITH", "10001"}, + null, 1, live}, + Key2 = {{i, "pdsRecord", "familyName_bin", "1972SMITH", "10001"}, + null, 2, tomb}, + Key3 = {{i, "pdsRecord", "familyName_bin", "1971SMITH", "10002"}, + null, 2, live}, + Key4 = {{i, "pdsRecord", "familyName_bin", "1972JONES", "10003"}, + null, 2, live}, KeyList = lists:sort([Key1, Key2, Key3, Key4]), - HelperFuns = {fun folder_indexkey/2, fun compare_indexkey/2, fun pointercheck_indexkey/1}, - ResultList = ["10002", "10003"], - ?_assertEqual(ResultList, termiterator(null, KeyList, [], HelperFuns, "1971", "1973")). + HelperFuns = {fun folder_indexkey/2, fun compare_indexkey/2, + fun pointercheck_indexkey/1, fun getnextslice/2}, + ?assertMatch(["10002", "10003"], + termiterator(KeyList, HelperFuns, {"1971", "1973", infinite})). + +iterateoverindexkeyswithpointer_test() -> + Key1 = {{i, "pdsRecord", "familyName_bin", "1972SMITH", "10001"}, + null, 1, live}, + Key2 = {{i, "pdsRecord", "familyName_bin", "1972SMITH", "10001"}, + null, 2, tomb}, + Key3 = {{i, "pdsRecord", "familyName_bin", "1971SMITH", "10002"}, + null, 2, live}, + Key4 = {{i, "pdsRecord", "familyName_bin", "1972JONES", "10003"}, + null, 2, live}, + Key5 = {{i, "pdsRecord", "familyName_bin", "1972ZAFRIDI", "10004"}, + null, 2, live}, + Key6 = {{i, "pdsRecord", "familyName_bin", "1972JONES", "10004"}, + null, 0, {pointer, {test, [Key5]}}}, + KeyList = lists:sort([Key1, Key2, Key3, Key4, Key6]), + HelperFuns = {fun folder_indexkey/2, fun compare_indexkey/2, + fun pointercheck_indexkey/1, fun getnextslice/2}, + ?assertMatch(["10002", "10003", "10004"], + termiterator(KeyList, HelperFuns, {"1971", "1973", infinite})), + ?assertMatch(["10002", "10003"], + termiterator(KeyList, HelperFuns, {"1971", "1973", 2})). + + diff --git a/test/lookup_test.erl b/test/lookup_test.erl index f8632f2..c8b87aa 100644 --- a/test/lookup_test.erl +++ b/test/lookup_test.erl @@ -1,7 +1,11 @@ -module(lookup_test). --export([go_dict/1, go_ets/1, go_gbtree/1, - go_arrayofdict/1, go_arrayofgbtree/1, go_arrayofdict_withcache/1]). +-export([go_dict/1, + go_ets/1, + go_gbtree/1, + go_arrayofdict/1, + go_arrayofgbtree/1, + go_arrayofdict_withcache/1]). -define(CACHE_SIZE, 512). From 46de8ad6a2aa0b0cf0bddcb366743b9c7c229b00 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 10 Jun 2015 08:14:37 +0100 Subject: [PATCH 005/167] Write initial block to bst file --- src/leveled_bst.erl | 123 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 113 insertions(+), 10 deletions(-) diff --git a/src/leveled_bst.erl b/src/leveled_bst.erl index f8b8687..706969d 100644 --- a/src/leveled_bst.erl +++ b/src/leveled_bst.erl @@ -1,7 +1,7 @@ %% %% This module provides functions for managing bst files - a modified version %% of sst files, to be used in leveleddb. -%% bst files are borken into the following sections: +%% bst files are broken into the following sections: %% - Header (fixed width 32 bytes - containing pointers and metadata) %% - Blocks (variable length) %% - Slots (variable length) @@ -16,6 +16,17 @@ %% - 14 bytes spare for future options %% - 4 bytes CRC (header) %% +%% A key in the file is a tuple of {Key, Value/Metadata, Sequence #, State} +%% - Keys are themselves tuples, and all entries must be added to the bst +%% in key order +%% - Metadata can be null or some fast-access information that may be required +%% in preference to the full value (e.g. vector clocks, hashes). This could +%% be a value instead of Metadata should the file be sued in an alternate +%% - Sequence numbers is the integer representing the order which the item +%% was added (which also acts as reference to find the cdb file under which +%% the value is stored) +%% - State can be tomb (for tombstone), active or {timestamp, TS} +%% %% The Blocks is a series of blocks of: %% - 4 byte block length %% - variable-length compressed list of 32 keys & values @@ -29,8 +40,9 @@ %% - 64 ordered variable-length key helpers pointing to first key in each %% block (in slot) of the form Key Length, Key, Block Position %% - 4 byte CRC for the slot +%% - ulitmately a slot covers 64 x 32 = 2048 keys %% -%% The slot index in the footer is made up of 128 keys and pointers at the +%% The slot index in the footer is made up of 64 keys and pointers at the %% the start of each slot %% - 64 x Key Length (4 byte), Key, Position (4 byte) indexes %% - 4 bytes CRC for the index @@ -42,20 +54,20 @@ -module(leveled_bst). --export([start_file/1, convert_header/1]). +-export([start_file/1, convert_header/1, append_slot/4]). -include_lib("eunit/include/eunit.hrl"). -define(WORD_SIZE, 4). -define(CURRENT_VERSION, {0,1}). --define(SLOT_COUNT, 128). +-define(SLOT_COUNT, 64). -define(BLOCK_SIZE, 32). --define(SLOT_SIZE, 32). +-define(SLOT_SIZE, 64). -record(metadata, {version = ?CURRENT_VERSION :: tuple(), mutable = false :: true | false, compressed = true :: true | false, - slot_array, + slot_index, open_slot :: integer(), cache :: tuple(), smallest_key :: tuple(), @@ -64,6 +76,11 @@ largest_sqn :: integer() }). +-record(object, {key :: tuple(), + value, + sequence_numb :: integer(), + state}). + %% Start a bare file with an initial header and no further details %% Return the {Handle, metadata record} start_file(FileName) when is_list(FileName) -> @@ -72,10 +89,10 @@ start_file(FileName) when is_list(FileName) -> start_file(Handle) -> Header = create_header(initial), {ok, _} = file:position(Handle, bof), - file:write(Handle, Header), + ok = file:write(Handle, Header), {Version, {M, C}, _, _} = convert_header(Header), FileMD = #metadata{version = Version, mutable = M, compressed = C, - slot_array = array:new(?SLOT_COUNT), open_slot = 0}, + slot_index = array:new(?SLOT_COUNT), open_slot = 0}, {Handle, FileMD}. @@ -120,14 +137,85 @@ convert_header_v01(Header) -> {{0, 1}, {M, C}, {FooterP, SlotLng, HlpLng}, none}. -% add_slot(Handle, FileMD, SlotArray) +%% Append a slot of blocks to the end file, and update the slot index in the +%% file metadata +append_slot(Handle, SortedKVList, SlotCount, FileMD) -> + {ok, _} = file:position(Handle, eof), + {KeyList, BlockIndexBin, BlockBin} = add_blocks(SortedKVList), + ok = file:write(Handle, BlockBin), + [TopObject|_] = SortedKVList, + BloomBin = leveled_rice:create_bloom(KeyList), + SlotIndex = array:set(SlotCount, + {TopObject#object.key, BloomBin, BlockIndexBin}, + FileMD#metadata.slot_index), + {Handle, FileMD#metadata{slot_index=SlotIndex}}. + + +add_blocks(SortedKVList) -> + add_blocks(SortedKVList, [], [], [], 0). + +add_blocks([], KeyList, BlockIndex, BlockBinList, _) -> + {KeyList, serialise_blockindex(BlockIndex), list_to_binary(BlockBinList)}; +add_blocks(SortedKVList, KeyList, BlockIndex, BlockBinList, Position) -> + case length(SortedKVList) of + KeyCount when KeyCount >= ?BLOCK_SIZE -> + {TopBlock, Rest} = lists:split(?BLOCK_SIZE, SortedKVList); + KeyCount -> + {TopBlock, Rest} = lists:split(KeyCount, SortedKVList) + end, + [TopKey|_] = TopBlock, + TopBin = serialise_block(TopBlock), + add_blocks(Rest, add_to_keylist(KeyList, TopBlock), + [{TopKey, Position}|BlockIndex], + [TopBin|BlockBinList], Position + byte_size(TopBin)). + +add_to_keylist(KeyList, []) -> + KeyList; +add_to_keylist(KeyList, [TopKV|Rest]) -> + add_to_keylist([map_keyforbloom(TopKV)|KeyList], Rest). + +map_keyforbloom(_Object=#object{key=Key}) -> + Key. + + +serialise_blockindex(BlockIndex) -> + serialise_blockindex(BlockIndex, <<>>). + +serialise_blockindex([], BlockBin) -> + BlockBin; +serialise_blockindex([TopIndex|Rest], BlockBin) -> + {Key, BlockPos} = TopIndex, + KeyBin = serialise_key(Key), + KeyLength = byte_size(KeyBin), + serialise_blockindex(Rest, + <>). + +serialise_block(Block) -> + term_to_binary(Block). + +serialise_key(Key) -> + term_to_binary(Key). %%%%%%%%%%%%%%%% % T E S T %%%%%%%%%%%%%%% +create_sample_kv(Prefix, Counter) -> + Key = {o, "Bucket1", lists:concat([Prefix, Counter])}, + Object = #object{key=Key, value=null, + sequence_numb=random:uniform(1000000), state=active}, + Object. + +create_ordered_kvlist(KeyList, 0) -> + KeyList; +create_ordered_kvlist(KeyList, Length) -> + KV = create_sample_kv("Key", Length), + create_ordered_kvlist([KV|KeyList], Length - 1). + + empty_header_test() -> Header = create_header(initial), ?assertMatch(32, byte_size(Header)), @@ -152,7 +240,22 @@ bad_header_test() -> record_onstartfile_test() -> {_, FileMD} = start_file("onstartfile.bst"), - ?assertMatch({0, 1}, FileMD#metadata.version). + ?assertMatch({0, 1}, FileMD#metadata.version), + ok = file:delete("onstartfile.bst"). + +append_initialblock_test() -> + {Handle, FileMD} = start_file("onstartfile.bst"), + KVList = create_ordered_kvlist([], 2048), + Key1 = {o, "Bucket1", "Key1"}, + [TopObj|_] = KVList, + ?assertMatch(Key1, TopObj#object.key), + {_, UpdFileMD} = append_slot(Handle, KVList, 0, FileMD), + {TopKey1, BloomBin, _} = array:get(0, UpdFileMD#metadata.slot_index), + io:format("top key of ~w~n", [TopKey1]), + ?assertMatch(Key1, TopKey1), + ?assertMatch(true, leveled_rice:check_key(Key1, BloomBin)), + ?assertMatch(false, leveled_rice:check_key("OtherKey", BloomBin)). + From b7ae91fb7184a586fde76de59beed5d9b1d0abc1 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 2 Jul 2015 14:22:45 +0100 Subject: [PATCH 006/167] Write initial block to bst file - part 2 --- src/leveled_bst.erl | 101 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 93 insertions(+), 8 deletions(-) diff --git a/src/leveled_bst.erl b/src/leveled_bst.erl index 706969d..cc1c7c4 100644 --- a/src/leveled_bst.erl +++ b/src/leveled_bst.erl @@ -42,8 +42,9 @@ %% - 4 byte CRC for the slot %% - ulitmately a slot covers 64 x 32 = 2048 keys %% -%% The slot index in the footer is made up of 64 keys and pointers at the -%% the start of each slot +%% The slot index in the footer is made up of up to 64 ordered keys and +%% pointers, with the key being a key at the start of each slot +%% - 1 byte value showing number of keys in slot index %% - 64 x Key Length (4 byte), Key, Position (4 byte) indexes %% - 4 bytes CRC for the index %% @@ -59,10 +60,13 @@ -include_lib("eunit/include/eunit.hrl"). -define(WORD_SIZE, 4). +-define(DWORD_SIZE, 8). -define(CURRENT_VERSION, {0,1}). -define(SLOT_COUNT, 64). -define(BLOCK_SIZE, 32). -define(SLOT_SIZE, 64). +-define(FOOTERPOS_HEADERPOS, 2). + -record(metadata, {version = ?CURRENT_VERSION :: tuple(), mutable = false :: true | false, @@ -141,16 +145,81 @@ convert_header_v01(Header) -> %% file metadata append_slot(Handle, SortedKVList, SlotCount, FileMD) -> - {ok, _} = file:position(Handle, eof), + {ok, SlotPos} = file:position(Handle, eof), {KeyList, BlockIndexBin, BlockBin} = add_blocks(SortedKVList), ok = file:write(Handle, BlockBin), [TopObject|_] = SortedKVList, BloomBin = leveled_rice:create_bloom(KeyList), SlotIndex = array:set(SlotCount, - {TopObject#object.key, BloomBin, BlockIndexBin}, + {TopObject#object.key, BloomBin, BlockIndexBin, SlotPos}, FileMD#metadata.slot_index), {Handle, FileMD#metadata{slot_index=SlotIndex}}. +append_slot_index(Handle, _FileMD=#metadata{slot_index=SlotIndex}) -> + {ok, FooterPos} = file:position(Handle, eof), + SlotBin1 = <>, + SlotBin2 = array:foldl(fun slot_folder_write/3, SlotBin1, SlotIndex), + CRC = erlang:crc32(SlotBin2), + SlotBin3 = <>, + ok = file:write(Handle, SlotBin3), + SlotLength = byte_size(SlotBin3), + Header = <>, + ok = file:pwrite(Handle, ?FOOTERPOS_HEADERPOS, Header). + +slot_folder_write(_Index, undefined, Bin) -> + Bin; +slot_folder_write(_Index, {ObjectKey, _, _, SlotPos}, Bin) -> + KeyBin = serialise_key(ObjectKey), + KeyLen = byte_size(KeyBin), + <>. + +slot_folder_read(<<>>, SlotIndex, SlotCount) -> + io:format("Slot index read with count=~w slots~n", [SlotCount]), + SlotIndex; +slot_folder_read(SlotIndexBin, SlotIndex, SlotCount) -> + <> = SlotIndexBin, + <> = Tail1, + slot_folder_read(Tail2, + array:set(SlotCount, {load_key(KeyBin), null, null, SlotPos}, SlotIndex), + SlotCount + 1). + +read_slot_index(SlotIndexBin) -> + <> = SlotIndexBin, + case erlang:crc32(SlotIndexBin2) of + CRC -> + <> = SlotIndexBin2, + CleanSlotIndex = array:new(SlotCount), + SlotIndex = slot_folder_read(SlotIndexBin3, CleanSlotIndex, 0), + {ok, SlotIndex}; + _ -> + {error, crc_wonky} + end. + +find_slot_index(Handle) -> + {ok, SlotIndexPtr} = file:pread(Handle, ?FOOTERPOS_HEADERPOS, ?DWORD_SIZE), + <> = SlotIndexPtr, + {ok, SlotIndexBin} = file:pread(Handle, FooterPos, SlotIndexLength), + SlotIndexBin. + + +read_blockindex(Handle, Position) -> + {ok, _FilePos} = file:position(Handle, Position), + {ok, <>} = file:read(Handle, 4), + io:format("block length is ~w~n", [BlockLength]), + {ok, BlockBin} = file:read(Handle, BlockLength), + CheckLessBlockLength = BlockLength - 4, + <> = BlockBin, + case erlang:crc32(Block) of + CRC -> + <> = Block, + <> = Tail, + {ok, BloomFilter, KeyHelper}; + _ -> + {error, crc_wonky} + end. + add_blocks(SortedKVList) -> add_blocks(SortedKVList, [], [], [], 0). @@ -198,6 +267,9 @@ serialise_block(Block) -> serialise_key(Key) -> term_to_binary(Key). +load_key(KeyBin) -> + binary_to_term(KeyBin). + %%%%%%%%%%%%%%%% % T E S T @@ -250,13 +322,26 @@ append_initialblock_test() -> [TopObj|_] = KVList, ?assertMatch(Key1, TopObj#object.key), {_, UpdFileMD} = append_slot(Handle, KVList, 0, FileMD), - {TopKey1, BloomBin, _} = array:get(0, UpdFileMD#metadata.slot_index), + {TopKey1, BloomBin, _, _} = array:get(0, UpdFileMD#metadata.slot_index), io:format("top key of ~w~n", [TopKey1]), ?assertMatch(Key1, TopKey1), ?assertMatch(true, leveled_rice:check_key(Key1, BloomBin)), - ?assertMatch(false, leveled_rice:check_key("OtherKey", BloomBin)). - - + ?assertMatch(false, leveled_rice:check_key("OtherKey", BloomBin)), + ok = file:delete("onstartfile.bst"). + +append_initialslotindex_test() -> + {Handle, FileMD} = start_file("onstartfile.bst"), + KVList = create_ordered_kvlist([], 2048), + {_, UpdFileMD} = append_slot(Handle, KVList, 0, FileMD), + append_slot_index(Handle, UpdFileMD), + SlotIndexBin = find_slot_index(Handle), + {ok, SlotIndex} = read_slot_index(SlotIndexBin), + io:format("slot index is ~w ~n", [SlotIndex]), + TopItem = array:get(0, SlotIndex), + io:format("top item in slot index is ~w~n", [TopItem]), + {ok, BloomFilter, KeyHelper} = read_blockindex(Handle, 32), + ?assertMatch(true, false), + ok = file:delete("onstartfile.bst"). From a95d77607e96a952ee63ba5527c65d285c589ac8 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 31 May 2016 17:21:14 +0100 Subject: [PATCH 007/167] Initial work on sft files Working on the delta-encoded segment filter, plus some initial performance testing. --- src/leveled_bst.erl | 13 +- src/leveled_sft.erl | 741 +++++++++++++++++++++++++++++++++++++++++++ test/lookup_test.erl | 80 ++++- 3 files changed, 828 insertions(+), 6 deletions(-) create mode 100644 src/leveled_sft.erl diff --git a/src/leveled_bst.erl b/src/leveled_bst.erl index cc1c7c4..9253667 100644 --- a/src/leveled_bst.erl +++ b/src/leveled_bst.erl @@ -3,17 +3,21 @@ %% of sst files, to be used in leveleddb. %% bst files are broken into the following sections: %% - Header (fixed width 32 bytes - containing pointers and metadata) +%% - Summaries (variable length) %% - Blocks (variable length) %% - Slots (variable length) %% - Footer (variable length - contains slot index and helper metadata) %% -%% The 32-byte header is made up of +%% The 64-byte header is made up of %% - 1 byte version (major 5 bits, minor 3 bits) - default 0.1 %% - 1 byte state bits (1 bit to indicate mutability, 1 for use of compression) +%% - 4 bytes summary length +%% - 4 bytes blocks length +%% - 4 bytes slots length %% - 4 bytes footer position %% - 4 bytes slot list length %% - 4 bytes helper length -%% - 14 bytes spare for future options +%% - 34 bytes spare for future options %% - 4 bytes CRC (header) %% %% A key in the file is a tuple of {Key, Value/Metadata, Sequence #, State} @@ -21,10 +25,9 @@ %% in key order %% - Metadata can be null or some fast-access information that may be required %% in preference to the full value (e.g. vector clocks, hashes). This could -%% be a value instead of Metadata should the file be sued in an alternate +%% be a value instead of Metadata should the file be used in an alternate %% - Sequence numbers is the integer representing the order which the item -%% was added (which also acts as reference to find the cdb file under which -%% the value is stored) +%% was added to the overall database %% - State can be tomb (for tombstone), active or {timestamp, TS} %% %% The Blocks is a series of blocks of: diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl new file mode 100644 index 0000000..cb253dc --- /dev/null +++ b/src/leveled_sft.erl @@ -0,0 +1,741 @@ +%% This module provides functions for managing sft files - a modified version +%% of sst files, to be used in leveleddb. +%% +%% sft files are segment filtered tables in that they are guarded by a quick +%% access filter that checks for the presence of key by segment id, with the +%% segment id being a hash in the range 0 - 1024 * 1024 +%% +%% This filter has a dual purpose +%% - a memory efficient way of discovering non-presence with low false positive +%% rate +%% - to make searching for all keys by hashtree segment more efficient (a +%% specific change to optimise behaviour for use with the incremental refresh) +%% of riak hashtrees +%% +%% All keys are not equal in sft files, keys are only expected in a specific +%% series of formats +%% - {o, Bucket, Key, State} - Object Keys +%% - {i, Bucket, IndexName, IndexTerm, Key, State} - Postings +%% The {Bucket, Key} part of all types of keys are hashed for segment filters. +%% For Postings the {Bucket, IndexName, IndexTerm} is also hashed. This +%% causes a false positive on lookup of a segment, but allows for the presence +%% of specific index terms to be checked +%% +%% The objects stored are a tuple of {Key, State, Value}, where +%% Key - as above +%% State - {SequenceNumber, active|tomb, ExpiryTimestamp | infinity} +%% Value - null (all postings) | [Object Metadata] (all object keys) +%% Keys should be unique in files. If more than two keys are candidate for +%% the same file the highest sequence number should be chosen. If the file +%% is at the basemenet level of a leveleddb database the objects with an +%% ExpiryTimestamp in the past should not be written, but at all other levels +%% keys should not be ignored because of a timestamp in the past. +%% tomb objects are written for deletions, and these tombstones may have an +%% Expirytimestamp which in effect is the time when the tombstone should be +%% reaped. +%% +%% sft files are broken into the following sections: +%% - Header (fixed width 80 bytes - containing pointers and metadata) +%% - Blocks (variable length) +%% - Slot Filter (variable length) +%% - Slot Index (variable length) +%% - Table Summary (variable length) +%% Each section should contain at the footer of the section a 4-byte CRC which +%% is to be checked only on the opening of the file +%% +%% The keys in the sft file are placed into the file in erlang term order. +%% There will normally be 256 slots of keys. The Slot Index is a gb_tree +%% acting as a helper to find the right slot to check when searching for a key +%% or range of keys. +%% The Key in the Slot Index is the Key at the start of the Slot. +%% The Value in the Slot Index is a record indicating: +%% - The starting position of the Slot within the Blocks (relative to the +%% starting position of the Blocks) +%% - The (relative) starting position of the Slot Filter for this Slot +%% - The number of blocks within the Slot +%% - The length of each of the Blocks within the Slot +%% +%% When checking for a Key in the sft file, the key should be hashed to the +%% segment, then the key should be looked-up in the Slot Index. The segment +%% ID can then be checked against the Slot Filter which will either return +%% not_present or [BlockIDs] +%% If a list of BlockIDs (normally of length 1) is returned the block should +%% be fetched using the starting position and length of the Block to find the +%% actual key (or not if the Slot Filter had returned a false positive) +%% +%% There will exist a Slot Filter for each entry in the Slot Index +%% The Slot Filter starts with some fixed length metadata +%% - 1 byte stating the expected number of keys in the block +%% - 1 byte stating the number of complete (i.e. containing the expected +%% number of keys) Blocks in the Slot +%% - 1 byte stating the number of keys in any incomplete Block (there can +%% only be 1 incomplete Block per Slot and it must be the last block) +%% - 3 bytes stating the largest segment ID in the Slot +%% - 1 byte stating the exponent used in the rice-encoding of the filter +%% The Filter itself is a rice-encoded list of Integers representing the +%% differences between the Segment IDs in the Slot with each entry being +%% appended by the minimal number of bits to represent the Block ID in which +%% an entry for that segment can be found. Where a segment exists more than +%% once then a 0 length will be used. +%% To use the filter code should roll over the filter incrementing the Segment +%% ID by each difference, and counting the keys by Block ID. This should +%% return one of: +%% mismatch - the final Segment Count didn't meet the largest Segment ID or +%% the per-block key counts don't add-up. There could have been a bit-flip, +%% so don't rely on the filter +%% no_match - everything added up but the counter never equalled the queried +%% Segment ID +%% {match, [BlockIDs]} - everything added up and the Segment may be +%% represented in the given blocks +%% +%% The makeup of a block +%% - A block is a list of 32 {Key, Value} pairs in Erlang term order +%% - The block is stored using standard compression in term_to_binary +%% May be improved by use of lz4 or schema-based binary_to_term +%% +%% The Table Summary may contain multiple summaries +%% The standard table summary contains: +%% - a count of keys by bucket and type of key (posting or object key) +%% - the total size of objects referred to by object keys +%% - the number of postings by index name +%% - the number of tombstones within the file +%% - the highest and lowest sequence number in the file +%% Summaries could be used for other summaries of table content in the future, +%% perhaps application-specific bloom filters + +%% The 80-byte header is made up of +%% - 1 byte version (major 5 bits, minor 3 bits) - default 0.1 +%% - 1 byte options (currently undefined) +%% - 1 byte Block Size - the expected number of keys in each block +%% - 1 byte Block Count - the expected number of blocks in each slot +%% - 2 byte Slot Count - the maximum number of slots in the file +%% - 6 bytes - spare +%% - 4 bytes - Blocks position +%% - 4 bytes - Blocks length +%% - 4 bytes - Slot Index position +%% - 4 bytes - Slot Index length +%% - 4 bytes - Slot Filter position +%% - 4 bytes - Slot Filter length +%% - 4 bytes - Table Summary position +%% - 4 bytes - Table summary length +%% - 24 bytes - spare +%% - 4 bytes - CRC32 +%% +%% The file body is written in the same order of events as the header (i.e. +%% Blocks first) +%% +%% Once open the file can be in the following states +%% - writing, the file is still being created +%% - available, the file may be read, but never again must be modified +%% - pending_deletion, the file can be closed and deleted once all outstanding +%% Snapshots have been started beyond a certain sequence number +%% +%% Level managers should only be aware of files in the available state. +%% Iterators may be aware of files in either available or pending_delete. +%% Level maintainers should control the file exclusively when in the writing +%% state, and send the event to trigger pending_delete with the a sequence +%% number equal to or higher than the number at the point it was no longer +%% active at any level. +%% +%% The format of the file is intended to support quick lookups, whilst +%% allowing for a new file to be written incrementally (so that all keys and +%% values need not be retained in memory) - perhaps n blocks at a time + + +-module(leveled_sft). + +-export([create_file/1, + generate_segment_filter/1, + serialise_segment_filter/1, + check_for_segments/3, + speedtest_check_forsegment/4, + generate_randomsegfilter/1]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(WORD_SIZE, 4). +-define(DWORD_SIZE, 8). +-define(CURRENT_VERSION, {0,1}). +-define(SLOT_COUNT, 256). +-define(BLOCK_SIZE, 32). +-define(BLOCK_COUNT, 4). +-define(FOOTERPOS_HEADERPOS, 2). +-define(MAX_SEG_HASH, 1048576). +-define(DIVISOR_BITS, 13). +-define(DIVISOR, 8092). + + +-record(state, {version = ?CURRENT_VERSION :: tuple(), + slot_index = gb_trees:empty() :: gb_trees:tree(), + next_position :: integer(), + smallest_sqn :: integer(), + largest_sqn :: integer()}). + + +%% Start a bare file with an initial header and no further details +%% Return the {Handle, metadata record} +create_file(FileName) when is_list(FileName) -> + {ok, Handle} = file:open(FileName, [binary, raw, read, write]), + create_file(Handle); +create_file(Handle) -> + Header = create_header(initial), + {ok, _} = file:position(Handle, bof), + ok = file:write(Handle, Header), + {ok, StartPos} = file:position(Handle), + FileMD = #state{next_position=StartPos}, + {Handle, FileMD}. + +create_header(initial) -> + {Major, Minor} = ?CURRENT_VERSION, + Version = <>, + Options = <<0:8>>, % Not thought of any options + {BlSize, BlCount, SlCount} = {?BLOCK_COUNT, ?BLOCK_SIZE, ?SLOT_COUNT}, + Settings = <>, + {SpareO, SpareL} = {<<0:48>>, <<0:192>>}, + Lengths = <<0:32, 0:32, 0:32, 0:32>>, + H1 = <>, + CRC32 = erlang:crc32(H1), + <

>. + +%% Take two potentially overlapping lists of keys and output a Block, +%% together with: +%% - block status (full, partial) +%% - the lowest and highest sequence numbers in the block +%% - the list of segment IDs in the block +%% - the remainders of the lists +%% The Key lists must be sorted in key order. The last key in a list may be +%% a pointer to request more keys for the file (otherwise it is assumed there +%% are no more keys) +%% +%% Level also to be passed in +%% This is either an integer (to be ignored) of {floor, os:timestamp()} +%% if this is the basement level of the LevelDB database and expired keys +%% and tombstone should be reaped + + +%% Do we need to check here that KeyList1 and KeyList2 are not just a [pointer] +%% Otherwise the pointer will never be expanded + +create_block(KeyList1, KeyList2, Level) -> + create_block(KeyList1, KeyList2, [], {0, 0}, [], Level). + +create_block(KeyList1, KeyList2, + BlockKeyList, {LSN, HSN}, SegmentList, _) + when length(BlockKeyList)==?BLOCK_SIZE -> + {BlockKeyList, full, {LSN, HSN}, SegmentList, KeyList1, KeyList2}; +create_block([], [], + BlockKeyList, {LSN, HSN}, SegmentList, _) -> + {BlockKeyList, partial, {LSN, HSN}, SegmentList, [], []}; +create_block(KeyList1, KeyList2, + BlockKeyList, {LSN, HSN}, SegmentList, Level) -> + case key_dominates(KeyList1, KeyList2, Level) of + {{next_key, TopKey}, Rem1, Rem2} -> + io:format("TopKey is ~w~n", [TopKey]), + {UpdLSN, UpdHSN} = update_sequencenumbers(TopKey, LSN, HSN), + NewBlockKeyList = lists:append(BlockKeyList, + [TopKey]), + NewSegmentList = lists:append(SegmentList, + [hash_for_segmentid(TopKey)]), + create_block(Rem1, Rem2, + NewBlockKeyList, {UpdLSN, UpdHSN}, + NewSegmentList, Level); + {skipped_key, Rem1, Rem2} -> + io:format("Key is skipped~n"), + create_block(Rem1, Rem2, + BlockKeyList, {LSN, HSN}, + SegmentList, Level) + end. + + + +%% Should return an index entry in the Slot Index. Each entry consists of: +%% - Start Key +%% - SegmentIDFilter for the (will eventually be replaced with a pointer) +%% - Serialised Slot (will eventually be replaced with a pointer) +%% - Length for each Block within the Serialised Slot +%% Additional information will also be provided +%% - {Low Seq Number, High Seq Number} within the slot +%% - End Key +%% - Whether the slot is full or partially filled +%% - Remainder of any KeyLists used to make the slot + + +%% create_slot(KeyList1, KeyList2, Level) +%% create_slot(KeyList1, KeyList2, Level, ?BLOCK_COUNT, null, <<>>, <<>>, []). + +%% create_slot(KL1, KL2, Level, 0, LowKey, SegFilter, SerialisedSlot, +%% LengthList, {LSN, HSN}, LastKey) -> +%% {{LowKey, SegFilter, SerialisedSlot, LengthList}, +%% {{LSN, HSN}, LastKey, full, KL1, KL2}}; +%% create_slot(KL1, KL2, Level, BlockCount, LowKey, SegFilter, SerialisedSlot, +%% LengthList, {LSN, HSN}, LastKey) -> +%% BlockDetails = create_block(KeyList1, KeyList2, Level), +%% {BlockKeyList, Status, {LSN, HSN}, SegmentList, KL1, KL2} = BlockDetails, +%% SerialisedBlock = serialise_block(BlockKeyList), +%% <>, + +%% case Status of +%% full -> + + + + +%% Compare the keys at the head of the list, and either skip that "best" key or +%% identify as the next key. +%% +%% The logic needs to change if the file is in the basement level, as keys with +%% expired timestamps need not be written at this level +%% +%% The best key is considered to be the lowest key in erlang term order. If +%% there are matching keys then the highest sequence number must be chosen and +%% any lower sequence numbers should be compacted out of existence + +key_dominates([H1|T1], [], Level) -> + {_, _, St1, _} = H1, + case maybe_reap_expiredkey(St1, Level) of + true -> + {skipped_key, maybe_expand_pointer(T1), []}; + false -> + {{next_key, H1}, maybe_expand_pointer(T1), []} + end; +key_dominates([], [H2|T2], Level) -> + {_, _, St2, _} = H2, + case maybe_reap_expiredkey(St2, Level) of + true -> + {skipped_key, [], maybe_expand_pointer(T2)}; + false -> + {{next_key, H2}, [], maybe_expand_pointer(T2)} + end; +key_dominates([H1|T1], [H2|T2], Level) -> + {K1, Sq1, St1, _} = H1, + {K2, Sq2, St2, _} = H2, + case K1 of + K2 -> + case Sq1 > Sq2 of + true -> + {skipped_key, [H1|T1], maybe_expand_pointer(T2)}; + false -> + {skipped_key, maybe_expand_pointer(T1), [H2|T2]} + end; + K1 when K1 < K2 -> + case maybe_reap_expiredkey(St1, Level) of + true -> + {skipped_key, maybe_expand_pointer(T1), [H2|T2]}; + false -> + {{next_key, H1}, maybe_expand_pointer(T1), [H2|T2]} + end; + _ -> + case maybe_reap_expiredkey(St2, Level) of + true -> + {skipped_key, [H1|T1], maybe_expand_pointer(T2)}; + false -> + {{next_key, H2}, [H1|T1], maybe_expand_pointer(T2)} + end + end. + + +maybe_reap_expiredkey({_, infinity}, _) -> + false; % key is not set to expire +maybe_reap_expiredkey({_, TS}, {basement, CurrTS}) when CurrTS > TS -> + true; % basement and ready to expire +maybe_reap_expiredkey(_, _) -> + false. + +%% Not worked out pointers yet +maybe_expand_pointer(Tail) -> + Tail. + +%% Update the sequence numbers +update_sequencenumbers({_, SN, _, _}, 0, 0) -> + {SN, SN}; +update_sequencenumbers({_, SN, _, _}, LSN, HSN) when SN < LSN -> + {SN, HSN}; +update_sequencenumbers({_, SN, _, _}, LSN, HSN) when SN > HSN -> + {LSN, SN}; +update_sequencenumbers({_, _, _, _}, LSN, HSN) -> + {LSN, HSN}. + + +%% The Segment filter is a compressed filter representing the keys in a +%% given slot. The filter is delta-compressed list of integers using rice +%% encoding extended by the reference to each integer having an extra two bits +%% to indicate the block - there are four blocks in each slot. +%% +%% So each delta is represented as +%% - variable length exponent ending in 0, +%% with 0 representing the exponent of 0, +%% 10 -> 2 ^ 13, +%% 110 -> 2^14, +%% 1110 -> 2^15 etc +%% - 13-bit fixed length remainder +%% - 2-bit block number +%% This gives about 2-bytes per key, with a 1:8000 (approx) false positive +%% ratio (when checking the key by hashing to the segment ID) +%% +%% Before the delta list are three 20-bit integers representing the highest +%% integer in each block. Plus two bytes to indicate how many hashes +%% there are in the slot +%% +%% To check for the presence of a segment in a slot, roll over the deltas +%% keeping a running total overall and the current highest segment ID seen +%% per block. Roll all the way through even if matches are found or passed +%% over to confirm that the totals match the expected value (hence creating +%% a natural checksum) +%% +%% The end-result is a 260-byte check for the presence of a key in a slot +%% returning the block in which the segment can be found, which may also be +%% used directly for checking for the presence of segments. +%% +%% This is more space efficient than the equivalent bloom filter and avoids +%% the calculation of many hash functions. + +generate_segment_filter(SegLists) -> + generate_segment_filter(merge_seglists(SegLists), + [], + [{0, 0}, {0, 1}, {0, 2}, {0, 3}]). + +%% to generate the segment filter needs a sorted list of {Delta, Block} pairs +%% as DeltaList and a list of {TopHash, Block} pairs as TopHashes + +generate_segment_filter([], DeltaList, TopHashes) -> + {lists:reverse(DeltaList), TopHashes}; +generate_segment_filter([NextSeg|SegTail], DeltaList, TopHashes) -> + {TopHash, _} = lists:max(TopHashes), + {NextSegHash, NextSegBlock} = NextSeg, + DeltaList2 = [{NextSegHash - TopHash, NextSegBlock}|DeltaList], + TopHashes2 = lists:keyreplace(NextSegBlock, 2, TopHashes, + {NextSegHash, NextSegBlock}), + generate_segment_filter(SegTail, DeltaList2, TopHashes2). + + +serialise_segment_filter({DeltaList, TopHashes}) -> + TopHashesBin = lists:foldl(fun({X, _}, Acc) -> + <> end, + <<>>, TopHashes), + Length = length(DeltaList), + HeaderBin = <>, + {Divisor, Factor} = {?DIVISOR, ?DIVISOR_BITS}, + F = fun({Delta, Block}, Acc) -> + Exponent = buildexponent(Delta div Divisor), + Remainder = Delta rem Divisor, + Block2Bit = Block, + <> end, + lists:foldl(F, HeaderBin, DeltaList). + + +buildexponent(Exponent) -> + buildexponent(Exponent, <<0:1>>). + +buildexponent(0, OutputBits) -> + OutputBits; +buildexponent(Exponent, OutputBits) -> + buildexponent(Exponent - 1, <<1:1, OutputBits/bitstring>>). + +merge_seglists({SegList1, SegList2, SegList3, SegList4}) -> + Stage1 = lists:foldl(fun(X, Acc) -> [{X, 0}|Acc] end, [], SegList1), + Stage2 = lists:foldl(fun(X, Acc) -> [{X, 1}|Acc] end, Stage1, SegList2), + Stage3 = lists:foldl(fun(X, Acc) -> [{X, 2}|Acc] end, Stage2, SegList3), + Stage4 = lists:foldl(fun(X, Acc) -> [{X, 3}|Acc] end, Stage3, SegList4), + lists:sort(Stage4). + +hash_for_segmentid(Key) -> + erlang:phash2(Key). + + +%% Check for a given list of segments in the filter, returning in normal +%% operations a TupleList of {SegmentID, [ListOFBlocks]} where the ListOfBlocks +%% are the block IDs which contain keys in that given segment +%% +%% If there is a failure - perhaps due to a bit flip of some sort an error +%% willl be returned (error_so_maybe_present) and all blocks should be checked +%% as the filter cannot be relied upon + +check_for_segments(SegFilter, SegmentList, CRCCheck) -> + case CRCCheck of + true -> + <> = SegFilter, + CheckSum = [T0, T1, T2, T3], + case safecheck_for_segments(SegRem, SegmentList, + [0, 0, 0, 0], + 0, Count, []) of + {error_so_maybe_present, Reason} -> + io:format("Segment filter failed due to ~s~n", [Reason]), + error_so_maybe_present; + {OutputCheck, BlockList} when OutputCheck == CheckSum, + BlockList == [] -> + not_present; + {OutputCheck, BlockList} when OutputCheck == CheckSum -> + {maybe_present, BlockList}; + {OutputCheck, _} -> + io:format("Segment filter failed due to CRC check~n + ~w did not match ~w~n", + [OutputCheck, CheckSum]), + error_so_maybe_present + end; + false -> + <<_:80/bitstring, Count:16/integer, SegRem/bitstring>> = SegFilter, + case quickcheck_for_segments(SegRem, SegmentList, + lists:max(SegmentList), + 0, Count, []) of + {error_so_maybe_present, Reason} -> + io:format("Segment filter failed due to ~s~n", [Reason]), + error_so_maybe_present; + BlockList when BlockList == [] -> + not_present; + BlockList -> + {maybe_present, BlockList} + end + end. + + +safecheck_for_segments(_, _, TopHashes, _, 0, BlockList) -> + {TopHashes, BlockList}; +safecheck_for_segments(Filter, SegmentList, TopHs, Acc, Count, BlockList) -> + case findexponent(Filter) of + {ok, Exp, FilterRem1} -> + case findremainder(FilterRem1, ?DIVISOR_BITS) of + {ok, Remainder, BlockID, FilterRem2} -> + {NextHash, BlockList2} = checkhash_forsegments(Acc, + Exp, + Remainder, + SegmentList, + BlockList, + BlockID), + TopHashes2 = setnth(BlockID, TopHs, NextHash), + safecheck_for_segments(FilterRem2, SegmentList, + TopHashes2, + NextHash, Count - 1, + BlockList2); + error -> + {error_so_maybe_present, "Remainder Check"} + end; + error -> + {error_so_maybe_present, "Exponent Check"} + end. + +quickcheck_for_segments(_, _, _, _, 0, BlockList) -> + BlockList; +quickcheck_for_segments(Filter, SegmentList, MaxSeg, Acc, Count, BlockList) -> + case findexponent(Filter) of + {ok, Exp, FilterRem1} -> + case findremainder(FilterRem1, ?DIVISOR_BITS) of + {ok, Remainder, BlockID, FilterRem2} -> + {NextHash, BlockList2} = checkhash_forsegments(Acc, + Exp, + Remainder, + SegmentList, + BlockList, + BlockID), + case NextHash > MaxSeg of + true -> + BlockList2; + false -> + quickcheck_for_segments(FilterRem2, SegmentList, + MaxSeg, + NextHash, Count - 1, + BlockList2) + end; + error -> + {error_so_maybe_present, "Remainder Check"} + end; + error -> + {error_so_maybe_present, "Exponent Check"} + end. + + +checkhash_forsegments(Acc, Exp, Remainder, SegmentList, BlockList, BlockID) -> + NextHash = Acc + ?DIVISOR * Exp + Remainder, + case lists:member(NextHash, SegmentList) of + true -> + {NextHash, [BlockID|BlockList]}; + false -> + {NextHash, BlockList} + end. + + +setnth(0, [_|Rest], New) -> [New|Rest]; +setnth(I, [E|Rest], New) -> [E|setnth(I-1, Rest, New)]. + + +findexponent(BitStr) -> + findexponent(BitStr, 0). + +findexponent(<<>>, _) -> + error; +findexponent(<>, Acc) -> + case H of + 1 -> findexponent(T, Acc + 1); + 0 -> {ok, Acc, T} + end. + + +findremainder(BitStr, Factor) -> + case BitStr of + <> -> + {ok, Remainder, BlockID, Tail}; + _ -> + error + end. + + + + +%%%%%%%%%%%%%%%% +% T E S T +%%%%%%%%%%%%%%% + + +speedtest_check_forsegment(_, 0, _, _) -> + true; +speedtest_check_forsegment(SegFilter, LoopCount, CRCCheck, IDsToCheck) -> + check_for_segments(SegFilter, gensegmentids(IDsToCheck), CRCCheck), + speedtest_check_forsegment(SegFilter, LoopCount - 1, CRCCheck, IDsToCheck). + +gensegmentids(Count) -> + gensegmentids([], Count). + +gensegmentids(GeneratedIDs, 0) -> + lists:sort(GeneratedIDs); +gensegmentids(GeneratedIDs, Count) -> + gensegmentids([random:uniform(1024*1024)|GeneratedIDs], Count - 1). + + +generate_randomsegfilter(BlockSize) -> + Block1 = gensegmentids(BlockSize), + Block2 = gensegmentids(BlockSize), + Block3 = gensegmentids(BlockSize), + Block4 = gensegmentids(BlockSize), + serialise_segment_filter(generate_segment_filter({Block1, + Block2, + Block3, + Block4})). + + +simple_create_block_test() -> + KeyList1 = [{{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}, + {{o, "Bucket1", "Key3"}, 2, {active, infinity}, null}], + KeyList2 = [{{o, "Bucket1", "Key2"}, 3, {active, infinity}, null}], + {MergedKeyList, ListStatus, SN, _, _, _} = create_block(KeyList1, + KeyList2, + 1), + ?assertMatch(partial, ListStatus), + [H1|T1] = MergedKeyList, + ?assertMatch(H1, {{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}), + [H2|T2] = T1, + ?assertMatch(H2, {{o, "Bucket1", "Key2"}, 3, {active, infinity}, null}), + ?assertMatch(T2, [{{o, "Bucket1", "Key3"}, 2, {active, infinity}, null}]), + io:format("SN is ~w~n", [SN]), + ?assertMatch(SN, {1,3}). + +dominate_create_block_test() -> + KeyList1 = [{{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}, + {{o, "Bucket1", "Key2"}, 2, {active, infinity}, null}], + KeyList2 = [{{o, "Bucket1", "Key2"}, 3, {tomb, infinity}, null}], + {MergedKeyList, ListStatus, SN, _, _, _} = create_block(KeyList1, + KeyList2, + 1), + ?assertMatch(partial, ListStatus), + [K1, K2] = MergedKeyList, + ?assertMatch(K1, {{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}), + ?assertMatch(K2, {{o, "Bucket1", "Key2"}, 3, {tomb, infinity}, null}), + ?assertMatch(SN, {1,3}). + +alternating_create_block_test() -> + KeyList1 = [{{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}, + {{o, "Bucket1", "Key3"}, 1, {active, infinity}, null}, + {{o, "Bucket1", "Key5"}, 1, {active, infinity}, null}, + {{o, "Bucket1", "Key7"}, 1, {active, infinity}, null}, + {{o, "Bucket1", "Key9"}, 1, {active, infinity}, null}, + {{o, "Bucket2", "Key1"}, 1, {active, infinity}, null}, + {{o, "Bucket2", "Key3"}, 1, {active, infinity}, null}, + {{o, "Bucket2", "Key5"}, 1, {active, infinity}, null}, + {{o, "Bucket2", "Key7"}, 1, {active, infinity}, null}, + {{o, "Bucket2", "Key9"}, 1, {active, infinity}, null}, + {{o, "Bucket3", "Key1"}, 1, {active, infinity}, null}, + {{o, "Bucket3", "Key3"}, 1, {active, infinity}, null}, + {{o, "Bucket3", "Key5"}, 1, {active, infinity}, null}, + {{o, "Bucket3", "Key7"}, 1, {active, infinity}, null}, + {{o, "Bucket3", "Key9"}, 1, {active, infinity}, null}, + {{o, "Bucket4", "Key1"}, 1, {active, infinity}, null}], + KeyList2 = [{{o, "Bucket1", "Key2"}, 1, {active, infinity}, null}, + {{o, "Bucket1", "Key4"}, 1, {active, infinity}, null}, + {{o, "Bucket1", "Key6"}, 1, {active, infinity}, null}, + {{o, "Bucket1", "Key8"}, 1, {active, infinity}, null}, + {{o, "Bucket1", "Key9a"}, 1, {active, infinity}, null}, + {{o, "Bucket1", "Key9b"}, 1, {active, infinity}, null}, + {{o, "Bucket1", "Key9c"}, 1, {active, infinity}, null}, + {{o, "Bucket1", "Key9d"}, 1, {active, infinity}, null}, + {{o, "Bucket2", "Key2"}, 1, {active, infinity}, null}, + {{o, "Bucket2", "Key4"}, 1, {active, infinity}, null}, + {{o, "Bucket2", "Key6"}, 1, {active, infinity}, null}, + {{o, "Bucket2", "Key8"}, 1, {active, infinity}, null}, + {{o, "Bucket3", "Key2"}, 1, {active, infinity}, null}, + {{o, "Bucket3", "Key4"}, 1, {active, infinity}, null}, + {{o, "Bucket3", "Key6"}, 1, {active, infinity}, null}, + {{o, "Bucket3", "Key8"}, 1, {active, infinity}, null}], + {MergedKeyList, ListStatus, _, _, _, _} = create_block(KeyList1, + KeyList2, + 1), + BlockSize = length(MergedKeyList), + io:format("Block size is ~w~n", [BlockSize]), + ?assertMatch(BlockSize, 32), + ?assertMatch(ListStatus, full), + K1 = lists:nth(1, MergedKeyList), + ?assertMatch(K1, {{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}), + K11 = lists:nth(11, MergedKeyList), + ?assertMatch(K11, {{o, "Bucket1", "Key9b"}, 1, {active, infinity}, null}), + K32 = lists:nth(32, MergedKeyList), + ?assertMatch(K32, {{o, "Bucket4", "Key1"}, 1, {active, infinity}, null}). + + +merge_seglists_test() -> + SegList1 = [0, 100, 200], + SegList2 = [50, 200], + SegList3 = [75, 10000], + SegList4 = [], + MergedList = merge_seglists({SegList1, SegList2, + SegList3, SegList4}), + ?assertMatch(MergedList, [{0, 0}, {50, 1}, {75, 2}, {100, 0}, + {200, 0}, {200,1}, {10000,2}]), + SegTerm = generate_segment_filter({SegList1, SegList2, + SegList3, SegList4}), + ?assertMatch(SegTerm, {[{0, 0}, {50, 1}, {25, 2}, {25, 0}, + {100, 0}, {0, 1}, {9800, 2}], + [{200, 0}, {200, 1}, {10000, 2},{0, 3}]}), + SegBin = serialise_segment_filter(SegTerm), + ExpectedTopHashes = <<200:20, 200:20, 10000:20, 0:20>>, + ExpectedDeltas = <<0:1, 0:13, 0:2, + 0:1, 50:13, 1:2, + 0:1, 25:13, 2:2, + 0:1, 25:13, 0:2, + 0:1, 100:13, 0:2, + 0:1, 0:13, 1:2, + 2:2, 1708:13, 2:2>>, + ExpectedResult = <>, + ?assertMatch(SegBin, ExpectedResult), + R1 = check_for_segments(SegBin, [100], true), + ?assertMatch(R1,{maybe_present, [0]}), + R2 = check_for_segments(SegBin, [900], true), + ?assertMatch(R2, not_present), + R3 = check_for_segments(SegBin, [200], true), + ?assertMatch(R3, {maybe_present, [1,0]}), + R4 = check_for_segments(SegBin, [0,900], true), + ?assertMatch(R4, {maybe_present, [0]}), + R5 = check_for_segments(SegBin, [100], false), + ?assertMatch(R5, {maybe_present, [0]}), + R6 = check_for_segments(SegBin, [900], false), + ?assertMatch(R6, not_present), + R7 = check_for_segments(SegBin, [200], false), + ?assertMatch(R7, {maybe_present, [1,0]}), + R8 = check_for_segments(SegBin, [0,900], false), + ?assertMatch(R8, {maybe_present, [0]}), + R9 = check_for_segments(SegBin, [1024*1024 - 1], false), + ?assertMatch(R9, not_present). + diff --git a/test/lookup_test.erl b/test/lookup_test.erl index c8b87aa..8afe7a4 100644 --- a/test/lookup_test.erl +++ b/test/lookup_test.erl @@ -5,7 +5,10 @@ go_gbtree/1, go_arrayofdict/1, go_arrayofgbtree/1, - go_arrayofdict_withcache/1]). + go_arrayofdict_withcache/1, + create_blocks/3, + size_testblocks/1, + test_testblocks/2]). -define(CACHE_SIZE, 512). @@ -242,4 +245,79 @@ merge_values(_, Value1, Value2) -> lists:append(Value1, Value2). +%% Some functions for testing options compressing term_to_binary +create_block(N, BlockType) -> + case BlockType of + keylist -> + create_block(N, BlockType, []); + keygbtree -> + create_block(N, BlockType, gb_trees:empty()) + end. + +create_block(0, _, KeyStruct) -> + KeyStruct; +create_block(N, BlockType, KeyStruct) -> + Bucket = <<"pdsRecord">>, + case N of + 20 -> + Key = lists:concat(["key-20-special"]); + _ -> + Key = lists:concat(["key-", N, "-", random:uniform(1000)]) + end, + SequenceNumber = random:uniform(1000000000), + Indexes = [{<<"DateOfBirth_int">>, random:uniform(10000)}, {<<"index1_bin">>, lists:concat([random:uniform(1000), "SomeCommonText"])}, {<<"index2_bin">>, <<"RepetitionRepetitionRepetition">>}], + case BlockType of + keylist -> + Term = {o, Bucket, Key, {Indexes, SequenceNumber}}, + create_block(N-1, BlockType, [Term|KeyStruct]); + keygbtree -> + create_block(N-1, BlockType, gb_trees:insert({o, Bucket, Key}, {Indexes, SequenceNumber}, KeyStruct)) + end. + + +create_blocks(N, Compression, BlockType) -> + create_blocks(N, Compression, BlockType, 10000, []). + +create_blocks(_, _, _, 0, BlockList) -> + BlockList; +create_blocks(N, Compression, BlockType, TestLoops, BlockList) -> + NewBlock = term_to_binary(create_block(N, BlockType), [{compressed, Compression}]), + create_blocks(N, Compression, BlockType, TestLoops - 1, [NewBlock|BlockList]). + +size_testblocks(BlockList) -> + size_testblocks(BlockList,0). + +size_testblocks([], Acc) -> + Acc; +size_testblocks([H|T], Acc) -> + size_testblocks(T, Acc + byte_size(H)). + +test_testblocks([], _) -> + true; +test_testblocks([H|T], BlockType) -> + Block = binary_to_term(H), + case findkey("key-20-special", Block, BlockType) of + true -> + test_testblocks(T, BlockType); + not_found -> + false + end. + +findkey(_, [], keylist) -> + not_found; +findkey(Key, [H|T], keylist) -> + case H of + {o, <<"pdsRecord">>, Key, _} -> + true; + _ -> + findkey(Key,T, keylist) + end; +findkey(Key, Tree, keygbtree) -> + case gb_trees:lookup({o, <<"pdsRecord">>, Key}, Tree) of + none -> + not_found; + _ -> + true + end. + \ No newline at end of file From cc16f90c9cc28a327f278415cbc2b4b23f7f92b7 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 10 Jun 2016 19:09:55 +0100 Subject: [PATCH 008/167] SFT file continued Writing of a slot --- src/leveled_sft.erl | 209 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 178 insertions(+), 31 deletions(-) diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index cb253dc..9835fd0 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -14,16 +14,18 @@ %% %% All keys are not equal in sft files, keys are only expected in a specific %% series of formats -%% - {o, Bucket, Key, State} - Object Keys -%% - {i, Bucket, IndexName, IndexTerm, Key, State} - Postings +%% - {o, Bucket, Key} - Object Keys +%% - {i, Bucket, IndexName, IndexTerm, Key} - Postings %% The {Bucket, Key} part of all types of keys are hashed for segment filters. %% For Postings the {Bucket, IndexName, IndexTerm} is also hashed. This %% causes a false positive on lookup of a segment, but allows for the presence %% of specific index terms to be checked %% -%% The objects stored are a tuple of {Key, State, Value}, where +%% The objects stored are a tuple of {Key, SequenceNumber, State, Value}, where %% Key - as above -%% State - {SequenceNumber, active|tomb, ExpiryTimestamp | infinity} +%% SequenceNumber - monotonically increasing counter of addition to the nursery +%% log +%% State - {active|tomb, ExpiryTimestamp | infinity} %% Value - null (all postings) | [Object Metadata] (all object keys) %% Keys should be unique in files. If more than two keys are candidate for %% the same file the highest sequence number should be chosen. If the file @@ -149,7 +151,8 @@ serialise_segment_filter/1, check_for_segments/3, speedtest_check_forsegment/4, - generate_randomsegfilter/1]). + generate_randomsegfilter/1, + create_slot/3]). -include_lib("eunit/include/eunit.hrl"). @@ -163,6 +166,7 @@ -define(MAX_SEG_HASH, 1048576). -define(DIVISOR_BITS, 13). -define(DIVISOR, 8092). +-define(COMPRESSION_LEVEL, 1). -record(state, {version = ?CURRENT_VERSION :: tuple(), @@ -216,9 +220,12 @@ create_header(initial) -> %% Do we need to check here that KeyList1 and KeyList2 are not just a [pointer] %% Otherwise the pointer will never be expanded +%% +%% Also this should return a partial block if the KeyLists have been exhausted +%% but the block is full create_block(KeyList1, KeyList2, Level) -> - create_block(KeyList1, KeyList2, [], {0, 0}, [], Level). + create_block(KeyList1, KeyList2, [], {infinity, 0}, [], Level). create_block(KeyList1, KeyList2, BlockKeyList, {LSN, HSN}, SegmentList, _) @@ -231,7 +238,6 @@ create_block(KeyList1, KeyList2, BlockKeyList, {LSN, HSN}, SegmentList, Level) -> case key_dominates(KeyList1, KeyList2, Level) of {{next_key, TopKey}, Rem1, Rem2} -> - io:format("TopKey is ~w~n", [TopKey]), {UpdLSN, UpdHSN} = update_sequencenumbers(TopKey, LSN, HSN), NewBlockKeyList = lists:append(BlockKeyList, [TopKey]), @@ -241,7 +247,6 @@ create_block(KeyList1, KeyList2, NewBlockKeyList, {UpdLSN, UpdHSN}, NewSegmentList, Level); {skipped_key, Rem1, Rem2} -> - io:format("Key is skipped~n"), create_block(Rem1, Rem2, BlockKeyList, {LSN, HSN}, SegmentList, Level) @@ -261,24 +266,61 @@ create_block(KeyList1, KeyList2, %% - Remainder of any KeyLists used to make the slot -%% create_slot(KeyList1, KeyList2, Level) -%% create_slot(KeyList1, KeyList2, Level, ?BLOCK_COUNT, null, <<>>, <<>>, []). +create_slot(KeyList1, KeyList2, Level) -> + create_slot(KeyList1, KeyList2, Level, ?BLOCK_COUNT, [], <<>>, [], + {null, infinity, 0, null, full}). -%% create_slot(KL1, KL2, Level, 0, LowKey, SegFilter, SerialisedSlot, -%% LengthList, {LSN, HSN}, LastKey) -> -%% {{LowKey, SegFilter, SerialisedSlot, LengthList}, -%% {{LSN, HSN}, LastKey, full, KL1, KL2}}; -%% create_slot(KL1, KL2, Level, BlockCount, LowKey, SegFilter, SerialisedSlot, -%% LengthList, {LSN, HSN}, LastKey) -> -%% BlockDetails = create_block(KeyList1, KeyList2, Level), -%% {BlockKeyList, Status, {LSN, HSN}, SegmentList, KL1, KL2} = BlockDetails, -%% SerialisedBlock = serialise_block(BlockKeyList), -%% <>, - -%% case Status of -%% full -> - +%% Keep adding blocks to the slot until either the block count is reached or +%% there is a partial block +create_slot(KL1, KL2, _, 0, SegLists, SerialisedSlot, LengthList, + {LowKey, LSN, HSN, LastKey, Status}) -> + {{LowKey, generate_segment_filter(SegLists), SerialisedSlot, LengthList}, + {{LSN, HSN}, LastKey, Status}, + KL1, KL2}; +create_slot(KL1, KL2, _, _, SegLists, SerialisedSlot, LengthList, + {LowKey, LSN, HSN, LastKey, partial}) -> + {{LowKey, generate_segment_filter(SegLists), SerialisedSlot, LengthList}, + {{LSN, HSN}, LastKey, partial}, + KL1, KL2}; +create_slot(KL1, KL2, Level, BlockCount, SegLists, SerialisedSlot, LengthList, + {LowKey, LSN, HSN, LastKey, _Status}) -> + {BlockKeyList, Status, + {LSNb, HSNb}, + SegmentList, KL1b, KL2b} = create_block(KL1, KL2, Level), + case LowKey of + null -> + [NewLowKeyV|_] = BlockKeyList, + TrackingMetadata = {strip_to_keyonly(NewLowKeyV), + min(LSN, LSNb), max(HSN, HSNb), + strip_to_keyonly(last(BlockKeyList, + {last, LastKey})), + Status}; + _ -> + TrackingMetadata = {LowKey, + min(LSN, LSNb), max(HSN, HSNb), + strip_to_keyonly(last(BlockKeyList, + {last, LastKey})), + Status} + end, + SerialisedBlock = serialise_block(BlockKeyList), + BlockLength = bit_size(SerialisedBlock), + SerialisedSlot2 = <>, + create_slot(KL1b, KL2b, Level, BlockCount - 1, SegLists ++ [SegmentList], + SerialisedSlot2, LengthList ++ [BlockLength], TrackingMetadata). + + +last([], {last, LastKey}) -> {keyonly, LastKey}; +last([E|Es], PrevLast) -> last(E, Es, PrevLast). + +last(_, [E|Es], PrevLast) -> last(E, Es, PrevLast); +last(E, [], _) -> E. + +strip_to_keyonly({keyonly, K}) -> K; +strip_to_keyonly({K, _, _, _}) -> K. + +serialise_block(BlockKeyList) -> + term_to_binary(BlockKeyList, [{compressed, ?COMPRESSION_LEVEL}]). %% Compare the keys at the head of the list, and either skip that "best" key or @@ -390,6 +432,12 @@ update_sequencenumbers({_, _, _, _}, LSN, HSN) -> %% This is more space efficient than the equivalent bloom filter and avoids %% the calculation of many hash functions. +generate_segment_filter([SegL1, []]) -> + generate_segment_filter({SegL1, [], [], []}); +generate_segment_filter([SegL1, SegL2, []]) -> + generate_segment_filter({SegL1, SegL2, [], []}); +generate_segment_filter([SegL1, SegL2, SegL3, SegL4]) -> + generate_segment_filter({SegL1, SegL2, SegL3, SegL4}); generate_segment_filter(SegLists) -> generate_segment_filter(merge_seglists(SegLists), [], @@ -441,8 +489,8 @@ merge_seglists({SegList1, SegList2, SegList3, SegList4}) -> Stage4 = lists:foldl(fun(X, Acc) -> [{X, 3}|Acc] end, Stage3, SegList4), lists:sort(Stage4). -hash_for_segmentid(Key) -> - erlang:phash2(Key). +hash_for_segmentid(KV) -> + erlang:phash2(strip_to_keyonly(KV), ?MAX_SEG_HASH). %% Check for a given list of segments in the filter, returning in normal @@ -616,6 +664,35 @@ generate_randomsegfilter(BlockSize) -> Block4})). +generate_randomkeys(Count) -> + generate_randomkeys(Count, []). + +generate_randomkeys(0, Acc) -> + Acc; +generate_randomkeys(Count, Acc) -> + RandKey = {{o, + lists:concat(["Bucket", random:uniform(1024)]), + lists:concat(["Key", random:uniform(1024)])}, + random:uniform(1024*1024), + {active, infinity}, null}, + generate_randomkeys(Count - 1, [RandKey|Acc]). + +generate_sequentialkeys(Count, Start) -> + generate_sequentialkeys(Count + Start, Start, []). + +generate_sequentialkeys(Target, Incr, Acc) when Incr =:= Target -> + Acc; +generate_sequentialkeys(Target, Incr, Acc) -> + KeyStr = string:right(integer_to_list(Incr), 8, $0), + NextKey = {{o, + "BucketSeq", + lists:concat(["Key", KeyStr])}, + 5, + {active, infinity}, null}, + generate_sequentialkeys(Target, Incr + 1, [NextKey|Acc]). + + + simple_create_block_test() -> KeyList1 = [{{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}, {{o, "Bucket1", "Key3"}, 2, {active, infinity}, null}], @@ -629,7 +706,6 @@ simple_create_block_test() -> [H2|T2] = T1, ?assertMatch(H2, {{o, "Bucket1", "Key2"}, 3, {active, infinity}, null}), ?assertMatch(T2, [{{o, "Bucket1", "Key3"}, 2, {active, infinity}, null}]), - io:format("SN is ~w~n", [SN]), ?assertMatch(SN, {1,3}). dominate_create_block_test() -> @@ -645,7 +721,7 @@ dominate_create_block_test() -> ?assertMatch(K2, {{o, "Bucket1", "Key2"}, 3, {tomb, infinity}, null}), ?assertMatch(SN, {1,3}). -alternating_create_block_test() -> +sample_keylist() -> KeyList1 = [{{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}, {{o, "Bucket1", "Key3"}, 1, {active, infinity}, null}, {{o, "Bucket1", "Key5"}, 1, {active, infinity}, null}, @@ -675,14 +751,17 @@ alternating_create_block_test() -> {{o, "Bucket2", "Key6"}, 1, {active, infinity}, null}, {{o, "Bucket2", "Key8"}, 1, {active, infinity}, null}, {{o, "Bucket3", "Key2"}, 1, {active, infinity}, null}, - {{o, "Bucket3", "Key4"}, 1, {active, infinity}, null}, - {{o, "Bucket3", "Key6"}, 1, {active, infinity}, null}, + {{o, "Bucket3", "Key4"}, 3, {active, infinity}, null}, + {{o, "Bucket3", "Key6"}, 2, {active, infinity}, null}, {{o, "Bucket3", "Key8"}, 1, {active, infinity}, null}], + {KeyList1, KeyList2}. + +alternating_create_block_test() -> + {KeyList1, KeyList2} = sample_keylist(), {MergedKeyList, ListStatus, _, _, _, _} = create_block(KeyList1, KeyList2, 1), BlockSize = length(MergedKeyList), - io:format("Block size is ~w~n", [BlockSize]), ?assertMatch(BlockSize, 32), ?assertMatch(ListStatus, full), K1 = lists:nth(1, MergedKeyList), @@ -738,4 +817,72 @@ merge_seglists_test() -> ?assertMatch(R8, {maybe_present, [0]}), R9 = check_for_segments(SegBin, [1024*1024 - 1], false), ?assertMatch(R9, not_present). + +createslot_stage1_test() -> + {KeyList1, KeyList2} = sample_keylist(), + Out = create_slot(KeyList1, KeyList2, 1), + {{LowKey, SegFilter, _SerialisedSlot, _LengthList}, + {{LSN, HSN}, LastKey, Status}, + KL1, KL2} = Out, + ?assertMatch(LowKey, {o, "Bucket1", "Key1"}), + ?assertMatch(LastKey, {o, "Bucket4", "Key1"}), + ?assertMatch(Status, partial), + ?assertMatch(KL1, []), + ?assertMatch(KL2, []), + R0 = check_for_segments(serialise_segment_filter(SegFilter), + [hash_for_segmentid({keyonly, {o, "Bucket1", "Key1"}})], + true), + ?assertMatch(R0, {maybe_present, [0]}), + R1 = check_for_segments(serialise_segment_filter(SegFilter), + [hash_for_segmentid({keyonly, {o, "Bucket1", "Key99"}})], + true), + ?assertMatch(R1, not_present), + ?assertMatch(LSN, 1), + ?assertMatch(HSN, 3). + +createslot_stage2_test() -> + Out = create_slot(lists:sort(generate_randomkeys(100)), + lists:sort(generate_randomkeys(100)), + 1), + {{_LowKey, _SegFilter, SerialisedSlot, LengthList}, + {{_LSN, _HSN}, _LastKey, Status}, + _KL1, _KL2} = Out, + ?assertMatch(Status, full), + Sum1 = lists:foldl(fun(X, Sum) -> Sum + X end, 0, LengthList), + Sum2 = bit_size(SerialisedSlot), + ?assertMatch(Sum1, Sum2). + + +createslot_stage3_test() -> + Out = create_slot(lists:sort(generate_sequentialkeys(100, 1)), + lists:sort(generate_sequentialkeys(100, 101)), + 1), + {{LowKey, SegFilter, SerialisedSlot, LengthList}, + {{_LSN, _HSN}, LastKey, Status}, + KL1, KL2} = Out, + ?assertMatch(Status, full), + Sum1 = lists:foldl(fun(X, Sum) -> Sum + X end, 0, LengthList), + Sum2 = bit_size(SerialisedSlot), + ?assertMatch(Sum1, Sum2), + ?assertMatch(LowKey, {o, "BucketSeq", "Key00000001"}), + ?assertMatch(LastKey, {o, "BucketSeq", "Key00000128"}), + ?assertMatch(KL1, []), + Rem = length(KL2), + ?assertMatch(Rem, 72), + R0 = check_for_segments(serialise_segment_filter(SegFilter), + [hash_for_segmentid({keyonly, {o, "BucketSeq", "Key00000100"}})], + true), + ?assertMatch(R0, {maybe_present, [3]}), + R1 = check_for_segments(serialise_segment_filter(SegFilter), + [hash_for_segmentid({keyonly, {o, "Bucket1", "Key99"}})], + true), + ?assertMatch(R1, not_present), + R2 = check_for_segments(serialise_segment_filter(SegFilter), + [hash_for_segmentid({keyonly, {o, "BucketSeq", "Key00000040"}})], + true), + ?assertMatch(R2, {maybe_present, [1]}), + R3 = check_for_segments(serialise_segment_filter(SegFilter), + [hash_for_segmentid({keyonly, {o, "BucketSeq", "Key00000004"}})], + true), + ?assertMatch(R3, {maybe_present, [0]}). From eedc29631435a7d4a279504cdc417eebc2a1e4c9 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 17 Jun 2016 15:14:25 +0100 Subject: [PATCH 009/167] Completing KeyLists on a block boundary Handle when writing a block empties the Key Lists but the block is full - don't go-on and create a second empty block --- src/leveled_sft.erl | 111 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 107 insertions(+), 4 deletions(-) diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 9835fd0..c55e8ce 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -160,6 +160,7 @@ -define(DWORD_SIZE, 8). -define(CURRENT_VERSION, {0,1}). -define(SLOT_COUNT, 256). +-define(SLOT_GROUPWRITE_COUNT, 32). -define(BLOCK_SIZE, 32). -define(BLOCK_COUNT, 4). -define(FOOTERPOS_HEADERPOS, 2). @@ -202,6 +203,72 @@ create_header(initial) -> CRC32 = erlang:crc32(H1), <

>. +%% Take a file handle at the sart position (after creating the header) and then +%% write the Key lists to the file slot by slot. +%% +%% Slots are created then written in bulk to impove I/O efficiency. Slots will +%% be written in groups of 32 + + +write_group(Handle, KL1, KL2, SlotIndex, SerialisedSlots, Level, WriteFun) -> + write_group(Handle, KL1, KL2, {0, 0}, + SlotIndex, SerialisedSlots, + {infinity, 0}, null, {last, null}, Level, WriteFun). + + +write_group(Handle, KL1, KL2, {SlotCount, SlotTotal}, + SlotIndex, SerialisedSlots, + {LSN, HSN}, LowKey, LastKey, Level, WriteFun) + when SlotCount =:= ?SLOT_GROUPWRITE_COUNT -> + UpdHandle = WriteFun(slot , {Handle, SerialisedSlots}), + case maxslots_bylevel(SlotTotal, Level) of + reached -> + UpdHandle; + continue -> + write_group(UpdHandle, KL1, KL2, 0, + SlotIndex, <<>>, + {LSN, HSN}, LowKey, LastKey, Level, WriteFun) + end; +write_group(Handle, KL1, KL2, {SlotCount, SlotTotal}, + SlotIndex, SerialisedSlots, + {LSN, HSN}, LowKey, LastKey, Level, WriteFun) -> + SlotOutput = create_slot(KL1, KL2, Level), + {{LowKey_Slot, SegFilter, SerialisedSlot, LengthList}, + {{LSN_Slot, HSN_Slot}, LastKey_Slot, Status}, + KL1rem, KL2rem} = SlotOutput, + UpdSlotIndex = lists:append(SlotIndex, + [{LowKey_Slot, SegFilter, LengthList}]), + UpdSlots = <>, + SNExtremes = {min(LSN_Slot, LSN), max(HSN_Slot, HSN)}, + FinalKey = case LastKey_Slot of null -> LastKey; _ -> LastKey_Slot end, + FirstKey = case LowKey of null -> LowKey_Slot; _ -> LowKey end, + case Status of + partial -> + UpdHandle = WriteFun(slot , {Handle, UpdSlots}), + WriteFun(finalise, {UpdHandle, UpdSlotIndex, SNExtremes, + {FirstKey, FinalKey}}); + full -> + write_group(Handle, KL1rem, KL2rem, {SlotCount + 1, SlotTotal + 1}, + UpdSlotIndex, UpdSlots, + SNExtremes, FirstKey, FinalKey, Level, WriteFun) + end. + + +sftwrite_function(slot, {Handle, _SerialisedSlots}) -> + Handle; +sftwrite_function(finalise, + {Handle, _UpdSlotIndex, _SNExtremes, _KeyExtremes}) -> + Handle. + +maxslots_bylevel(SlotTotal, _Level) -> + case SlotTotal of + ?SLOT_COUNT -> + reached; + X when X < ?SLOT_COUNT -> + continue + end. + + %% Take two potentially overlapping lists of keys and output a Block, %% together with: %% - block status (full, partial) @@ -230,7 +297,12 @@ create_block(KeyList1, KeyList2, Level) -> create_block(KeyList1, KeyList2, BlockKeyList, {LSN, HSN}, SegmentList, _) when length(BlockKeyList)==?BLOCK_SIZE -> - {BlockKeyList, full, {LSN, HSN}, SegmentList, KeyList1, KeyList2}; + case {KeyList1, KeyList2} of + {[], []} -> + {BlockKeyList, complete, {LSN, HSN}, SegmentList, [], []}; + _ -> + {BlockKeyList, full, {LSN, HSN}, SegmentList, KeyList1, KeyList2} + end; create_block([], [], BlockKeyList, {LSN, HSN}, SegmentList, _) -> {BlockKeyList, partial, {LSN, HSN}, SegmentList, [], []}; @@ -283,6 +355,11 @@ create_slot(KL1, KL2, _, _, SegLists, SerialisedSlot, LengthList, {{LowKey, generate_segment_filter(SegLists), SerialisedSlot, LengthList}, {{LSN, HSN}, LastKey, partial}, KL1, KL2}; +create_slot(KL1, KL2, _, _, SegLists, SerialisedSlot, LengthList, + {LowKey, LSN, HSN, LastKey, complete}) -> + {{LowKey, generate_segment_filter(SegLists), SerialisedSlot, LengthList}, + {{LSN, HSN}, LastKey, partial}, + KL1, KL2}; create_slot(KL1, KL2, Level, BlockCount, SegLists, SerialisedSlot, LengthList, {LowKey, LSN, HSN, LastKey, _Status}) -> {BlockKeyList, Status, @@ -304,10 +381,11 @@ create_slot(KL1, KL2, Level, BlockCount, SegLists, SerialisedSlot, LengthList, Status} end, SerialisedBlock = serialise_block(BlockKeyList), + % io:format("Serialised Block to be added ~w based on BlockKeyList ~w~n", [SerialisedBlock, BlockKeyList]), BlockLength = bit_size(SerialisedBlock), SerialisedSlot2 = <>, create_slot(KL1b, KL2b, Level, BlockCount - 1, SegLists ++ [SegmentList], - SerialisedSlot2, LengthList ++ [BlockLength], TrackingMetadata). + SerialisedSlot2, LengthList ++ [BlockLength], TrackingMetadata). last([], {last, LastKey}) -> {keyonly, LastKey}; @@ -432,6 +510,8 @@ update_sequencenumbers({_, _, _, _}, LSN, HSN) -> %% This is more space efficient than the equivalent bloom filter and avoids %% the calculation of many hash functions. +generate_segment_filter([SegL1]) -> + generate_segment_filter({SegL1, [], [], []}); generate_segment_filter([SegL1, []]) -> generate_segment_filter({SegL1, [], [], []}); generate_segment_filter([SegL1, SegL2, []]) -> @@ -763,13 +843,16 @@ alternating_create_block_test() -> 1), BlockSize = length(MergedKeyList), ?assertMatch(BlockSize, 32), - ?assertMatch(ListStatus, full), + ?assertMatch(ListStatus, complete), K1 = lists:nth(1, MergedKeyList), ?assertMatch(K1, {{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}), K11 = lists:nth(11, MergedKeyList), ?assertMatch(K11, {{o, "Bucket1", "Key9b"}, 1, {active, infinity}, null}), K32 = lists:nth(32, MergedKeyList), - ?assertMatch(K32, {{o, "Bucket4", "Key1"}, 1, {active, infinity}, null}). + ?assertMatch(K32, {{o, "Bucket4", "Key1"}, 1, {active, infinity}, null}), + HKey = {{o, "Bucket1", "Key0"}, 1, {active, infinity}, null}, + {_, ListStatus2, _, _, _, _} = create_block([HKey|KeyList1], KeyList2, 1), + ?assertMatch(ListStatus2, full). merge_seglists_test() -> @@ -886,3 +969,23 @@ createslot_stage3_test() -> [hash_for_segmentid({keyonly, {o, "BucketSeq", "Key00000004"}})], true), ?assertMatch(R3, {maybe_present, [0]}). + + + +testwrite_function(slot, {Handle, SerialisedSlots}) -> + lists:append(Handle, [SerialisedSlots]); +testwrite_function(finalise, {Handle, UpdSlotIndex, SNExtremes, KeyExtremes}) -> + {Handle, UpdSlotIndex, SNExtremes, KeyExtremes}. + +writegroup_stage1_test() -> + {KL1, KL2} = sample_keylist(), + Output = write_group([], KL1, KL2, [], <<>>, 1, fun testwrite_function/2), + {Handle, UpdSlotIndex, SNExtremes, KeyExtremes} = Output, + ?assertMatch(SNExtremes, {1,3}), + ?assertMatch(KeyExtremes, {{o, "Bucket1", "Key1"}, {o, "Bucket4", "Key1"}}), + [TopIndex|[]] = UpdSlotIndex, + {TopKey, _SegFilter, LengthList} = TopIndex, + ?assertMatch(TopKey, {o, "Bucket1", "Key1"}), + TotalLength = lists:foldl(fun(X, Acc) -> Acc + X end, 0, LengthList), + ActualLength = lists:foldl(fun(X, Acc) -> Acc + bit_size(X) end, 0, Handle), + ?assertMatch(TotalLength, ActualLength). From 27dc026176ec89895c10a4a65c37548ff838e5e8 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 6 Jul 2016 10:52:47 +0100 Subject: [PATCH 010/167] Write a SFT File With some initial test support --- src/leveled_sft.erl | 226 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 207 insertions(+), 19 deletions(-) diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index c55e8ce..ef26814 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -168,13 +168,21 @@ -define(DIVISOR_BITS, 13). -define(DIVISOR, 8092). -define(COMPRESSION_LEVEL, 1). +-define(HEADER_LENGTH, 56). -record(state, {version = ?CURRENT_VERSION :: tuple(), - slot_index = gb_trees:empty() :: gb_trees:tree(), + slot_index :: list(), next_position :: integer(), smallest_sqn :: integer(), - largest_sqn :: integer()}). + highest_sqn :: integer(), + smallest_key :: string(), + highest_key :: string(), + slots_pointer :: integer(), + index_pointer :: integer(), + filter_pointer :: integer(), + summary_pointer :: integer(), + summary_length :: integer()}). %% Start a bare file with an initial header and no further details @@ -186,10 +194,25 @@ create_file(Handle) -> Header = create_header(initial), {ok, _} = file:position(Handle, bof), ok = file:write(Handle, Header), - {ok, StartPos} = file:position(Handle), + {ok, StartPos} = file:position(Handle, cur), FileMD = #state{next_position=StartPos}, {Handle, FileMD}. + +%% The 56-byte header is made up of +%% - 1 byte version (major 5 bits, minor 3 bits) - default 0.1 +%% - 1 byte options (currently undefined) +%% - 1 byte Block Size - the expected number of keys in each block +%% - 1 byte Block Count - the expected number of blocks in each slot +%% - 2 byte Slot Count - the maximum number of slots in the file +%% - 6 bytes - spare +%% - 4 bytes - Blocks length +%% - 4 bytes - Slot Index length +%% - 4 bytes - Slot Filter length +%% - 4 bytes - Table summary length +%% - 24 bytes - spare +%% - 4 bytes - CRC32 + create_header(initial) -> {Major, Minor} = ?CURRENT_VERSION, Version = <>, @@ -203,13 +226,95 @@ create_header(initial) -> CRC32 = erlang:crc32(H1), <

>. + +%% Take a file handle with a previously created header and complete it based on +%% the two key lists KL1 and KL2 + +complete_file(Handle, FileMD, KL1, KL2, Level) -> + {UpdHandle, + PointerList, + {LowSQN, HighSQN}, + {LowKey, HighKey}} = write_group(Handle, KL1, KL2, [], <<>>, Level, + fun sftwrite_function/2), + {ok, HeaderLengths} = file:pread(UpdHandle, 12, 16), + <> = HeaderLengths, + {UpdHandle, FileMD#state{slot_index=PointerList, + smallest_sqn=LowSQN, + highest_sqn=HighSQN, + smallest_key=LowKey, + highest_key=HighKey, + slots_pointer=?HEADER_LENGTH, + index_pointer=?HEADER_LENGTH + Blnth, + filter_pointer=?HEADER_LENGTH + Blnth + Ilnth, + summary_pointer=?HEADER_LENGTH + Blnth + Ilnth + Flnth, + summary_length=Slnth}}. + +%% Fetch a Key and Value from a file, returns +%% {value, KV} or not_present + +fetch_keyvalue(Handle, FileMD, Key) -> + {_NearestKey, {FilterLen, PointerF}, + {LengthList, PointerB}} = get_nearestkey(FileMD#state.slot_index, Key), + {ok, SegFilter} = file:pread(Handle, + PointerF + FileMD#state.filter_pointer, + FilterLen), + SegID = hash_for_segmentid({keyonly, Key}), + case check_for_segments(SegFilter, [SegID], true) of + {maybe_present, BlockList} -> + fetch_keyvalue_fromblock(BlockList, + Key, + LengthList, + Handle, + PointerB + FileMD#state.slots_pointer); + not_present -> + not_present; + error_so_maybe_present -> + fetch_keyvalue_fromblock(lists:seq(0,3), + Key, + LengthList, + Handle, + PointerB + FileMD#state.slots_pointer) + end. + + +fetch_keyvalue_fromblock([], _Key, _LengthList, _Handle, _StartOfSlot) -> + not_present; +fetch_keyvalue_fromblock([BlockNumber|T], Key, LengthList, Handle, StartOfSlot) -> + Start = lists:sum(lists:sublist(LengthList, BlockNumber)), + Length = lists:nth(BlockNumber + 1, LengthList), + {ok, BlockToCheckBin} = file:pread(Handle, Start + StartOfSlot, Length), + BlockToCheck = binary_to_term(BlockToCheckBin), + Result = lists:keyfind(Key, 1, BlockToCheck), + case Result of + false -> + fetch_keyvalue_fromblock(T, Key, LengthList, Handle, StartOfSlot); + KV -> + KV + end. + + + + +get_nearestkey(KVList, Key) -> + get_nearestkey(KVList, Key, not_found). + +get_nearestkey([], _KeyToFind, PrevV) -> + PrevV; +get_nearestkey([{K, _FilterInfo, _SlotInfo}|_T], KeyToFind, PrevV) when K > KeyToFind -> + PrevV; +get_nearestkey([Result|T], KeyToFind, _) -> + get_nearestkey(T, KeyToFind, Result). + + %% Take a file handle at the sart position (after creating the header) and then %% write the Key lists to the file slot by slot. %% %% Slots are created then written in bulk to impove I/O efficiency. Slots will %% be written in groups of 32 - write_group(Handle, KL1, KL2, SlotIndex, SerialisedSlots, Level, WriteFun) -> write_group(Handle, KL1, KL2, {0, 0}, SlotIndex, SerialisedSlots, @@ -220,7 +325,7 @@ write_group(Handle, KL1, KL2, {SlotCount, SlotTotal}, SlotIndex, SerialisedSlots, {LSN, HSN}, LowKey, LastKey, Level, WriteFun) when SlotCount =:= ?SLOT_GROUPWRITE_COUNT -> - UpdHandle = WriteFun(slot , {Handle, SerialisedSlots}), + UpdHandle = WriteFun(slots , {Handle, SerialisedSlots}), case maxslots_bylevel(SlotTotal, Level) of reached -> UpdHandle; @@ -244,9 +349,14 @@ write_group(Handle, KL1, KL2, {SlotCount, SlotTotal}, FirstKey = case LowKey of null -> LowKey_Slot; _ -> LowKey end, case Status of partial -> - UpdHandle = WriteFun(slot , {Handle, UpdSlots}), - WriteFun(finalise, {UpdHandle, UpdSlotIndex, SNExtremes, - {FirstKey, FinalKey}}); + UpdHandle = WriteFun(slots , {Handle, UpdSlots}), + ConvSlotIndex = convert_slotindex(UpdSlotIndex), + FinHandle = WriteFun(finalise, {UpdHandle, + ConvSlotIndex, + SNExtremes, + {FirstKey, FinalKey}}), + {_, PointerIndex} = ConvSlotIndex, + {FinHandle, PointerIndex, SNExtremes, {FirstKey, FinalKey}}; full -> write_group(Handle, KL1rem, KL2rem, {SlotCount + 1, SlotTotal + 1}, UpdSlotIndex, UpdSlots, @@ -254,11 +364,58 @@ write_group(Handle, KL1, KL2, {SlotCount, SlotTotal}, end. -sftwrite_function(slot, {Handle, _SerialisedSlots}) -> +%% Take a slot index, and remove the SegFilters replacing with pointers +%% Return a tuple of the accumulated slot filters, and a pointer-based slot-index + +convert_slotindex(SlotIndex) -> + SlotFun = fun({LowKey, SegFilter, LengthList}, + {FilterAcc, SlotIndexAcc, PointerF, PointerB}) -> + FilterOut = serialise_segment_filter(SegFilter), + FilterLen = byte_size(FilterOut), + {<>, + lists:append(SlotIndexAcc, [{LowKey, + {FilterLen, PointerF}, + {LengthList, PointerB}}]), + PointerF + FilterLen, + PointerB + lists:sum(LengthList)} end, + {SlotFilters, PointerIndex, _FLength, _BLength} = lists:foldl(SlotFun, + {<<>>, [], 0, 0}, + SlotIndex), + {SlotFilters, PointerIndex}. + +sftwrite_function(slots, {Handle, SerialisedSlots}) -> + ok = file:write(Handle, SerialisedSlots), Handle; sftwrite_function(finalise, - {Handle, _UpdSlotIndex, _SNExtremes, _KeyExtremes}) -> - Handle. + {Handle, + {SlotFilters, PointerIndex}, + _SNExtremes, + _KeyExtremes}) -> + {ok, Position} = file:position(Handle, cur), + + SlotsLength = Position - ?HEADER_LENGTH, + SerialisedIndex = term_to_binary(PointerIndex), + IndexLength = byte_size(SerialisedIndex), + + %% Write Index + ok = file:write(Handle, SerialisedIndex), + %% Write Filter + ok = file:write(Handle, SlotFilters), + %% Write Lengths into header + ok = file:pwrite(Handle, 12, <>), + Handle; +sftwrite_function(finalise, + {Handle, + SlotIndex, + SNExtremes, + KeyExtremes}) -> + {SlotFilters, PointerIndex} = convert_slotindex(SlotIndex), + sftwrite_function(finalise, + {Handle, + {SlotFilters, PointerIndex}, + SNExtremes, + KeyExtremes}). maxslots_bylevel(SlotTotal, _Level) -> case SlotTotal of @@ -269,6 +426,7 @@ maxslots_bylevel(SlotTotal, _Level) -> end. + %% Take two potentially overlapping lists of keys and output a Block, %% together with: %% - block status (full, partial) @@ -551,9 +709,16 @@ serialise_segment_filter({DeltaList, TopHashes}) -> <> end, - lists:foldl(F, HeaderBin, DeltaList). + pad_binary(lists:foldl(F, HeaderBin, DeltaList)). +pad_binary(BitString) -> + Pad = 8 - bit_size(BitString) rem 8, + case Pad of + 8 -> BitString; + _ -> <> + end. + buildexponent(Exponent) -> buildexponent(Exponent, <<0:1>>). @@ -880,7 +1045,8 @@ merge_seglists_test() -> 2:2, 1708:13, 2:2>>, ExpectedResult = <>, + ExpectedDeltas/bitstring, + 0:7/integer>>, ?assertMatch(SegBin, ExpectedResult), R1 = check_for_segments(SegBin, [100], true), ?assertMatch(R1,{maybe_present, [0]}), @@ -972,20 +1138,42 @@ createslot_stage3_test() -> -testwrite_function(slot, {Handle, SerialisedSlots}) -> +testwrite_function(slots, {Handle, SerialisedSlots}) -> lists:append(Handle, [SerialisedSlots]); -testwrite_function(finalise, {Handle, UpdSlotIndex, SNExtremes, KeyExtremes}) -> - {Handle, UpdSlotIndex, SNExtremes, KeyExtremes}. +testwrite_function(finalise, {Handle, ConvSlotIndex, SNExtremes, KeyExtremes}) -> + {Handle, ConvSlotIndex, SNExtremes, KeyExtremes}. writegroup_stage1_test() -> {KL1, KL2} = sample_keylist(), Output = write_group([], KL1, KL2, [], <<>>, 1, fun testwrite_function/2), - {Handle, UpdSlotIndex, SNExtremes, KeyExtremes} = Output, + {{Handle, {_, PointerIndex}, SNExtremes, KeyExtremes}, + PointerIndex, SNExtremes, KeyExtremes} = Output, ?assertMatch(SNExtremes, {1,3}), ?assertMatch(KeyExtremes, {{o, "Bucket1", "Key1"}, {o, "Bucket4", "Key1"}}), - [TopIndex|[]] = UpdSlotIndex, - {TopKey, _SegFilter, LengthList} = TopIndex, + [TopIndex|[]] = PointerIndex, + {TopKey, _SegFilter, {LengthList, _Total}} = TopIndex, ?assertMatch(TopKey, {o, "Bucket1", "Key1"}), TotalLength = lists:foldl(fun(X, Acc) -> Acc + X end, 0, LengthList), ActualLength = lists:foldl(fun(X, Acc) -> Acc + bit_size(X) end, 0, Handle), ?assertMatch(TotalLength, ActualLength). + +initial_create_header_test() -> + Output = create_header(initial), + ?assertMatch(?HEADER_LENGTH, byte_size(Output)). + +initial_create_file_test() -> + Filename = "test1.sft", + {KL1, KL2} = sample_keylist(), + {Handle, FileMD} = create_file(Filename), + {UpdHandle, UpdFileMD} = complete_file(Handle, FileMD, KL1, KL2, 1), + Result1 = fetch_keyvalue(UpdHandle, UpdFileMD, {o, "Bucket1", "Key8"}), + io:format("Result is ~w~n", [Result1]), + ?assertMatch(Result1, {{o, "Bucket1", "Key8"}, 1, {active, infinity}, null}), + Result2 = fetch_keyvalue(UpdHandle, UpdFileMD, {o, "Bucket1", "Key88"}), + io:format("Result is ~w~n", [Result2]), + ?assertMatch(Result2, not_present), + ok = file:close(UpdHandle), + ok = file:delete(Filename). + +%% big_create_file_test() -> + \ No newline at end of file From 71a65382883cf00d64b7c57379f664500681b4c8 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 6 Jul 2016 16:09:08 +0100 Subject: [PATCH 011/167] Improved testing of file creation Rsolved some off-by-one errors, and ability to support KeyLists larger than the keys supported in a file --- src/leveled_sft.erl | 95 ++++++++++++++++++++++++++++++++------------- 1 file changed, 69 insertions(+), 26 deletions(-) diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index ef26814..a28155e 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -328,9 +328,12 @@ write_group(Handle, KL1, KL2, {SlotCount, SlotTotal}, UpdHandle = WriteFun(slots , {Handle, SerialisedSlots}), case maxslots_bylevel(SlotTotal, Level) of reached -> - UpdHandle; + complete_write(UpdHandle, + SlotIndex, + {LSN, HSN}, {LowKey, LastKey}, + WriteFun); continue -> - write_group(UpdHandle, KL1, KL2, 0, + write_group(UpdHandle, KL1, KL2, {0, SlotTotal}, SlotIndex, <<>>, {LSN, HSN}, LowKey, LastKey, Level, WriteFun) end; @@ -350,13 +353,10 @@ write_group(Handle, KL1, KL2, {SlotCount, SlotTotal}, case Status of partial -> UpdHandle = WriteFun(slots , {Handle, UpdSlots}), - ConvSlotIndex = convert_slotindex(UpdSlotIndex), - FinHandle = WriteFun(finalise, {UpdHandle, - ConvSlotIndex, - SNExtremes, - {FirstKey, FinalKey}}), - {_, PointerIndex} = ConvSlotIndex, - {FinHandle, PointerIndex, SNExtremes, {FirstKey, FinalKey}}; + complete_write(UpdHandle, + UpdSlotIndex, + SNExtremes, {FirstKey, FinalKey}, + WriteFun); full -> write_group(Handle, KL1rem, KL2rem, {SlotCount + 1, SlotTotal + 1}, UpdSlotIndex, UpdSlots, @@ -364,8 +364,19 @@ write_group(Handle, KL1, KL2, {SlotCount, SlotTotal}, end. +complete_write(Handle, SlotIndex, SNExtremes, {FirstKey, FinalKey}, WriteFun) -> + ConvSlotIndex = convert_slotindex(SlotIndex), + FinHandle = WriteFun(finalise, {Handle, + ConvSlotIndex, + SNExtremes, + {FirstKey, FinalKey}}), + {_, PointerIndex} = ConvSlotIndex, + {FinHandle, PointerIndex, SNExtremes, {FirstKey, FinalKey}}. + + %% Take a slot index, and remove the SegFilters replacing with pointers -%% Return a tuple of the accumulated slot filters, and a pointer-based slot-index +%% Return a tuple of the accumulated slot filters, and a pointer-based +%% slot-index convert_slotindex(SlotIndex) -> SlotFun = fun({LowKey, SegFilter, LengthList}, @@ -418,6 +429,7 @@ sftwrite_function(finalise, KeyExtremes}). maxslots_bylevel(SlotTotal, _Level) -> + io:format("Slot total of ~w~n", [SlotTotal]), case SlotTotal of ?SLOT_COUNT -> reached; @@ -540,10 +552,11 @@ create_slot(KL1, KL2, Level, BlockCount, SegLists, SerialisedSlot, LengthList, end, SerialisedBlock = serialise_block(BlockKeyList), % io:format("Serialised Block to be added ~w based on BlockKeyList ~w~n", [SerialisedBlock, BlockKeyList]), - BlockLength = bit_size(SerialisedBlock), + BlockLength = byte_size(SerialisedBlock), SerialisedSlot2 = <>, create_slot(KL1b, KL2b, Level, BlockCount - 1, SegLists ++ [SegmentList], - SerialisedSlot2, LengthList ++ [BlockLength], TrackingMetadata). + SerialisedSlot2, LengthList ++ [BlockLength], + TrackingMetadata). last([], {last, LastKey}) -> {keyonly, LastKey}; @@ -670,10 +683,10 @@ update_sequencenumbers({_, _, _, _}, LSN, HSN) -> generate_segment_filter([SegL1]) -> generate_segment_filter({SegL1, [], [], []}); -generate_segment_filter([SegL1, []]) -> - generate_segment_filter({SegL1, [], [], []}); -generate_segment_filter([SegL1, SegL2, []]) -> +generate_segment_filter([SegL1, SegL2]) -> generate_segment_filter({SegL1, SegL2, [], []}); +generate_segment_filter([SegL1, SegL2, SegL3]) -> + generate_segment_filter({SegL1, SegL2, SegL3, []}); generate_segment_filter([SegL1, SegL2, SegL3, SegL4]) -> generate_segment_filter({SegL1, SegL2, SegL3, SegL4}); generate_segment_filter(SegLists) -> @@ -1099,7 +1112,7 @@ createslot_stage2_test() -> _KL1, _KL2} = Out, ?assertMatch(Status, full), Sum1 = lists:foldl(fun(X, Sum) -> Sum + X end, 0, LengthList), - Sum2 = bit_size(SerialisedSlot), + Sum2 = byte_size(SerialisedSlot), ?assertMatch(Sum1, Sum2). @@ -1112,7 +1125,7 @@ createslot_stage3_test() -> KL1, KL2} = Out, ?assertMatch(Status, full), Sum1 = lists:foldl(fun(X, Sum) -> Sum + X end, 0, LengthList), - Sum2 = bit_size(SerialisedSlot), + Sum2 = byte_size(SerialisedSlot), ?assertMatch(Sum1, Sum2), ?assertMatch(LowKey, {o, "BucketSeq", "Key00000001"}), ?assertMatch(LastKey, {o, "BucketSeq", "Key00000128"}), @@ -1140,8 +1153,8 @@ createslot_stage3_test() -> testwrite_function(slots, {Handle, SerialisedSlots}) -> lists:append(Handle, [SerialisedSlots]); -testwrite_function(finalise, {Handle, ConvSlotIndex, SNExtremes, KeyExtremes}) -> - {Handle, ConvSlotIndex, SNExtremes, KeyExtremes}. +testwrite_function(finalise, {Handle, C_SlotIndex, SNExtremes, KeyExtremes}) -> + {Handle, C_SlotIndex, SNExtremes, KeyExtremes}. writegroup_stage1_test() -> {KL1, KL2} = sample_keylist(), @@ -1149,12 +1162,15 @@ writegroup_stage1_test() -> {{Handle, {_, PointerIndex}, SNExtremes, KeyExtremes}, PointerIndex, SNExtremes, KeyExtremes} = Output, ?assertMatch(SNExtremes, {1,3}), - ?assertMatch(KeyExtremes, {{o, "Bucket1", "Key1"}, {o, "Bucket4", "Key1"}}), + ?assertMatch(KeyExtremes, {{o, "Bucket1", "Key1"}, + {o, "Bucket4", "Key1"}}), [TopIndex|[]] = PointerIndex, {TopKey, _SegFilter, {LengthList, _Total}} = TopIndex, ?assertMatch(TopKey, {o, "Bucket1", "Key1"}), - TotalLength = lists:foldl(fun(X, Acc) -> Acc + X end, 0, LengthList), - ActualLength = lists:foldl(fun(X, Acc) -> Acc + bit_size(X) end, 0, Handle), + TotalLength = lists:foldl(fun(X, Acc) -> Acc + X end, + 0, LengthList), + ActualLength = lists:foldl(fun(X, Acc) -> Acc + byte_size(X) end, + 0, Handle), ?assertMatch(TotalLength, ActualLength). initial_create_header_test() -> @@ -1162,18 +1178,45 @@ initial_create_header_test() -> ?assertMatch(?HEADER_LENGTH, byte_size(Output)). initial_create_file_test() -> - Filename = "test1.sft", + Filename = "../test/test1.sft", {KL1, KL2} = sample_keylist(), {Handle, FileMD} = create_file(Filename), {UpdHandle, UpdFileMD} = complete_file(Handle, FileMD, KL1, KL2, 1), Result1 = fetch_keyvalue(UpdHandle, UpdFileMD, {o, "Bucket1", "Key8"}), io:format("Result is ~w~n", [Result1]), - ?assertMatch(Result1, {{o, "Bucket1", "Key8"}, 1, {active, infinity}, null}), + ?assertMatch(Result1, {{o, "Bucket1", "Key8"}, + 1, {active, infinity}, null}), Result2 = fetch_keyvalue(UpdHandle, UpdFileMD, {o, "Bucket1", "Key88"}), io:format("Result is ~w~n", [Result2]), ?assertMatch(Result2, not_present), ok = file:close(UpdHandle), ok = file:delete(Filename). -%% big_create_file_test() -> - \ No newline at end of file +big_create_file_test() -> + Filename = "../test/bigtest1.sft", + {KL1, KL2} = {lists:sort(generate_randomkeys(50000)), + lists:sort(generate_randomkeys(50000))}, + {InitHandle, InitFileMD} = create_file(Filename), + {Handle, FileMD} = complete_file(InitHandle, InitFileMD, KL1, KL2, 1), + [{K1, Sq1, St1, V1}|_] = KL1, + [{K2, Sq2, St2, V2}|_] = KL2, + Result1 = fetch_keyvalue(Handle, FileMD, K1), + Result2 = fetch_keyvalue(Handle, FileMD, K2), + io:format("Results are:~n~w~n~w~n", [Result1, Result2]), + ?assertMatch(Result1, {K1, Sq1, St1, V1}), + ?assertMatch(Result2, {K2, Sq2, St2, V2}), + FailedFinds = lists:foldl(fun(K, Acc) -> + {Kn, _, _, _} = K, + Rn = fetch_keyvalue(Handle, FileMD, Kn), + case Rn of + {Kn, _, _, _} -> Acc; + _ -> Acc + 1 + end + end, + 0, + lists:sublist(KL2, 1000)), + ?assertMatch(FailedFinds, 0), + Result3 = fetch_keyvalue(Handle, FileMD, {o, "Bucket1024", "Key1024Alt"}), + ?assertMatch(Result3, not_present), + ok = file:close(Handle), + ok = file:delete(Filename). From 3b954aea43ef177f10fa6956a62aa688942e5874 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 6 Jul 2016 18:09:40 +0100 Subject: [PATCH 012/167] Some refinements Minor amendments to formatting and outputs --- src/leveled_sft.erl | 107 ++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 54 deletions(-) diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index a28155e..988d5a7 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -105,20 +105,16 @@ %% Summaries could be used for other summaries of table content in the future, %% perhaps application-specific bloom filters -%% The 80-byte header is made up of +%% The 56-byte header is made up of %% - 1 byte version (major 5 bits, minor 3 bits) - default 0.1 %% - 1 byte options (currently undefined) %% - 1 byte Block Size - the expected number of keys in each block %% - 1 byte Block Count - the expected number of blocks in each slot %% - 2 byte Slot Count - the maximum number of slots in the file %% - 6 bytes - spare -%% - 4 bytes - Blocks position %% - 4 bytes - Blocks length -%% - 4 bytes - Slot Index position %% - 4 bytes - Slot Index length -%% - 4 bytes - Slot Filter position %% - 4 bytes - Slot Filter length -%% - 4 bytes - Table Summary position %% - 4 bytes - Table summary length %% - 24 bytes - spare %% - 4 bytes - CRC32 @@ -168,7 +164,7 @@ -define(DIVISOR_BITS, 13). -define(DIVISOR, 8092). -define(COMPRESSION_LEVEL, 1). --define(HEADER_LENGTH, 56). +-define(HEADER_LEN, 56). -record(state, {version = ?CURRENT_VERSION :: tuple(), @@ -181,8 +177,8 @@ slots_pointer :: integer(), index_pointer :: integer(), filter_pointer :: integer(), - summary_pointer :: integer(), - summary_length :: integer()}). + summ_pointer :: integer(), + summ_length :: integer()}). %% Start a bare file with an initial header and no further details @@ -199,20 +195,6 @@ create_file(Handle) -> {Handle, FileMD}. -%% The 56-byte header is made up of -%% - 1 byte version (major 5 bits, minor 3 bits) - default 0.1 -%% - 1 byte options (currently undefined) -%% - 1 byte Block Size - the expected number of keys in each block -%% - 1 byte Block Count - the expected number of blocks in each slot -%% - 2 byte Slot Count - the maximum number of slots in the file -%% - 6 bytes - spare -%% - 4 bytes - Blocks length -%% - 4 bytes - Slot Index length -%% - 4 bytes - Slot Filter length -%% - 4 bytes - Table summary length -%% - 24 bytes - spare -%% - 4 bytes - CRC32 - create_header(initial) -> {Major, Minor} = ?CURRENT_VERSION, Version = <>, @@ -226,34 +208,47 @@ create_header(initial) -> CRC32 = erlang:crc32(H1), <

>. +%% Open a file returning a handle and metadata which can be used in fetch and +%% iterator requests +open_file(Filename) -> + {ok, _Handle} = file:open(Filename, [binary, raw, read, write]). + %% Need to write other metadata somewhere + %% ... probably in summmary + %% ... is there a need for two levels of summary? + %% Take a file handle with a previously created header and complete it based on %% the two key lists KL1 and KL2 complete_file(Handle, FileMD, KL1, KL2, Level) -> - {UpdHandle, + {{UpdHandle, PointerList, {LowSQN, HighSQN}, - {LowKey, HighKey}} = write_group(Handle, KL1, KL2, [], <<>>, Level, + {LowKey, HighKey}}, + KeyRemainders} = write_group(Handle, KL1, KL2, [], <<>>, Level, fun sftwrite_function/2), {ok, HeaderLengths} = file:pread(UpdHandle, 12, 16), - <> = HeaderLengths, - {UpdHandle, FileMD#state{slot_index=PointerList, - smallest_sqn=LowSQN, - highest_sqn=HighSQN, - smallest_key=LowKey, - highest_key=HighKey, - slots_pointer=?HEADER_LENGTH, - index_pointer=?HEADER_LENGTH + Blnth, - filter_pointer=?HEADER_LENGTH + Blnth + Ilnth, - summary_pointer=?HEADER_LENGTH + Blnth + Ilnth + Flnth, - summary_length=Slnth}}. + <> = HeaderLengths, + {UpdHandle, + FileMD#state{slot_index=PointerList, + smallest_sqn=LowSQN, + highest_sqn=HighSQN, + smallest_key=LowKey, + highest_key=HighKey, + slots_pointer=?HEADER_LEN, + index_pointer=?HEADER_LEN + Blen, + filter_pointer=?HEADER_LEN + Blen + Ilen, + summ_pointer=?HEADER_LEN + Blen + Ilen + Flen, + summ_length=Slen}, + KeyRemainders}. %% Fetch a Key and Value from a file, returns %% {value, KV} or not_present +%% The key must be pre-checked to ensure it is in the valid range for the file +%% A key out of range may fail fetch_keyvalue(Handle, FileMD, Key) -> {_NearestKey, {FilterLen, PointerF}, @@ -296,14 +291,13 @@ fetch_keyvalue_fromblock([BlockNumber|T], Key, LengthList, Handle, StartOfSlot) end. - - get_nearestkey(KVList, Key) -> get_nearestkey(KVList, Key, not_found). get_nearestkey([], _KeyToFind, PrevV) -> PrevV; -get_nearestkey([{K, _FilterInfo, _SlotInfo}|_T], KeyToFind, PrevV) when K > KeyToFind -> +get_nearestkey([{K, _FilterInfo, _SlotInfo}|_T], KeyToFind, PrevV) + when K > KeyToFind -> PrevV; get_nearestkey([Result|T], KeyToFind, _) -> get_nearestkey(T, KeyToFind, Result). @@ -328,10 +322,11 @@ write_group(Handle, KL1, KL2, {SlotCount, SlotTotal}, UpdHandle = WriteFun(slots , {Handle, SerialisedSlots}), case maxslots_bylevel(SlotTotal, Level) of reached -> - complete_write(UpdHandle, + {complete_write(UpdHandle, SlotIndex, {LSN, HSN}, {LowKey, LastKey}, - WriteFun); + WriteFun), + {KL1, KL2}}; continue -> write_group(UpdHandle, KL1, KL2, {0, SlotTotal}, SlotIndex, <<>>, @@ -353,10 +348,11 @@ write_group(Handle, KL1, KL2, {SlotCount, SlotTotal}, case Status of partial -> UpdHandle = WriteFun(slots , {Handle, UpdSlots}), - complete_write(UpdHandle, + {complete_write(UpdHandle, UpdSlotIndex, SNExtremes, {FirstKey, FinalKey}, - WriteFun); + WriteFun), + {KL1rem, KL2rem}}; full -> write_group(Handle, KL1rem, KL2rem, {SlotCount + 1, SlotTotal + 1}, UpdSlotIndex, UpdSlots, @@ -364,7 +360,9 @@ write_group(Handle, KL1, KL2, {SlotCount, SlotTotal}, end. -complete_write(Handle, SlotIndex, SNExtremes, {FirstKey, FinalKey}, WriteFun) -> +complete_write(Handle, SlotIndex, + SNExtremes, {FirstKey, FinalKey}, + WriteFun) -> ConvSlotIndex = convert_slotindex(SlotIndex), FinHandle = WriteFun(finalise, {Handle, ConvSlotIndex, @@ -404,7 +402,7 @@ sftwrite_function(finalise, _KeyExtremes}) -> {ok, Position} = file:position(Handle, cur), - SlotsLength = Position - ?HEADER_LENGTH, + SlotsLength = Position - ?HEADER_LEN, SerialisedIndex = term_to_binary(PointerIndex), IndexLength = byte_size(SerialisedIndex), @@ -429,7 +427,6 @@ sftwrite_function(finalise, KeyExtremes}). maxslots_bylevel(SlotTotal, _Level) -> - io:format("Slot total of ~w~n", [SlotTotal]), case SlotTotal of ?SLOT_COUNT -> reached; @@ -551,7 +548,6 @@ create_slot(KL1, KL2, Level, BlockCount, SegLists, SerialisedSlot, LengthList, Status} end, SerialisedBlock = serialise_block(BlockKeyList), - % io:format("Serialised Block to be added ~w based on BlockKeyList ~w~n", [SerialisedBlock, BlockKeyList]), BlockLength = byte_size(SerialisedBlock), SerialisedSlot2 = <>, create_slot(KL1b, KL2b, Level, BlockCount - 1, SegLists ++ [SegmentList], @@ -1159,8 +1155,9 @@ testwrite_function(finalise, {Handle, C_SlotIndex, SNExtremes, KeyExtremes}) -> writegroup_stage1_test() -> {KL1, KL2} = sample_keylist(), Output = write_group([], KL1, KL2, [], <<>>, 1, fun testwrite_function/2), - {{Handle, {_, PointerIndex}, SNExtremes, KeyExtremes}, - PointerIndex, SNExtremes, KeyExtremes} = Output, + {{{Handle, {_, PointerIndex}, SNExtremes, KeyExtremes}, + PointerIndex, SNExtremes, KeyExtremes}, + {_KL1Rem, _KL2Rem}} = Output, ?assertMatch(SNExtremes, {1,3}), ?assertMatch(KeyExtremes, {{o, "Bucket1", "Key1"}, {o, "Bucket4", "Key1"}}), @@ -1175,13 +1172,13 @@ writegroup_stage1_test() -> initial_create_header_test() -> Output = create_header(initial), - ?assertMatch(?HEADER_LENGTH, byte_size(Output)). + ?assertMatch(?HEADER_LEN, byte_size(Output)). initial_create_file_test() -> Filename = "../test/test1.sft", {KL1, KL2} = sample_keylist(), {Handle, FileMD} = create_file(Filename), - {UpdHandle, UpdFileMD} = complete_file(Handle, FileMD, KL1, KL2, 1), + {UpdHandle, UpdFileMD, {[], []}} = complete_file(Handle, FileMD, KL1, KL2, 1), Result1 = fetch_keyvalue(UpdHandle, UpdFileMD, {o, "Bucket1", "Key8"}), io:format("Result is ~w~n", [Result1]), ?assertMatch(Result1, {{o, "Bucket1", "Key8"}, @@ -1197,7 +1194,9 @@ big_create_file_test() -> {KL1, KL2} = {lists:sort(generate_randomkeys(50000)), lists:sort(generate_randomkeys(50000))}, {InitHandle, InitFileMD} = create_file(Filename), - {Handle, FileMD} = complete_file(InitHandle, InitFileMD, KL1, KL2, 1), + {Handle, FileMD, {_KL1Rem, _KL2Rem}} = complete_file(InitHandle, + InitFileMD, + KL1, KL2, 1), [{K1, Sq1, St1, V1}|_] = KL1, [{K2, Sq2, St2, V2}|_] = KL2, Result1 = fetch_keyvalue(Handle, FileMD, K1), From 9dae89395829bcad51d4bee15efeef96d148d238 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 7 Jul 2016 11:43:26 +0100 Subject: [PATCH 013/167] Read-only opening Completing the file also closes it and leads to a read-only opening of the file. --- src/leveled_sft.erl | 151 ++++++++++++++++++++++---------------------- 1 file changed, 77 insertions(+), 74 deletions(-) diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 988d5a7..39183d8 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -178,20 +178,19 @@ index_pointer :: integer(), filter_pointer :: integer(), summ_pointer :: integer(), - summ_length :: integer()}). + summ_length :: integer(), + filename :: string()}). %% Start a bare file with an initial header and no further details %% Return the {Handle, metadata record} create_file(FileName) when is_list(FileName) -> {ok, Handle} = file:open(FileName, [binary, raw, read, write]), - create_file(Handle); -create_file(Handle) -> Header = create_header(initial), {ok, _} = file:position(Handle, bof), ok = file:write(Handle, Header), {ok, StartPos} = file:position(Handle, cur), - FileMD = #state{next_position=StartPos}, + FileMD = #state{next_position=StartPos, filename=FileName}, {Handle, FileMD}. @@ -210,40 +209,42 @@ create_header(initial) -> %% Open a file returning a handle and metadata which can be used in fetch and %% iterator requests +%% The handle should be read-only as these are immutable files, a file cannot +%% be opened for writing keys, it can only be created to write keys -open_file(Filename) -> - {ok, _Handle} = file:open(Filename, [binary, raw, read, write]). - %% Need to write other metadata somewhere - %% ... probably in summmary - %% ... is there a need for two levels of summary? +open_file(FileMD) -> + Filename = FileMD#state.filename, + {ok, Handle} = file:open(Filename, [binary, raw, read]), + {ok, HeaderLengths} = file:pread(Handle, 12, 16), + <> = HeaderLengths, + {ok, SummaryBin} = file:pread(Handle, ?HEADER_LEN + Blen + Ilen + Flen, Slen), + {{LowSQN, HighSQN}, {LowKey, HighKey}} = binary_to_term(SummaryBin), + {ok, SlotIndexBin} = file:pread(Handle, ?HEADER_LEN + Blen, Ilen), + SlotIndex = binary_to_term(SlotIndexBin), + {Handle, FileMD#state{slot_index=SlotIndex, + smallest_sqn=LowSQN, + highest_sqn=HighSQN, + smallest_key=LowKey, + highest_key=HighKey, + slots_pointer=?HEADER_LEN, + index_pointer=?HEADER_LEN + Blen, + filter_pointer=?HEADER_LEN + Blen + Ilen, + summ_pointer=?HEADER_LEN + Blen + Ilen + Flen, + summ_length=Slen}}. %% Take a file handle with a previously created header and complete it based on %% the two key lists KL1 and KL2 complete_file(Handle, FileMD, KL1, KL2, Level) -> - {{UpdHandle, - PointerList, - {LowSQN, HighSQN}, - {LowKey, HighKey}}, - KeyRemainders} = write_group(Handle, KL1, KL2, [], <<>>, Level, - fun sftwrite_function/2), - {ok, HeaderLengths} = file:pread(UpdHandle, 12, 16), - <> = HeaderLengths, - {UpdHandle, - FileMD#state{slot_index=PointerList, - smallest_sqn=LowSQN, - highest_sqn=HighSQN, - smallest_key=LowKey, - highest_key=HighKey, - slots_pointer=?HEADER_LEN, - index_pointer=?HEADER_LEN + Blen, - filter_pointer=?HEADER_LEN + Blen + Ilen, - summ_pointer=?HEADER_LEN + Blen + Ilen + Flen, - summ_length=Slen}, - KeyRemainders}. + {ok, KeyRemainders} = write_keys(Handle, + KL1, KL2, [], <<>>, + Level, + fun sftwrite_function/2), + {ReadHandle, UpdFileMD} = open_file(FileMD), + {ReadHandle, UpdFileMD, KeyRemainders}. %% Fetch a Key and Value from a file, returns %% {value, KV} or not_present @@ -259,14 +260,17 @@ fetch_keyvalue(Handle, FileMD, Key) -> SegID = hash_for_segmentid({keyonly, Key}), case check_for_segments(SegFilter, [SegID], true) of {maybe_present, BlockList} -> + io:format("Filter has returned maybe~n"), fetch_keyvalue_fromblock(BlockList, Key, LengthList, Handle, PointerB + FileMD#state.slots_pointer); not_present -> + io:format("Filter has returned no~n"), not_present; error_so_maybe_present -> + io:format("Filter has returned error~n"), fetch_keyvalue_fromblock(lists:seq(0,3), Key, LengthList, @@ -309,30 +313,30 @@ get_nearestkey([Result|T], KeyToFind, _) -> %% Slots are created then written in bulk to impove I/O efficiency. Slots will %% be written in groups of 32 -write_group(Handle, KL1, KL2, SlotIndex, SerialisedSlots, Level, WriteFun) -> - write_group(Handle, KL1, KL2, {0, 0}, +write_keys(Handle, KL1, KL2, SlotIndex, SerialisedSlots, Level, WriteFun) -> + write_keys(Handle, KL1, KL2, {0, 0}, SlotIndex, SerialisedSlots, {infinity, 0}, null, {last, null}, Level, WriteFun). -write_group(Handle, KL1, KL2, {SlotCount, SlotTotal}, +write_keys(Handle, KL1, KL2, {SlotCount, SlotTotal}, SlotIndex, SerialisedSlots, {LSN, HSN}, LowKey, LastKey, Level, WriteFun) when SlotCount =:= ?SLOT_GROUPWRITE_COUNT -> UpdHandle = WriteFun(slots , {Handle, SerialisedSlots}), case maxslots_bylevel(SlotTotal, Level) of reached -> - {complete_write(UpdHandle, - SlotIndex, - {LSN, HSN}, {LowKey, LastKey}, - WriteFun), - {KL1, KL2}}; + {complete_keywrite(UpdHandle, + SlotIndex, + {LSN, HSN}, {LowKey, LastKey}, + WriteFun), + {KL1, KL2}}; continue -> - write_group(UpdHandle, KL1, KL2, {0, SlotTotal}, + write_keys(UpdHandle, KL1, KL2, {0, SlotTotal}, SlotIndex, <<>>, {LSN, HSN}, LowKey, LastKey, Level, WriteFun) end; -write_group(Handle, KL1, KL2, {SlotCount, SlotTotal}, +write_keys(Handle, KL1, KL2, {SlotCount, SlotTotal}, SlotIndex, SerialisedSlots, {LSN, HSN}, LowKey, LastKey, Level, WriteFun) -> SlotOutput = create_slot(KL1, KL2, Level), @@ -348,28 +352,26 @@ write_group(Handle, KL1, KL2, {SlotCount, SlotTotal}, case Status of partial -> UpdHandle = WriteFun(slots , {Handle, UpdSlots}), - {complete_write(UpdHandle, - UpdSlotIndex, - SNExtremes, {FirstKey, FinalKey}, - WriteFun), - {KL1rem, KL2rem}}; + {complete_keywrite(UpdHandle, + UpdSlotIndex, + SNExtremes, {FirstKey, FinalKey}, + WriteFun), + {KL1rem, KL2rem}}; full -> - write_group(Handle, KL1rem, KL2rem, {SlotCount + 1, SlotTotal + 1}, + write_keys(Handle, KL1rem, KL2rem, {SlotCount + 1, SlotTotal + 1}, UpdSlotIndex, UpdSlots, SNExtremes, FirstKey, FinalKey, Level, WriteFun) end. -complete_write(Handle, SlotIndex, +complete_keywrite(Handle, SlotIndex, SNExtremes, {FirstKey, FinalKey}, WriteFun) -> ConvSlotIndex = convert_slotindex(SlotIndex), - FinHandle = WriteFun(finalise, {Handle, - ConvSlotIndex, - SNExtremes, - {FirstKey, FinalKey}}), - {_, PointerIndex} = ConvSlotIndex, - {FinHandle, PointerIndex, SNExtremes, {FirstKey, FinalKey}}. + WriteFun(finalise, {Handle, + ConvSlotIndex, + SNExtremes, + {FirstKey, FinalKey}}). %% Take a slot index, and remove the SegFilters replacing with pointers @@ -398,22 +400,26 @@ sftwrite_function(slots, {Handle, SerialisedSlots}) -> sftwrite_function(finalise, {Handle, {SlotFilters, PointerIndex}, - _SNExtremes, - _KeyExtremes}) -> + SNExtremes, + KeyExtremes}) -> {ok, Position} = file:position(Handle, cur), - SlotsLength = Position - ?HEADER_LEN, - SerialisedIndex = term_to_binary(PointerIndex), - IndexLength = byte_size(SerialisedIndex), - - %% Write Index - ok = file:write(Handle, SerialisedIndex), - %% Write Filter - ok = file:write(Handle, SlotFilters), + BlocksLength = Position - ?HEADER_LEN, + Index = term_to_binary(PointerIndex), + IndexLength = byte_size(Index), + FilterLength = byte_size(SlotFilters), + Summary = term_to_binary({SNExtremes, KeyExtremes}), + SummaryLength = byte_size(Summary), + %% Write Index, Filter and Summary + ok = file:write(Handle, <>), %% Write Lengths into header - ok = file:pwrite(Handle, 12, <>), - Handle; + ok = file:pwrite(Handle, 12, <>), + file:close(Handle); sftwrite_function(finalise, {Handle, SlotIndex, @@ -1152,12 +1158,10 @@ testwrite_function(slots, {Handle, SerialisedSlots}) -> testwrite_function(finalise, {Handle, C_SlotIndex, SNExtremes, KeyExtremes}) -> {Handle, C_SlotIndex, SNExtremes, KeyExtremes}. -writegroup_stage1_test() -> +writekeys_stage1_test() -> {KL1, KL2} = sample_keylist(), - Output = write_group([], KL1, KL2, [], <<>>, 1, fun testwrite_function/2), - {{{Handle, {_, PointerIndex}, SNExtremes, KeyExtremes}, - PointerIndex, SNExtremes, KeyExtremes}, - {_KL1Rem, _KL2Rem}} = Output, + {FunOut, {_KL1Rem, _KL2Rem}} = write_keys([], KL1, KL2, [], <<>>, 1, fun testwrite_function/2), + {Handle, {_, PointerIndex}, SNExtremes, KeyExtremes} = FunOut, ?assertMatch(SNExtremes, {1,3}), ?assertMatch(KeyExtremes, {{o, "Bucket1", "Key1"}, {o, "Bucket4", "Key1"}}), @@ -1201,7 +1205,6 @@ big_create_file_test() -> [{K2, Sq2, St2, V2}|_] = KL2, Result1 = fetch_keyvalue(Handle, FileMD, K1), Result2 = fetch_keyvalue(Handle, FileMD, K2), - io:format("Results are:~n~w~n~w~n", [Result1, Result2]), ?assertMatch(Result1, {K1, Sq1, St1, V1}), ?assertMatch(Result2, {K2, Sq2, St2, V2}), FailedFinds = lists:foldl(fun(K, Acc) -> From 45c10613e74a40f0ac4a664a5791ad66b5689881 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 12 Jul 2016 19:42:50 +0100 Subject: [PATCH 014/167] Iterator support added Initial support for iterators --- src/leveled_sft.erl | 226 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 202 insertions(+), 24 deletions(-) diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 39183d8..ef0377e 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -165,6 +165,7 @@ -define(DIVISOR, 8092). -define(COMPRESSION_LEVEL, 1). -define(HEADER_LEN, 56). +-define(ITERATOR_SCANWIDTH, 1). -record(state, {version = ?CURRENT_VERSION :: tuple(), @@ -254,38 +255,132 @@ complete_file(Handle, FileMD, KL1, KL2, Level) -> fetch_keyvalue(Handle, FileMD, Key) -> {_NearestKey, {FilterLen, PointerF}, {LengthList, PointerB}} = get_nearestkey(FileMD#state.slot_index, Key), - {ok, SegFilter} = file:pread(Handle, - PointerF + FileMD#state.filter_pointer, - FilterLen), + {ok, SegFilter} = file:pread(Handle, + PointerF + FileMD#state.filter_pointer, + FilterLen), SegID = hash_for_segmentid({keyonly, Key}), case check_for_segments(SegFilter, [SegID], true) of {maybe_present, BlockList} -> - io:format("Filter has returned maybe~n"), fetch_keyvalue_fromblock(BlockList, Key, LengthList, Handle, PointerB + FileMD#state.slots_pointer); not_present -> - io:format("Filter has returned no~n"), not_present; error_so_maybe_present -> - io:format("Filter has returned error~n"), - fetch_keyvalue_fromblock(lists:seq(0,3), + fetch_keyvalue_fromblock(lists:seq(0,length(LengthList)), Key, LengthList, Handle, PointerB + FileMD#state.slots_pointer) end. +%% Fetches a range of keys returning a list of {Key, SeqN} tuples +fetch_range_keysonly(Handle, FileMD, StartKey, EndKey) -> + fetch_range(Handle, FileMD, StartKey, EndKey, [], fun acc_list_keysonly/2). + +fetch_range_keysonly(Handle, FileMD, StartKey, EndKey, ScanWidth) -> + fetch_range(Handle, FileMD, StartKey, EndKey, [], fun acc_list_keysonly/2, ScanWidth). + +acc_list_keysonly(null, empty) -> + []; +acc_list_keysonly(null, RList) -> + RList; +acc_list_keysonly(R, RList) -> + lists:append(RList, [strip_to_key_seqn_only(R)]). + +%% Iterate keys, returning a batch of keys & values in a range +%% - the iterator can have a ScanWidth which is how many slots should be +%% scanned by the iterator before returning a result +%% - batches can be ended with a pointer to indicate there are potentially +%% further values in the range +%% - a list of functions can be provided, which should either return true +%% or false, and these can be used to filter the results from the query, +%% for example to ignore keys above a certain sequence number, to ignore +%% keys not matching a certain regular expression, or to ignore keys not +%% a member of a particular partition +%% - An Accumulator and an Accumulator function can be passed. The function +%% needs to handle being passed (KV, Acc) to add the current result to the +%% Accumulator. The functional should handle KV=null, Acc=empty to initiate +%% the accumulator, and KV=null to leave the Accumulator unchanged. +%% Flexibility with accumulators is such that keys-only can be returned rather +%% than keys and values, or other entirely different accumulators can be +%% used - e.g. counters, hash-lists to build bloom filters etc + +fetch_range(Handle, FileMD, StartKey, EndKey, FunList, AccFun) -> + fetch_range(Handle, FileMD, StartKey, EndKey, FunList, AccFun, ?ITERATOR_SCANWIDTH). + +fetch_range(Handle, FileMD, StartKey, EndKey, FunList, AccFun, ScanWidth) -> + fetch_range(Handle, FileMD, StartKey, EndKey, FunList, AccFun, ScanWidth, empty). + +fetch_range(_Handle, _FileMD, StartKey, _EndKey, _FunList, _AccFun, 0, Acc) -> + {partial, Acc, StartKey}; +fetch_range(Handle, FileMD, StartKey, EndKey, FunList, AccFun, ScanWidth, Acc) -> + %% get_nearestkey gets the last key in the index <= StartKey, or the next + %% key along if {next, StartKey} is passed + case get_nearestkey(FileMD#state.slot_index, StartKey) of + {NearestKey, _Filter, {LengthList, PointerB}} -> + fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, FunList, AccFun, ScanWidth, + LengthList, 0, PointerB + FileMD#state.slots_pointer, + AccFun(null, Acc)); + not_found -> + {complete, Acc} + end. + +fetch_range(Handle, FileMD, _StartKey, NearestKey, EndKey, FunList, AccFun, ScanWidth, + LengthList, BlockNumber, _Pointer, Acc) + when length(LengthList) == BlockNumber -> + %% Reached the end of the slot. Move the start key on one to scan a new slot + fetch_range(Handle, FileMD, {next, NearestKey}, EndKey, FunList, AccFun, ScanWidth - 1, Acc); +fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, FunList, AccFun, ScanWidth, + LengthList, BlockNumber, Pointer, Acc) -> + Block = fetch_block(Handle, LengthList, BlockNumber, Pointer), + Results = scan_block(Block, StartKey, EndKey, FunList, AccFun, Acc), + case Results of + {partial, Acc1, StartKey} -> + %% Move on to the next block + fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, FunList, AccFun, ScanWidth, + LengthList, BlockNumber + 1, Pointer, Acc1); + {complete, Acc1} -> + {complete, Acc1} + end. + +scan_block([], StartKey, _EndKey, _FunList, _AccFun, Acc) -> + {partial, Acc, StartKey}; +scan_block([HeadKV|T], StartKey, EndKey, FunList, AccFun, Acc) -> + K = strip_to_keyonly(HeadKV), + case K of + K when K < StartKey -> + scan_block(T, StartKey, EndKey, FunList, AccFun, Acc); + K when K > EndKey -> + {complete, Acc}; + _ -> + case applyfuns(FunList, HeadKV) of + include -> + %% Add result to the accumulator + scan_block(T, StartKey, EndKey, FunList, + AccFun, AccFun(HeadKV, Acc)); + skip -> + scan_block(T, StartKey, EndKey, FunList, + AccFun, Acc) + end + end. + +applyfuns([], _KV) -> + include; +applyfuns([HeadFun|OtherFuns], KV) -> + case HeadFun(KV) of + true -> + applyfuns(OtherFuns, KV); + false -> + skip + end. fetch_keyvalue_fromblock([], _Key, _LengthList, _Handle, _StartOfSlot) -> not_present; fetch_keyvalue_fromblock([BlockNumber|T], Key, LengthList, Handle, StartOfSlot) -> - Start = lists:sum(lists:sublist(LengthList, BlockNumber)), - Length = lists:nth(BlockNumber + 1, LengthList), - {ok, BlockToCheckBin} = file:pread(Handle, Start + StartOfSlot, Length), - BlockToCheck = binary_to_term(BlockToCheckBin), + BlockToCheck = fetch_block(Handle, LengthList, BlockNumber, StartOfSlot), Result = lists:keyfind(Key, 1, BlockToCheck), case Result of false -> @@ -294,17 +389,48 @@ fetch_keyvalue_fromblock([BlockNumber|T], Key, LengthList, Handle, StartOfSlot) KV end. - +fetch_block(Handle, LengthList, BlockNumber, StartOfSlot) -> + Start = lists:sum(lists:sublist(LengthList, BlockNumber)), + Length = lists:nth(BlockNumber + 1, LengthList), + {ok, BlockToCheckBin} = file:pread(Handle, Start + StartOfSlot, Length), + binary_to_term(BlockToCheckBin). + +%% Need to deal with either Key or {next, Key} get_nearestkey(KVList, Key) -> - get_nearestkey(KVList, Key, not_found). - -get_nearestkey([], _KeyToFind, PrevV) -> + case Key of + {first, K} -> + get_firstkeytomatch(KVList, K, not_found); + {next, K} -> + get_nextkeyaftermatch(KVList, K, not_found); + _ -> + get_firstkeytomatch(KVList, Key, not_found) + end. + +get_firstkeytomatch([], _KeyToFind, PrevV) -> PrevV; -get_nearestkey([{K, _FilterInfo, _SlotInfo}|_T], KeyToFind, PrevV) +get_firstkeytomatch([{K, FilterInfo, SlotInfo}|_T], KeyToFind, PrevV) when K > KeyToFind -> - PrevV; -get_nearestkey([Result|T], KeyToFind, _) -> - get_nearestkey(T, KeyToFind, Result). + case PrevV of + not_found -> + {K, FilterInfo, SlotInfo}; + _ -> + PrevV + end; +get_firstkeytomatch([{K, FilterInfo, SlotInfo}|T], KeyToFind, _PrevV) -> + get_firstkeytomatch(T, KeyToFind, {K, FilterInfo, SlotInfo}). + +get_nextkeyaftermatch([], _KeyToFind, _PrevV) -> + not_found; +get_nextkeyaftermatch([{K, FilterInfo, SlotInfo}|T], KeyToFind, PrevV) + when K >= KeyToFind -> + case PrevV of + not_found -> + get_nextkeyaftermatch(T, KeyToFind, next); + next -> + {K, FilterInfo, SlotInfo} + end; +get_nextkeyaftermatch([_KTuple|T], KeyToFind, PrevV) -> + get_nextkeyaftermatch(T, KeyToFind, PrevV). %% Take a file handle at the sart position (after creating the header) and then @@ -570,6 +696,8 @@ last(E, [], _) -> E. strip_to_keyonly({keyonly, K}) -> K; strip_to_keyonly({K, _, _, _}) -> K. +strip_to_key_seqn_only({K, SeqN, _, _}) -> {K, SeqN}. + serialise_block(BlockKeyList) -> term_to_binary(BlockKeyList, [{compressed, ?COMPRESSION_LEVEL}]). @@ -933,7 +1061,7 @@ generate_randomkeys(Count, Acc) -> RandKey = {{o, lists:concat(["Bucket", random:uniform(1024)]), lists:concat(["Key", random:uniform(1024)])}, - random:uniform(1024*1024), + Count + 1, {active, infinity}, null}, generate_randomkeys(Count - 1, [RandKey|Acc]). @@ -1195,7 +1323,7 @@ initial_create_file_test() -> big_create_file_test() -> Filename = "../test/bigtest1.sft", - {KL1, KL2} = {lists:sort(generate_randomkeys(50000)), + {KL1, KL2} = {lists:sort(generate_randomkeys(2000)), lists:sort(generate_randomkeys(50000))}, {InitHandle, InitFileMD} = create_file(Filename), {Handle, FileMD, {_KL1Rem, _KL2Rem}} = complete_file(InitHandle, @@ -1207,18 +1335,68 @@ big_create_file_test() -> Result2 = fetch_keyvalue(Handle, FileMD, K2), ?assertMatch(Result1, {K1, Sq1, St1, V1}), ?assertMatch(Result2, {K2, Sq2, St2, V2}), + SubList = lists:sublist(KL2, 1000), FailedFinds = lists:foldl(fun(K, Acc) -> {Kn, _, _, _} = K, Rn = fetch_keyvalue(Handle, FileMD, Kn), case Rn of - {Kn, _, _, _} -> Acc; - _ -> Acc + 1 + {Kn, _, _, _} -> + Acc; + _ -> + Acc + 1 end end, 0, - lists:sublist(KL2, 1000)), + SubList), + io:format("FailedFinds of ~w~n", [FailedFinds]), ?assertMatch(FailedFinds, 0), Result3 = fetch_keyvalue(Handle, FileMD, {o, "Bucket1024", "Key1024Alt"}), ?assertMatch(Result3, not_present), ok = file:close(Handle), ok = file:delete(Filename). + +initial_iterator_test() -> + Filename = "../test/test2.sft", + {KL1, KL2} = sample_keylist(), + {Handle, FileMD} = create_file(Filename), + {UpdHandle, UpdFileMD, {[], []}} = complete_file(Handle, FileMD, KL1, KL2, 1), + Result1 = fetch_range_keysonly(UpdHandle, UpdFileMD, + {o, "Bucket1", "Key8"}, + {o, "Bucket1", "Key9d"}), + io:format("Result returned of ~w~n", [Result1]), + ?assertMatch(Result1, {complete, [{{o, "Bucket1", "Key8"}, 1}, + {{o, "Bucket1", "Key9"}, 1}, + {{o, "Bucket1", "Key9a"}, 1}, + {{o, "Bucket1", "Key9b"}, 1}, + {{o, "Bucket1", "Key9c"}, 1}, + {{o, "Bucket1", "Key9d"}, 1}]}), + Result2 = fetch_range_keysonly(UpdHandle, UpdFileMD, + {o, "Bucket1", "Key8"}, + {o, "Bucket1", "Key9b"}), + ?assertMatch(Result2, {complete, [{{o, "Bucket1", "Key8"}, 1}, + {{o, "Bucket1", "Key9"}, 1}, + {{o, "Bucket1", "Key9a"}, 1}, + {{o, "Bucket1", "Key9b"}, 1}]}), + ok = file:close(UpdHandle), + ok = file:delete(Filename). + +big_iterator_test() -> + Filename = "../test/bigtest1.sft", + {KL1, KL2} = {lists:sort(generate_randomkeys(10000)), []}, + {InitHandle, InitFileMD} = create_file(Filename), + {Handle, FileMD, {KL1Rem, KL2Rem}} = complete_file(InitHandle, + InitFileMD, + KL1, KL2, 1), + io:format("Remainder lengths are ~w and ~w ~n", [length(KL1Rem), length(KL2Rem)]), + {complete, Result1} = fetch_range_keysonly(Handle, FileMD, {o, "Bucket0000", "Key0000"}, + {o, "Bucket9999", "Key9999"}, + 256), + NumFoundKeys1 = length(Result1), + NumAddedKeys = 10000 - length(KL1Rem), + ?assertMatch(NumFoundKeys1, NumAddedKeys), + {partial, Result2, _} = fetch_range_keysonly(Handle, FileMD, {o, "Bucket0000", "Key0000"}, + {o, "Bucket9999", "Key9999"}, + 32), + NumFoundKeys2 = length(Result2), + ?assertMatch(NumFoundKeys2, 32 * 128). + \ No newline at end of file From 392830c8399db394e531be28f8ce5c7de725842f Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 12 Jul 2016 19:44:16 +0100 Subject: [PATCH 015/167] Improve test clean-up Remove file created in a test --- src/leveled_sft.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index ef0377e..0b44db9 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -1398,5 +1398,7 @@ big_iterator_test() -> {o, "Bucket9999", "Key9999"}, 32), NumFoundKeys2 = length(Result2), - ?assertMatch(NumFoundKeys2, 32 * 128). + ?assertMatch(NumFoundKeys2, 32 * 128), + ok = file:close(Handle), + ok = file:delete(Filename). \ No newline at end of file From d96ac87fb5bd5b9b516407828a2cfaad0d839a00 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 12 Jul 2016 19:45:22 +0100 Subject: [PATCH 016/167] Add gitignore Ignore beam files --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1ef2775 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.beam \ No newline at end of file From a07ea27dd81519753ca77ef705335a603996cdb2 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 22 Jul 2016 16:57:28 +0100 Subject: [PATCH 017/167] Add merging of files Add first function of worker - the ability to merge multiple files together from different levels --- src/leveled_sft.erl | 288 +++++++++++++++++++++++++++++++++++++---- src/leveled_worker.erl | 161 +++++++++++++++++++++++ 2 files changed, 422 insertions(+), 27 deletions(-) create mode 100644 src/leveled_worker.erl diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 0b44db9..ef1fd9d 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -142,13 +142,24 @@ -module(leveled_sft). --export([create_file/1, - generate_segment_filter/1, - serialise_segment_filter/1, - check_for_segments/3, +-behaviour(gen_server). + +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3, speedtest_check_forsegment/4, generate_randomsegfilter/1, - create_slot/3]). + generate_randomkeys/1, + strip_to_keyonly/1, + sft_new/4, + sft_open/1, + sft_get/2, + sft_getkeyrange/4, + sft_close/1, + sft_clear/1]). -include_lib("eunit/include/eunit.hrl"). @@ -166,6 +177,7 @@ -define(COMPRESSION_LEVEL, 1). -define(HEADER_LEN, 56). -define(ITERATOR_SCANWIDTH, 1). +-define(MERGE_SCANWIDTH, 8). -record(state, {version = ?CURRENT_VERSION :: tuple(), @@ -180,19 +192,168 @@ filter_pointer :: integer(), summ_pointer :: integer(), summ_length :: integer(), - filename :: string()}). + filename :: string(), + handle :: file:fd()}). + + +%%%============================================================================ +%%% API +%%%============================================================================ + +sft_new(Filename, KL1, KL2, Level) -> + {ok, Pid} = gen_server:start(?MODULE, [], []), + Reply = gen_server:call(Pid, {sft_new, Filename, KL1, KL2, Level}, infinity), + {ok, Pid, Reply}. + +sft_open(Filename) -> + {ok, Pid} = gen_server:start(?MODULE, [], []), + case gen_server:call(Pid, {sft_open, Filename}, infinity) of + ok -> + {ok, Pid}; + Error -> + Error + end. + +sft_get(Pid, Key) -> + file_request(Pid, {get_kv, Key}). + +sft_getkeyrange(Pid, StartKey, EndKey, ScanWidth) -> + file_request(Pid, {get_keyrange, StartKey, EndKey, ScanWidth}). + +sft_getkvrange(Pid, StartKey, EndKey, ScanWidth) -> + file_request(Pid, {get_kvrange, StartKey, EndKey, ScanWidth}). + +sft_close(Pid) -> + file_request(Pid, close). + +sft_clear(Pid) -> + file_request(Pid, clear). + + +%%%============================================================================ +%%% API helper functions +%%%============================================================================ + +%% This saftey measure of checking the Pid is alive before perfoming any ops +%% is copied from the bitcask source code. +%% +%% It is not clear at present if this is necessary. + +file_request(Pid, Request) -> + case check_pid(Pid) of + ok -> + try + gen_server:call(Pid, Request, infinity) + catch + exit:{normal,_} when Request == file_close -> + %% Honest race condition in bitcask_eqc PULSE test. + ok; + exit:{noproc,_} when Request == file_close -> + %% Honest race condition in bitcask_eqc PULSE test. + ok; + X1:X2 -> + exit({file_request_error, self(), Request, X1, X2}) + end; + Error -> + Error + end. + +check_pid(Pid) -> + IsPid = is_pid(Pid), + IsAlive = IsPid andalso is_process_alive(Pid), + case {IsAlive, IsPid} of + {true, _} -> + ok; + {false, true} -> + %% Same result as `file' module when accessing closed FD + {error, einval}; + _ -> + %% Same result as `file' module when providing wrong arg + {error, badarg} + end. + +%%%============================================================================ +%%% gen_server callbacks +%%%============================================================================ + +init([]) -> + {ok, #state{}}. + +handle_call({sft_new, Filename, KL1, KL2, Level}, _From, State) -> + case create_file(Filename) of + {error, Reason} -> + {reply, {error, Reason}, State}; + {Handle, FileMD} -> + io:format("Creating file with inputs of size ~w ~w~n", + [length(KL1), length(KL2)]), + {ReadHandle, UpdFileMD, KeyRemainders} = complete_file(Handle, + FileMD, + KL1, KL2, + Level), + {KL1Rem, KL2Rem} = KeyRemainders, + io:format("File created with remainders of size ~w ~w~n", + [length(KL1Rem), length(KL2Rem)]), + {reply, {KeyRemainders, + UpdFileMD#state.smallest_key, + UpdFileMD#state.highest_key}, + UpdFileMD#state{handle=ReadHandle}} + end; +handle_call({sft_open, Filename}, _From, _State) -> + {_Handle, FileMD} = open_file(Filename), + {reply, {FileMD#state.smallest_key, FileMD#state.highest_key}, FileMD}; +handle_call({get_kv, Key}, _From, State) -> + Reply = fetch_keyvalue(State#state.handle, State, Key), + {reply, Reply, State}; +handle_call({get_keyrange, StartKey, EndKey, ScanWidth}, _From, State) -> + Reply = fetch_range_keysonly(State#state.handle, State, + StartKey, EndKey, + ScanWidth), + {reply, Reply, State}; +handle_call({get_kvrange, StartKey, EndKey, ScanWidth}, _From, State) -> + Reply = fetch_range_kv(State#state.handle, State, + StartKey, EndKey, + ScanWidth), + {reply, Reply, State}; +handle_call(close, _From, State) -> + {reply, true, State}; +handle_call(clear, _From, State) -> + ok = file:close(State#state.handle), + ok = file:delete(State#state.filename), + {reply, true, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%============================================================================ +%%% Internal functions +%%%============================================================================ %% Start a bare file with an initial header and no further details %% Return the {Handle, metadata record} create_file(FileName) when is_list(FileName) -> - {ok, Handle} = file:open(FileName, [binary, raw, read, write]), - Header = create_header(initial), - {ok, _} = file:position(Handle, bof), - ok = file:write(Handle, Header), - {ok, StartPos} = file:position(Handle, cur), - FileMD = #state{next_position=StartPos, filename=FileName}, - {Handle, FileMD}. + io:format("Opening file with filename ~s~n", [FileName]), + case file:open(FileName, [binary, raw, read, write]) of + {ok, Handle} -> + Header = create_header(initial), + {ok, _} = file:position(Handle, bof), + ok = file:write(Handle, Header), + {ok, StartPos} = file:position(Handle, cur), + FileMD = #state{next_position=StartPos, filename=FileName}, + {Handle, FileMD}; + {error, Reason} -> + io:format("Error opening filename ~s with reason ~s", [FileName, Reason]), + {error, Reason} + end. create_header(initial) -> @@ -234,14 +395,17 @@ open_file(FileMD) -> index_pointer=?HEADER_LEN + Blen, filter_pointer=?HEADER_LEN + Blen + Ilen, summ_pointer=?HEADER_LEN + Blen + Ilen + Flen, - summ_length=Slen}}. + summ_length=Slen, + handle=Handle}}. %% Take a file handle with a previously created header and complete it based on %% the two key lists KL1 and KL2 complete_file(Handle, FileMD, KL1, KL2, Level) -> {ok, KeyRemainders} = write_keys(Handle, - KL1, KL2, [], <<>>, + maybe_expand_pointer(KL1), + maybe_expand_pointer(KL2), + [], <<>>, Level, fun sftwrite_function/2), {ReadHandle, UpdFileMD} = open_file(FileMD), @@ -283,6 +447,10 @@ fetch_range_keysonly(Handle, FileMD, StartKey, EndKey) -> fetch_range_keysonly(Handle, FileMD, StartKey, EndKey, ScanWidth) -> fetch_range(Handle, FileMD, StartKey, EndKey, [], fun acc_list_keysonly/2, ScanWidth). +%% Fetches a range of keys returning the full tuple, including value +fetch_range_kv(Handle, FileMD, StartKey, EndKey, ScanWidth) -> + fetch_range(Handle, FileMD, StartKey, EndKey, [], fun acc_list_kv/2, ScanWidth). + acc_list_keysonly(null, empty) -> []; acc_list_keysonly(null, RList) -> @@ -290,6 +458,13 @@ acc_list_keysonly(null, RList) -> acc_list_keysonly(R, RList) -> lists:append(RList, [strip_to_key_seqn_only(R)]). +acc_list_kv(null, empty) -> + []; +acc_list_kv(null, RList) -> + RList; +acc_list_kv(R, RList) -> + lists:append(RList, [R]). + %% Iterate keys, returning a batch of keys & values in a range %% - the iterator can have a ScanWidth which is how many slots should be %% scanned by the iterator before returning a result @@ -351,9 +526,9 @@ scan_block([], StartKey, _EndKey, _FunList, _AccFun, Acc) -> scan_block([HeadKV|T], StartKey, EndKey, FunList, AccFun, Acc) -> K = strip_to_keyonly(HeadKV), case K of - K when K < StartKey -> + K when K < StartKey, StartKey /= all -> scan_block(T, StartKey, EndKey, FunList, AccFun, Acc); - K when K > EndKey -> + K when K > EndKey, EndKey /= all -> {complete, Acc}; _ -> case applyfuns(FunList, HeadKV) of @@ -367,6 +542,7 @@ scan_block([HeadKV|T], StartKey, EndKey, FunList, AccFun, Acc) -> end end. + applyfuns([], _KV) -> include; applyfuns([HeadFun|OtherFuns], KV) -> @@ -396,6 +572,16 @@ fetch_block(Handle, LengthList, BlockNumber, StartOfSlot) -> binary_to_term(BlockToCheckBin). %% Need to deal with either Key or {next, Key} +get_nearestkey(KVList, all) -> + case KVList of + [] -> + not_found; + [H|_Tail] -> + H; + _ -> + io:format("KVList issue ~w~n", [KVList]), + error + end; get_nearestkey(KVList, Key) -> case Key of {first, K} -> @@ -486,7 +672,14 @@ write_keys(Handle, KL1, KL2, {SlotCount, SlotTotal}, full -> write_keys(Handle, KL1rem, KL2rem, {SlotCount + 1, SlotTotal + 1}, UpdSlotIndex, UpdSlots, - SNExtremes, FirstKey, FinalKey, Level, WriteFun) + SNExtremes, FirstKey, FinalKey, Level, WriteFun); + complete -> + UpdHandle = WriteFun(slots , {Handle, UpdSlots}), + {complete_keywrite(UpdHandle, + UpdSlotIndex, + SNExtremes, {FirstKey, FinalKey}, + WriteFun), + {KL1rem, KL2rem}} end. @@ -712,6 +905,7 @@ serialise_block(BlockKeyList) -> %% there are matching keys then the highest sequence number must be chosen and %% any lower sequence numbers should be compacted out of existence + key_dominates([H1|T1], [], Level) -> {_, _, St1, _} = H1, case maybe_reap_expiredkey(St1, Level) of @@ -763,10 +957,35 @@ maybe_reap_expiredkey({_, TS}, {basement, CurrTS}) when CurrTS > TS -> maybe_reap_expiredkey(_, _) -> false. -%% Not worked out pointers yet -maybe_expand_pointer(Tail) -> - Tail. +%% When a list is provided it may include a pointer to gain another batch of +%% entries from the same file, or a new batch of entries from another file +%% +%% This resultant list should include the Tail of any pointers added at the +%% end of the list +maybe_expand_pointer([]) -> + []; +maybe_expand_pointer([H|Tail]) -> + case H of + {next, SFTPid, StartKey} -> + io:format("Scanning further on PID ~w ~w~n", [SFTPid, StartKey]), + QResult = sft_getkvrange(SFTPid, StartKey, all, ?MERGE_SCANWIDTH), + Acc = pointer_append_queryresults(QResult, SFTPid), + lists:append(Acc, Tail); + _ -> + [H|Tail] + end. + + +pointer_append_queryresults(Results, QueryPid) -> + case Results of + {complete, Acc} -> + Acc; + {partial, Acc, StartKey} -> + lists:append(Acc, [{next, QueryPid, StartKey}]) + end. + + %% Update the sequence numbers update_sequencenumbers({_, SN, _, _}, 0, 0) -> {SN, SN}; @@ -1020,11 +1239,9 @@ findremainder(BitStr, Factor) -> - -%%%%%%%%%%%%%%%% -% T E S T -%%%%%%%%%%%%%%% - +%%%============================================================================ +%%% Test +%%%============================================================================ speedtest_check_forsegment(_, 0, _, _) -> true; @@ -1288,7 +1505,8 @@ testwrite_function(finalise, {Handle, C_SlotIndex, SNExtremes, KeyExtremes}) -> writekeys_stage1_test() -> {KL1, KL2} = sample_keylist(), - {FunOut, {_KL1Rem, _KL2Rem}} = write_keys([], KL1, KL2, [], <<>>, 1, fun testwrite_function/2), + {FunOut, {_KL1Rem, _KL2Rem}} = write_keys([], KL1, KL2, [], <<>>, 1, + fun testwrite_function/2), {Handle, {_, PointerIndex}, SNExtremes, KeyExtremes} = FunOut, ?assertMatch(SNExtremes, {1,3}), ?assertMatch(KeyExtremes, {{o, "Bucket1", "Key1"}, @@ -1377,6 +1595,17 @@ initial_iterator_test() -> {{o, "Bucket1", "Key9"}, 1}, {{o, "Bucket1", "Key9a"}, 1}, {{o, "Bucket1", "Key9b"}, 1}]}), + Result3 = fetch_range_keysonly(UpdHandle, UpdFileMD, + {o, "Bucket3", "Key4"}, + all), + {partial, RL3, _} = Result3, + ?assertMatch(RL3, [{{o, "Bucket3", "Key4"}, 3}, + {{o, "Bucket3", "Key5"}, 1}, + {{o, "Bucket3", "Key6"}, 2}, + {{o, "Bucket3", "Key7"}, 1}, + {{o, "Bucket3", "Key8"}, 1}, + {{o, "Bucket3", "Key9"}, 1}, + {{o, "Bucket4", "Key1"}, 1}]), ok = file:close(UpdHandle), ok = file:delete(Filename). @@ -1399,6 +1628,11 @@ big_iterator_test() -> 32), NumFoundKeys2 = length(Result2), ?assertMatch(NumFoundKeys2, 32 * 128), + {partial, Result3, _} = fetch_range_keysonly(Handle, FileMD, {o, "Bucket0000", "Key0000"}, + {o, "Bucket9999", "Key9999"}, + 4), + NumFoundKeys3 = length(Result3), + ?assertMatch(NumFoundKeys3, 4 * 128), ok = file:close(Handle), ok = file:delete(Filename). \ No newline at end of file diff --git a/src/leveled_worker.erl b/src/leveled_worker.erl new file mode 100644 index 0000000..cdeed7b --- /dev/null +++ b/src/leveled_worker.erl @@ -0,0 +1,161 @@ +%% Controlling asynchronour work in leveleddb to manage compaction within a +%% level and cleaning out of old files across a level + + +-module(leveled_worker). + +-export([merge_file/3, perform_merge/3]). + +-include_lib("eunit/include/eunit.hrl"). + + +merge_file(_FileToMerge, _ManifestMgr, _Level) -> + %% CandidateList = leveled_manifest:get_manifest_atlevel(ManifestMgr, Level), + %% [Adds, Removes] = perform_merge(FileToMerge, CandidateList, Level), + %%leveled_manifest:update_manifest_atlevel(ManifestMgr, Level, Adds, Removes), + ok. + + +%% Assumption is that there is a single SFT from a higher level that needs +%% to be merged into multiple SFTs at a lower level. This should create an +%% entirely new set of SFTs, and the calling process can then update the +%% manifest. +%% +%% Once the FileToMerge has been emptied, the remainder of the candidate list +%% needs to be placed in a remainder SFT that may be of a sub-optimal (small) +%% size. This stops the need to perpetually roll over the whole level if the +%% level consists of already full files. Some smartness may be required when +%% selecting the candidate list so that small files just outside the candidate +%% list be included to avoid a proliferation of small files. +%% +%% FileToMerge should be a tuple of {FileName, Pid} where the Pid is the Pid of +%% the gen_server leveled_sft process representing the file. +%% +%% CandidateList should be a list of {StartKey, EndKey, Pid} tuples +%% representing different gen_server leveled_sft processes, sorted by StartKey. +%% +%% The level is the level which the new files should be created at. + +perform_merge(FileToMerge, CandidateList, Level) -> + {Filename, UpperSFTPid} = FileToMerge, + MergeID = generate_merge_id(Filename, Level), + io:format("Merge to be commenced for FileToMerge=~s with MergeID=~s~n", + [Filename, MergeID]), + PointerList = lists:map(fun(P) -> {next, P, all} end, CandidateList), + do_merge([{next, UpperSFTPid, all}], + PointerList, Level, MergeID, 0, []). + +do_merge([], [], Level, MergeID, FileCounter, OutList) -> + io:format("Merge completed with MergeID=~s Level=~w and FileCounter=~w~n", + [MergeID, Level, FileCounter]), + OutList; +do_merge(KL1, KL2, Level, MergeID, FileCounter, OutList) -> + FileName = lists:flatten(io_lib:format("../test/~s_~w.sft", [MergeID, FileCounter])), + io:format("File to be created as part of MergeID=~s Filename=~s~n", [MergeID, FileName]), + case leveled_sft:sft_new(FileName, KL1, KL2, Level) of + {ok, _Pid, {error, Reason}} -> + io:format("Exiting due to error~w~n", [Reason]); + {ok, Pid, Reply} -> + {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, + do_merge(KL1Rem, KL2Rem, Level, MergeID, FileCounter + 1, + lists:append(OutList, [{SmallestKey, HighestKey, Pid}])) + end. + + +generate_merge_id(Filename, Level) -> + <> = crypto:rand_bytes(14), + FileID = erlang:phash2(Filename, 256), + B = FileID * 256 + Level, + Str = io_lib:format("~8.16.0b-~4.16.0b-4~3.16.0b-~4.16.0b-~12.16.0b", + [A, B, C band 16#0fff, D band 16#3fff bor 16#8000, E]), + list_to_binary(Str). + + + + +%%%============================================================================ +%%% Test +%%%============================================================================ + + +generate_randomkeys(Count, BucketRangeLow, BucketRangeHigh) -> + generate_randomkeys(Count, [], BucketRangeLow, BucketRangeHigh). + +generate_randomkeys(0, Acc, _BucketLow, _BucketHigh) -> + Acc; +generate_randomkeys(Count, Acc, BucketLow, BRange) -> + BNumber = string:right(integer_to_list(BucketLow + random:uniform(BRange)), + 4, $0), + KNumber = string:right(integer_to_list(random:uniform(1000)), 4, $0), + RandKey = {{o, + "Bucket" ++ BNumber, + "Key" ++ KNumber}, + Count + 1, + {active, infinity}, null}, + generate_randomkeys(Count - 1, [RandKey|Acc], BucketLow, BRange). + +choose_pid_toquery([{StartKey, EndKey, Pid}|_T], Key) when Key >= StartKey, + EndKey >= Key -> + Pid; +choose_pid_toquery([_H|T], Key) -> + choose_pid_toquery(T, Key). + + +find_randomkeys(_FList, 0, _Source) -> + ok; +find_randomkeys(FList, Count, Source) -> + K1 = leveled_sft:strip_to_keyonly(lists:nth(random:uniform(length(Source)), + Source)), + P1 = choose_pid_toquery(FList, K1), + FoundKV = leveled_sft:sft_get(P1, K1), + case FoundKV of + not_present -> + io:format("Failed to find ~w in ~w~n", [K1, P1]), + ?assertMatch(true, false); + _ -> + Found = leveled_sft:strip_to_keyonly(FoundKV), + io:format("success finding ~w in ~w~n", [K1, P1]), + ?assertMatch(K1, Found) + end, + find_randomkeys(FList, Count - 1, Source). + + +merge_file_test() -> + KL1_L1 = lists:sort(generate_randomkeys(16000, 0, 1000)), + {ok, PidL1_1, _} = leveled_sft:sft_new("../test/KL1_L1.sft", + KL1_L1, [], 1), + KL1_L2 = lists:sort(generate_randomkeys(16000, 0, 250)), + {ok, PidL2_1, _} = leveled_sft:sft_new("../test/KL1_L2.sft", + KL1_L2, [], 2), + KL2_L2 = lists:sort(generate_randomkeys(16000, 250, 250)), + {ok, PidL2_2, _} = leveled_sft:sft_new("../test/KL2_L2.sft", + KL2_L2, [], 2), + KL3_L2 = lists:sort(generate_randomkeys(16000, 500, 250)), + {ok, PidL2_3, _} = leveled_sft:sft_new("../test/KL3_L2.sft", + KL3_L2, [], 2), + KL4_L2 = lists:sort(generate_randomkeys(16000, 750, 250)), + {ok, PidL2_4, _} = leveled_sft:sft_new("../test/KL4_L2.sft", + KL4_L2, [], 2), + Result = perform_merge({"../test/KL1_L1.sft", PidL1_1}, + [PidL2_1, PidL2_2, PidL2_3, PidL2_4], + 2), + lists:foreach(fun({{o, B1, K1}, {o, B2, K2}, R}) -> + io:format("Result of ~s ~s and ~s ~s with Pid ~w~n", + [B1, K1, B2, K2, R]) end, + Result), + io:format("Finding keys in KL1_L1~n"), + ok = find_randomkeys(Result, 50, KL1_L1), + io:format("Finding keys in KL1_L2~n"), + ok = find_randomkeys(Result, 50, KL1_L2), + io:format("Finding keys in KL2_L2~n"), + ok = find_randomkeys(Result, 50, KL2_L2), + io:format("Finding keys in KL3_L2~n"), + ok = find_randomkeys(Result, 50, KL3_L2), + io:format("Finding keys in KL4_L2~n"), + ok = find_randomkeys(Result, 50, KL4_L2), + leveled_sft:sft_clear(PidL1_1), + leveled_sft:sft_clear(PidL2_1), + leveled_sft:sft_clear(PidL2_2), + leveled_sft:sft_clear(PidL2_3), + leveled_sft:sft_clear(PidL2_4), + lists:foreach(fun({_StK, _EndK, Pid}) -> leveled_sft:sft_clear(Pid) end, Result). \ No newline at end of file From a2d873a06d5ce8c14e0b3f177e5864fc7f818106 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 27 Jul 2016 18:03:44 +0100 Subject: [PATCH 018/167] Add first draft of manager Start to build up functions for the keymanager --- src/leveled_keymanager.erl | 220 +++++++++++++++++++++++++++++++++++++ src/leveled_worker.erl | 2 +- 2 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 src/leveled_keymanager.erl diff --git a/src/leveled_keymanager.erl b/src/leveled_keymanager.erl new file mode 100644 index 0000000..b08b3ae --- /dev/null +++ b/src/leveled_keymanager.erl @@ -0,0 +1,220 @@ +%% The manager is responsible for controlling access to the store and +%% maintaining both an in-memory view and a persisted state of all the sft +%% files in use across the store. +%% +%% The store is divided into many levels +%% L0: May contain one, and only one sft file PID which is the most recent file +%% added to the top of the store. Access to the store will be stalled when a +%% second file is added whilst one still remains at this level. The target +%% size of L0 is therefore 0. +%% L1 - Ln: May contain multiple non-overlapping PIDs managing sft files. +%% Compaction work should be sheduled if the number of files exceeds the target +%% size of the level, where the target size is 8 ^ n. +%% +%% The most recent revision of a Key can be found by checking each level until +%% the key is found. To check a level the write file must be sought from the +%% manifest for that level, and then a call is made to that level. +%% +%% If a compaction change takes the size of a level beyond the target size, +%% then compaction work for that level + 1 should be added to the compaction +%% work queue. +%% Compaction work is fetched from the compaction worker because: +%% - it has timed out due to a period of inactivity +%% - it has been triggered by the a cast to indicate the arrival of high +%% priority compaction work +%% The compaction worker will always call the level manager to find out the +%% highest priority work currently in the queue before proceeding. +%% +%% When the compaction worker picks work off the queue it will take the current +%% manifest for the level and level - 1. The compaction worker will choose +%% which file to compact from level - 1, and once the compaction is complete +%% will call to the manager with the new version of the manifest to be written. +%% Once the new version of the manifest had been persisted, the state of any +%% deleted files will be changed to pending deletion. In pending deletion they +%% will call the manifets manager on a timeout to confirm that they are no +%% longer in use (by any iterators). +%% +%% If there is an iterator request, the manager will simply handoff a copy of +%% the manifest, and register the interest of the iterator at the manifest +%% sequence number at the time of the request. Iterators should de-register +%% themselves from the manager on completion. Iterators should be +%% automatically release after a timeout period. A file can be deleted if +%% there are no registered iterators from before the point the file was +%% removed from the manifest. + +-module(leveled_keymanager). + +%% -behaviour(gen_server). + +-export([return_work/2, commit_manifest_change/7]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(LEVEL_SCALEFACTOR, [0, 8, 64, 512, + 4096, 32768, 262144, infinity]). +-define(MAX_LEVELS, 8). +-define(MAX_WORK_WAIT, 300). + +-record(state, {level_fileref :: list(), + ongoing_work :: list(), + manifest_sqn :: integer(), + registered_iterators :: list(), + unreferenced_files :: list()}). + + +%% Work out what the current work queue should be +%% +%% The work queue should have a lower level work at the front, and no work +%% should be added to the queue if a compaction worker has already been asked +%% to look at work at that level + +return_work(State, From) -> + OngoingWork = State#state.ongoing_work, + WorkQueue = assess_workqueue([], + 0, + State#state.level_fileref, + OngoingWork), + case length(WorkQueue) of + L when L > 0 -> + [{SrcLevel, SrcManifest, SnkManifest}|OtherWork] = WorkQueue, + UpdatedWork = lists:append(OngoingWork, + [{SrcLevel, From, os:timestamp()}, + {SrcLevel + 1, From, os:timestamp()}]), + io:format("Work at Level ~w to be scheduled for ~w with ~w queue + items outstanding", [SrcLevel, From, length(OtherWork)]), + {State#state{ongoing_work=UpdatedWork}, + {SrcLevel, SrcManifest, SnkManifest}}; + _ -> + {State, none} + end. + + +assess_workqueue(WorkQ, ?MAX_LEVELS - 1, _LevelFileRef, _OngoingWork) -> + WorkQ; +assess_workqueue(WorkQ, LevelToAssess, LevelFileRef, OngoingWork)-> + MaxFiles = get_item(LevelToAssess + 1, ?LEVEL_SCALEFACTOR, 0), + FileCount = length(get_item(LevelToAssess + 1, LevelFileRef, [])), + NewWQ = maybe_append_work(WorkQ, LevelToAssess, LevelFileRef, MaxFiles, + FileCount, OngoingWork), + assess_workqueue(NewWQ, LevelToAssess + 1, LevelFileRef, OngoingWork). + + +maybe_append_work(WorkQ, Level, LevelFileRef, + MaxFiles, FileCount, OngoingWork) + when FileCount > MaxFiles -> + io:format("Outstanding compaction work items of ~w at level ~w~n", + [FileCount - MaxFiles, Level]), + case lists:keyfind(Level, 1, OngoingWork) of + {Level, Pid, TS} -> + io:format("Work will not be added to queue due to + outstanding work with ~w assigned at ~w~n", [Pid, TS]), + WorkQ; + false -> + lists:append(WorkQ, [{Level, + get_item(Level + 1, LevelFileRef, []), + get_item(Level + 2, LevelFileRef, [])}]) + end; +maybe_append_work(WorkQ, Level, _LevelFileRef, + _MaxFiles, FileCount, _OngoingWork) -> + io:format("No compaction work due to file count ~w at level ~w~n", + [FileCount, Level]), + WorkQ. + + +get_item(Index, List, Default) when Index > length(List) -> + Default; +get_item(Index, List, _Default) -> + lists:nth(Index, List). + + +%% Request a manifest change +%% Should be passed the +%% - {SrcLevel, NewSrcManifest, NewSnkManifest, ClearedFiles, MergeID, From, +%% State} +%% To complete a manifest change need to: +%% - Update the Manifest Sequence Number (msn) +%% - Confirm this Pid has a current element of manifest work outstanding at +%% that level +%% - Rename the manifest file created under the MergeID at the sink Level +%% to be the current manifest file (current..sink) +%% (Note than on startup if the highest msn in all the current. files for that +%% level is a sink file, it must be confirmed that th elevel above is at the +%% same or higher msn. If not the next lowest current..sink must be +%% chosen. This avoids inconsistency on crash between these steps - although +%% the inconsistency would have been survivable) +%% - Rename the manifest file created under the MergeID at the source levl +%% to the current manifest file (current..src) +%% - Update the state of the LevelFileRef lists +%% - Add the ClearedFiles to the list of files to be cleared (as a tuple with +%% the new msn) + + +commit_manifest_change(SrcLevel, NewSrcMan, NewSnkMan, ClearedFiles, + MergeID, From, State) -> + NewMSN = State#state.manifest_sqn + 1, + OngoingWork = State#state.ongoing_work, + SnkLevel = SrcLevel + 1, + case {lists:keyfind(SrcLevel, 1, OngoingWork), + lists:keyfind(SrcLevel + 1, 1, OngoingWork)} of + {{SrcLevel, From, TS}, {SnkLevel, From, TS}} -> + io:format("Merge ~s was a success in ~w microseconds", + [MergeID, timer:diff_now(os:timestamp(), TS)]), + _OutstandingWork = lists:keydelete(SnkLevel, 1, + lists:keydelete(SrcLevel, 1, OngoingWork)), + rename_manifest_file(MergeID, sink, NewMSN, SnkLevel), + rename_manifest_file(MergeID, src, NewMSN, SrcLevel), + _NewLFR = update_levelfileref(NewSrcMan, + NewSnkMan, + SrcLevel, + State#state.level_fileref), + _UnreferencedFiles = update_deletions(ClearedFiles, + NewMSN, + State#state.unreferenced_files), + ok; + _ -> + error + end. + + + +rename_manifest_file(_MergeID, _SrcOrSink, _NewMSN, _Level) -> + ok. + +update_levelfileref(_NewSrcMan, _NewSinkMan, _SrcLevel, CurrLFR) -> + CurrLFR. + +update_deletions(_ClearedFiles, _NewMSN, UnreferencedFiles) -> + UnreferencedFiles. + +%%%============================================================================ +%%% Test +%%%============================================================================ + + +compaction_work_assessment_test() -> + L0 = [{{o, "B1", "K1"}, {o, "B3", "K3"}, dummy_pid}], + L1 = [{{o, "B1", "K1"}, {o, "B2", "K2"}, dummy_pid}, + {{o, "B2", "K3"}, {o, "B4", "K4"}, dummy_pid}], + LevelFileRef = [L0, L1], + OngoingWork1 = [], + WorkQ1 = assess_workqueue([], 0, LevelFileRef, OngoingWork1), + ?assertMatch(WorkQ1, [{0, L0, L1}]), + OngoingWork2 = [{0, dummy_pid, os:timestamp()}], + WorkQ2 = assess_workqueue([], 0, LevelFileRef, OngoingWork2), + ?assertMatch(WorkQ2, []), + L1Alt = lists:append(L1, + [{{o, "B5", "K0001"}, {o, "B5", "K9999"}, dummy_pid}, + {{o, "B6", "K0001"}, {o, "B6", "K9999"}, dummy_pid}, + {{o, "B7", "K0001"}, {o, "B7", "K9999"}, dummy_pid}, + {{o, "B8", "K0001"}, {o, "B8", "K9999"}, dummy_pid}, + {{o, "B9", "K0001"}, {o, "B9", "K9999"}, dummy_pid}, + {{o, "BA", "K0001"}, {o, "BA", "K9999"}, dummy_pid}, + {{o, "BB", "K0001"}, {o, "BB", "K9999"}, dummy_pid}]), + WorkQ3 = assess_workqueue([], 0, [[], L1Alt], OngoingWork1), + ?assertMatch(WorkQ3, [{1, L1Alt, []}]), + WorkQ4 = assess_workqueue([], 0, [[], L1Alt], OngoingWork2), + ?assertMatch(WorkQ4, [{1, L1Alt, []}]), + OngoingWork3 = lists:append(OngoingWork2, [{1, dummy_pid, os:timestamp()}]), + WorkQ5 = assess_workqueue([], 0, [[], L1Alt], OngoingWork3), + ?assertMatch(WorkQ5, []). + diff --git a/src/leveled_worker.erl b/src/leveled_worker.erl index cdeed7b..50b3faf 100644 --- a/src/leveled_worker.erl +++ b/src/leveled_worker.erl @@ -158,4 +158,4 @@ merge_file_test() -> leveled_sft:sft_clear(PidL2_2), leveled_sft:sft_clear(PidL2_3), leveled_sft:sft_clear(PidL2_4), - lists:foreach(fun({_StK, _EndK, Pid}) -> leveled_sft:sft_clear(Pid) end, Result). \ No newline at end of file + lists:foreach(fun({_StK, _EndK, Pid}) -> leveled_sft:sft_clear(Pid) end, Result). From c1f6a042d91c9315eda605e5bd3571c8a4aa8b35 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 28 Jul 2016 17:22:50 +0100 Subject: [PATCH 019/167] Renaming Delete some old working files and adopt a new naming convention. The keymanager actor has now been replaced by a concierge, to reflect that this management role is performed at the front of house --- src/leveled_bst.erl | 350 ------------------ ...d_keymanager.erl => leveled_concierge.erl} | 121 +++--- ...ed_worker.erl => leveled_housekeeping.erl} | 2 +- 3 files changed, 80 insertions(+), 393 deletions(-) delete mode 100644 src/leveled_bst.erl rename src/{leveled_keymanager.erl => leveled_concierge.erl} (65%) rename src/{leveled_worker.erl => leveled_housekeeping.erl} (99%) diff --git a/src/leveled_bst.erl b/src/leveled_bst.erl deleted file mode 100644 index 9253667..0000000 --- a/src/leveled_bst.erl +++ /dev/null @@ -1,350 +0,0 @@ -%% -%% This module provides functions for managing bst files - a modified version -%% of sst files, to be used in leveleddb. -%% bst files are broken into the following sections: -%% - Header (fixed width 32 bytes - containing pointers and metadata) -%% - Summaries (variable length) -%% - Blocks (variable length) -%% - Slots (variable length) -%% - Footer (variable length - contains slot index and helper metadata) -%% -%% The 64-byte header is made up of -%% - 1 byte version (major 5 bits, minor 3 bits) - default 0.1 -%% - 1 byte state bits (1 bit to indicate mutability, 1 for use of compression) -%% - 4 bytes summary length -%% - 4 bytes blocks length -%% - 4 bytes slots length -%% - 4 bytes footer position -%% - 4 bytes slot list length -%% - 4 bytes helper length -%% - 34 bytes spare for future options -%% - 4 bytes CRC (header) -%% -%% A key in the file is a tuple of {Key, Value/Metadata, Sequence #, State} -%% - Keys are themselves tuples, and all entries must be added to the bst -%% in key order -%% - Metadata can be null or some fast-access information that may be required -%% in preference to the full value (e.g. vector clocks, hashes). This could -%% be a value instead of Metadata should the file be used in an alternate -%% - Sequence numbers is the integer representing the order which the item -%% was added to the overall database -%% - State can be tomb (for tombstone), active or {timestamp, TS} -%% -%% The Blocks is a series of blocks of: -%% - 4 byte block length -%% - variable-length compressed list of 32 keys & values -%% - 4 byte CRC for block -%% There will be up to 4000 blocks in a single bst file -%% -%% The slots is a series of references -%% - 4 byte bloom-filter length -%% - 4 byte key-helper length -%% - a variable-length compressed bloom filter for all keys in slot (approx 3KB) -%% - 64 ordered variable-length key helpers pointing to first key in each -%% block (in slot) of the form Key Length, Key, Block Position -%% - 4 byte CRC for the slot -%% - ulitmately a slot covers 64 x 32 = 2048 keys -%% -%% The slot index in the footer is made up of up to 64 ordered keys and -%% pointers, with the key being a key at the start of each slot -%% - 1 byte value showing number of keys in slot index -%% - 64 x Key Length (4 byte), Key, Position (4 byte) indexes -%% - 4 bytes CRC for the index -%% -%% The format of the file is intended to support quick lookups, whilst -%% allowing for a new file to be written incrementally (so that all keys and -%% values need not be retained in memory) - perhaps n blocks at a time - - --module(leveled_bst). - --export([start_file/1, convert_header/1, append_slot/4]). - --include_lib("eunit/include/eunit.hrl"). - --define(WORD_SIZE, 4). --define(DWORD_SIZE, 8). --define(CURRENT_VERSION, {0,1}). --define(SLOT_COUNT, 64). --define(BLOCK_SIZE, 32). --define(SLOT_SIZE, 64). --define(FOOTERPOS_HEADERPOS, 2). - - --record(metadata, {version = ?CURRENT_VERSION :: tuple(), - mutable = false :: true | false, - compressed = true :: true | false, - slot_index, - open_slot :: integer(), - cache :: tuple(), - smallest_key :: tuple(), - largest_key :: tuple(), - smallest_sqn :: integer(), - largest_sqn :: integer() - }). - --record(object, {key :: tuple(), - value, - sequence_numb :: integer(), - state}). - -%% Start a bare file with an initial header and no further details -%% Return the {Handle, metadata record} -start_file(FileName) when is_list(FileName) -> - {ok, Handle} = file:open(FileName, [binary, raw, read, write]), - start_file(Handle); -start_file(Handle) -> - Header = create_header(initial), - {ok, _} = file:position(Handle, bof), - ok = file:write(Handle, Header), - {Version, {M, C}, _, _} = convert_header(Header), - FileMD = #metadata{version = Version, mutable = M, compressed = C, - slot_index = array:new(?SLOT_COUNT), open_slot = 0}, - {Handle, FileMD}. - - -create_header(initial) -> - {Major, Minor} = ?CURRENT_VERSION, - Version = <>, - State = <<0:6, 1:1, 1:1>>, % Mutable and compressed - Lengths = <<0:32, 0:32, 0:32>>, - Options = <<0:112>>, - H1 = <>, - CRC32 = erlang:crc32(H1), - <

>. - - -convert_header(Header) -> - <> = Header, - case erlang:crc32(H1) of - CRC32 -> - <> = H1, - case {Major, Minor} of - {0, 1} -> - convert_header_v01(H1); - _ -> - unknown_version - end; - _ -> - crc_mismatch - end. - -convert_header_v01(Header) -> - <<_:8, 0:6, Mutable:1, Comp:1, - FooterP:32/integer, SlotLng:32/integer, HlpLng:32/integer, - _/binary>> = Header, - case Mutable of - 1 -> M = true; - 0 -> M = false - end, - case Comp of - 1 -> C = true; - 0 -> C = false - end, - {{0, 1}, {M, C}, {FooterP, SlotLng, HlpLng}, none}. - - -%% Append a slot of blocks to the end file, and update the slot index in the -%% file metadata - -append_slot(Handle, SortedKVList, SlotCount, FileMD) -> - {ok, SlotPos} = file:position(Handle, eof), - {KeyList, BlockIndexBin, BlockBin} = add_blocks(SortedKVList), - ok = file:write(Handle, BlockBin), - [TopObject|_] = SortedKVList, - BloomBin = leveled_rice:create_bloom(KeyList), - SlotIndex = array:set(SlotCount, - {TopObject#object.key, BloomBin, BlockIndexBin, SlotPos}, - FileMD#metadata.slot_index), - {Handle, FileMD#metadata{slot_index=SlotIndex}}. - -append_slot_index(Handle, _FileMD=#metadata{slot_index=SlotIndex}) -> - {ok, FooterPos} = file:position(Handle, eof), - SlotBin1 = <>, - SlotBin2 = array:foldl(fun slot_folder_write/3, SlotBin1, SlotIndex), - CRC = erlang:crc32(SlotBin2), - SlotBin3 = <>, - ok = file:write(Handle, SlotBin3), - SlotLength = byte_size(SlotBin3), - Header = <>, - ok = file:pwrite(Handle, ?FOOTERPOS_HEADERPOS, Header). - -slot_folder_write(_Index, undefined, Bin) -> - Bin; -slot_folder_write(_Index, {ObjectKey, _, _, SlotPos}, Bin) -> - KeyBin = serialise_key(ObjectKey), - KeyLen = byte_size(KeyBin), - <>. - -slot_folder_read(<<>>, SlotIndex, SlotCount) -> - io:format("Slot index read with count=~w slots~n", [SlotCount]), - SlotIndex; -slot_folder_read(SlotIndexBin, SlotIndex, SlotCount) -> - <> = SlotIndexBin, - <> = Tail1, - slot_folder_read(Tail2, - array:set(SlotCount, {load_key(KeyBin), null, null, SlotPos}, SlotIndex), - SlotCount + 1). - -read_slot_index(SlotIndexBin) -> - <> = SlotIndexBin, - case erlang:crc32(SlotIndexBin2) of - CRC -> - <> = SlotIndexBin2, - CleanSlotIndex = array:new(SlotCount), - SlotIndex = slot_folder_read(SlotIndexBin3, CleanSlotIndex, 0), - {ok, SlotIndex}; - _ -> - {error, crc_wonky} - end. - -find_slot_index(Handle) -> - {ok, SlotIndexPtr} = file:pread(Handle, ?FOOTERPOS_HEADERPOS, ?DWORD_SIZE), - <> = SlotIndexPtr, - {ok, SlotIndexBin} = file:pread(Handle, FooterPos, SlotIndexLength), - SlotIndexBin. - - -read_blockindex(Handle, Position) -> - {ok, _FilePos} = file:position(Handle, Position), - {ok, <>} = file:read(Handle, 4), - io:format("block length is ~w~n", [BlockLength]), - {ok, BlockBin} = file:read(Handle, BlockLength), - CheckLessBlockLength = BlockLength - 4, - <> = BlockBin, - case erlang:crc32(Block) of - CRC -> - <> = Block, - <> = Tail, - {ok, BloomFilter, KeyHelper}; - _ -> - {error, crc_wonky} - end. - - -add_blocks(SortedKVList) -> - add_blocks(SortedKVList, [], [], [], 0). - -add_blocks([], KeyList, BlockIndex, BlockBinList, _) -> - {KeyList, serialise_blockindex(BlockIndex), list_to_binary(BlockBinList)}; -add_blocks(SortedKVList, KeyList, BlockIndex, BlockBinList, Position) -> - case length(SortedKVList) of - KeyCount when KeyCount >= ?BLOCK_SIZE -> - {TopBlock, Rest} = lists:split(?BLOCK_SIZE, SortedKVList); - KeyCount -> - {TopBlock, Rest} = lists:split(KeyCount, SortedKVList) - end, - [TopKey|_] = TopBlock, - TopBin = serialise_block(TopBlock), - add_blocks(Rest, add_to_keylist(KeyList, TopBlock), - [{TopKey, Position}|BlockIndex], - [TopBin|BlockBinList], Position + byte_size(TopBin)). - -add_to_keylist(KeyList, []) -> - KeyList; -add_to_keylist(KeyList, [TopKV|Rest]) -> - add_to_keylist([map_keyforbloom(TopKV)|KeyList], Rest). - -map_keyforbloom(_Object=#object{key=Key}) -> - Key. - - -serialise_blockindex(BlockIndex) -> - serialise_blockindex(BlockIndex, <<>>). - -serialise_blockindex([], BlockBin) -> - BlockBin; -serialise_blockindex([TopIndex|Rest], BlockBin) -> - {Key, BlockPos} = TopIndex, - KeyBin = serialise_key(Key), - KeyLength = byte_size(KeyBin), - serialise_blockindex(Rest, - <>). - -serialise_block(Block) -> - term_to_binary(Block). - -serialise_key(Key) -> - term_to_binary(Key). - -load_key(KeyBin) -> - binary_to_term(KeyBin). - - -%%%%%%%%%%%%%%%% -% T E S T -%%%%%%%%%%%%%%% - -create_sample_kv(Prefix, Counter) -> - Key = {o, "Bucket1", lists:concat([Prefix, Counter])}, - Object = #object{key=Key, value=null, - sequence_numb=random:uniform(1000000), state=active}, - Object. - -create_ordered_kvlist(KeyList, 0) -> - KeyList; -create_ordered_kvlist(KeyList, Length) -> - KV = create_sample_kv("Key", Length), - create_ordered_kvlist([KV|KeyList], Length - 1). - - -empty_header_test() -> - Header = create_header(initial), - ?assertMatch(32, byte_size(Header)), - <> = Header, - ?assertMatch({0, 1}, {Major, Minor}), - {Version, State, Lengths, Options} = convert_header(Header), - ?assertMatch({0, 1}, Version), - ?assertMatch({true, true}, State), - ?assertMatch({0, 0, 0}, Lengths), - ?assertMatch(none, Options). - -bad_header_test() -> - Header = create_header(initial), - <<_:1/binary, Rest/binary >> = Header, - HdrDetails1 = convert_header(<<0:5/integer, 2:3/integer, Rest/binary>>), - ?assertMatch(crc_mismatch, HdrDetails1), - <<_:1/binary, RestToCRC:27/binary, _:32/integer>> = Header, - NewHdr1 = <<0:5/integer, 2:3/integer, RestToCRC/binary>>, - CRC32 = erlang:crc32(NewHdr1), - NewHdr2 = <>, - ?assertMatch(unknown_version, convert_header(NewHdr2)). - -record_onstartfile_test() -> - {_, FileMD} = start_file("onstartfile.bst"), - ?assertMatch({0, 1}, FileMD#metadata.version), - ok = file:delete("onstartfile.bst"). - -append_initialblock_test() -> - {Handle, FileMD} = start_file("onstartfile.bst"), - KVList = create_ordered_kvlist([], 2048), - Key1 = {o, "Bucket1", "Key1"}, - [TopObj|_] = KVList, - ?assertMatch(Key1, TopObj#object.key), - {_, UpdFileMD} = append_slot(Handle, KVList, 0, FileMD), - {TopKey1, BloomBin, _, _} = array:get(0, UpdFileMD#metadata.slot_index), - io:format("top key of ~w~n", [TopKey1]), - ?assertMatch(Key1, TopKey1), - ?assertMatch(true, leveled_rice:check_key(Key1, BloomBin)), - ?assertMatch(false, leveled_rice:check_key("OtherKey", BloomBin)), - ok = file:delete("onstartfile.bst"). - -append_initialslotindex_test() -> - {Handle, FileMD} = start_file("onstartfile.bst"), - KVList = create_ordered_kvlist([], 2048), - {_, UpdFileMD} = append_slot(Handle, KVList, 0, FileMD), - append_slot_index(Handle, UpdFileMD), - SlotIndexBin = find_slot_index(Handle), - {ok, SlotIndex} = read_slot_index(SlotIndexBin), - io:format("slot index is ~w ~n", [SlotIndex]), - TopItem = array:get(0, SlotIndex), - io:format("top item in slot index is ~w~n", [TopItem]), - {ok, BloomFilter, KeyHelper} = read_blockindex(Handle, 32), - ?assertMatch(true, false), - ok = file:delete("onstartfile.bst"). - - - diff --git a/src/leveled_keymanager.erl b/src/leveled_concierge.erl similarity index 65% rename from src/leveled_keymanager.erl rename to src/leveled_concierge.erl index b08b3ae..4147a3a 100644 --- a/src/leveled_keymanager.erl +++ b/src/leveled_concierge.erl @@ -1,4 +1,4 @@ -%% The manager is responsible for controlling access to the store and +%% The concierge is responsible for controlling access to the store and %% maintaining both an in-memory view and a persisted state of all the sft %% files in use across the store. %% @@ -34,15 +34,18 @@ %% will call the manifets manager on a timeout to confirm that they are no %% longer in use (by any iterators). %% -%% If there is an iterator request, the manager will simply handoff a copy of -%% the manifest, and register the interest of the iterator at the manifest -%% sequence number at the time of the request. Iterators should de-register -%% themselves from the manager on completion. Iterators should be +%% If there is a iterator/snapshot request, the concierge will simply handoff a +%% copy of the manifest, and register the interest of the iterator at the +%% manifest sequence number at the time of the request. Iterators should +%% de-register themselves from the manager on completion. Iterators should be %% automatically release after a timeout period. A file can be deleted if %% there are no registered iterators from before the point the file was %% removed from the manifest. +%% + + --module(leveled_keymanager). +-module(leveled_concierge). %% -behaviour(gen_server). @@ -50,16 +53,19 @@ -include_lib("eunit/include/eunit.hrl"). --define(LEVEL_SCALEFACTOR, [0, 8, 64, 512, - 4096, 32768, 262144, infinity]). +-define(LEVEL_SCALEFACTOR, [{0, 0}, {1, 8}, {2, 64}, {3, 512}, + {4, 4096}, {5, 32768}, {6, 262144}, {7, infinity}]). -define(MAX_LEVELS, 8). -define(MAX_WORK_WAIT, 300). +-define(MANIFEST_FP, "manifest"). +-define(FILES_FP, "files"). -record(state, {level_fileref :: list(), ongoing_work :: list(), manifest_sqn :: integer(), registered_iterators :: list(), - unreferenced_files :: list()}). + unreferenced_files :: list(), + root_path :: string()}). %% Work out what the current work queue should be @@ -92,8 +98,8 @@ return_work(State, From) -> assess_workqueue(WorkQ, ?MAX_LEVELS - 1, _LevelFileRef, _OngoingWork) -> WorkQ; assess_workqueue(WorkQ, LevelToAssess, LevelFileRef, OngoingWork)-> - MaxFiles = get_item(LevelToAssess + 1, ?LEVEL_SCALEFACTOR, 0), - FileCount = length(get_item(LevelToAssess + 1, LevelFileRef, [])), + MaxFiles = get_item(LevelToAssess, ?LEVEL_SCALEFACTOR, 0), + FileCount = length(get_item(LevelToAssess, LevelFileRef, [])), NewWQ = maybe_append_work(WorkQ, LevelToAssess, LevelFileRef, MaxFiles, FileCount, OngoingWork), assess_workqueue(NewWQ, LevelToAssess + 1, LevelFileRef, OngoingWork). @@ -111,8 +117,8 @@ maybe_append_work(WorkQ, Level, LevelFileRef, WorkQ; false -> lists:append(WorkQ, [{Level, - get_item(Level + 1, LevelFileRef, []), - get_item(Level + 2, LevelFileRef, [])}]) + get_item(Level, LevelFileRef, []), + get_item(Level + 1, LevelFileRef, [])}]) end; maybe_append_work(WorkQ, Level, _LevelFileRef, _MaxFiles, FileCount, _OngoingWork) -> @@ -121,10 +127,13 @@ maybe_append_work(WorkQ, Level, _LevelFileRef, WorkQ. -get_item(Index, List, Default) when Index > length(List) -> - Default; -get_item(Index, List, _Default) -> - lists:nth(Index, List). +get_item(Index, List, Default) -> + case lists:keysearch(Index, 1, List) of + {value, {Index, Value}} -> + Value; + false -> + Default + end. %% Request a manifest change @@ -135,15 +144,16 @@ get_item(Index, List, _Default) -> %% - Update the Manifest Sequence Number (msn) %% - Confirm this Pid has a current element of manifest work outstanding at %% that level -%% - Rename the manifest file created under the MergeID at the sink Level -%% to be the current manifest file (current..sink) -%% (Note than on startup if the highest msn in all the current. files for that -%% level is a sink file, it must be confirmed that th elevel above is at the -%% same or higher msn. If not the next lowest current..sink must be -%% chosen. This avoids inconsistency on crash between these steps - although -%% the inconsistency would have been survivable) -%% - Rename the manifest file created under the MergeID at the source levl -%% to the current manifest file (current..src) +%% - Rename the manifest file created under the MergeID (.) +%% at the sink Level to be the current manifest file (current_.) +%% -------- NOTE -------- +%% If there is a crash between these two points, the K/V data that has been +%% merged from the source level will now be in both the source and the sink +%% level. Therefore in store operations this potential duplication must be +%% handled. +%% -------- NOTE -------- +%% - Rename the manifest file created under the MergeID (.) +%% at the source level to the current manifest file (current_.) %% - Update the state of the LevelFileRef lists %% - Add the ClearedFiles to the list of files to be cleared (as a tuple with %% the new msn) @@ -153,38 +163,65 @@ commit_manifest_change(SrcLevel, NewSrcMan, NewSnkMan, ClearedFiles, MergeID, From, State) -> NewMSN = State#state.manifest_sqn + 1, OngoingWork = State#state.ongoing_work, + RootPath = State#state.root_path, SnkLevel = SrcLevel + 1, case {lists:keyfind(SrcLevel, 1, OngoingWork), lists:keyfind(SrcLevel + 1, 1, OngoingWork)} of {{SrcLevel, From, TS}, {SnkLevel, From, TS}} -> io:format("Merge ~s was a success in ~w microseconds", [MergeID, timer:diff_now(os:timestamp(), TS)]), - _OutstandingWork = lists:keydelete(SnkLevel, 1, + OutstandingWork = lists:keydelete(SnkLevel, 1, lists:keydelete(SrcLevel, 1, OngoingWork)), - rename_manifest_file(MergeID, sink, NewMSN, SnkLevel), - rename_manifest_file(MergeID, src, NewMSN, SrcLevel), - _NewLFR = update_levelfileref(NewSrcMan, + ok = rename_manifest_files(RootPath, MergeID, + NewMSN, SrcLevel, SnkLevel), + NewLFR = update_levelfileref(NewSrcMan, NewSnkMan, SrcLevel, State#state.level_fileref), - _UnreferencedFiles = update_deletions(ClearedFiles, + UnreferencedFiles = update_deletions(ClearedFiles, NewMSN, State#state.unreferenced_files), - ok; + io:format("Merge ~s has been commmitted at sequence number ~w~n", + [MergeID, NewMSN]), + {ok, State#state{ongoing_work=OutstandingWork, + manifest_sqn=NewMSN, + level_fileref=NewLFR, + unreferenced_files=UnreferencedFiles}}; _ -> - error + io:format("Merge commit ~s not matched to known work~n", + [MergeID]), + {error, State} end. -rename_manifest_file(_MergeID, _SrcOrSink, _NewMSN, _Level) -> +rename_manifest_files(RootPath, MergeID, NewMSN, SrcLevel, SnkLevel) -> + ManifestFP = RootPath ++ "/" ++ ?MANIFEST_FP ++ "/", + ok = file:rename(ManifestFP ++ MergeID + ++ "." ++ integer_to_list(SnkLevel), + ManifestFP ++ "current_" ++ integer_to_list(SnkLevel) + ++ "." ++ integer_to_list(NewMSN)), + ok = file:rename(ManifestFP ++ MergeID + ++ "." ++ integer_to_list(SrcLevel), + ManifestFP ++ "current_" ++ integer_to_list(SrcLevel) + ++ "." ++ integer_to_list(NewMSN)), ok. -update_levelfileref(_NewSrcMan, _NewSinkMan, _SrcLevel, CurrLFR) -> - CurrLFR. +update_levelfileref(NewSrcMan, NewSinkMan, SrcLevel, CurrLFR) -> + lists:keyreplace(SrcLevel + 1, + 1, + lists:keyreplace(SrcLevel, + 1, + CurrLFR, + {SrcLevel, NewSrcMan}), + {SrcLevel + 1, NewSinkMan}). -update_deletions(_ClearedFiles, _NewMSN, UnreferencedFiles) -> - UnreferencedFiles. +update_deletions([], _NewMSN, UnreferencedFiles) -> + UnreferencedFiles; +update_deletions([ClearedFile|Tail], MSN, UnreferencedFiles) -> + update_deletions(Tail, + MSN, + lists:append(UnreferencedFiles, [{ClearedFile, MSN}])). %%%============================================================================ %%% Test @@ -195,7 +232,7 @@ compaction_work_assessment_test() -> L0 = [{{o, "B1", "K1"}, {o, "B3", "K3"}, dummy_pid}], L1 = [{{o, "B1", "K1"}, {o, "B2", "K2"}, dummy_pid}, {{o, "B2", "K3"}, {o, "B4", "K4"}, dummy_pid}], - LevelFileRef = [L0, L1], + LevelFileRef = [{0, L0}, {1, L1}], OngoingWork1 = [], WorkQ1 = assess_workqueue([], 0, LevelFileRef, OngoingWork1), ?assertMatch(WorkQ1, [{0, L0, L1}]), @@ -210,11 +247,11 @@ compaction_work_assessment_test() -> {{o, "B9", "K0001"}, {o, "B9", "K9999"}, dummy_pid}, {{o, "BA", "K0001"}, {o, "BA", "K9999"}, dummy_pid}, {{o, "BB", "K0001"}, {o, "BB", "K9999"}, dummy_pid}]), - WorkQ3 = assess_workqueue([], 0, [[], L1Alt], OngoingWork1), + WorkQ3 = assess_workqueue([], 0, [{0, []}, {1, L1Alt}], OngoingWork1), ?assertMatch(WorkQ3, [{1, L1Alt, []}]), - WorkQ4 = assess_workqueue([], 0, [[], L1Alt], OngoingWork2), + WorkQ4 = assess_workqueue([], 0, [{0, []}, {1, L1Alt}], OngoingWork2), ?assertMatch(WorkQ4, [{1, L1Alt, []}]), OngoingWork3 = lists:append(OngoingWork2, [{1, dummy_pid, os:timestamp()}]), - WorkQ5 = assess_workqueue([], 0, [[], L1Alt], OngoingWork3), + WorkQ5 = assess_workqueue([], 0, [{0, []}, {1, L1Alt}], OngoingWork3), ?assertMatch(WorkQ5, []). diff --git a/src/leveled_worker.erl b/src/leveled_housekeeping.erl similarity index 99% rename from src/leveled_worker.erl rename to src/leveled_housekeeping.erl index 50b3faf..3d6b8c1 100644 --- a/src/leveled_worker.erl +++ b/src/leveled_housekeeping.erl @@ -2,7 +2,7 @@ %% level and cleaning out of old files across a level --module(leveled_worker). +-module(leveled_housekeeping). -export([merge_file/3, perform_merge/3]). From 28f612426a05a312608a9fdd8941f0c8de21cc0e Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 29 Jul 2016 17:19:30 +0100 Subject: [PATCH 020/167] Reformat of CDB CDB was failing tests (was it always this way?). There has been a little bit of a patch-up of the test, but there are still some potentially outstanding issues with scanning over a file when attempting to read beyond the end of the file. Tabbing reformatting and general tidy. Concierge documentation development ongoing. --- src/leveled_cdb.erl | 1370 ++++++++++++++++++++----------------- src/leveled_concierge.erl | 77 ++- 2 files changed, 814 insertions(+), 633 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index da25d1c..46004e0 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -9,7 +9,7 @@ %% - Support for merging of multiple CDB files with a key-checking function to %% allow for compaction %% - Automatic adding of a helper object that will keep a small proportion of -%% keys to be used when checking to see if the cdb file is a candidate for +%% keys to be used when checking to see if the cdb file is a candidate for %% compaction %% - The ability to scan a database and accumulate all the Key, Values to %% rebuild in-memory tables on startup @@ -46,17 +46,27 @@ -module(leveled_cdb). --export([from_dict/2, - create/2, - dump/1, - get/2, - get_mem/3, - put/4, - open_active_file/1, - get_nextkey/1, - get_nextkey/2, - fold/3, - fold_keys/3]). +-behaviour(gen_server). + +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3, + cdb_open_writer/1, + cdb_open_reader/1, + from_dict/2, + create/2, + dump/1, + get/2, + get_mem/3, + put/4, + open_active_file/1, + get_nextkey/1, + get_nextkey/2, + fold/3, + fold_keys/3]). -include_lib("eunit/include/eunit.hrl"). @@ -66,13 +76,94 @@ -define(MAX_FILE_SIZE, 3221225472). -define(BASE_POSITION, 2048). -%% +-record(state, {hashtree, + last_position :: integer(), + smallest_sqn :: integer(), + highest_sqn :: integer(), + filename :: string(), + handle :: file:fd(), + writer :: boolean}). + + +%%%============================================================================ +%%% API +%%%============================================================================ + +cdb_open_writer(Filename) -> + {ok, Pid} = gen_server:start(?MODULE, [], []), + case gen_server:call(Pid, {cdb_open_writer, Filename}, infinity) of + ok -> + {ok, Pid}; + Error -> + Error + end. + +cdb_open_reader(Filename) -> + {ok, Pid} = gen_server:start(?MODULE, [], []), + case gen_server:call(Pid, {cdb_open_reader, Filename}, infinity) of + ok -> + {ok, Pid}; + Error -> + Error + end. + +%cdb_get(Pid, Key) -> +% gen_server:call(Pid, {cdb_get, Key}, infinity). +% +%cdb_put(Pid, Key, Value) -> +% gen_server:call(Pid, {cdb_put, Key, Value}, infinity). +% +%cdb_close(Pid) -> +% gen_server:call(Pid, cdb_close, infinity). + + +%%%============================================================================ +%%% gen_server callbacks +%%%============================================================================ + +init([]) -> + {ok, #state{}}. + +handle_call({cdb_open_writer, Filename}, _From, State) -> + io:format("Opening file for writing with filename ~s~n", [Filename]), + {LastPosition, HashTree} = open_active_file(Filename), + {ok, Handle} = file:open(Filename, [binary, raw, read, + write, delayed_write]), + {reply, ok, State#state{handle=Handle, + last_position=LastPosition, + filename=Filename, + hashtree=HashTree, + writer=true}}; +handle_call({cdb_open_reader, Filename}, _From, State) -> + io:format("Opening file for reading with filename ~s~n", [Filename]), + {ok, Handle} = file:open(Filename, [binary, raw, read]), + {reply, ok, State#state{handle=Handle, + filename=Filename, + writer=false}}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%============================================================================ +%%% Internal functions +%%%============================================================================ + + %% from_dict(FileName,ListOfKeyValueTuples) %% Given a filename and a dictionary, create a cdb %% using the key value pairs from the dict. from_dict(FileName,Dict) -> - KeyValueList = dict:to_list(Dict), - create(FileName, KeyValueList). + KeyValueList = dict:to_list(Dict), + create(FileName, KeyValueList). %% %% create(FileName,ListOfKeyValueTuples) -> ok @@ -80,10 +171,10 @@ from_dict(FileName,Dict) -> %% this function creates a CDB %% create(FileName,KeyValueList) -> - {ok, Handle} = file:open(FileName, [binary, raw, read, write]), - {ok, _} = file:position(Handle, {bof, ?BASE_POSITION}), - {BasePos, HashTree} = write_key_value_pairs(Handle, KeyValueList), - close_file(Handle, HashTree, BasePos). + {ok, Handle} = file:open(FileName, [binary, raw, read, write]), + {ok, _} = file:position(Handle, {bof, ?BASE_POSITION}), + {BasePos, HashTree} = write_key_value_pairs(Handle, KeyValueList), + close_file(Handle, HashTree, BasePos). %% %% dump(FileName) -> List @@ -91,38 +182,38 @@ create(FileName,KeyValueList) -> %% of {key,value} tuples from the CDB. %% dump(FileName) -> - dump(FileName, ?CRC_CHECK). + dump(FileName, ?CRC_CHECK). dump(FileName, CRCCheck) -> - {ok, Handle} = file:open(FileName, [binary, raw, read]), - Fn = fun(Index, Acc) -> - {ok, _} = file:position(Handle, ?DWORD_SIZE * Index), - {_, Count} = read_next_2_integers(Handle), - Acc + Count - end, - NumberOfPairs = lists:foldl(Fn, 0, lists:seq(0,255)) bsr 1, - io:format("Count of keys in db is ~w~n", [NumberOfPairs]), - - {ok, _} = file:position(Handle, {bof, 2048}), - Fn1 = fun(_I,Acc) -> - {KL,VL} = read_next_2_integers(Handle), - Key = read_next_term(Handle, KL), - io:format("Key read of ~w~n", [Key]), - case read_next_term(Handle, VL, crc, CRCCheck) of - {false, _} -> - {ok, CurrLoc} = file:position(Handle, cur), - Return = {crc_wonky, get(Handle, Key)}; - {_, Value} -> - {ok, CurrLoc} = file:position(Handle, cur), - Return = case get(Handle, Key) of - {Key,Value} -> {Key ,Value}; - X -> {wonky, X} - end + {ok, Handle} = file:open(FileName, [binary, raw, read]), + Fn = fun(Index, Acc) -> + {ok, _} = file:position(Handle, ?DWORD_SIZE * Index), + {_, Count} = read_next_2_integers(Handle), + Acc + Count end, - {ok, _} = file:position(Handle, CurrLoc), - [Return | Acc] - end, - lists:foldr(Fn1,[],lists:seq(0,NumberOfPairs-1)). + NumberOfPairs = lists:foldl(Fn, 0, lists:seq(0,255)) bsr 1, + io:format("Count of keys in db is ~w~n", [NumberOfPairs]), + {ok, _} = file:position(Handle, {bof, 2048}), + Fn1 = fun(_I,Acc) -> + {KL,VL} = read_next_2_integers(Handle), + Key = read_next_term(Handle, KL), + io:format("Key read of ~w~n", [Key]), + case read_next_term(Handle, VL, crc, CRCCheck) of + {false, _} -> + {ok, CurrLoc} = file:position(Handle, cur), + Return = {crc_wonky, get(Handle, Key)}; + {_, Value} -> + {ok, CurrLoc} = file:position(Handle, cur), + Return = + case get(Handle, Key) of + {Key,Value} -> {Key ,Value}; + X -> {wonky, X} + end + end, + {ok, _} = file:position(Handle, CurrLoc), + [Return | Acc] + end, + lists:foldr(Fn1,[],lists:seq(0,NumberOfPairs-1)). %% Open an active file - one for which it is assumed the hash tables have not %% yet been written @@ -134,21 +225,21 @@ dump(FileName, CRCCheck) -> %% tuples as the write_key_value_pairs function, and the current position, and %% the file handle open_active_file(FileName) when is_list(FileName) -> - {ok, Handle} = file:open(FileName, [binary, raw, read, write]), - {ok, Position} = file:position(Handle, {bof, 256*?DWORD_SIZE}), - {LastPosition, HashTree} = scan_over_file(Handle, Position), - case file:position(Handle, eof) of - {ok, LastPosition} -> - ok = file:close(Handle); - {ok, _} -> - LogDetails = [LastPosition, file:position(Handle, eof)], - io:format("File to be truncated at last position of" - "~w with end of file at ~w~n", LogDetails), - {ok, LastPosition} = file:position(Handle, LastPosition), - ok = file:truncate(Handle), - ok = file:close(Handle) - end, - {LastPosition, HashTree}. + {ok, Handle} = file:open(FileName, [binary, raw, read, write]), + {ok, Position} = file:position(Handle, {bof, 256*?DWORD_SIZE}), + {LastPosition, HashTree} = scan_over_file(Handle, Position), + case file:position(Handle, eof) of + {ok, LastPosition} -> + ok = file:close(Handle); + {ok, _} -> + LogDetails = [LastPosition, file:position(Handle, eof)], + io:format("File to be truncated at last position of" + "~w with end of file at ~w~n", LogDetails), + {ok, LastPosition} = file:position(Handle, LastPosition), + ok = file:truncate(Handle), + ok = file:close(Handle) + end, + {LastPosition, HashTree}. %% put(Handle, Key, Value, {LastPosition, HashDict}) -> {NewPosition, KeyDict} %% Append to an active file a new key/value pair returning an updated @@ -175,68 +266,67 @@ put(Handle, Key, Value, {LastPosition, HashTree}) -> %% Given a filename and a key, returns a key and value tuple. %% get(FileNameOrHandle, Key) -> - get(FileNameOrHandle, Key, ?CRC_CHECK). + get(FileNameOrHandle, Key, ?CRC_CHECK). get(FileName, Key, CRCCheck) when is_list(FileName), is_list(Key) -> - {ok,Handle} = file:open(FileName,[binary, raw, read]), - get(Handle,Key, CRCCheck); - + {ok,Handle} = file:open(FileName,[binary, raw, read]), + get(Handle,Key, CRCCheck); get(Handle, Key, CRCCheck) when is_tuple(Handle), is_list(Key) -> - Hash = hash(Key), - Index = hash_to_index(Hash), - {ok,_} = file:position(Handle, {bof, ?DWORD_SIZE * Index}), - % Get location of hashtable and number of entries in the hash - {HashTable, Count} = read_next_2_integers(Handle), - % If the count is 0 for that index - key must be missing - case Count of - 0 -> - missing; - _ -> - % Get starting slot in hashtable - {ok, FirstHashPosition} = file:position(Handle, {bof, HashTable}), - Slot = hash_to_slot(Hash, Count), - {ok, _} = file:position(Handle, {cur, Slot * ?DWORD_SIZE}), - LastHashPosition = HashTable + ((Count-1) * ?DWORD_SIZE), - LocList = lists:seq(FirstHashPosition, LastHashPosition, ?DWORD_SIZE), - % Split list around starting slot. - {L1, L2} = lists:split(Slot, LocList), - search_hash_table(Handle, lists:append(L2, L1), Hash, Key, CRCCheck) - end. + Hash = hash(Key), + Index = hash_to_index(Hash), + {ok,_} = file:position(Handle, {bof, ?DWORD_SIZE * Index}), + % Get location of hashtable and number of entries in the hash + {HashTable, Count} = read_next_2_integers(Handle), + % If the count is 0 for that index - key must be missing + case Count of + 0 -> + missing; + _ -> + % Get starting slot in hashtable + {ok, FirstHashPosition} = file:position(Handle, {bof, HashTable}), + Slot = hash_to_slot(Hash, Count), + {ok, _} = file:position(Handle, {cur, Slot * ?DWORD_SIZE}), + LastHashPosition = HashTable + ((Count-1) * ?DWORD_SIZE), + LocList = lists:seq(FirstHashPosition, LastHashPosition, ?DWORD_SIZE), + % Split list around starting slot. + {L1, L2} = lists:split(Slot, LocList), + search_hash_table(Handle, lists:append(L2, L1), Hash, Key, CRCCheck) + end. %% Get a Key/Value pair from an active CDB file (with no hash table written) %% This requires a key dictionary to be passed in (mapping keys to positions) %% Will return {Key, Value} or missing get_mem(Key, Filename, HashTree) when is_list(Filename) -> - {ok, Handle} = file:open(Filename, [binary, raw, read]), - get_mem(Key, Handle, HashTree); + {ok, Handle} = file:open(Filename, [binary, raw, read]), + get_mem(Key, Handle, HashTree); get_mem(Key, Handle, HashTree) -> - extract_kvpair(Handle, get_hashtree(Key, HashTree), Key). + extract_kvpair(Handle, get_hashtree(Key, HashTree), Key). %% Get the next key at a position in the file (or the first key if no position %% is passed). Will return both a key and the next position get_nextkey(Filename) when is_list(Filename) -> - {ok, Handle} = file:open(Filename, [binary, raw, read]), - get_nextkey(Handle); + {ok, Handle} = file:open(Filename, [binary, raw, read]), + get_nextkey(Handle); get_nextkey(Handle) -> - {ok, _} = file:position(Handle, bof), - {FirstHashPosition, _} = read_next_2_integers(Handle), - get_nextkey(Handle, {256 * ?DWORD_SIZE, FirstHashPosition}). + {ok, _} = file:position(Handle, bof), + {FirstHashPosition, _} = read_next_2_integers(Handle), + get_nextkey(Handle, {256 * ?DWORD_SIZE, FirstHashPosition}). get_nextkey(Handle, {Position, FirstHashPosition}) -> - {ok, Position} = file:position(Handle, Position), - case read_next_2_integers(Handle) of - {KeyLength, ValueLength} -> - NextKey = read_next_term(Handle, KeyLength), - NextPosition = Position + KeyLength + ValueLength + ?DWORD_SIZE, - case NextPosition of - FirstHashPosition -> - {NextKey, nomorekeys}; - _ -> - {NextKey, Handle, {NextPosition, FirstHashPosition}} - end; - eof -> - nomorekeys - end. + {ok, Position} = file:position(Handle, Position), + case read_next_2_integers(Handle) of + {KeyLength, ValueLength} -> + NextKey = read_next_term(Handle, KeyLength), + NextPosition = Position + KeyLength + ValueLength + ?DWORD_SIZE, + case NextPosition of + FirstHashPosition -> + {NextKey, nomorekeys}; + _ -> + {NextKey, Handle, {NextPosition, FirstHashPosition}} + end; + eof -> + nomorekeys +end. %% Fold over all of the objects in the file, applying FoldFun to each object @@ -244,52 +334,66 @@ get_nextkey(Handle, {Position, FirstHashPosition}) -> %% set to true fold(FileName, FoldFun, Acc0) when is_list(FileName) -> - {ok, Handle} = file:open(FileName, [binary, raw, read]), - fold(Handle, FoldFun, Acc0); + {ok, Handle} = file:open(FileName, [binary, raw, read]), + fold(Handle, FoldFun, Acc0); fold(Handle, FoldFun, Acc0) -> - {ok, _} = file:position(Handle, bof), - {FirstHashPosition, _} = read_next_2_integers(Handle), - fold(Handle, FoldFun, Acc0, {256 * ?DWORD_SIZE, FirstHashPosition}, false). + {ok, _} = file:position(Handle, bof), + {FirstHashPosition, _} = read_next_2_integers(Handle), + fold(Handle, FoldFun, Acc0, {256 * ?DWORD_SIZE, FirstHashPosition}, false). fold(Handle, FoldFun, Acc0, {Position, FirstHashPosition}, KeyOnly) -> - {ok, Position} = file:position(Handle, Position), - case Position of - FirstHashPosition -> - Acc0; - _ -> - case read_next_2_integers(Handle) of - {KeyLength, ValueLength} -> - NextKey = read_next_term(Handle, KeyLength), - NextPosition = Position + KeyLength + ValueLength + ?DWORD_SIZE, - case KeyOnly of - true -> - fold(Handle, FoldFun, FoldFun(NextKey, Acc0), - {NextPosition, FirstHashPosition}, KeyOnly); - false -> - case read_next_term(Handle, ValueLength, crc, ?CRC_CHECK) of - {false, _} -> - io:format("Skipping value for Key ~w as CRC check failed~n", - [NextKey]), - fold(Handle, FoldFun, Acc0, - {NextPosition, FirstHashPosition}, KeyOnly); - {_, Value} -> - fold(Handle, FoldFun, FoldFun(NextKey, Value, Acc0), - {NextPosition, FirstHashPosition}, KeyOnly) - end - end; - eof -> - Acc0 - end - end. + {ok, Position} = file:position(Handle, Position), + case Position of + FirstHashPosition -> + Acc0; + _ -> + case read_next_2_integers(Handle) of + {KeyLength, ValueLength} -> + NextKey = read_next_term(Handle, KeyLength), + NextPosition = Position + + KeyLength + ValueLength + + ?DWORD_SIZE, + case KeyOnly of + true -> + fold(Handle, + FoldFun, + FoldFun(NextKey, Acc0), + {NextPosition, FirstHashPosition}, + KeyOnly); + false -> + case read_next_term(Handle, + ValueLength, + crc, + ?CRC_CHECK) of + {false, _} -> + io:format("Skipping value for Key ~w as CRC + check failed~n", [NextKey]), + fold(Handle, + FoldFun, + Acc0, + {NextPosition, FirstHashPosition}, + KeyOnly); + {_, Value} -> + fold(Handle, + FoldFun, + FoldFun(NextKey, Value, Acc0), + {NextPosition, FirstHashPosition}, + KeyOnly) + end + end; + eof -> + Acc0 + end + end. fold_keys(FileName, FoldFun, Acc0) when is_list(FileName) -> - {ok, Handle} = file:open(FileName, [binary, raw, read]), - fold_keys(Handle, FoldFun, Acc0); + {ok, Handle} = file:open(FileName, [binary, raw, read]), + fold_keys(Handle, FoldFun, Acc0); fold_keys(Handle, FoldFun, Acc0) -> - {ok, _} = file:position(Handle, bof), - {FirstHashPosition, _} = read_next_2_integers(Handle), - fold(Handle, FoldFun, Acc0, {256 * ?DWORD_SIZE, FirstHashPosition}, true). + {ok, _} = file:position(Handle, bof), + {FirstHashPosition, _} = read_next_2_integers(Handle), + fold(Handle, FoldFun, Acc0, {256 * ?DWORD_SIZE, FirstHashPosition}, true). %%%%%%%%%%%%%%%%%%%% @@ -302,24 +406,24 @@ fold_keys(Handle, FoldFun, Acc0) -> %% Base Pos should be at the end of the KV pairs written (the position for) %% the hash tables close_file(Handle, HashTree, BasePos) -> - {ok, BasePos} = file:position(Handle, BasePos), - L2 = write_hash_tables(Handle, HashTree), - write_top_index_table(Handle, BasePos, L2), - file:close(Handle). + {ok, BasePos} = file:position(Handle, BasePos), + L2 = write_hash_tables(Handle, HashTree), + write_top_index_table(Handle, BasePos, L2), + file:close(Handle). %% Fetch a list of positions by passing a key to the HashTree get_hashtree(Key, HashTree) -> - Hash = hash(Key), - Index = hash_to_index(Hash), - Tree = array:get(Index, HashTree), - case gb_trees:lookup(Hash, Tree) of - {value, List} -> - List; - _ -> - [] - end. + Hash = hash(Key), + Index = hash_to_index(Hash), + Tree = array:get(Index, HashTree), + case gb_trees:lookup(Hash, Tree) of + {value, List} -> + List; + _ -> + [] + end. %% Add to hash tree - this is an array of 256 gb_trees that contains the Hash %% and position of objects which have been added to an open CDB file @@ -337,100 +441,117 @@ put_hashtree(Key, Position, HashTree) -> %% Function to extract a Key-Value pair given a file handle and a position %% Will confirm that the key matches and do a CRC check when requested extract_kvpair(Handle, Positions, Key) -> - extract_kvpair(Handle, Positions, Key, ?CRC_CHECK). + extract_kvpair(Handle, Positions, Key, ?CRC_CHECK). extract_kvpair(_, [], _, _) -> - missing; + missing; extract_kvpair(Handle, [Position|Rest], Key, Check) -> - {ok, _} = file:position(Handle, Position), - {KeyLength, ValueLength} = read_next_2_integers(Handle), - case read_next_term(Handle, KeyLength) of - Key -> % If same key as passed in, then found! - case read_next_term(Handle, ValueLength, crc, Check) of - {false, _} -> - crc_wonky; - {_, Value} -> - {Key,Value} - end; - _ -> - extract_kvpair(Handle, Rest, Key, Check) - end. + {ok, _} = file:position(Handle, Position), + {KeyLength, ValueLength} = read_next_2_integers(Handle), + case read_next_term(Handle, KeyLength) of + Key -> % If same key as passed in, then found! + case read_next_term(Handle, ValueLength, crc, Check) of + {false, _} -> + crc_wonky; + {_, Value} -> + {Key,Value} + end; + _ -> + extract_kvpair(Handle, Rest, Key, Check) + end. %% Scan through the file until there is a failure to crc check an input, and %% at that point return the position and the key dictionary scanned so far scan_over_file(Handle, Position) -> - HashTree = array:new(256, {default, gb_trees:empty()}), - scan_over_file(Handle, Position, HashTree). + HashTree = array:new(256, {default, gb_trees:empty()}), + scan_over_file(Handle, Position, HashTree). scan_over_file(Handle, Position, HashTree) -> - case saferead_keyvalue(Handle) of - false -> - {Position, HashTree}; - {Key, ValueAsBin, KeyLength, ValueLength} -> - case crccheck_value(ValueAsBin) of - true -> - NewPosition = Position + KeyLength + ValueLength + ?DWORD_SIZE, - scan_over_file(Handle, NewPosition, - put_hashtree(Key, Position, HashTree)); + case saferead_keyvalue(Handle) of false -> - io:format("CRC check returned false on key of ~w ~n", [Key]), - {Position, HashTree} - end; - eof -> - {Position, HashTree} - end. + {Position, HashTree}; + {Key, ValueAsBin, KeyLength, ValueLength} -> + case crccheck_value(ValueAsBin) of + true -> + NewPosition = Position + KeyLength + ValueLength + + ?DWORD_SIZE, + scan_over_file(Handle, + NewPosition, + put_hashtree(Key, Position, HashTree)); + false -> + io:format("CRC check returned false on key of ~w ~n", + [Key]), + {Position, HashTree} + end; + eof -> + {Position, HashTree} + end. %% Read the Key/Value at this point, returning {ok, Key, Value} %% catch expected exceptiosn associated with file corruption (or end) and %% return eof saferead_keyvalue(Handle) -> - case read_next_2_integers(Handle) of - {error, einval} -> - false; - eof -> - false; - {KeyL, ValueL} -> - case read_next_term(Handle, KeyL) of + case read_next_2_integers(Handle) of {error, einval} -> - false; + false; eof -> - false; - Key -> - case file:read(Handle, ValueL) of - {error, einval} -> - false; - eof -> - false; - {ok, Value} -> - {Key, Value, KeyL, ValueL} - end - end - end. + false; + {KeyL, ValueL} -> + io:format("KeyL ~w ValueL ~w~n", [KeyL, ValueL]), + case safe_read_next_term(Handle, KeyL) of + {error, einval} -> + false; + eof -> + false; + false -> + false; + Key -> + io:format("Found Key of ~s~n", [Key]), + case file:read(Handle, ValueL) of + {error, einval} -> + false; + eof -> + false; + {ok, Value} -> + {Key, Value, KeyL, ValueL} + end + end + end. + + +safe_read_next_term(Handle, Length) -> + try read_next_term(Handle, Length) of + Term -> + Term + catch + error:badarg -> + false + end. %% The first four bytes of the value are the crc check crccheck_value(Value) when byte_size(Value) >4 -> - << Hash:32/integer, Tail/bitstring>> = Value, - case calc_crc(Tail) of - Hash -> - true; - _ -> - io:format("CRC check failed due to mismatch ~n"), - false - end; + << Hash:32/integer, Tail/bitstring>> = Value, + case calc_crc(Tail) of + Hash -> + true; + _ -> + io:format("CRC check failed due to mismatch ~n"), + false + end; crccheck_value(_) -> - io:format("CRC check failed due to size ~n"), - false. + io:format("CRC check failed due to size ~n"), + false. %% Run a crc check filling out any values which don't fit on byte boundary calc_crc(Value) -> - case bit_size(Value) rem 8 of - 0 -> - erlang:crc32(Value); - N -> - M = 8 - N, - erlang:crc32(<>) - end. + case bit_size(Value) rem 8 of + 0 -> + erlang:crc32(Value); + N -> + M = 8 - N, + erlang:crc32(<>) + end. %% %% to_dict(FileName) @@ -443,70 +564,69 @@ calc_crc(Value) -> %% dictionary() = dict() %% to_dict(FileName) -> - KeyValueList = dump(FileName), - dict:from_list(KeyValueList). + KeyValueList = dump(FileName), + dict:from_list(KeyValueList). read_next_term(Handle, Length) -> - case file:read(Handle, Length) of - {ok, Bin} -> - binary_to_term(Bin); - ReadError -> - ReadError - end. + case file:read(Handle, Length) of + {ok, Bin} -> + binary_to_term(Bin); + ReadError -> + ReadError + end. %% Read next string where the string has a CRC prepended - stripping the crc %% and checking if requested read_next_term(Handle, Length, crc, Check) -> - case Check of - true -> - {ok, <>} = file:read(Handle, Length), - case calc_crc(Bin) of - CRC -> - {true, binary_to_term(Bin)}; + case Check of + true -> + {ok, <>} = file:read(Handle, Length), + case calc_crc(Bin) of + CRC -> + {true, binary_to_term(Bin)}; + _ -> + {false, binary_to_term(Bin)} + end; _ -> - {false, binary_to_term(Bin)} - end; - _ -> - {ok, _} = file:position(Handle, {cur, 4}), - {ok, Bin} = file:read(Handle, Length - 4), - {unchecked, binary_to_term(Bin)} - end. + {ok, _} = file:position(Handle, {cur, 4}), + {ok, Bin} = file:read(Handle, Length - 4), + {unchecked, binary_to_term(Bin)} + end. %% Used for reading lengths %% Note that the endian_flip is required to make the file format compatible %% with CDB read_next_2_integers(Handle) -> - case file:read(Handle,?DWORD_SIZE) of - {ok, <>} -> - {endian_flip(Int1), endian_flip(Int2)}; - ReadError - -> - ReadError - end. + case file:read(Handle,?DWORD_SIZE) of + {ok, <>} -> + {endian_flip(Int1), endian_flip(Int2)}; + ReadError -> + ReadError + end. %% Seach the hash table for the matching hash and key. Be prepared for %% multiple keys to have the same hash value. search_hash_table(_Handle, [], _Hash, _Key, _CRCCHeck) -> - missing; + missing; search_hash_table(Handle, [Entry|RestOfEntries], Hash, Key, CRCCheck) -> - {ok, _} = file:position(Handle, Entry), - {StoredHash, DataLoc} = read_next_2_integers(Handle), - case StoredHash of - Hash -> - KV = extract_kvpair(Handle, [DataLoc], Key, CRCCheck), - case KV of - missing -> - search_hash_table(Handle, RestOfEntries, Hash, Key, CRCCheck); + {ok, _} = file:position(Handle, Entry), + {StoredHash, DataLoc} = read_next_2_integers(Handle), + case StoredHash of + Hash -> + KV = extract_kvpair(Handle, [DataLoc], Key, CRCCheck), + case KV of + missing -> + search_hash_table(Handle, RestOfEntries, Hash, Key, CRCCheck); + _ -> + KV + end; + 0 -> + % Hash is 0 so key must be missing as 0 found before Hash matched + missing; _ -> - KV - end; - 0 -> - % Hash is 0 so key must be missing as 0 found before Hash matched - missing; - _ -> - search_hash_table(Handle, RestOfEntries, Hash, Key, CRCCheck) - end. + search_hash_table(Handle, RestOfEntries, Hash, Key, CRCCheck) + end. % Write Key and Value tuples into the CDB. Each tuple consists of a % 4 byte key length, a 4 byte value length, the actual key followed @@ -517,53 +637,53 @@ search_hash_table(Handle, [Entry|RestOfEntries], Hash, Key, CRCCheck) -> % values being a list of the hash and the position of the % key/value binary in the file. write_key_value_pairs(Handle, KeyValueList) -> - {ok, Position} = file:position(Handle, cur), - HashTree = array:new(256, {default, gb_trees:empty()}), - write_key_value_pairs(Handle, KeyValueList, {Position, HashTree}). + {ok, Position} = file:position(Handle, cur), + HashTree = array:new(256, {default, gb_trees:empty()}), + write_key_value_pairs(Handle, KeyValueList, {Position, HashTree}). write_key_value_pairs(_, [], Acc) -> - Acc; + Acc; write_key_value_pairs(Handle, [HeadPair|TailList], Acc) -> - {Key, Value} = HeadPair, - {Handle, NewPosition, HashTree} = put(Handle, Key, Value, Acc), - write_key_value_pairs(Handle, TailList, {NewPosition, HashTree}). + {Key, Value} = HeadPair, + {Handle, NewPosition, HashTree} = put(Handle, Key, Value, Acc), + write_key_value_pairs(Handle, TailList, {NewPosition, HashTree}). %% Write the actual hashtables at the bottom of the file. Each hash table %% entry is a doubleword in length. The first word is the hash value %% corresponding to a key and the second word is a file pointer to the %% corresponding {key,value} tuple. write_hash_tables(Handle, HashTree) -> - Seq = lists:seq(0, 255), - {ok, StartPos} = file:position(Handle, cur), - write_hash_tables(Seq, Handle, HashTree, StartPos, []). + Seq = lists:seq(0, 255), + {ok, StartPos} = file:position(Handle, cur), + write_hash_tables(Seq, Handle, HashTree, StartPos, []). write_hash_tables([], Handle, _, StartPos, IndexList) -> - {ok, EndPos} = file:position(Handle, cur), - ok = file:advise(Handle, StartPos, EndPos - StartPos, will_need), - IndexList; + {ok, EndPos} = file:position(Handle, cur), + ok = file:advise(Handle, StartPos, EndPos - StartPos, will_need), + IndexList; write_hash_tables([Index|Rest], Handle, HashTree, StartPos, IndexList) -> - Tree = array:get(Index, HashTree), - case gb_trees:keys(Tree) of - [] -> - write_hash_tables(Rest, Handle, HashTree, StartPos, IndexList); - _ -> - HashList = gb_trees:to_list(Tree), - BinList = build_binaryhashlist(HashList, []), - IndexLength = length(BinList) * 2, - SlotList = lists:duplicate(IndexLength, <<0:32, 0:32>>), - - Fn = fun({Hash, Binary}, AccSlotList) -> - Slot1 = find_open_slot(AccSlotList, Hash), - {L1, [<<0:32, 0:32>>|L2]} = lists:split(Slot1, AccSlotList), - lists:append(L1, [Binary|L2]) - end, - NewSlotList = lists:foldl(Fn, SlotList, BinList), - - {ok, CurrPos} = file:position(Handle, cur), - file:write(Handle, NewSlotList), - write_hash_tables(Rest, Handle, HashTree, StartPos, - [{Index, CurrPos, IndexLength}|IndexList]) - end. + Tree = array:get(Index, HashTree), + case gb_trees:keys(Tree) of + [] -> + write_hash_tables(Rest, Handle, HashTree, StartPos, IndexList); + _ -> + HashList = gb_trees:to_list(Tree), + BinList = build_binaryhashlist(HashList, []), + IndexLength = length(BinList) * 2, + SlotList = lists:duplicate(IndexLength, <<0:32, 0:32>>), + + Fn = fun({Hash, Binary}, AccSlotList) -> + Slot1 = find_open_slot(AccSlotList, Hash), + {L1, [<<0:32, 0:32>>|L2]} = lists:split(Slot1, AccSlotList), + lists:append(L1, [Binary|L2]) + end, + + NewSlotList = lists:foldl(Fn, SlotList, BinList), + {ok, CurrPos} = file:position(Handle, cur), + file:write(Handle, NewSlotList), + write_hash_tables(Rest, Handle, HashTree, StartPos, + [{Index, CurrPos, IndexLength}|IndexList]) + end. %% The list created from the original HashTree may have duplicate positions %% e.g. {Key, [Value1, Value2]}. Before any writing is done it is necessary @@ -572,31 +692,31 @@ write_hash_tables([Index|Rest], Handle, HashTree, StartPos, IndexList) -> %% This function creates {Hash, Binary} pairs on a list where there is a unique %% entry for eveyr Key/Value build_binaryhashlist([], BinList) -> - BinList; + BinList; build_binaryhashlist([{Hash, [Position|TailP]}|TailKV], BinList) -> - HashLE = endian_flip(Hash), - PosLE = endian_flip(Position), - NewBin = <>, - case TailP of - [] -> - build_binaryhashlist(TailKV, [{Hash, NewBin}|BinList]); - _ -> - build_binaryhashlist([{Hash, TailP}|TailKV], [{Hash, NewBin}|BinList]) - end. + HashLE = endian_flip(Hash), + PosLE = endian_flip(Position), + NewBin = <>, + case TailP of + [] -> + build_binaryhashlist(TailKV, [{Hash, NewBin}|BinList]); + _ -> + build_binaryhashlist([{Hash, TailP}|TailKV], [{Hash, NewBin}|BinList]) + end. %% Slot is zero based because it comes from a REM find_open_slot(List, Hash) -> - Len = length(List), - Slot = hash_to_slot(Hash, Len), - Seq = lists:seq(1, Len), - {CL1, CL2} = lists:split(Slot, Seq), - {L1, L2} = lists:split(Slot, List), - find_open_slot1(lists:append(CL2, CL1), lists:append(L2, L1)). + Len = length(List), + Slot = hash_to_slot(Hash, Len), + Seq = lists:seq(1, Len), + {CL1, CL2} = lists:split(Slot, Seq), + {L1, L2} = lists:split(Slot, List), + find_open_slot1(lists:append(CL2, CL1), lists:append(L2, L1)). find_open_slot1([Slot|_RestOfSlots], [<<0:32,0:32>>|_RestOfEntries]) -> - Slot - 1; + Slot - 1; find_open_slot1([_|RestOfSlots], [_|RestOfEntries]) -> - find_open_slot1(RestOfSlots, RestOfEntries). + find_open_slot1(RestOfSlots, RestOfEntries). %% Write the top most 255 doubleword entries. First word is the @@ -606,71 +726,71 @@ find_open_slot1([_|RestOfSlots], [_|RestOfEntries]) -> write_top_index_table(Handle, BasePos, List) -> % fold function to find any missing index tuples, and add one a replacement % in this case with a count of 0. Also orders the list by index - FnMakeIndex = fun(I, Acc) -> - case lists:keysearch(I, 1, List) of - {value, Tuple} -> - [Tuple|Acc]; - false -> - [{I, BasePos, 0}|Acc] - end - end, - % Fold function to write the index entries - FnWriteIndex = fun({Index, Pos, Count}, CurrPos) -> - {ok, _} = file:position(Handle, ?DWORD_SIZE * Index), - case Count == 0 of - true -> - PosLE = endian_flip(CurrPos), - NextPos = CurrPos; - false -> - PosLE = endian_flip(Pos), - NextPos = Pos + (Count * ?DWORD_SIZE) - end, - CountLE = endian_flip(Count), - Bin = <>, - file:write(Handle, Bin), - NextPos - end, - - Seq = lists:seq(0, 255), - CompleteList = lists:keysort(1, lists:foldl(FnMakeIndex, [], Seq)), - lists:foldl(FnWriteIndex, BasePos, CompleteList), - ok = file:advise(Handle, 0, ?DWORD_SIZE * 256, will_need). + FnMakeIndex = fun(I, Acc) -> + case lists:keysearch(I, 1, List) of + {value, Tuple} -> + [Tuple|Acc]; + false -> + [{I, BasePos, 0}|Acc] + end + end, + % Fold function to write the index entries + FnWriteIndex = fun({Index, Pos, Count}, CurrPos) -> + {ok, _} = file:position(Handle, ?DWORD_SIZE * Index), + case Count == 0 of + true -> + PosLE = endian_flip(CurrPos), + NextPos = CurrPos; + false -> + PosLE = endian_flip(Pos), + NextPos = Pos + (Count * ?DWORD_SIZE) + end, + CountLE = endian_flip(Count), + Bin = <>, + file:write(Handle, Bin), + NextPos + end, + + Seq = lists:seq(0, 255), + CompleteList = lists:keysort(1, lists:foldl(FnMakeIndex, [], Seq)), + lists:foldl(FnWriteIndex, BasePos, CompleteList), + ok = file:advise(Handle, 0, ?DWORD_SIZE * 256, will_need). endian_flip(Int) -> - <> = <>, - X. + <> = <>, + X. hash(Key) -> - BK = term_to_binary(Key), - H = 5381, - hash1(H, BK) band 16#FFFFFFFF. + BK = term_to_binary(Key), + H = 5381, + hash1(H, BK) band 16#FFFFFFFF. hash1(H, <<>>) -> - H; + H; hash1(H, <>) -> - H1 = H * 33, - H2 = H1 bxor B, - hash1(H2, Rest). + H1 = H * 33, + H2 = H1 bxor B, + hash1(H2, Rest). % Get the least significant 8 bits from the hash. hash_to_index(Hash) -> - Hash band 255. + Hash band 255. hash_to_slot(Hash,L) -> - (Hash bsr 8) rem L. + (Hash bsr 8) rem L. %% Create a binary of the LengthKeyLengthValue, adding a CRC check %% at the front of the value key_value_to_record({Key, Value}) -> - BK = term_to_binary(Key), - BV = term_to_binary(Value), - LK = byte_size(BK), - LV = byte_size(BV), - LK_FL = endian_flip(LK), - LV_FL = endian_flip(LV + 4), - CRC = calc_crc(BV), - <>. + BK = term_to_binary(Key), + BV = term_to_binary(Value), + LK = byte_size(BK), + LV = byte_size(BV), + LK_FL = endian_flip(LK), + LV_FL = endian_flip(LV + 4), + CRC = calc_crc(BV), + <>. %%%%%%%%%%%%%%%% @@ -679,307 +799,307 @@ key_value_to_record({Key, Value}) -> -ifdef(TEST). write_key_value_pairs_1_test() -> - {ok,Handle} = file:open("test.cdb",write), - {_, HashTree} = write_key_value_pairs(Handle,[{"key1","value1"},{"key2","value2"}]), - Hash1 = hash("key1"), - Index1 = hash_to_index(Hash1), - Hash2 = hash("key2"), - Index2 = hash_to_index(Hash2), - R0 = array:new(256, {default, gb_trees:empty()}), - R1 = array:set(Index1, gb_trees:insert(Hash1, [0], array:get(Index1, R0)), R0), - R2 = array:set(Index2, gb_trees:insert(Hash2, [30], array:get(Index2, R1)), R1), - io:format("HashTree is ~w~n", [HashTree]), - io:format("Expected HashTree is ~w~n", [R2]), - ?assertMatch(R2, HashTree), - ok = file:delete("test.cdb"). + {ok,Handle} = file:open("../test/test.cdb",write), + {_, HashTree} = write_key_value_pairs(Handle,[{"key1","value1"},{"key2","value2"}]), + Hash1 = hash("key1"), + Index1 = hash_to_index(Hash1), + Hash2 = hash("key2"), + Index2 = hash_to_index(Hash2), + R0 = array:new(256, {default, gb_trees:empty()}), + R1 = array:set(Index1, gb_trees:insert(Hash1, [0], array:get(Index1, R0)), R0), + R2 = array:set(Index2, gb_trees:insert(Hash2, [30], array:get(Index2, R1)), R1), + io:format("HashTree is ~w~n", [HashTree]), + io:format("Expected HashTree is ~w~n", [R2]), + ?assertMatch(R2, HashTree), + ok = file:delete("../test/test.cdb"). write_hash_tables_1_test() -> - {ok, Handle} = file:open("test.cdb",write), - R0 = array:new(256, {default, gb_trees:empty()}), - R1 = array:set(64, gb_trees:insert(6383014720, [18], array:get(64, R0)), R0), - R2 = array:set(67, gb_trees:insert(6383014723, [0], array:get(67, R1)), R1), - Result = write_hash_tables(Handle, R2), - io:format("write hash tables result of ~w ~n", [Result]), - ?assertMatch(Result,[{67,16,2},{64,0,2}]), - ok = file:delete("test.cdb"). + {ok, Handle} = file:open("../test/testx.cdb",write), + R0 = array:new(256, {default, gb_trees:empty()}), + R1 = array:set(64, gb_trees:insert(6383014720, [18], array:get(64, R0)), R0), + R2 = array:set(67, gb_trees:insert(6383014723, [0], array:get(67, R1)), R1), + Result = write_hash_tables(Handle, R2), + io:format("write hash tables result of ~w ~n", [Result]), + ?assertMatch(Result,[{67,16,2},{64,0,2}]), + ok = file:delete("../test/testx.cdb"). find_open_slot_1_test() -> - List = [<<1:32,1:32>>,<<0:32,0:32>>,<<1:32,1:32>>,<<1:32,1:32>>], - Slot = find_open_slot(List,0), - ?assertMatch(Slot,1). + List = [<<1:32,1:32>>,<<0:32,0:32>>,<<1:32,1:32>>,<<1:32,1:32>>], + Slot = find_open_slot(List,0), + ?assertMatch(Slot,1). find_open_slot_2_test() -> - List = [<<0:32,0:32>>,<<0:32,0:32>>,<<1:32,1:32>>,<<1:32,1:32>>], - Slot = find_open_slot(List,0), - ?assertMatch(Slot,0). + List = [<<0:32,0:32>>,<<0:32,0:32>>,<<1:32,1:32>>,<<1:32,1:32>>], + Slot = find_open_slot(List,0), + ?assertMatch(Slot,0). find_open_slot_3_test() -> - List = [<<1:32,1:32>>,<<1:32,1:32>>,<<1:32,1:32>>,<<0:32,0:32>>], - Slot = find_open_slot(List,2), - ?assertMatch(Slot,3). + List = [<<1:32,1:32>>,<<1:32,1:32>>,<<1:32,1:32>>,<<0:32,0:32>>], + Slot = find_open_slot(List,2), + ?assertMatch(Slot,3). find_open_slot_4_test() -> - List = [<<0:32,0:32>>,<<1:32,1:32>>,<<1:32,1:32>>,<<1:32,1:32>>], - Slot = find_open_slot(List,1), - ?assertMatch(Slot,0). + List = [<<0:32,0:32>>,<<1:32,1:32>>,<<1:32,1:32>>,<<1:32,1:32>>], + Slot = find_open_slot(List,1), + ?assertMatch(Slot,0). find_open_slot_5_test() -> - List = [<<1:32,1:32>>,<<1:32,1:32>>,<<0:32,0:32>>,<<1:32,1:32>>], - Slot = find_open_slot(List,3), - ?assertMatch(Slot,2). + List = [<<1:32,1:32>>,<<1:32,1:32>>,<<0:32,0:32>>,<<1:32,1:32>>], + Slot = find_open_slot(List,3), + ?assertMatch(Slot,2). full_1_test() -> - List1 = lists:sort([{"key1","value1"},{"key2","value2"}]), - create("simple.cdb",lists:sort([{"key1","value1"},{"key2","value2"}])), - List2 = lists:sort(dump("simple.cdb")), - ?assertMatch(List1,List2), - ok = file:delete("simple.cdb"). + List1 = lists:sort([{"key1","value1"},{"key2","value2"}]), + create("../test/simple.cdb",lists:sort([{"key1","value1"},{"key2","value2"}])), + List2 = lists:sort(dump("../test/simple.cdb")), + ?assertMatch(List1,List2), + ok = file:delete("../test/simple.cdb"). full_2_test() -> - List1 = lists:sort([{lists:flatten(io_lib:format("~s~p",[Prefix,Plug])), - lists:flatten(io_lib:format("value~p",[Plug]))} - || Plug <- lists:seq(1,2000), - Prefix <- ["dsd","so39ds","oe9%#*(","020dkslsldclsldowlslf%$#", - "tiep4||","qweq"]]), - create("full.cdb",List1), - List2 = lists:sort(dump("full.cdb")), - ?assertMatch(List1,List2), - ok = file:delete("full.cdb"). + List1 = lists:sort([{lists:flatten(io_lib:format("~s~p",[Prefix,Plug])), + lists:flatten(io_lib:format("value~p",[Plug]))} + || Plug <- lists:seq(1,2000), + Prefix <- ["dsd","so39ds","oe9%#*(","020dkslsldclsldowlslf%$#", + "tiep4||","qweq"]]), + create("../test/full.cdb",List1), + List2 = lists:sort(dump("../test/full.cdb")), + ?assertMatch(List1,List2), + ok = file:delete("../test/full.cdb"). from_dict_test() -> - D = dict:new(), - D1 = dict:store("a","b",D), - D2 = dict:store("c","d",D1), - ok = from_dict("from_dict_test.cdb",D2), - io:format("Store created ~n", []), - KVP = lists:sort(dump("from_dict_test.cdb")), - D3 = lists:sort(dict:to_list(D2)), - io:format("KVP is ~w~n", [KVP]), - io:format("D3 is ~w~n", [D3]), - ?assertMatch(KVP, D3), - ok = file:delete("from_dict_test.cdb"). + D = dict:new(), + D1 = dict:store("a","b",D), + D2 = dict:store("c","d",D1), + ok = from_dict("../test/from_dict_test.cdb",D2), + io:format("Store created ~n", []), + KVP = lists:sort(dump("../test/from_dict_test.cdb")), + D3 = lists:sort(dict:to_list(D2)), + io:format("KVP is ~w~n", [KVP]), + io:format("D3 is ~w~n", [D3]), + ?assertMatch(KVP, D3), + ok = file:delete("../test/from_dict_test.cdb"). to_dict_test() -> - D = dict:new(), - D1 = dict:store("a","b",D), - D2 = dict:store("c","d",D1), - ok = from_dict("from_dict_test.cdb",D2), - Dict = to_dict("from_dict_test.cdb"), - D3 = lists:sort(dict:to_list(D2)), - D4 = lists:sort(dict:to_list(Dict)), - ?assertMatch(D4,D3), - ok = file:delete("from_dict_test.cdb"). + D = dict:new(), + D1 = dict:store("a","b",D), + D2 = dict:store("c","d",D1), + ok = from_dict("../test/from_dict_test1.cdb",D2), + Dict = to_dict("../test/from_dict_test1.cdb"), + D3 = lists:sort(dict:to_list(D2)), + D4 = lists:sort(dict:to_list(Dict)), + ?assertMatch(D4,D3), + ok = file:delete("../test/from_dict_test1.cdb"). crccheck_emptyvalue_test() -> - ?assertMatch(false, crccheck_value(<<>>)). + ?assertMatch(false, crccheck_value(<<>>)). crccheck_shortvalue_test() -> - Value = <<128,128,32>>, - ?assertMatch(false, crccheck_value(Value)). + Value = <<128,128,32>>, + ?assertMatch(false, crccheck_value(Value)). crccheck_justshortvalue_test() -> - Value = <<128,128,32,64>>, - ?assertMatch(false, crccheck_value(Value)). + Value = <<128,128,32,64>>, + ?assertMatch(false, crccheck_value(Value)). crccheck_correctvalue_test() -> - Value = term_to_binary("some text as value"), - Hash = erlang:crc32(Value), - ValueOnDisk = <>, - ?assertMatch(true, crccheck_value(ValueOnDisk)). + Value = term_to_binary("some text as value"), + Hash = erlang:crc32(Value), + ValueOnDisk = <>, + ?assertMatch(true, crccheck_value(ValueOnDisk)). crccheck_wronghash_test() -> - Value = term_to_binary("some text as value"), - Hash = erlang:crc32(Value) + 1, - ValueOnDisk = <>, - ?assertMatch(false, crccheck_value(ValueOnDisk)). + Value = term_to_binary("some text as value"), + Hash = erlang:crc32(Value) + 1, + ValueOnDisk = <>, + ?assertMatch(false, crccheck_value(ValueOnDisk)). crccheck_truncatedvalue_test() -> - Value = term_to_binary("some text as value"), - Hash = erlang:crc32(Value), - ValueOnDisk = <>, - Size = bit_size(ValueOnDisk) - 1, - <> = ValueOnDisk, - ?assertMatch(false, crccheck_value(TruncatedValue)). + Value = term_to_binary("some text as value"), + Hash = erlang:crc32(Value), + ValueOnDisk = <>, + Size = bit_size(ValueOnDisk) - 1, + <> = ValueOnDisk, + ?assertMatch(false, crccheck_value(TruncatedValue)). activewrite_singlewrite_test() -> - Key = "0002", - Value = "some text as new value", - InitialD = dict:new(), - InitialD1 = dict:store("0001", "Initial value", InitialD), - ok = from_dict("test_mem.cdb", InitialD1), - io:format("New db file created ~n", []), - {LastPosition, KeyDict} = open_active_file("test_mem.cdb"), - io:format("File opened as new active file " - "with LastPosition=~w ~n", [LastPosition]), - {_, _, UpdKeyDict} = put("test_mem.cdb", Key, Value, {LastPosition, KeyDict}), - io:format("New key and value added to active file ~n", []), - ?assertMatch({Key, Value}, get_mem(Key, "test_mem.cdb", UpdKeyDict)), - ok = file:delete("test_mem.cdb"). + Key = "0002", + Value = "some text as new value", + InitialD = dict:new(), + InitialD1 = dict:store("0001", "Initial value", InitialD), + ok = from_dict("../test/test_mem.cdb", InitialD1), + io:format("New db file created ~n", []), + {LastPosition, KeyDict} = open_active_file("../test/test_mem.cdb"), + io:format("File opened as new active file " + "with LastPosition=~w ~n", [LastPosition]), + {_, _, UpdKeyDict} = put("../test/test_mem.cdb", Key, Value, {LastPosition, KeyDict}), + io:format("New key and value added to active file ~n", []), + ?assertMatch({Key, Value}, get_mem(Key, "../test/test_mem.cdb", UpdKeyDict)), + ok = file:delete("../test/test_mem.cdb"). search_hash_table_findinslot_test() -> - Key1 = "key1", % this is in slot 3 if count is 8 - D = dict:from_list([{Key1, "value1"}, {"K2", "V2"}, {"K3", "V3"}, - {"K4", "V4"}, {"K5", "V5"}, {"K6", "V6"}, {"K7", "V7"}, - {"K8", "V8"}]), - ok = from_dict("hashtable1_test.cdb",D), - {ok, Handle} = file:open("hashtable1_test.cdb", [binary, raw, read, write]), - Hash = hash(Key1), - Index = hash_to_index(Hash), - {ok, _} = file:position(Handle, {bof, ?DWORD_SIZE*Index}), - {HashTable, Count} = read_next_2_integers(Handle), - io:format("Count of ~w~n", [Count]), - {ok, FirstHashPosition} = file:position(Handle, {bof, HashTable}), - Slot = hash_to_slot(Hash, Count), - io:format("Slot of ~w~n", [Slot]), - {ok, _} = file:position(Handle, {cur, Slot * ?DWORD_SIZE}), - {ReadH3, ReadP3} = read_next_2_integers(Handle), - {ReadH4, ReadP4} = read_next_2_integers(Handle), - io:format("Slot 1 has Hash ~w Position ~w~n", [ReadH3, ReadP3]), - io:format("Slot 2 has Hash ~w Position ~w~n", [ReadH4, ReadP4]), - ?assertMatch(0, ReadH4), - ?assertMatch({"key1", "value1"}, get(Handle, Key1)), - {ok, _} = file:position(Handle, FirstHashPosition), - FlipH3 = endian_flip(ReadH3), - FlipP3 = endian_flip(ReadP3), - RBin = <>, - io:format("Replacement binary of ~w~n", [RBin]), - {ok, OldBin} = file:pread(Handle, - FirstHashPosition + (Slot -1) * ?DWORD_SIZE, 16), - io:format("Bin to be replaced is ~w ~n", [OldBin]), - ok = file:pwrite(Handle, FirstHashPosition + (Slot -1) * ?DWORD_SIZE, RBin), - ok = file:close(Handle), - io:format("Find key following change to hash table~n"), - ?assertMatch(missing, get("hashtable1_test.cdb", Key1)), - ok = file:delete("hashtable1_test.cdb"). + Key1 = "key1", % this is in slot 3 if count is 8 + D = dict:from_list([{Key1, "value1"}, {"K2", "V2"}, {"K3", "V3"}, + {"K4", "V4"}, {"K5", "V5"}, {"K6", "V6"}, {"K7", "V7"}, + {"K8", "V8"}]), + ok = from_dict("../test/hashtable1_test.cdb",D), + {ok, Handle} = file:open("../test/hashtable1_test.cdb", [binary, raw, read, write]), + Hash = hash(Key1), + Index = hash_to_index(Hash), + {ok, _} = file:position(Handle, {bof, ?DWORD_SIZE*Index}), + {HashTable, Count} = read_next_2_integers(Handle), + io:format("Count of ~w~n", [Count]), + {ok, FirstHashPosition} = file:position(Handle, {bof, HashTable}), + Slot = hash_to_slot(Hash, Count), + io:format("Slot of ~w~n", [Slot]), + {ok, _} = file:position(Handle, {cur, Slot * ?DWORD_SIZE}), + {ReadH3, ReadP3} = read_next_2_integers(Handle), + {ReadH4, ReadP4} = read_next_2_integers(Handle), + io:format("Slot 1 has Hash ~w Position ~w~n", [ReadH3, ReadP3]), + io:format("Slot 2 has Hash ~w Position ~w~n", [ReadH4, ReadP4]), + ?assertMatch(0, ReadH4), + ?assertMatch({"key1", "value1"}, get(Handle, Key1)), + {ok, _} = file:position(Handle, FirstHashPosition), + FlipH3 = endian_flip(ReadH3), + FlipP3 = endian_flip(ReadP3), + RBin = <>, + io:format("Replacement binary of ~w~n", [RBin]), + {ok, OldBin} = file:pread(Handle, + FirstHashPosition + (Slot -1) * ?DWORD_SIZE, 16), + io:format("Bin to be replaced is ~w ~n", [OldBin]), + ok = file:pwrite(Handle, FirstHashPosition + (Slot -1) * ?DWORD_SIZE, RBin), + ok = file:close(Handle), + io:format("Find key following change to hash table~n"), + ?assertMatch(missing, get("../test/hashtable1_test.cdb", Key1)), + ok = file:delete("../test/hashtable1_test.cdb"). getnextkey_inclemptyvalue_test() -> - L = [{"K9", "V9"}, {"K2", "V2"}, {"K3", ""}, - {"K4", "V4"}, {"K5", "V5"}, {"K6", "V6"}, {"K7", "V7"}, - {"K8", "V8"}, {"K1", "V1"}], - ok = create("hashtable1_test.cdb", L), - {FirstKey, Handle, P1} = get_nextkey("hashtable1_test.cdb"), - io:format("Next position details of ~w~n", [P1]), - ?assertMatch("K9", FirstKey), - {SecondKey, Handle, P2} = get_nextkey(Handle, P1), - ?assertMatch("K2", SecondKey), - {ThirdKeyNoValue, Handle, P3} = get_nextkey(Handle, P2), - ?assertMatch("K3", ThirdKeyNoValue), - {_, Handle, P4} = get_nextkey(Handle, P3), - {_, Handle, P5} = get_nextkey(Handle, P4), - {_, Handle, P6} = get_nextkey(Handle, P5), - {_, Handle, P7} = get_nextkey(Handle, P6), - {_, Handle, P8} = get_nextkey(Handle, P7), - {LastKey, Info} = get_nextkey(Handle, P8), - ?assertMatch(nomorekeys, Info), - ?assertMatch("K1", LastKey), - ok = file:delete("hashtable1_test.cdb"). + L = [{"K9", "V9"}, {"K2", "V2"}, {"K3", ""}, + {"K4", "V4"}, {"K5", "V5"}, {"K6", "V6"}, {"K7", "V7"}, + {"K8", "V8"}, {"K1", "V1"}], + ok = create("../test/hashtable2_test.cdb", L), + {FirstKey, Handle, P1} = get_nextkey("../test/hashtable2_test.cdb"), + io:format("Next position details of ~w~n", [P1]), + ?assertMatch("K9", FirstKey), + {SecondKey, Handle, P2} = get_nextkey(Handle, P1), + ?assertMatch("K2", SecondKey), + {ThirdKeyNoValue, Handle, P3} = get_nextkey(Handle, P2), + ?assertMatch("K3", ThirdKeyNoValue), + {_, Handle, P4} = get_nextkey(Handle, P3), + {_, Handle, P5} = get_nextkey(Handle, P4), + {_, Handle, P6} = get_nextkey(Handle, P5), + {_, Handle, P7} = get_nextkey(Handle, P6), + {_, Handle, P8} = get_nextkey(Handle, P7), + {LastKey, Info} = get_nextkey(Handle, P8), + ?assertMatch(nomorekeys, Info), + ?assertMatch("K1", LastKey), + ok = file:delete("../test/hashtable2_test.cdb"). newactivefile_test() -> - {LastPosition, _} = open_active_file("activefile_test.cdb"), - ?assertMatch(256 * ?DWORD_SIZE, LastPosition), - Response = get_nextkey("activefile_test.cdb"), - ?assertMatch(nomorekeys, Response), - ok = file:delete("activefile_test.cdb"). + {LastPosition, _} = open_active_file("../test/activefile_test.cdb"), + ?assertMatch(256 * ?DWORD_SIZE, LastPosition), + Response = get_nextkey("../test/activefile_test.cdb"), + ?assertMatch(nomorekeys, Response), + ok = file:delete("../test/activefile_test.cdb"). emptyvalue_fromdict_test() -> - D = dict:new(), - D1 = dict:store("K1", "V1", D), - D2 = dict:store("K2", "", D1), - D3 = dict:store("K3", "V3", D2), - D4 = dict:store("K4", "", D3), - ok = from_dict("from_dict_test_ev.cdb",D4), - io:format("Store created ~n", []), - KVP = lists:sort(dump("from_dict_test_ev.cdb")), - D_Result = lists:sort(dict:to_list(D4)), - io:format("KVP is ~w~n", [KVP]), - io:format("D_Result is ~w~n", [D_Result]), - ?assertMatch(KVP, D_Result), - ok = file:delete("from_dict_test_ev.cdb"). + D = dict:new(), + D1 = dict:store("K1", "V1", D), + D2 = dict:store("K2", "", D1), + D3 = dict:store("K3", "V3", D2), + D4 = dict:store("K4", "", D3), + ok = from_dict("../test/from_dict_test_ev.cdb",D4), + io:format("Store created ~n", []), + KVP = lists:sort(dump("../test/from_dict_test_ev.cdb")), + D_Result = lists:sort(dict:to_list(D4)), + io:format("KVP is ~w~n", [KVP]), + io:format("D_Result is ~w~n", [D_Result]), + ?assertMatch(KVP, D_Result), + ok = file:delete("../test/from_dict_test_ev.cdb"). fold_test() -> - K1 = {"Key1", 1}, - V1 = 2, - K2 = {"Key1", 2}, - V2 = 4, - K3 = {"Key1", 3}, - V3 = 8, - K4 = {"Key1", 4}, - V4 = 16, - K5 = {"Key1", 5}, - V5 = 32, - D = dict:from_list([{K1, V1}, {K2, V2}, {K3, V3}, {K4, V4}, {K5, V5}]), - ok = from_dict("fold_test.cdb", D), - FromSN = 2, - FoldFun = fun(K, V, Acc) -> - {_Key, Seq} = K, - if Seq > FromSN -> - Acc + V; - true -> - Acc - end - end, - ?assertMatch(56, fold("fold_test.cdb", FoldFun, 0)), - ok = file:delete("fold_test.cdb"). + K1 = {"Key1", 1}, + V1 = 2, + K2 = {"Key1", 2}, + V2 = 4, + K3 = {"Key1", 3}, + V3 = 8, + K4 = {"Key1", 4}, + V4 = 16, + K5 = {"Key1", 5}, + V5 = 32, + D = dict:from_list([{K1, V1}, {K2, V2}, {K3, V3}, {K4, V4}, {K5, V5}]), + ok = from_dict("../test/fold_test.cdb", D), + FromSN = 2, + FoldFun = fun(K, V, Acc) -> + {_Key, Seq} = K, + if Seq > FromSN -> + Acc + V; + true -> + Acc + end + end, + ?assertMatch(56, fold("../test/fold_test.cdb", FoldFun, 0)), + ok = file:delete("../test/fold_test.cdb"). fold_keys_test() -> - K1 = {"Key1", 1}, - V1 = 2, - K2 = {"Key2", 2}, - V2 = 4, - K3 = {"Key3", 3}, - V3 = 8, - K4 = {"Key4", 4}, - V4 = 16, - K5 = {"Key5", 5}, - V5 = 32, - D = dict:from_list([{K1, V1}, {K2, V2}, {K3, V3}, {K4, V4}, {K5, V5}]), - ok = from_dict("fold_keys_test.cdb", D), - FromSN = 2, - FoldFun = fun(K, Acc) -> + K1 = {"Key1", 1}, + V1 = 2, + K2 = {"Key2", 2}, + V2 = 4, + K3 = {"Key3", 3}, + V3 = 8, + K4 = {"Key4", 4}, + V4 = 16, + K5 = {"Key5", 5}, + V5 = 32, + D = dict:from_list([{K1, V1}, {K2, V2}, {K3, V3}, {K4, V4}, {K5, V5}]), + ok = from_dict("../test/fold_keys_test.cdb", D), + FromSN = 2, + FoldFun = fun(K, Acc) -> {Key, Seq} = K, if Seq > FromSN -> - lists:append(Acc, [Key]); - true -> - Acc + lists:append(Acc, [Key]); + true -> + Acc end - end, - Result = fold_keys("fold_keys_test.cdb", FoldFun, []), - ?assertMatch(["Key3", "Key4", "Key5"], lists:sort(Result)), - ok = file:delete("fold_keys_test.cdb"). + end, + Result = fold_keys("../test/fold_keys_test.cdb", FoldFun, []), + ?assertMatch(["Key3", "Key4", "Key5"], lists:sort(Result)), + ok = file:delete("../test/fold_keys_test.cdb"). fold2_test() -> - K1 = {"Key1", 1}, - V1 = 2, - K2 = {"Key1", 2}, - V2 = 4, - K3 = {"Key1", 3}, - V3 = 8, - K4 = {"Key1", 4}, - V4 = 16, - K5 = {"Key1", 5}, - V5 = 32, - K6 = {"Key2", 1}, - V6 = 64, - D = dict:from_list([{K1, V1}, {K2, V2}, {K3, V3}, + K1 = {"Key1", 1}, + V1 = 2, + K2 = {"Key1", 2}, + V2 = 4, + K3 = {"Key1", 3}, + V3 = 8, + K4 = {"Key1", 4}, + V4 = 16, + K5 = {"Key1", 5}, + V5 = 32, + K6 = {"Key2", 1}, + V6 = 64, + D = dict:from_list([{K1, V1}, {K2, V2}, {K3, V3}, {K4, V4}, {K5, V5}, {K6, V6}]), - ok = from_dict("fold2_test.cdb", D), - FoldFun = fun(K, V, Acc) -> - {Key, Seq} = K, - case dict:find(Key, Acc) of - error -> - dict:store(Key, {Seq, V}, Acc); - {ok, {LSN, _V}} when Seq > LSN -> - dict:store(Key, {Seq, V}, Acc); - _ -> - Acc - end - end, - RD = dict:new(), - RD1 = dict:store("Key1", {5, 32}, RD), - RD2 = dict:store("Key2", {1, 64}, RD1), - Result = fold("fold2_test.cdb", FoldFun, dict:new()), - ?assertMatch(RD2, Result), - ok = file:delete("fold2_test.cdb"). + ok = from_dict("../test/fold2_test.cdb", D), + FoldFun = fun(K, V, Acc) -> + {Key, Seq} = K, + case dict:find(Key, Acc) of + error -> + dict:store(Key, {Seq, V}, Acc); + {ok, {LSN, _V}} when Seq > LSN -> + dict:store(Key, {Seq, V}, Acc); + _ -> + Acc + end + end, + RD = dict:new(), + RD1 = dict:store("Key1", {5, 32}, RD), + RD2 = dict:store("Key2", {1, 64}, RD1), + Result = fold("../test/fold2_test.cdb", FoldFun, dict:new()), + ?assertMatch(RD2, Result), + ok = file:delete("../test/fold2_test.cdb"). -endif. diff --git a/src/leveled_concierge.erl b/src/leveled_concierge.erl index 4147a3a..8abb892 100644 --- a/src/leveled_concierge.erl +++ b/src/leveled_concierge.erl @@ -1,3 +1,71 @@ +%% -------- Overview --------- +%% +%% The eleveleddb is based on the LSM-tree similar to leveldb, except that: +%% - Values are kept seperately to Keys & Metadata +%% - Different file formats are used for value store (based on constant +%% database), and key store (based on sst) +%% - It is not intended to be general purpose, but be specifically suited for +%% use as a Riak backend in specific circumstances (relatively large values, +%% and frequent use of iterators) +%% - The Value store is an extended nursery log in leveldb terms. It is keyed +%% on the sequence number of the write +%% - The Key Store is a LSM tree, where the key is the actaul object key, and +%% the value is the metadata of the object including the sequence number +%% +%% -------- Concierge & Manifest --------- +%% +%% The concierge is responsible for opening up the store, and keeps a manifest +%% of where items can be found. The manifest keeps a mapping of: +%% - Sequence Number ranges and the PID of the Value Store file that contains +%% that range +%% - Key ranges to PID mappings for each leval of the KeyStore +%% +%% -------- GET -------- +%% +%% A GET request for Key and Metadata requires a lookup in the KeyStore only. +%% - The concierge should consult the manifest for the lowest level to find +%% the PID which may contain the Key +%% - The concierge should ask the file owner if the Key is present, if not +%% present lower levels should be consulted until the objetc is found +%% +%% If a value is required, when the Key/Metadata has been fetched from the +%% KeyStore, the sequence number should be tkane, and matched in the ValueStore +%% manifest to find the right value. +%% +%% For recent PUTs the Key/Metadata is added into memory, and there is an +%% in-memory hash table for the entries in the most recent ValueStore CDB. +%% +%% -------- PUT -------- +%% +%% A PUT request must be persisted to the open (and append only) CDB file which +%% acts as a transaction log to persist the change. The Key & Metadata needs +%% also to be placed in memory. +%% +%% Once the CDB file is full, the managing process should be requested to +%% complete the lookup hash, and a new CDB file be started. +%% +%% Once the in-memory +%% +%% -------- Snapshots (Key Only) -------- +%% +%% If there is a iterator/snapshot request, the concierge will simply handoff a +%% copy of the manifest, and register the interest of the iterator at the +%% manifest sequence number at the time of the request. Iterators should +%% de-register themselves from the manager on completion. Iterators should be +%% automatically release after a timeout period. A file can be deleted if +%% there are no registered iterators from before the point the file was +%% removed from the manifest. +%% +%% -------- Snapshots (Key & Value) -------- +%% +%% +%% +%% -------- Special Ops -------- +%% +%% e.g. Get all for SegmentID/Partition +%% +%% -------- KeyStore --------- +%% %% The concierge is responsible for controlling access to the store and %% maintaining both an in-memory view and a persisted state of all the sft %% files in use across the store. @@ -34,14 +102,7 @@ %% will call the manifets manager on a timeout to confirm that they are no %% longer in use (by any iterators). %% -%% If there is a iterator/snapshot request, the concierge will simply handoff a -%% copy of the manifest, and register the interest of the iterator at the -%% manifest sequence number at the time of the request. Iterators should -%% de-register themselves from the manager on completion. Iterators should be -%% automatically release after a timeout period. A file can be deleted if -%% there are no registered iterators from before the point the file was -%% removed from the manifest. -%% + From b5db1b4e14c612d199255b3e9f8eba51334d1911 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 29 Jul 2016 17:48:11 +0100 Subject: [PATCH 021/167] CDB to gen_server First draft to make CDB a gen_Server --- src/leveled_cdb.erl | 43 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 46004e0..ff52acf 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -56,6 +56,8 @@ code_change/3, cdb_open_writer/1, cdb_open_reader/1, + cdb_get/2, + cdb_put/3, from_dict/2, create/2, dump/1, @@ -107,11 +109,11 @@ cdb_open_reader(Filename) -> Error end. -%cdb_get(Pid, Key) -> -% gen_server:call(Pid, {cdb_get, Key}, infinity). -% -%cdb_put(Pid, Key, Value) -> -% gen_server:call(Pid, {cdb_put, Key, Value}, infinity). +cdb_get(Pid, Key) -> + gen_server:call(Pid, {cdb_get, Key}, infinity). + +cdb_put(Pid, Key, Value) -> + gen_server:call(Pid, {cdb_put, Key, Value}, infinity). % %cdb_close(Pid) -> % gen_server:call(Pid, cdb_close, infinity). @@ -139,7 +141,36 @@ handle_call({cdb_open_reader, Filename}, _From, State) -> {ok, Handle} = file:open(Filename, [binary, raw, read]), {reply, ok, State#state{handle=Handle, filename=Filename, - writer=false}}. + writer=false}}; +handle_call({cdb_get, Key}, _From, State) -> + case State#state.writer of + true -> + {reply, + get_mem(Key, State#state.handle, State#state.hashtree), + State}; + false -> + {reply, + get(State#state.handle, Key), + State} + end; +handle_call({cdb_put, Key, Value}, _From, State) -> + case State#state.writer of + true -> + Result = put(State#state.handle, + Key, Value, + {State#state.last_position, State#state.hashtree}), + {UpdHandle, NewPosition, HashTree} = Result, + {reply, + ok, + State#state{handle=UpdHandle, + last_position=NewPosition, + hashtree=HashTree}}; + false -> + {reply, + {error, read_only}, + State} + end. + handle_cast(_Msg, State) -> {noreply, State}. From 04da89127291c2b22c920eb7e2d373d108738566 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 29 Jul 2016 17:52:44 +0100 Subject: [PATCH 022/167] Tidy up removing old files --- src/onstartfile.bst | Bin 32 -> 0 bytes src/rice.erl | 155 -------------------------------------------- 2 files changed, 155 deletions(-) delete mode 100644 src/onstartfile.bst delete mode 100644 src/rice.erl diff --git a/src/onstartfile.bst b/src/onstartfile.bst deleted file mode 100644 index 72153f261c71b68b435b2d014289c0010926cd82..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32 QcmZQ%X21!I0+*Wu00Z3tW&i*H diff --git a/src/rice.erl b/src/rice.erl deleted file mode 100644 index 68dd78a..0000000 --- a/src/rice.erl +++ /dev/null @@ -1,155 +0,0 @@ --module(rice). --export([encode/1, - encode/2, - checkforhash/2, - converttohash/1]). --include_lib("eunit/include/eunit.hrl"). - -%% Factor is the power of 2 representing the expected normal gap size between -%% members of the hash, and therefore the size of the bitstring to represent the -%% remainder for the gap -%% -%% The encoded output should contain a single byte which is the Factor, followed -%% by a series of exponents and remainders. -%% -%% The exponent is n 1's followed by a 0, where n * (2 ^ Factor) + remainder -%% represents the gap to the next hash -%% -%% The size passed in should be the maximum possible value of the hash. -%% If this isn't provided - assumes 2^32 - the default for phash2 - -encode(HashList) -> - encode(HashList, 4 * 1024 * 1024 * 1024). - -encode(HashList, Size) -> - SortedHashList = lists:usort(HashList), - ExpectedGapSize = Size div length(SortedHashList), - Factor = findpowerundergap(ExpectedGapSize), - riceencode(SortedHashList, Factor). - -%% Outcome may be suboptimal if lists have not been de-duplicated -%% Will fail on an unsorted list - -riceencode(HashList, Factor) when Factor<256 -> - Divisor = powtwo(Factor), - riceencode(HashList, Factor, Divisor, <<>>, 0). - -riceencode([], Factor, _, BitStrAcc, _) -> - Prefix = binary:encode_unsigned(Factor), - <>; -riceencode([HeadHash|TailList], Factor, Divisor, BitStrAcc, LastHash) -> - HashGap = HeadHash - LastHash, - case HashGap of - 0 -> - riceencode(TailList, Factor, Divisor, BitStrAcc, HeadHash); - N when N > 0 -> - Exponent = buildexponent(HashGap div Divisor), - Remainder = HashGap rem Divisor, - ExpandedBitStrAcc = <>, - riceencode(TailList, Factor, Divisor, ExpandedBitStrAcc, HeadHash) - end. - - -%% Checking for a hash needs to roll through the compressed bloom, decoding until -%% the member is found (match!), passed (not matched) or the end of the encoded -%% bitstring has been reached (not matched) - -checkforhash(HashToCheck, BitStr) -> - <> = BitStr, - Divisor = powtwo(Factor), - checkforhash(HashToCheck, RiceEncodedBitStr, Factor, Divisor, 0). - -checkforhash(_, <<>>, _, _, _) -> - false; -checkforhash(HashToCheck, BitStr, Factor, Divisor, Acc) -> - [Exponent, BitStrTail] = findexponent(BitStr), - [Remainder, BitStrTail2] = findremainder(BitStrTail, Factor), - NextHash = Acc + Divisor * Exponent + Remainder, - case NextHash of - HashToCheck -> true; - N when N>HashToCheck -> false; - _ -> checkforhash(HashToCheck, BitStrTail2, Factor, Divisor, NextHash) - end. - - -%% Exported functions - currently used only in testing - -converttohash(ItemList) -> - converttohash(ItemList, []). - -converttohash([], HashList) -> - HashList; -converttohash([H|T], HashList) -> - converttohash(T, [erlang:phash2(H)|HashList]). - - - -%% Helper functions - -buildexponent(Exponent) -> - buildexponent(Exponent, <<0:1>>). - -buildexponent(0, OutputBits) -> - OutputBits; -buildexponent(Exponent, OutputBits) -> - buildexponent(Exponent - 1, <<1:1, OutputBits/bitstring>>). - - -findexponent(BitStr) -> - findexponent(BitStr, 0). - -findexponent(BitStr, Acc) -> - <> = BitStr, - case H of - <<1:1>> -> findexponent(T, Acc + 1); - <<0:1>> -> [Acc, T] - end. - - -findremainder(BitStr, Factor) -> - <> = BitStr, - [Remainder, BitStrTail]. - - -powtwo(N) -> powtwo(N, 1). - -powtwo(0, Acc) -> - Acc; -powtwo(N, Acc) -> - powtwo(N-1, Acc * 2). - -%% Helper method for finding the factor of two which provides the most -%% efficient compression given an average gap size - -findpowerundergap(GapSize) -> findpowerundergap(GapSize, 1, 0). - -findpowerundergap(GapSize, Acc, Counter) -> - case Acc of - N when N > GapSize -> Counter - 1; - _ -> findpowerundergap(GapSize, Acc * 2, Counter + 1) - end. - - -%% Unit tests - -findpowerundergap_test_() -> - [ - ?_assertEqual(9, findpowerundergap(700)), - ?_assertEqual(9, findpowerundergap(512)), - ?_assertEqual(8, findpowerundergap(511))]. - -encode_test_() -> - [ - ?_assertEqual(<<9, 6, 44, 4:5>>, encode([24,924], 1024)), - ?_assertEqual(<<9, 6, 44, 4:5>>, encode([24,24,924], 1024)), - ?_assertEqual(<<9, 6, 44, 4:5>>, encode([24,924,924], 1024)) - ]. - -check_test_() -> - [ - ?_assertEqual(true, checkforhash(924, <<9, 6, 44, 4:5>>)), - ?_assertEqual(true, checkforhash(24, <<9, 6, 44, 4:5>>)), - ?_assertEqual(false, checkforhash(23, <<9, 6, 44, 4:5>>)), - ?_assertEqual(false, checkforhash(923, <<9, 6, 44, 4:5>>)), - ?_assertEqual(false, checkforhash(925, <<9, 6, 44, 4:5>>)) - ]. From 2bdb5fba6cc23400a574c2a774faeb7e6430ae9c Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 2 Aug 2016 13:44:48 +0100 Subject: [PATCH 023/167] Re-naming Naming things is hard. This change renames things based on the Bookie/Inker/Penciller terminology --- src/leveled_bookie.erl | 104 ++++++++++++++++ src/leveled_cdb.erl | 54 +++++---- ...led_housekeeping.erl => leveled_clerk.erl} | 2 +- src/leveled_inker.erl | 8 ++ ...ed_concierge.erl => leveled_penciller.erl} | 111 ++++-------------- src/leveled_sft.erl | 9 +- 6 files changed, 170 insertions(+), 118 deletions(-) create mode 100644 src/leveled_bookie.erl rename src/{leveled_housekeeping.erl => leveled_clerk.erl} (99%) create mode 100644 src/leveled_inker.erl rename src/{leveled_concierge.erl => leveled_penciller.erl} (72%) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl new file mode 100644 index 0000000..321c7f8 --- /dev/null +++ b/src/leveled_bookie.erl @@ -0,0 +1,104 @@ +%% -------- Overview --------- +%% +%% The eleveleddb is based on the LSM-tree similar to leveldb, except that: +%% - Keys, Metadata and Values are not persisted together - the Keys and +%% Metadata are kept in a tree-based ledger, whereas the values are stored +%% only in a sequential Journal. +%% - Different file formats are used for Journal (based on constant +%% database), and the ledger (sft, based on sst) +%% - It is not intended to be general purpose, but be specifically suited for +%% use as a Riak backend in specific circumstances (relatively large values, +%% and frequent use of iterators) +%% - The Journal is an extended nursery log in leveldb terms. It is keyed +%% on the sequence number of the write +%% - The ledger is a LSM tree, where the key is the actaul object key, and +%% the value is the metadata of the object including the sequence number +%% +%% +%% -------- The actors --------- +%% +%% The store is fronted by a Bookie, who takes support from different actors: +%% - An Inker who persists new data into the jornal, and returns items from +%% the journal based on sequence number +%% - A Penciller who periodically redraws the ledger +%% - One or more Clerks, who may be used by either the inker or the penciller +%% to fulfill background tasks +%% +%% Both the Inker and the Penciller maintain a manifest of the files which +%% represent the current state of the Journal and the Ledger repsectively. +%% For the Inker the manifest maps ranges of sequence numbers to cdb files. +%% For the Penciller the manifest maps key ranges to files at each level of +%% the Ledger. +%% +%% -------- PUT -------- +%% +%% A PUT request consists of +%% - A primary Key +%% - Metadata associated with the primary key (2i, vector clock, object size) +%% - A value +%% - A set of secondary key changes which should be made as part of the commit +%% +%% The Bookie takes the place request and passes it first to the Inker to add +%% the request to the ledger. +%% +%% The inker will pass the request to the current (append only) CDB journal +%% fileto persist the change. The call should return either 'ok' or 'roll'. +%% 'roll' indicates that the CDB file has insufficient capacity for +%% this write. + +%% In resonse to a 'roll', the inker should: +%% - start a new active journal file with an open_write_request, and then; +%% - call to PUT the object in this file; +%% - reply to the bookie, but then in the background +%% - close the previously active journal file (writing the hashtree), and move +%% it to the historic journal +%% +%% Once the object has been persisted to the Journal, the Key and Metadata can +%% be added to the ledger. Initially this will be added to the Bookie's +%% in-memory view of recent changes only. +%% +%% The Bookie's memory consists of up to two in-memory ets tables +%% - the 'cmem' (current in-memory table) which is always subject to potential +%% change; +%% - the 'imem' (the immutable in-memory table) which is awaiting persistence +%% to the disk-based lsm-tree by the Penciller. +%% +%% The key and metadata should be written to the cmem store if it has +%% sufficient capacity, but this potentially should include the secondary key +%% changes which have been made as part of the transaction. +%% +%% If there is insufficient space in the cmem, the cmem should be converted +%% into the imem, and a new cmem be created. This requires the previous imem +%% to have been cleared from state due to compaction into the persisted Ledger +%% by the Penciller - otherwise the PUT is blocked. On creation of an imem, +%% the compaction process for that imem by the Penciller should be triggered. +%% +%% This completes the non-deferrable work associated with a PUT +%% +%% -------- Snapshots (Key & Metadata Only) -------- +%% +%% If there is a snapshot request (e.g. to iterate over the keys) the Bookie +%% must first produce a tree representing the results of the request which are +%% present in its in-memory view of the ledger. The Bookie then requests +%% a copy of the current Ledger manifest from the Penciller, and the Penciller +%5 should interest of the iterator at the manifest sequence number at the time +%% of the request. +%% +%% Iterators should de-register themselves from the Penciller on completion. +%% Iterators should be automatically release after a timeout period. A file +%% can only be deleted from the Ledger if it is no longer in the manifest, and +%% there are no registered iterators from before the point the file was +%% removed from the manifest. +%% +%% Snapshots may be non-recent, if recency is unimportant. Non-recent +%% snapshots do no require the Bookie to return the results of the in-memory +%% table, the Penciller alone cna be asked. +%% +%% -------- Special Ops -------- +%% +%% e.g. Get all for SegmentID/Partition +%% + + +-module(leveled_bookie). + diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index ff52acf..e2c266f 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -58,17 +58,7 @@ cdb_open_reader/1, cdb_get/2, cdb_put/3, - from_dict/2, - create/2, - dump/1, - get/2, - get_mem/3, - put/4, - open_active_file/1, - get_nextkey/1, - get_nextkey/2, - fold/3, - fold_keys/3]). + cdb_close/1]). -include_lib("eunit/include/eunit.hrl"). @@ -114,9 +104,9 @@ cdb_get(Pid, Key) -> cdb_put(Pid, Key, Value) -> gen_server:call(Pid, {cdb_put, Key, Value}, infinity). -% -%cdb_close(Pid) -> -% gen_server:call(Pid, cdb_close, infinity). + +cdb_close(Pid) -> + gen_server:call(Pid, cdb_close, infinity). %%%============================================================================ @@ -159,18 +149,30 @@ handle_call({cdb_put, Key, Value}, _From, State) -> Result = put(State#state.handle, Key, Value, {State#state.last_position, State#state.hashtree}), - {UpdHandle, NewPosition, HashTree} = Result, - {reply, - ok, - State#state{handle=UpdHandle, - last_position=NewPosition, - hashtree=HashTree}}; + case Result of + roll -> + {reply, roll, State}; + {UpdHandle, NewPosition, HashTree} -> + {reply, ok, State#state{handle=UpdHandle, + last_position=NewPosition, + hashtree=HashTree}} + end; false -> {reply, {error, read_only}, State} - end. - + end; +handle_call(cdb_close, _From, State) -> + case State#state.writer of + true -> + ok = close_file(State#state.handle, + State#state.hashtree, + State#state.last_position); + false -> + ok = file:close(State#state.handle) + end, + {stop, normal, ok, State}. + handle_cast(_Msg, State) -> {noreply, State}. @@ -178,8 +180,8 @@ handle_cast(_Msg, State) -> handle_info(_Info, State) -> {noreply, State}. -terminate(_Reason, _State) -> - ok. +terminate(_Reason, State) -> + file:close(State#state.handle). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -264,8 +266,8 @@ open_active_file(FileName) when is_list(FileName) -> ok = file:close(Handle); {ok, _} -> LogDetails = [LastPosition, file:position(Handle, eof)], - io:format("File to be truncated at last position of" - "~w with end of file at ~w~n", LogDetails), + io:format("File to be truncated at last position of ~w " + "with end of file at ~w~n", LogDetails), {ok, LastPosition} = file:position(Handle, LastPosition), ok = file:truncate(Handle), ok = file:close(Handle) diff --git a/src/leveled_housekeeping.erl b/src/leveled_clerk.erl similarity index 99% rename from src/leveled_housekeeping.erl rename to src/leveled_clerk.erl index 3d6b8c1..807a254 100644 --- a/src/leveled_housekeeping.erl +++ b/src/leveled_clerk.erl @@ -2,7 +2,7 @@ %% level and cleaning out of old files across a level --module(leveled_housekeeping). +-module(leveled_clerk). -export([merge_file/3, perform_merge/3]). diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl new file mode 100644 index 0000000..c192433 --- /dev/null +++ b/src/leveled_inker.erl @@ -0,0 +1,8 @@ +%% -------- Inker --------- +%% +%% +%% +%% -------- Ledger --------- +%% +%% + diff --git a/src/leveled_concierge.erl b/src/leveled_penciller.erl similarity index 72% rename from src/leveled_concierge.erl rename to src/leveled_penciller.erl index 8abb892..2ac3384 100644 --- a/src/leveled_concierge.erl +++ b/src/leveled_penciller.erl @@ -1,87 +1,28 @@ -%% -------- Overview --------- +%% -------- Penciller --------- %% -%% The eleveleddb is based on the LSM-tree similar to leveldb, except that: -%% - Values are kept seperately to Keys & Metadata -%% - Different file formats are used for value store (based on constant -%% database), and key store (based on sst) -%% - It is not intended to be general purpose, but be specifically suited for -%% use as a Riak backend in specific circumstances (relatively large values, -%% and frequent use of iterators) -%% - The Value store is an extended nursery log in leveldb terms. It is keyed -%% on the sequence number of the write -%% - The Key Store is a LSM tree, where the key is the actaul object key, and -%% the value is the metadata of the object including the sequence number +%% The penciller is repsonsible for writing and re-writing the ledger - a +%% persisted, ordered view of non-recent Keys and Metadata which have been +%% added to the store. +%% - The penciller maintains a manifest of all the files within the current +%% Ledger. +%% - The Penciller queues re-write (compaction) work up to be managed by Clerks +%% - The Penciller mainatins a register of iterators who have requested +%% snapshots of the Ledger +%% - The accepts new dumps (in the form of immutable ets tables) from the +%% Bookie, and calls the Bookie once the process of pencilling this data in +%% the Ledger is complete - and the Bookie is free to forget about the data %% -%% -------- Concierge & Manifest --------- +%% -------- Ledger --------- %% -%% The concierge is responsible for opening up the store, and keeps a manifest -%% of where items can be found. The manifest keeps a mapping of: -%% - Sequence Number ranges and the PID of the Value Store file that contains -%% that range -%% - Key ranges to PID mappings for each leval of the KeyStore -%% -%% -------- GET -------- -%% -%% A GET request for Key and Metadata requires a lookup in the KeyStore only. -%% - The concierge should consult the manifest for the lowest level to find -%% the PID which may contain the Key -%% - The concierge should ask the file owner if the Key is present, if not -%% present lower levels should be consulted until the objetc is found -%% -%% If a value is required, when the Key/Metadata has been fetched from the -%% KeyStore, the sequence number should be tkane, and matched in the ValueStore -%% manifest to find the right value. -%% -%% For recent PUTs the Key/Metadata is added into memory, and there is an -%% in-memory hash table for the entries in the most recent ValueStore CDB. -%% -%% -------- PUT -------- -%% -%% A PUT request must be persisted to the open (and append only) CDB file which -%% acts as a transaction log to persist the change. The Key & Metadata needs -%% also to be placed in memory. -%% -%% Once the CDB file is full, the managing process should be requested to -%% complete the lookup hash, and a new CDB file be started. -%% -%% Once the in-memory -%% -%% -------- Snapshots (Key Only) -------- -%% -%% If there is a iterator/snapshot request, the concierge will simply handoff a -%% copy of the manifest, and register the interest of the iterator at the -%% manifest sequence number at the time of the request. Iterators should -%% de-register themselves from the manager on completion. Iterators should be -%% automatically release after a timeout period. A file can be deleted if -%% there are no registered iterators from before the point the file was -%% removed from the manifest. -%% -%% -------- Snapshots (Key & Value) -------- -%% -%% -%% -%% -------- Special Ops -------- -%% -%% e.g. Get all for SegmentID/Partition -%% -%% -------- KeyStore --------- -%% -%% The concierge is responsible for controlling access to the store and -%% maintaining both an in-memory view and a persisted state of all the sft -%% files in use across the store. -%% -%% The store is divided into many levels -%% L0: May contain one, and only one sft file PID which is the most recent file -%% added to the top of the store. Access to the store will be stalled when a -%% second file is added whilst one still remains at this level. The target -%% size of L0 is therefore 0. +%% The Ledger is divided into many levels %% L1 - Ln: May contain multiple non-overlapping PIDs managing sft files. %% Compaction work should be sheduled if the number of files exceeds the target %% size of the level, where the target size is 8 ^ n. %% %% The most recent revision of a Key can be found by checking each level until -%% the key is found. To check a level the write file must be sought from the -%% manifest for that level, and then a call is made to that level. +%% the key is found. To check a level the correct file must be sought from the +%% manifest for that level, and then a call is made to that file. If the Key +%% is not present then every level should be checked. %% %% If a compaction change takes the size of a level beyond the target size, %% then compaction work for that level + 1 should be added to the compaction @@ -93,20 +34,19 @@ %% The compaction worker will always call the level manager to find out the %% highest priority work currently in the queue before proceeding. %% -%% When the compaction worker picks work off the queue it will take the current -%% manifest for the level and level - 1. The compaction worker will choose -%% which file to compact from level - 1, and once the compaction is complete -%% will call to the manager with the new version of the manifest to be written. +%% When the clerk picks work off the queue it will take the current manifest +%% for the level and level - 1. The clerk will choose which file to compact +%% from level - 1, and once the compaction is complete will call to the +%% Penciller with the new version of the manifest to be written. +%% %% Once the new version of the manifest had been persisted, the state of any %% deleted files will be changed to pending deletion. In pending deletion they -%% will call the manifets manager on a timeout to confirm that they are no -%% longer in use (by any iterators). +%% will call the Penciller on a timeout to confirm that they are no longer in +%% use (by any iterators). %% - - --module(leveled_concierge). +-module(leveled_penciller). %% -behaviour(gen_server). @@ -315,4 +255,3 @@ compaction_work_assessment_test() -> OngoingWork3 = lists:append(OngoingWork2, [{1, dummy_pid, os:timestamp()}]), WorkQ5 = assess_workqueue([], 0, [{0, []}, {1, L1Alt}], OngoingWork3), ?assertMatch(WorkQ5, []). - diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index ef1fd9d..04e08ba 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -150,10 +150,6 @@ handle_info/2, terminate/2, code_change/3, - speedtest_check_forsegment/4, - generate_randomsegfilter/1, - generate_randomkeys/1, - strip_to_keyonly/1, sft_new/4, sft_open/1, sft_get/2, @@ -1296,7 +1292,10 @@ generate_sequentialkeys(Target, Incr, Acc) -> {active, infinity}, null}, generate_sequentialkeys(Target, Incr + 1, [NextKey|Acc]). - +dummy_test() -> + R = speedtest_check_forsegment(a, 0, b, c), + ?assertMatch(R, true), + _ = generate_randomsegfilter(8). simple_create_block_test() -> KeyList1 = [{{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}, From 33f1efd5762771ee4d41e131b8a5170ac0102999 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 2 Aug 2016 17:51:43 +0100 Subject: [PATCH 024/167] Work on descriptions Add further descriptions of roles following name changes. Attempt to simplify manifest management in the Penciller by assuming there is only one Penciller's Clerk active - and so only one piece of work can be ongoing --- src/leveled_bookie.erl | 18 +-- src/leveled_inker.erl | 3 + src/leveled_penciller.erl | 267 ++++++++++++++++++++++++-------------- 3 files changed, 173 insertions(+), 115 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 321c7f8..84f7b5a 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -57,21 +57,9 @@ %% be added to the ledger. Initially this will be added to the Bookie's %% in-memory view of recent changes only. %% -%% The Bookie's memory consists of up to two in-memory ets tables -%% - the 'cmem' (current in-memory table) which is always subject to potential -%% change; -%% - the 'imem' (the immutable in-memory table) which is awaiting persistence -%% to the disk-based lsm-tree by the Penciller. -%% -%% The key and metadata should be written to the cmem store if it has -%% sufficient capacity, but this potentially should include the secondary key -%% changes which have been made as part of the transaction. -%% -%% If there is insufficient space in the cmem, the cmem should be converted -%% into the imem, and a new cmem be created. This requires the previous imem -%% to have been cleared from state due to compaction into the persisted Ledger -%% by the Penciller - otherwise the PUT is blocked. On creation of an imem, -%% the compaction process for that imem by the Penciller should be triggered. +%% The Bookie's memory consists of an in-memory ets table. Periodically, the +%% current table is pushed to the Penciller for eventual persistence, and a +%% new tabble is started. %% %% This completes the non-deferrable work associated with a PUT %% diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index c192433..7564ed4 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -6,3 +6,6 @@ %% %% + +-module(leveled_inker). + diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 2ac3384..0e1cfaa 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -1,4 +1,4 @@ -%% -------- Penciller --------- +%% -------- PENCILLER --------- %% %% The penciller is repsonsible for writing and re-writing the ledger - a %% persisted, ordered view of non-recent Keys and Metadata which have been @@ -12,10 +12,16 @@ %% Bookie, and calls the Bookie once the process of pencilling this data in %% the Ledger is complete - and the Bookie is free to forget about the data %% -%% -------- Ledger --------- +%% -------- LEDGER --------- %% %% The Ledger is divided into many levels -%% L1 - Ln: May contain multiple non-overlapping PIDs managing sft files. +%% L0: ETS tables are received from the Bookie and merged into a single ETS +%% table, until that table is the size of a SFT file, and it is then persisted +%% as a SFT file at this level. Once the persistence is completed, the ETS +%% table can be dropped. There can be only one SFT file at Level 0, so +%% the work to merge that file to the lower level must be the highest priority, +%% as otherwise the database will stall. +%% L1 TO L7: May contain multiple non-overlapping PIDs managing sft files. %% Compaction work should be sheduled if the number of files exceeds the target %% size of the level, where the target size is 8 ^ n. %% @@ -27,12 +33,14 @@ %% If a compaction change takes the size of a level beyond the target size, %% then compaction work for that level + 1 should be added to the compaction %% work queue. -%% Compaction work is fetched from the compaction worker because: +%% Compaction work is fetched by the Pencllier's Clerk because: %% - it has timed out due to a period of inactivity %% - it has been triggered by the a cast to indicate the arrival of high %% priority compaction work -%% The compaction worker will always call the level manager to find out the -%% highest priority work currently in the queue before proceeding. +%% The Penciller's Clerk (which performs compaction worker) will always call +%% the Penciller to find out the highest priority work currently in the queue +%% whenever it has either completed work, or a timeout has occurred since it +%% was informed there was no work to do. %% %% When the clerk picks work off the queue it will take the current manifest %% for the level and level - 1. The clerk will choose which file to compact @@ -44,13 +52,95 @@ %% will call the Penciller on a timeout to confirm that they are no longer in %% use (by any iterators). %% +%% ---------- PUSH ---------- +%% +%% The Penciller must support the PUSH of an ETS table from the Bookie. The +%% call to PUSH should be immediately acknowledged, and then work should be +%% completed to merge the ETS table into the L0 ETS table. +%% +%% The Penciller MUST NOT accept a new PUSH if the Clerk has commenced the +%% conversion of the current ETS table into a SFT file, but not completed this +%% change. This should prompt a stall. +%% +%% ---------- FETCH ---------- +%% +%% On request to fetch a key the Penciller should look first in the L0 ETS +%% table, and then look in the SFT files Level by Level, consulting the +%% Manifest to determine which file should be checked at each level. +%% +%% ---------- SNAPSHOT ---------- +%% +%% Iterators may request a snapshot of the database. To provide a snapshot +%% the Penciller must snapshot the ETS table, and then send this with a copy +%% of the manifest. +%% +%% Iterators requesting snapshots are registered by the Penciller, so that SFT +%% files valid at the point of the snapshot until either the iterator is +%% completed or has timed out. +%% +%% ---------- ON STARTUP ---------- +%% +%% On Startup the Bookie with ask the Penciller to initiate the Ledger first. +%% To initiate the Ledger the must consult the manifest, and then start a SFT +%% management process for each file in the manifest. +%% +%% The penciller should then try and read any persisted ETS table in the +%% on_shutdown folder. The Penciller must then discover the highest sequence +%% number in the ledger, and respond to the Bookie with that sequence number. +%% +%% The Bookie will ask the Inker for any Keys seen beyond that sequence number +%% before the startup of the overall store can be completed. +%% +%% ---------- ON SHUTDOWN ---------- +%% +%% On a controlled shutdown the Penciller should attempt to write any in-memory +%% ETS table to disk into the special ..on_shutdown folder +%% +%% ---------- FOLDER STRUCTURE ---------- +%% +%% The following folders are used by the Penciller +%% $ROOT/ledger_manifest/ - used for keeping manifest files +%% $ROOT/ledger_onshutdown/ - containing the persisted view of the ETS table +%% written on controlled shutdown +%% $ROOT/ledger_files/ - containing individual SFT files +%% +%% In larger stores there could be a large number of files in the ledger_file +%% folder - perhaps o(1000). It is assumed that modern file systems should +%% handle this efficiently. +%% +%% ---------- COMPACTION & MANIFEST UPDATES ---------- +%% +%% The Penciller can have one and only one Clerk for performing compaction +%% work. When the Clerk has requested and taken work, it should perform the +%5 compaction work starting the new SFT process to manage the new Ledger state +%% and then write a new manifest file that represents that state with using +%% The MergeID as the filename .pnd. +%% +%% Prior to completing the work the previous manifest file should be renamed +%% to the filename .bak, and any .bak files other than the +%% the most recent n files should be deleted. +%% +%% The Penciller on accepting the change should rename the manifest file to +%% '.crr'. +%% +%% On startup, the Penciller should look first for a *.crr file, and if +%% one is not present it should promot the most recently modified *.bak - +%% checking first that all files referenced in it are still present. +%% +%% The pace at which the store can accept updates will be dependent on the +%% speed at which the Penciller's Clerk can merge files at lower levels plus +%% the time it takes to merge from Level 0. As if a clerk has commenced +%% compaction work at a lower level and then immediately a L0 SFT file is +%% written the Penciller will need to wait for this compaction work to +%% complete and the L0 file to be compacted before the ETS table can be +%% allowed to again reach capacity -module(leveled_penciller). %% -behaviour(gen_server). --export([return_work/2, commit_manifest_change/7]). +-export([return_work/2, commit_manifest_change/5]). -include_lib("eunit/include/eunit.hrl"). @@ -58,15 +148,22 @@ {4, 4096}, {5, 32768}, {6, 262144}, {7, infinity}]). -define(MAX_LEVELS, 8). -define(MAX_WORK_WAIT, 300). --define(MANIFEST_FP, "manifest"). --define(FILES_FP, "files"). +-define(MANIFEST_FP, "ledger_manifest"). +-define(FILES_FP, "ledger_files"). +-define(SHUTDOWN_FP, "ledger_onshutdown"). +-define(CURRENT_FILEX, "crr"). +-define(PENDING_FILEX, "pnd"). +-define(BACKUP_FILEX, "bak"). +-define(ARCHIVE_FILEX, "arc"). --record(state, {level_fileref :: list(), +-record(state, {manifest :: list(), ongoing_work :: list(), manifest_sqn :: integer(), registered_iterators :: list(), unreferenced_files :: list(), - root_path :: string()}). + root_path :: string(), + mem :: ets:tid()}). + %% Work out what the current work queue should be @@ -76,37 +173,42 @@ %% to look at work at that level return_work(State, From) -> - OngoingWork = State#state.ongoing_work, - WorkQueue = assess_workqueue([], - 0, - State#state.level_fileref, - OngoingWork), - case length(WorkQueue) of - L when L > 0 -> - [{SrcLevel, SrcManifest, SnkManifest}|OtherWork] = WorkQueue, - UpdatedWork = lists:append(OngoingWork, - [{SrcLevel, From, os:timestamp()}, - {SrcLevel + 1, From, os:timestamp()}]), - io:format("Work at Level ~w to be scheduled for ~w with ~w queue - items outstanding", [SrcLevel, From, length(OtherWork)]), - {State#state{ongoing_work=UpdatedWork}, - {SrcLevel, SrcManifest, SnkManifest}}; - _ -> + case State#state.ongoing_work of + [] -> + WorkQueue = assess_workqueue([], + 0, + State#state.manifest, + []), + case length(WorkQueue) of + L when L > 0 -> + [{SrcLevel, Manifest}|OtherWork] = WorkQueue, + io:format("Work at Level ~w to be scheduled for ~w + with ~w queue items outstanding~n", + [SrcLevel, From, length(OtherWork)]), + {State#state{ongoing_work={SrcLevel, From, os:timestamp()}}, + {SrcLevel, Manifest}}; + _ -> + {State, none} + end; + [{SrcLevel, OtherFrom, _TS}|T] -> + io:format("Ongoing work requested by ~w but work + outstanding from Level ~w and Clerk ~w with + ~w other items outstanding~n", + [From, SrcLevel, OtherFrom, length(T)]), {State, none} end. - -assess_workqueue(WorkQ, ?MAX_LEVELS - 1, _LevelFileRef, _OngoingWork) -> +assess_workqueue(WorkQ, ?MAX_LEVELS - 1, _Manifest, _OngoingWork) -> WorkQ; -assess_workqueue(WorkQ, LevelToAssess, LevelFileRef, OngoingWork)-> +assess_workqueue(WorkQ, LevelToAssess, Manifest, OngoingWork)-> MaxFiles = get_item(LevelToAssess, ?LEVEL_SCALEFACTOR, 0), - FileCount = length(get_item(LevelToAssess, LevelFileRef, [])), - NewWQ = maybe_append_work(WorkQ, LevelToAssess, LevelFileRef, MaxFiles, + FileCount = length(get_item(LevelToAssess, Manifest, [])), + NewWQ = maybe_append_work(WorkQ, LevelToAssess, Manifest, MaxFiles, FileCount, OngoingWork), - assess_workqueue(NewWQ, LevelToAssess + 1, LevelFileRef, OngoingWork). + assess_workqueue(NewWQ, LevelToAssess + 1, Manifest, OngoingWork). -maybe_append_work(WorkQ, Level, LevelFileRef, +maybe_append_work(WorkQ, Level, Manifest, MaxFiles, FileCount, OngoingWork) when FileCount > MaxFiles -> io:format("Outstanding compaction work items of ~w at level ~w~n", @@ -117,11 +219,9 @@ maybe_append_work(WorkQ, Level, LevelFileRef, outstanding work with ~w assigned at ~w~n", [Pid, TS]), WorkQ; false -> - lists:append(WorkQ, [{Level, - get_item(Level, LevelFileRef, []), - get_item(Level + 1, LevelFileRef, [])}]) + lists:append(WorkQ, [{Level, Manifest}]) end; -maybe_append_work(WorkQ, Level, _LevelFileRef, +maybe_append_work(WorkQ, Level, _Manifest, _MaxFiles, FileCount, _OngoingWork) -> io:format("No compaction work due to file count ~w at level ~w~n", [FileCount, Level]), @@ -139,55 +239,37 @@ get_item(Index, List, Default) -> %% Request a manifest change %% Should be passed the -%% - {SrcLevel, NewSrcManifest, NewSnkManifest, ClearedFiles, MergeID, From, -%% State} +%% - {SrcLevel, NewManifest, ClearedFiles, MergeID, From, State} %% To complete a manifest change need to: %% - Update the Manifest Sequence Number (msn) %% - Confirm this Pid has a current element of manifest work outstanding at %% that level -%% - Rename the manifest file created under the MergeID (.) -%% at the sink Level to be the current manifest file (current_.) -%% -------- NOTE -------- -%% If there is a crash between these two points, the K/V data that has been -%% merged from the source level will now be in both the source and the sink -%% level. Therefore in store operations this potential duplication must be -%% handled. -%% -------- NOTE -------- -%% - Rename the manifest file created under the MergeID (.) -%% at the source level to the current manifest file (current_.) +%% - Rename the manifest file created under the MergeID (.manifest) +%% to the filename current.manifest %% - Update the state of the LevelFileRef lists %% - Add the ClearedFiles to the list of files to be cleared (as a tuple with %% the new msn) -commit_manifest_change(SrcLevel, NewSrcMan, NewSnkMan, ClearedFiles, - MergeID, From, State) -> +commit_manifest_change(NewManifest, ClearedFiles, MergeID, From, State) -> NewMSN = State#state.manifest_sqn + 1, OngoingWork = State#state.ongoing_work, RootPath = State#state.root_path, - SnkLevel = SrcLevel + 1, - case {lists:keyfind(SrcLevel, 1, OngoingWork), - lists:keyfind(SrcLevel + 1, 1, OngoingWork)} of - {{SrcLevel, From, TS}, {SnkLevel, From, TS}} -> - io:format("Merge ~s was a success in ~w microseconds", - [MergeID, timer:diff_now(os:timestamp(), TS)]), - OutstandingWork = lists:keydelete(SnkLevel, 1, - lists:keydelete(SrcLevel, 1, OngoingWork)), - ok = rename_manifest_files(RootPath, MergeID, - NewMSN, SrcLevel, SnkLevel), - NewLFR = update_levelfileref(NewSrcMan, - NewSnkMan, - SrcLevel, - State#state.level_fileref), - UnreferencedFiles = update_deletions(ClearedFiles, - NewMSN, - State#state.unreferenced_files), + UnreferencedFiles = State#state.unreferenced_files, + case OngoingWork of + {SrcLevel, From, TS} -> + io:format("Merge ~s completed in ~w microseconds at Level ~w~n", + [MergeID, timer:diff_now(os:timestamp(), TS), SrcLevel]), + ok = rename_manifest_files(RootPath, MergeID), + UnreferencedFilesUpd = update_deletions(ClearedFiles, + NewMSN, + UnreferencedFiles), io:format("Merge ~s has been commmitted at sequence number ~w~n", [MergeID, NewMSN]), - {ok, State#state{ongoing_work=OutstandingWork, + {ok, State#state{ongoing_work=null, manifest_sqn=NewMSN, - level_fileref=NewLFR, - unreferenced_files=UnreferencedFiles}}; + manifest=NewManifest, + unreferenced_files=UnreferencedFilesUpd}}; _ -> io:format("Merge commit ~s not matched to known work~n", [MergeID]), @@ -196,27 +278,14 @@ commit_manifest_change(SrcLevel, NewSrcMan, NewSnkMan, ClearedFiles, -rename_manifest_files(RootPath, MergeID, NewMSN, SrcLevel, SnkLevel) -> +rename_manifest_files(RootPath, MergeID) -> ManifestFP = RootPath ++ "/" ++ ?MANIFEST_FP ++ "/", ok = file:rename(ManifestFP ++ MergeID - ++ "." ++ integer_to_list(SnkLevel), - ManifestFP ++ "current_" ++ integer_to_list(SnkLevel) - ++ "." ++ integer_to_list(NewMSN)), - ok = file:rename(ManifestFP ++ MergeID - ++ "." ++ integer_to_list(SrcLevel), - ManifestFP ++ "current_" ++ integer_to_list(SrcLevel) - ++ "." ++ integer_to_list(NewMSN)), + ++ "." ++ ?PENDING_FILEX, + ManifestFP ++ MergeID + ++ "." ++ ?CURRENT_FILEX), ok. -update_levelfileref(NewSrcMan, NewSinkMan, SrcLevel, CurrLFR) -> - lists:keyreplace(SrcLevel + 1, - 1, - lists:keyreplace(SrcLevel, - 1, - CurrLFR, - {SrcLevel, NewSrcMan}), - {SrcLevel + 1, NewSinkMan}). - update_deletions([], _NewMSN, UnreferencedFiles) -> UnreferencedFiles; update_deletions([ClearedFile|Tail], MSN, UnreferencedFiles) -> @@ -233,12 +302,12 @@ compaction_work_assessment_test() -> L0 = [{{o, "B1", "K1"}, {o, "B3", "K3"}, dummy_pid}], L1 = [{{o, "B1", "K1"}, {o, "B2", "K2"}, dummy_pid}, {{o, "B2", "K3"}, {o, "B4", "K4"}, dummy_pid}], - LevelFileRef = [{0, L0}, {1, L1}], + Manifest = [{0, L0}, {1, L1}], OngoingWork1 = [], - WorkQ1 = assess_workqueue([], 0, LevelFileRef, OngoingWork1), - ?assertMatch(WorkQ1, [{0, L0, L1}]), + WorkQ1 = assess_workqueue([], 0, Manifest, OngoingWork1), + ?assertMatch(WorkQ1, [{0, Manifest}]), OngoingWork2 = [{0, dummy_pid, os:timestamp()}], - WorkQ2 = assess_workqueue([], 0, LevelFileRef, OngoingWork2), + WorkQ2 = assess_workqueue([], 0, Manifest, OngoingWork2), ?assertMatch(WorkQ2, []), L1Alt = lists:append(L1, [{{o, "B5", "K0001"}, {o, "B5", "K9999"}, dummy_pid}, @@ -248,10 +317,8 @@ compaction_work_assessment_test() -> {{o, "B9", "K0001"}, {o, "B9", "K9999"}, dummy_pid}, {{o, "BA", "K0001"}, {o, "BA", "K9999"}, dummy_pid}, {{o, "BB", "K0001"}, {o, "BB", "K9999"}, dummy_pid}]), - WorkQ3 = assess_workqueue([], 0, [{0, []}, {1, L1Alt}], OngoingWork1), - ?assertMatch(WorkQ3, [{1, L1Alt, []}]), - WorkQ4 = assess_workqueue([], 0, [{0, []}, {1, L1Alt}], OngoingWork2), - ?assertMatch(WorkQ4, [{1, L1Alt, []}]), - OngoingWork3 = lists:append(OngoingWork2, [{1, dummy_pid, os:timestamp()}]), - WorkQ5 = assess_workqueue([], 0, [{0, []}, {1, L1Alt}], OngoingWork3), - ?assertMatch(WorkQ5, []). + Manifest3 = [{0, []}, {1, L1Alt}], + WorkQ3 = assess_workqueue([], 0, Manifest3, OngoingWork1), + ?assertMatch(WorkQ3, [{1, Manifest3}]), + WorkQ4 = assess_workqueue([], 0, Manifest3, OngoingWork2), + ?assertMatch(WorkQ4, [{1, Manifest3}]). From 75996b90ca2e7bec779b0514cb14aaa3aacb1e5c Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 2 Aug 2016 17:54:13 +0100 Subject: [PATCH 025/167] Remove file Not needed yet --- src/eleveleddb.app.src | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 src/eleveleddb.app.src diff --git a/src/eleveleddb.app.src b/src/eleveleddb.app.src deleted file mode 100644 index 1be8c00..0000000 --- a/src/eleveleddb.app.src +++ /dev/null @@ -1,17 +0,0 @@ -{application, eleveleddb, - [ - {description, ""}, - {vsn, "0.0.1"}, - {modules, []}, - {registered, []}, - {applications, [ - kernel, - stdlib - ]}, - {mod, {eleveleddb_app, []}}, - {env, [ - %% Default max file size (in bytes) - {max_file_size, 32#80000000}, % 4GB default - - ]} - ]}. \ No newline at end of file From 718425633a5bf7b72c3b19bae3c412747fb6c08a Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 9 Aug 2016 16:09:29 +0100 Subject: [PATCH 026/167] Penciller accepting push Standardise on record definitions between modules to make easier - then add functionality to pushing to penciller as bookie would do. Some initial manual testing of this seems OK. --- include/leveled.hrl | 21 ++ src/leveled_clerk.erl | 226 +++++++++++++++++---- src/leveled_penciller.erl | 400 +++++++++++++++++++++++++++++--------- src/leveled_sft.erl | 103 +++++++++- 4 files changed, 612 insertions(+), 138 deletions(-) create mode 100644 include/leveled.hrl diff --git a/include/leveled.hrl b/include/leveled.hrl new file mode 100644 index 0000000..fdf779c --- /dev/null +++ b/include/leveled.hrl @@ -0,0 +1,21 @@ + +-record(sft_options, + {wait = true :: boolean(), + expire_tombstones = false :: boolean()}). + +-record(penciller_work, + {next_sqn :: integer(), + clerk :: pid(), + src_level :: integer(), + manifest :: list(), + start_time :: tuple(), + ledger_filepath :: string(), + manifest_file :: string(), + new_manifest :: list(), + unreferenced_files :: list()}). + +-record(manifest_entry, + {start_key :: tuple(), + end_key :: tuple(), + owner :: pid(), + filename :: string()}). \ No newline at end of file diff --git a/src/leveled_clerk.erl b/src/leveled_clerk.erl index 807a254..3197eb4 100644 --- a/src/leveled_clerk.erl +++ b/src/leveled_clerk.erl @@ -1,19 +1,156 @@ -%% Controlling asynchronour work in leveleddb to manage compaction within a +%% Controlling asynchronous work in leveleddb to manage compaction within a %% level and cleaning out of old files across a level -module(leveled_clerk). --export([merge_file/3, perform_merge/3]). +-behaviour(gen_server). + +-include("../include/leveled.hrl"). + +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + clerk_new/0, + clerk_prompt/2, + code_change/3, + perform_merge/4]). -include_lib("eunit/include/eunit.hrl"). +-record(state, {owner :: pid()}). -merge_file(_FileToMerge, _ManifestMgr, _Level) -> - %% CandidateList = leveled_manifest:get_manifest_atlevel(ManifestMgr, Level), - %% [Adds, Removes] = perform_merge(FileToMerge, CandidateList, Level), - %%leveled_manifest:update_manifest_atlevel(ManifestMgr, Level, Adds, Removes), +%%%============================================================================ +%%% API +%%%============================================================================ + +clerk_new() -> + {ok, Pid} = gen_server:start(?MODULE, [], []), + ok = gen_server:call(Pid, register, infinity), + {ok, Pid}. + + +clerk_prompt(Pid, penciller) -> + gen_server:cast(Pid, penciller_prompt, infinity). + +%%%============================================================================ +%%% gen_server callbacks +%%%============================================================================ + +init([]) -> + {ok, #state{}}. + +handle_call(register, From, State) -> + {noreply, State#state{owner=From}}. + +handle_cast({penciller_prompt, From}, State) -> + case leveled_penciller:pcl_workforclerk(State#state.owner) of + none -> + io:format("Work prompted but none needed~n"), + {noreply, State}; + WI -> + {NewManifest, FilesToDelete} = merge(WI), + UpdWI = WI#penciller_work{new_manifest=NewManifest, + unreferenced_files=FilesToDelete}, + leveled_penciller:pcl_requestmanifestchange(From, UpdWI), + {noreply, State} + end. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + + +%%%============================================================================ +%%% Internal functions +%%%============================================================================ + +merge(WI) -> + SrcLevel = WI#penciller_work.src_level, + {Selection, UpdMFest1} = select_filetomerge(SrcLevel, + WI#penciller_work.manifest), + {{StartKey, EndKey}, SrcFile} = Selection, + SrcFilename = leveled_sft:sft_getfilename(SrcFile), + SinkFiles = get_item(SrcLevel + 1, UpdMFest1, []), + SplitLists = lists:splitwith(fun(Ref) -> + case {Ref#manifest_entry.start_key, + Ref#manifest_entry.end_key} of + {_, EK} when StartKey > EK -> + false; + {SK, _} when EndKey < SK -> + false; + _ -> + true + end end, + SinkFiles), + {Candidates, Others} = SplitLists, + + %% TODO: + %% Need to work out if this is the top level + %% And then tell merge process to create files at the top level + %% Which will include the reaping of expired tombstones + + io:format("Merge from level ~w to merge into ~w files below", + [SrcLevel, length(Candidates)]), + + MergedFiles = case length(Candidates) of + 0 -> + %% If no overlapping candiates, manifest change only required + %% + %% TODO: need to think still about simply renaming when at + %% lower level + [SrcFile]; + _ -> + perform_merge({SrcFile, SrcFilename}, + Candidates, + SrcLevel, + {WI#penciller_work.ledger_filepath, + WI#penciller_work.next_sqn}) + end, + + NewLevel = lists:sort(lists:append(MergedFiles, Others)), + UpdMFest2 = lists:keyreplace(SrcLevel + 1, + 1, + UpdMFest1, + {SrcLevel, NewLevel}), + + {ok, Handle} = file:open(WI#penciller_work.manifest_file, + [binary, raw, write]), + ok = file:write(Handle, term_to_binary(UpdMFest2)), + ok = file:close(Handle), + {UpdMFest2, Candidates}. + + +%% An algorithm for discovering which files to merge .... +%% We can find the most optimal file: +%% - The one with the most overlapping data below? +%% - The one that overlaps with the fewest files below? +%% - The smallest file? +%% We could try and be fair in some way (merge oldest first) +%% Ultimately, there is alack of certainty that being fair or optimal is +%% genuinely better - ultimately every file has to be compacted. +%% +%% Hence, the initial implementation is to select files to merge at random + +select_filetomerge(SrcLevel, Manifest) -> + {SrcLevel, LevelManifest} = lists:keyfind(SrcLevel, 1, Manifest), + Selected = lists:nth(random:uniform(length(LevelManifest)), + LevelManifest), + UpdManifest = lists:keyreplace(SrcLevel, + 1, + Manifest, + {SrcLevel, + lists:delete(Selected, + LevelManifest)}), + {Selected, UpdManifest}. + %% Assumption is that there is a single SFT from a higher level that needs @@ -36,41 +173,45 @@ merge_file(_FileToMerge, _ManifestMgr, _Level) -> %% %% The level is the level which the new files should be created at. -perform_merge(FileToMerge, CandidateList, Level) -> +perform_merge(FileToMerge, CandidateList, Level, {Filepath, MSN}) -> {Filename, UpperSFTPid} = FileToMerge, - MergeID = generate_merge_id(Filename, Level), - io:format("Merge to be commenced for FileToMerge=~s with MergeID=~s~n", - [Filename, MergeID]), + io:format("Merge to be commenced for FileToMerge=~s with MSN=~w~n", + [Filename, MSN]), PointerList = lists:map(fun(P) -> {next, P, all} end, CandidateList), do_merge([{next, UpperSFTPid, all}], - PointerList, Level, MergeID, 0, []). + PointerList, Level, {Filepath, MSN}, 0, []). -do_merge([], [], Level, MergeID, FileCounter, OutList) -> - io:format("Merge completed with MergeID=~s Level=~w and FileCounter=~w~n", - [MergeID, Level, FileCounter]), +do_merge([], [], Level, {_Filepath, MSN}, FileCounter, OutList) -> + io:format("Merge completed with MSN=~w Level=~w and FileCounter=~w~n", + [MSN, Level, FileCounter]), OutList; -do_merge(KL1, KL2, Level, MergeID, FileCounter, OutList) -> - FileName = lists:flatten(io_lib:format("../test/~s_~w.sft", [MergeID, FileCounter])), - io:format("File to be created as part of MergeID=~s Filename=~s~n", [MergeID, FileName]), +do_merge(KL1, KL2, Level, {Filepath, MSN}, FileCounter, OutList) -> + FileName = lists:flatten(io_lib:format(Filepath ++ "_~w_~w.sft", + [Level, FileCounter])), + io:format("File to be created as part of MSN=~w Filename=~s~n", + [MSN, FileName]), case leveled_sft:sft_new(FileName, KL1, KL2, Level) of {ok, _Pid, {error, Reason}} -> io:format("Exiting due to error~w~n", [Reason]); {ok, Pid, Reply} -> {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, - do_merge(KL1Rem, KL2Rem, Level, MergeID, FileCounter + 1, - lists:append(OutList, [{SmallestKey, HighestKey, Pid}])) + ExtMan = lists:append(OutList, + [#manifest_entry{start_key=SmallestKey, + end_key=HighestKey, + owner=Pid, + filename=FileName}]), + do_merge(KL1Rem, KL2Rem, Level, {Filepath, MSN}, + FileCounter + 1, ExtMan) end. -generate_merge_id(Filename, Level) -> - <> = crypto:rand_bytes(14), - FileID = erlang:phash2(Filename, 256), - B = FileID * 256 + Level, - Str = io_lib:format("~8.16.0b-~4.16.0b-4~3.16.0b-~4.16.0b-~12.16.0b", - [A, B, C band 16#0fff, D band 16#3fff bor 16#8000, E]), - list_to_binary(Str). - - +get_item(Index, List, Default) -> + case lists:keysearch(Index, 1, List) of + {value, {Index, Value}} -> + Value; + false -> + Default + end. %%%============================================================================ @@ -94,9 +235,10 @@ generate_randomkeys(Count, Acc, BucketLow, BRange) -> {active, infinity}, null}, generate_randomkeys(Count - 1, [RandKey|Acc], BucketLow, BRange). -choose_pid_toquery([{StartKey, EndKey, Pid}|_T], Key) when Key >= StartKey, - EndKey >= Key -> - Pid; +choose_pid_toquery([ManEntry|_T], Key) when + Key >= ManEntry#manifest_entry.start_key, + ManEntry#manifest_entry.end_key >= Key -> + ManEntry#manifest_entry.owner; choose_pid_toquery([_H|T], Key) -> choose_pid_toquery(T, Key). @@ -138,10 +280,12 @@ merge_file_test() -> KL4_L2, [], 2), Result = perform_merge({"../test/KL1_L1.sft", PidL1_1}, [PidL2_1, PidL2_2, PidL2_3, PidL2_4], - 2), - lists:foreach(fun({{o, B1, K1}, {o, B2, K2}, R}) -> + 2, {"../test/", 99}), + lists:foreach(fun(ManEntry) -> + {o, B1, K1} = ManEntry#manifest_entry.start_key, + {o, B2, K2} = ManEntry#manifest_entry.end_key, io:format("Result of ~s ~s and ~s ~s with Pid ~w~n", - [B1, K1, B2, K2, R]) end, + [B1, K1, B2, K2, ManEntry#manifest_entry.owner]) end, Result), io:format("Finding keys in KL1_L1~n"), ok = find_randomkeys(Result, 50, KL1_L1), @@ -158,4 +302,16 @@ merge_file_test() -> leveled_sft:sft_clear(PidL2_2), leveled_sft:sft_clear(PidL2_3), leveled_sft:sft_clear(PidL2_4), - lists:foreach(fun({_StK, _EndK, Pid}) -> leveled_sft:sft_clear(Pid) end, Result). + lists:foreach(fun(ManEntry) -> + leveled_sft:sft_clear(ManEntry#manifest_entry.owner) end, + Result). + + +select_merge_file_test() -> + L0 = [{{o, "B1", "K1"}, {o, "B3", "K3"}, dummy_pid}], + L1 = [{{o, "B1", "K1"}, {o, "B2", "K2"}, dummy_pid}, + {{o, "B2", "K3"}, {o, "B4", "K4"}, dummy_pid}], + Manifest = [{0, L0}, {1, L1}], + {FileRef, NewManifest} = select_filetomerge(0, Manifest), + ?assertMatch(FileRef, {{o, "B1", "K1"}, {o, "B3", "K3"}, dummy_pid}), + ?assertMatch(NewManifest, [{0, []}, {1, L1}]). \ No newline at end of file diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 0e1cfaa..5e75e4a 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -114,18 +114,14 @@ %% work. When the Clerk has requested and taken work, it should perform the %5 compaction work starting the new SFT process to manage the new Ledger state %% and then write a new manifest file that represents that state with using -%% The MergeID as the filename .pnd. -%% -%% Prior to completing the work the previous manifest file should be renamed -%% to the filename .bak, and any .bak files other than the -%% the most recent n files should be deleted. +%% the next Manifest sequence number as the filename: +%% - nonzero_.pnd %% -%% The Penciller on accepting the change should rename the manifest file to -%% '.crr'. +%% The Penciller on accepting the change should rename the manifest file to - +%% - nonzero_.crr %% -%% On startup, the Penciller should look first for a *.crr file, and if -%% one is not present it should promot the most recently modified *.bak - -%% checking first that all files referenced in it are still present. +%% On startup, the Penciller should look for the nonzero_*.crr file with the +%% highest such manifest sequence number. %% %% The pace at which the store can accept updates will be dependent on the %% speed at which the Penciller's Clerk can merge files at lower levels plus @@ -134,13 +130,36 @@ %% written the Penciller will need to wait for this compaction work to %% complete and the L0 file to be compacted before the ETS table can be %% allowed to again reach capacity +%% +%% The writing of L0 files do not require the involvement of the clerk. +%% The L0 files are prompted directly by the penciller when the in-memory ets +%% table has reached capacity. When there is a next push into memory the +%% penciller calls to check that the file is now active (which may pause if the +%% write is ongoing the acceptence of the push), and if so it can clear the ets +%% table and build a new table starting with the remainder, and the keys from +%% the latest push. +%% -module(leveled_penciller). -%% -behaviour(gen_server). +-behaviour(gen_server). --export([return_work/2, commit_manifest_change/5]). +-include("../include/leveled.hrl"). + +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3, + pcl_new/0, + pcl_start/1, + pcl_pushmem/2, + pcl_fetch/2, + pcl_workforclerk/1, + pcl_requestmanifestchange/2, + commit_manifest_change/3]). -include_lib("eunit/include/eunit.hrl"). @@ -155,14 +174,186 @@ -define(PENDING_FILEX, "pnd"). -define(BACKUP_FILEX, "bak"). -define(ARCHIVE_FILEX, "arc"). +-define(MEMTABLE, mem). +-define(MAX_TABLESIZE, 32000). +-define(L0PEND_RESET, {false, [], none}). --record(state, {manifest :: list(), - ongoing_work :: list(), - manifest_sqn :: integer(), - registered_iterators :: list(), - unreferenced_files :: list(), - root_path :: string(), - mem :: ets:tid()}). +-record(state, {manifest = [] :: list(), + ongoing_work = [] :: list(), + manifest_sqn = 0 :: integer(), + levelzero_sqn =0 :: integer(), + registered_iterators = [] :: list(), + unreferenced_files = [] :: list(), + root_path = "../test/" :: string(), + table_size = 0 :: integer(), + clerk :: pid(), + levelzero_pending = {false, [], none} :: tuple(), + memtable}). + + +%%%============================================================================ +%%% API +%%%============================================================================ + +pcl_new() -> + gen_server:start(?MODULE, [], []). + +pcl_start(_RootDir) -> + %% TODO + %% Need to call startup to rebuild from disk + ok. + +pcl_pushmem(Pid, DumpList) -> + %% Bookie to dump memory onto penciller + gen_server:call(Pid, {push_mem, DumpList}, infinity). + +pcl_fetch(Pid, Key) -> + gen_server:call(Pid, {fetch, Key}, infinity). + +pcl_workforclerk(Pid) -> + gen_server:call(Pid, work_for_clerk, infinity). + +pcl_requestmanifestchange(Pid, WorkItem) -> + gen_server:call(Pid, {manifest_change, WorkItem}, infinity). + +%%%============================================================================ +%%% gen_server callbacks +%%%============================================================================ + +init([]) -> + TID = ets:new(?MEMTABLE, [ordered_set, private]), + {ok, #state{memtable=TID}}. + +handle_call({push_mem, DumpList}, _From, State) -> + {TableSize, Manifest, L0Pend} = case State#state.levelzero_pending of + {true, Remainder, {StartKey, EndKey, Pid}} -> + %% Need to handle not error scenarios? + %% N.B. Sync call - so will be ready + ok = leveled_sft:sft_checkready(Pid), + %% Reset ETS, but re-insert any remainder + true = ets:delete_all_objects(State#state.memtable), + true = ets:insert(State#state.memtable, Remainder), + {length(Remainder), + lists:keystore(0, + 1, + State#state.manifest, + {0, [{StartKey, EndKey, Pid}]}), + ?L0PEND_RESET}; + {false, _, _} -> + {State#state.table_size, + State#state.manifest, + State#state.levelzero_pending}; + Unexpected -> + io:format("Unexpected value of ~w~n", [Unexpected]), + error + end, + case do_push_to_mem(DumpList, TableSize, State#state.memtable) of + {twist, ApproxTableSize} -> + {reply, ok, State#state{table_size=ApproxTableSize, + manifest=Manifest, + levelzero_pending=L0Pend}}; + {roll, ApproxTableSize} -> + case {get_item(0, Manifest, []), L0Pend} of + {[], ?L0PEND_RESET} -> + L0SN = State#state.levelzero_sqn + 1, + FileName = State#state.root_path + ++ ?FILES_FP ++ "/" + ++ integer_to_list(L0SN), + SFT = leveled_sft:sft_new(FileName, + ets:tab2list(State#state.memtable), + [], + 0, + #sft_options{wait=false}), + {ok, L0Pid, Reply} = SFT, + {{KL1Rem, []}, L0StartKey, L0EndKey} = Reply, + {reply, ok, State#state{levelzero_pending={true, + KL1Rem, + {L0StartKey, + L0EndKey, + L0Pid}}, + table_size=ApproxTableSize, + levelzero_sqn=L0SN}}; + _ -> + io:format("Memory has exceeded limit but L0 file is still + awaiting compaction ~n"), + {reply, pause, State#state{table_size=ApproxTableSize, + manifest=Manifest, + levelzero_pending=L0Pend}} + end + end; +handle_call({fetch, Key}, _From, State) -> + {reply, fetch(Key, State#state.manifest, State#state.memtable), State}; +handle_call(work_for_clerk, From, State) -> + {UpdState, Work} = return_work(State, From), + {reply, Work, UpdState}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + + +%%%============================================================================ +%%% Internal functions +%%%============================================================================ + + +fetch(Key, Manifest, TID) -> + case ets:lookup(TID, Key) of + [Object] -> + Object; + [] -> + fetch(Key, Manifest, 0, fun leveled_sft:sft_get/2) + end. + +fetch(_Key, _Manifest, ?MAX_LEVELS + 1, _FetchFun) -> + not_present; +fetch(Key, Manifest, Level, FetchFun) -> + LevelManifest = get_item(Level, Manifest, []), + case lists:foldl(fun(File, Acc) -> + case Acc of + not_present when + Key >= File#manifest_entry.start_key, + File#manifest_entry.end_key >= Key -> + File#manifest_entry.owner; + PidFound -> + PidFound + end end, + not_present, + LevelManifest) of + not_present -> + fetch(Key, Manifest, Level + 1, FetchFun); + FileToCheck -> + case FetchFun(FileToCheck, Key) of + not_present -> + fetch(Key, Manifest, Level + 1, FetchFun); + ObjectFound -> + ObjectFound + end + end. + +do_push_to_mem(DumpList, TableSize, MemTable) -> + ets:insert(MemTable, DumpList), + case TableSize + length(DumpList) of + ApproxTableSize when ApproxTableSize > ?MAX_TABLESIZE -> + case ets:info(MemTable, size) of + ActTableSize when ActTableSize > ?MAX_TABLESIZE -> + {roll, ActTableSize}; + ActTableSize -> + io:format("Table size is actually ~w~n", [ActTableSize]), + {twist, ActTableSize} + end; + ApproxTableSize -> + io:format("Table size is approximately ~w~n", [ApproxTableSize]), + {twist, ApproxTableSize} + end. @@ -173,56 +364,70 @@ %% to look at work at that level return_work(State, From) -> - case State#state.ongoing_work of - [] -> - WorkQueue = assess_workqueue([], - 0, - State#state.manifest, - []), - case length(WorkQueue) of - L when L > 0 -> - [{SrcLevel, Manifest}|OtherWork] = WorkQueue, - io:format("Work at Level ~w to be scheduled for ~w - with ~w queue items outstanding~n", - [SrcLevel, From, length(OtherWork)]), - {State#state{ongoing_work={SrcLevel, From, os:timestamp()}}, - {SrcLevel, Manifest}}; - _ -> + WorkQueue = assess_workqueue([], + 0, + State#state.manifest), + case length(WorkQueue) of + L when L > 0 -> + [{SrcLevel, Manifest}|OtherWork] = WorkQueue, + io:format("Work at Level ~w to be scheduled for ~w + with ~w queue items outstanding~n", + [SrcLevel, From, length(OtherWork)]), + case State#state.ongoing_work of + [] -> + %% No work currently outstanding + %% Can allocate work + NextSQN = State#state.manifest_sqn + 1, + FP = filepath(State#state.root_path, + NextSQN, + new_merge_files), + ManFile = filepath(State#state.root_path, + NextSQN, + pending_manifest), + WI = #penciller_work{next_sqn=NextSQN, + clerk=From, + src_level=SrcLevel, + manifest=Manifest, + start_time = os:timestamp(), + ledger_filepath = FP, + manifest_file = ManFile}, + {State#state{ongoing_work=[WI]}, WI}; + [OutstandingWork] -> + %% Still awaiting a response + io:format("Ongoing work requested by ~w but work + outstanding from Level ~w and Clerk ~w + at sequence number ~w~n", + [From, + OutstandingWork#penciller_work.src_level, + OutstandingWork#penciller_work.clerk, + OutstandingWork#penciller_work.next_sqn]), {State, none} end; - [{SrcLevel, OtherFrom, _TS}|T] -> - io:format("Ongoing work requested by ~w but work - outstanding from Level ~w and Clerk ~w with - ~w other items outstanding~n", - [From, SrcLevel, OtherFrom, length(T)]), + _ -> {State, none} end. -assess_workqueue(WorkQ, ?MAX_LEVELS - 1, _Manifest, _OngoingWork) -> + + + +assess_workqueue(WorkQ, ?MAX_LEVELS - 1, _Manifest) -> WorkQ; -assess_workqueue(WorkQ, LevelToAssess, Manifest, OngoingWork)-> +assess_workqueue(WorkQ, LevelToAssess, Manifest)-> MaxFiles = get_item(LevelToAssess, ?LEVEL_SCALEFACTOR, 0), FileCount = length(get_item(LevelToAssess, Manifest, [])), NewWQ = maybe_append_work(WorkQ, LevelToAssess, Manifest, MaxFiles, - FileCount, OngoingWork), - assess_workqueue(NewWQ, LevelToAssess + 1, Manifest, OngoingWork). + FileCount), + assess_workqueue(NewWQ, LevelToAssess + 1, Manifest). maybe_append_work(WorkQ, Level, Manifest, - MaxFiles, FileCount, OngoingWork) + MaxFiles, FileCount) when FileCount > MaxFiles -> io:format("Outstanding compaction work items of ~w at level ~w~n", [FileCount - MaxFiles, Level]), - case lists:keyfind(Level, 1, OngoingWork) of - {Level, Pid, TS} -> - io:format("Work will not be added to queue due to - outstanding work with ~w assigned at ~w~n", [Pid, TS]), - WorkQ; - false -> - lists:append(WorkQ, [{Level, Manifest}]) - end; + lists:append(WorkQ, [{Level, Manifest}]); maybe_append_work(WorkQ, Level, _Manifest, - _MaxFiles, FileCount, _OngoingWork) -> + _MaxFiles, FileCount) -> io:format("No compaction work due to file count ~w at level ~w~n", [FileCount, Level]), WorkQ. @@ -238,54 +443,65 @@ get_item(Index, List, Default) -> %% Request a manifest change -%% Should be passed the -%% - {SrcLevel, NewManifest, ClearedFiles, MergeID, From, State} -%% To complete a manifest change need to: -%% - Update the Manifest Sequence Number (msn) -%% - Confirm this Pid has a current element of manifest work outstanding at -%% that level -%% - Rename the manifest file created under the MergeID (.manifest) -%% to the filename current.manifest -%% - Update the state of the LevelFileRef lists -%% - Add the ClearedFiles to the list of files to be cleared (as a tuple with -%% the new msn) +%% The clerk should have completed the work, and created a new manifest +%% and persisted the new view of the manifest +%% +%% To complete the change of manifest: +%% - the state of the manifest file needs to be changed from pending to current +%% - the list of unreferenced files needs to be updated on State +%% - the current manifest needs to be update don State +%% - the list of ongoing work needs to be cleared of this item -commit_manifest_change(NewManifest, ClearedFiles, MergeID, From, State) -> +commit_manifest_change(ReturnedWorkItem, From, State) -> NewMSN = State#state.manifest_sqn + 1, - OngoingWork = State#state.ongoing_work, + [SentWorkItem] = State#state.ongoing_work, RootPath = State#state.root_path, UnreferencedFiles = State#state.unreferenced_files, - case OngoingWork of - {SrcLevel, From, TS} -> - io:format("Merge ~s completed in ~w microseconds at Level ~w~n", - [MergeID, timer:diff_now(os:timestamp(), TS), SrcLevel]), - ok = rename_manifest_files(RootPath, MergeID), - UnreferencedFilesUpd = update_deletions(ClearedFiles, + + case {SentWorkItem#penciller_work.next_sqn, + SentWorkItem#penciller_work.clerk} of + {NewMSN, From} -> + MTime = timer:diff_now(os:timestamp(), + SentWorkItem#penciller_work.start_time), + io:format("Merge to sqn ~w completed in ~w microseconds + at Level ~w~n", + [SentWorkItem#penciller_work.next_sqn, + MTime, + SentWorkItem#penciller_work.src_level]), + ok = rename_manifest_files(RootPath, NewMSN), + FilesToDelete = ReturnedWorkItem#penciller_work.unreferenced_files, + UnreferencedFilesUpd = update_deletions(FilesToDelete, NewMSN, UnreferencedFiles), - io:format("Merge ~s has been commmitted at sequence number ~w~n", - [MergeID, NewMSN]), + io:format("Merge has been commmitted at sequence number ~w~n", + [NewMSN]), + NewManifest = ReturnedWorkItem#penciller_work.new_manifest, {ok, State#state{ongoing_work=null, manifest_sqn=NewMSN, manifest=NewManifest, unreferenced_files=UnreferencedFilesUpd}}; - _ -> - io:format("Merge commit ~s not matched to known work~n", - [MergeID]), + {MaybeWrongMSN, MaybeWrongClerk} -> + io:format("Merge commit from ~w at sqn ~w not matched to expected + clerk ~w or sqn ~w~n", + [From, NewMSN, MaybeWrongClerk, MaybeWrongMSN]), {error, State} - end. - + end. -rename_manifest_files(RootPath, MergeID) -> - ManifestFP = RootPath ++ "/" ++ ?MANIFEST_FP ++ "/", - ok = file:rename(ManifestFP ++ MergeID - ++ "." ++ ?PENDING_FILEX, - ManifestFP ++ MergeID - ++ "." ++ ?CURRENT_FILEX), - ok. +rename_manifest_files(RootPath, NewMSN) -> + file:rename(filepath(RootPath, NewMSN, pending_manifest), + filepath(RootPath, NewMSN, current_manifest)). +filepath(RootPath, NewMSN, pending_manifest) -> + RootPath ++ "/" ++ ?MANIFEST_FP ++ "/" ++ "nonzero_" + ++ integer_to_list(NewMSN) ++ "." ++ ?PENDING_FILEX; +filepath(RootPath, NewMSN, current_manifest) -> + RootPath ++ "/" ++ ?MANIFEST_FP ++ "/" ++ "nonzero_" + ++ integer_to_list(NewMSN) ++ "." ++ ?CURRENT_FILEX; +filepath(RootPath, NewMSN, new_merge_files) -> + RootPath ++ "/" ++ ?FILES_FP ++ "/" ++ integer_to_list(NewMSN). + update_deletions([], _NewMSN, UnreferencedFiles) -> UnreferencedFiles; update_deletions([ClearedFile|Tail], MSN, UnreferencedFiles) -> @@ -303,12 +519,8 @@ compaction_work_assessment_test() -> L1 = [{{o, "B1", "K1"}, {o, "B2", "K2"}, dummy_pid}, {{o, "B2", "K3"}, {o, "B4", "K4"}, dummy_pid}], Manifest = [{0, L0}, {1, L1}], - OngoingWork1 = [], - WorkQ1 = assess_workqueue([], 0, Manifest, OngoingWork1), + WorkQ1 = assess_workqueue([], 0, Manifest), ?assertMatch(WorkQ1, [{0, Manifest}]), - OngoingWork2 = [{0, dummy_pid, os:timestamp()}], - WorkQ2 = assess_workqueue([], 0, Manifest, OngoingWork2), - ?assertMatch(WorkQ2, []), L1Alt = lists:append(L1, [{{o, "B5", "K0001"}, {o, "B5", "K9999"}, dummy_pid}, {{o, "B6", "K0001"}, {o, "B6", "K9999"}, dummy_pid}, @@ -318,7 +530,5 @@ compaction_work_assessment_test() -> {{o, "BA", "K0001"}, {o, "BA", "K9999"}, dummy_pid}, {{o, "BB", "K0001"}, {o, "BB", "K9999"}, dummy_pid}]), Manifest3 = [{0, []}, {1, L1Alt}], - WorkQ3 = assess_workqueue([], 0, Manifest3, OngoingWork1), - ?assertMatch(WorkQ3, [{1, Manifest3}]), - WorkQ4 = assess_workqueue([], 0, Manifest3, OngoingWork2), - ?assertMatch(WorkQ4, [{1, Manifest3}]). + WorkQ3 = assess_workqueue([], 0, Manifest3), + ?assertMatch(WorkQ3, [{1, Manifest3}]). diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 04e08ba..2896796 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -143,6 +143,7 @@ -module(leveled_sft). -behaviour(gen_server). +-include("../include/leveled.hrl"). -export([init/1, handle_call/3, @@ -151,14 +152,20 @@ terminate/2, code_change/3, sft_new/4, + sft_new/5, sft_open/1, sft_get/2, sft_getkeyrange/4, sft_close/1, - sft_clear/1]). + sft_clear/1, + sft_checkready/1, + sft_getfilename/1, + strip_to_keyonly/1, + generate_randomkeys/1]). -include_lib("eunit/include/eunit.hrl"). + -define(WORD_SIZE, 4). -define(DWORD_SIZE, 8). -define(CURRENT_VERSION, {0,1}). @@ -174,6 +181,7 @@ -define(HEADER_LEN, 56). -define(ITERATOR_SCANWIDTH, 1). -define(MERGE_SCANWIDTH, 8). +-define(MAX_KEYS, ?SLOT_COUNT * ?BLOCK_COUNT * ?BLOCK_SIZE). -record(state, {version = ?CURRENT_VERSION :: tuple(), @@ -189,7 +197,9 @@ summ_pointer :: integer(), summ_length :: integer(), filename :: string(), - handle :: file:fd()}). + handle :: file:fd(), + background_complete=false :: boolean(), + background_failure="Unknown" :: string()}). %%%============================================================================ @@ -197,10 +207,23 @@ %%%============================================================================ sft_new(Filename, KL1, KL2, Level) -> + sft_new(Filename, KL1, KL2, Level, #sft_options{}). + +sft_new(Filename, KL1, KL2, Level, Options) -> {ok, Pid} = gen_server:start(?MODULE, [], []), - Reply = gen_server:call(Pid, {sft_new, Filename, KL1, KL2, Level}, infinity), + Reply = case Options#sft_options.wait of + true -> + gen_server:call(Pid, + {sft_new, Filename, KL1, KL2, Level}, + infinity); + false -> + gen_server:call(Pid, + {sft_new, Filename, KL1, KL2, Level, background}, + infinity) + end, {ok, Pid, Reply}. + sft_open(Filename) -> {ok, Pid} = gen_server:start(?MODULE, [], []), case gen_server:call(Pid, {sft_open, Filename}, infinity) of @@ -225,6 +248,11 @@ sft_close(Pid) -> sft_clear(Pid) -> file_request(Pid, clear). +sft_checkready(Pid) -> + gen_server:call(Pid, background_complete, infinity). + +sft_getfilename(Pid) -> + gen_server:call(Pid, get_filename, infinty). %%%============================================================================ %%% API helper functions @@ -275,6 +303,47 @@ check_pid(Pid) -> init([]) -> {ok, #state{}}. +handle_call({sft_new, Filename, KL1, [], Level, background}, From, State) -> + {ListForFile, KL1Rem} = case length(KL1) of + L when L >= ?MAX_KEYS -> + lists:split(?MAX_KEYS, KL1); + _ -> + {KL1, []} + end, + StartKey = strip_to_keyonly(lists:nth(1, ListForFile)), + EndKey = strip_to_keyonly(lists:last(ListForFile)), + Ext = filename:extension(Filename), + Components = filename:split(Filename), + {TmpFilename, PrmFilename} = case Ext of + [] -> + {filename:join(Components) ++ ".pnd", filename:join(Components) ++ ".sft"}; + Ext -> + %% This seems unnecessarily hard + DN = filename:dirname(Filename), + FP = lists:last(Components), + FP_NOEXT = lists:sublist(FP, 1, 1 + length(FP) - length(Ext)), + {DN ++ "/" ++ FP_NOEXT ++ ".pnd", DN ++ "/" ++ FP_NOEXT ++ ".sft"} + end, + gen_server:reply(From, {{KL1Rem, []}, StartKey, EndKey}), + case create_file(TmpFilename) of + {error, Reason} -> + {noreply, State#state{background_complete=false, + background_failure=Reason}}; + {Handle, FileMD} -> + io:format("Creating file in background with input of size ~w~n", + [length(ListForFile)]), + % Key remainders must match to empty + Rename = {true, TmpFilename, PrmFilename}, + {ReadHandle, UpdFileMD, {[], []}} = complete_file(Handle, + FileMD, + ListForFile, + [], + Level, + Rename), + {noreply, UpdFileMD#state{handle=ReadHandle, + filename=PrmFilename, + background_complete=true}} + end; handle_call({sft_new, Filename, KL1, KL2, Level}, _From, State) -> case create_file(Filename) of {error, Reason} -> @@ -292,7 +361,7 @@ handle_call({sft_new, Filename, KL1, KL2, Level}, _From, State) -> {reply, {KeyRemainders, UpdFileMD#state.smallest_key, UpdFileMD#state.highest_key}, - UpdFileMD#state{handle=ReadHandle}} + UpdFileMD#state{handle=ReadHandle, filename=Filename}} end; handle_call({sft_open, Filename}, _From, _State) -> {_Handle, FileMD} = open_file(Filename), @@ -315,8 +384,17 @@ handle_call(close, _From, State) -> handle_call(clear, _From, State) -> ok = file:close(State#state.handle), ok = file:delete(State#state.filename), - {reply, true, State}. - + {reply, true, State}; +handle_call(background_complete, _From, State) -> + case State#state.background_complete of + true -> + {reply, ok, State}; + false -> + {reply, {error, State#state.background_failure}, State} + end; +handle_call(get_filename, _from, State) -> + {reply, State#state.filename, State}. + handle_cast(_Msg, State) -> {noreply, State}. @@ -338,6 +416,7 @@ code_change(_OldVsn, State, _Extra) -> %% Return the {Handle, metadata record} create_file(FileName) when is_list(FileName) -> io:format("Opening file with filename ~s~n", [FileName]), + ok = filelib:ensure_dir(FileName), case file:open(FileName, [binary, raw, read, write]) of {ok, Handle} -> Header = create_header(initial), @@ -396,15 +475,23 @@ open_file(FileMD) -> %% Take a file handle with a previously created header and complete it based on %% the two key lists KL1 and KL2 - complete_file(Handle, FileMD, KL1, KL2, Level) -> + complete_file(Handle, FileMD, KL1, KL2, Level, false). + +complete_file(Handle, FileMD, KL1, KL2, Level, Rename) -> {ok, KeyRemainders} = write_keys(Handle, maybe_expand_pointer(KL1), maybe_expand_pointer(KL2), [], <<>>, Level, fun sftwrite_function/2), - {ReadHandle, UpdFileMD} = open_file(FileMD), + {ReadHandle, UpdFileMD} = case Rename of + false -> + open_file(FileMD); + {true, OldName, NewName} -> + ok = file:rename(OldName, NewName), + open_file(FileMD#state{filename=NewName}) + end, {ReadHandle, UpdFileMD, KeyRemainders}. %% Fetch a Key and Value from a file, returns From c269eb3c52719a294119b3c0493ab35860dfbfdf Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 10 Aug 2016 13:02:08 +0100 Subject: [PATCH 027/167] Initial level merging Some basic merging in the lsm tree. --- src/leveled_clerk.erl | 109 ++++++++++++++++++++++---------------- src/leveled_penciller.erl | 100 +++++++++++++++++++++++++--------- src/leveled_sft.erl | 70 ++++++++++++++++-------- 3 files changed, 186 insertions(+), 93 deletions(-) diff --git a/src/leveled_clerk.erl b/src/leveled_clerk.erl index 3197eb4..bf5e252 100644 --- a/src/leveled_clerk.erl +++ b/src/leveled_clerk.erl @@ -13,7 +13,7 @@ handle_cast/2, handle_info/2, terminate/2, - clerk_new/0, + clerk_new/1, clerk_prompt/2, code_change/3, perform_merge/4]). @@ -26,14 +26,15 @@ %%% API %%%============================================================================ -clerk_new() -> +clerk_new(Owner) -> {ok, Pid} = gen_server:start(?MODULE, [], []), - ok = gen_server:call(Pid, register, infinity), + ok = gen_server:call(Pid, {register, Owner}, infinity), {ok, Pid}. clerk_prompt(Pid, penciller) -> - gen_server:cast(Pid, penciller_prompt, infinity). + gen_server:cast(Pid, penciller_prompt), + ok. %%%============================================================================ %%% gen_server callbacks @@ -42,10 +43,10 @@ clerk_prompt(Pid, penciller) -> init([]) -> {ok, #state{}}. -handle_call(register, From, State) -> - {noreply, State#state{owner=From}}. +handle_call({register, Owner}, _From, State) -> + {reply, ok, State#state{owner=Owner}}. -handle_cast({penciller_prompt, From}, State) -> +handle_cast(penciller_prompt, State) -> case leveled_penciller:pcl_workforclerk(State#state.owner) of none -> io:format("Work prompted but none needed~n"), @@ -54,7 +55,9 @@ handle_cast({penciller_prompt, From}, State) -> {NewManifest, FilesToDelete} = merge(WI), UpdWI = WI#penciller_work{new_manifest=NewManifest, unreferenced_files=FilesToDelete}, - leveled_penciller:pcl_requestmanifestchange(From, UpdWI), + leveled_penciller:pcl_requestmanifestchange(State#state.owner, + UpdWI), + mark_for_delete(FilesToDelete, State#state.owner), {noreply, State} end. @@ -74,58 +77,69 @@ code_change(_OldVsn, State, _Extra) -> merge(WI) -> SrcLevel = WI#penciller_work.src_level, - {Selection, UpdMFest1} = select_filetomerge(SrcLevel, + {SrcF, UpdMFest1} = select_filetomerge(SrcLevel, WI#penciller_work.manifest), - {{StartKey, EndKey}, SrcFile} = Selection, - SrcFilename = leveled_sft:sft_getfilename(SrcFile), SinkFiles = get_item(SrcLevel + 1, UpdMFest1, []), - SplitLists = lists:splitwith(fun(Ref) -> - case {Ref#manifest_entry.start_key, - Ref#manifest_entry.end_key} of - {_, EK} when StartKey > EK -> - false; - {SK, _} when EndKey < SK -> - false; - _ -> - true - end end, - SinkFiles), - {Candidates, Others} = SplitLists, + Splits = lists:splitwith(fun(Ref) -> + case {Ref#manifest_entry.start_key, + Ref#manifest_entry.end_key} of + {_, EK} when SrcF#manifest_entry.start_key > EK -> + false; + {SK, _} when SrcF#manifest_entry.end_key < SK -> + false; + _ -> + true + end end, + SinkFiles), + {Candidates, Others} = Splits, %% TODO: %% Need to work out if this is the top level %% And then tell merge process to create files at the top level %% Which will include the reaping of expired tombstones - - io:format("Merge from level ~w to merge into ~w files below", + io:format("Merge from level ~w to merge into ~w files below~n", [SrcLevel, length(Candidates)]), - + MergedFiles = case length(Candidates) of 0 -> %% If no overlapping candiates, manifest change only required %% %% TODO: need to think still about simply renaming when at %% lower level - [SrcFile]; + [SrcF]; _ -> - perform_merge({SrcFile, SrcFilename}, + perform_merge({SrcF#manifest_entry.owner, + SrcF#manifest_entry.filename}, Candidates, SrcLevel, {WI#penciller_work.ledger_filepath, WI#penciller_work.next_sqn}) - end, + end, + case MergedFiles of + error -> + merge_failure; + _ -> + NewLevel = lists:sort(lists:append(MergedFiles, Others)), + UpdMFest2 = lists:keystore(SrcLevel + 1, + 1, + UpdMFest1, + {SrcLevel + 1, NewLevel}), + + ok = filelib:ensure_dir(WI#penciller_work.manifest_file), + {ok, Handle} = file:open(WI#penciller_work.manifest_file, + [binary, raw, write]), + ok = file:write(Handle, term_to_binary(UpdMFest2)), + ok = file:close(Handle), + {UpdMFest2, Candidates} + end. - NewLevel = lists:sort(lists:append(MergedFiles, Others)), - UpdMFest2 = lists:keyreplace(SrcLevel + 1, - 1, - UpdMFest1, - {SrcLevel, NewLevel}), + +mark_for_delete([], _Penciller) -> + ok; +mark_for_delete([Head|Tail], Penciller) -> + leveled_sft:sft_setfordelete(Head#manifest_entry.owner, Penciller), + mark_for_delete(Tail, Penciller). - {ok, Handle} = file:open(WI#penciller_work.manifest_file, - [binary, raw, write]), - ok = file:write(Handle, term_to_binary(UpdMFest2)), - ok = file:close(Handle), - {UpdMFest2, Candidates}. %% An algorithm for discovering which files to merge .... @@ -173,11 +187,12 @@ select_filetomerge(SrcLevel, Manifest) -> %% %% The level is the level which the new files should be created at. -perform_merge(FileToMerge, CandidateList, Level, {Filepath, MSN}) -> - {Filename, UpperSFTPid} = FileToMerge, +perform_merge({UpperSFTPid, Filename}, CandidateList, Level, {Filepath, MSN}) -> io:format("Merge to be commenced for FileToMerge=~s with MSN=~w~n", [Filename, MSN]), - PointerList = lists:map(fun(P) -> {next, P, all} end, CandidateList), + PointerList = lists:map(fun(P) -> + {next, P#manifest_entry.owner, all} end, + CandidateList), do_merge([{next, UpperSFTPid, all}], PointerList, Level, {Filepath, MSN}, 0, []). @@ -187,12 +202,14 @@ do_merge([], [], Level, {_Filepath, MSN}, FileCounter, OutList) -> OutList; do_merge(KL1, KL2, Level, {Filepath, MSN}, FileCounter, OutList) -> FileName = lists:flatten(io_lib:format(Filepath ++ "_~w_~w.sft", - [Level, FileCounter])), + [Level + 1, FileCounter])), io:format("File to be created as part of MSN=~w Filename=~s~n", [MSN, FileName]), + TS1 = os:timestamp(), case leveled_sft:sft_new(FileName, KL1, KL2, Level) of {ok, _Pid, {error, Reason}} -> - io:format("Exiting due to error~w~n", [Reason]); + io:format("Exiting due to error~w~n", [Reason]), + error; {ok, Pid, Reply} -> {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, ExtMan = lists:append(OutList, @@ -200,6 +217,8 @@ do_merge(KL1, KL2, Level, {Filepath, MSN}, FileCounter, OutList) -> end_key=HighestKey, owner=Pid, filename=FileName}]), + MTime = timer:now_diff(os:timestamp(), TS1), + io:format("file creation took ~w microseconds ~n", [MTime]), do_merge(KL1Rem, KL2Rem, Level, {Filepath, MSN}, FileCounter + 1, ExtMan) end. @@ -278,7 +297,7 @@ merge_file_test() -> KL4_L2 = lists:sort(generate_randomkeys(16000, 750, 250)), {ok, PidL2_4, _} = leveled_sft:sft_new("../test/KL4_L2.sft", KL4_L2, [], 2), - Result = perform_merge({"../test/KL1_L1.sft", PidL1_1}, + Result = perform_merge({PidL1_1, "../test/KL1_L1.sft"}, [PidL2_1, PidL2_2, PidL2_3, PidL2_4], 2, {"../test/", 99}), lists:foreach(fun(ManEntry) -> diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 5e75e4a..fb7432d 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -159,7 +159,7 @@ pcl_fetch/2, pcl_workforclerk/1, pcl_requestmanifestchange/2, - commit_manifest_change/3]). + pcl_confirmdelete/2]). -include_lib("eunit/include/eunit.hrl"). @@ -181,7 +181,7 @@ -record(state, {manifest = [] :: list(), ongoing_work = [] :: list(), manifest_sqn = 0 :: integer(), - levelzero_sqn =0 :: integer(), + levelzero_sqn = 0 :: integer(), registered_iterators = [] :: list(), unreferenced_files = [] :: list(), root_path = "../test/" :: string(), @@ -214,7 +214,10 @@ pcl_workforclerk(Pid) -> gen_server:call(Pid, work_for_clerk, infinity). pcl_requestmanifestchange(Pid, WorkItem) -> - gen_server:call(Pid, {manifest_change, WorkItem}, infinity). + gen_server:cast(Pid, {manifest_change, WorkItem}). + +pcl_confirmdelete(Pid, FileName) -> + gen_server:call(Pid, {confirm_delete, FileName}). %%%============================================================================ %%% gen_server callbacks @@ -222,22 +225,27 @@ pcl_requestmanifestchange(Pid, WorkItem) -> init([]) -> TID = ets:new(?MEMTABLE, [ordered_set, private]), - {ok, #state{memtable=TID}}. + {ok, Clerk} = leveled_clerk:clerk_new(self()), + {ok, #state{memtable=TID, clerk=Clerk}}. handle_call({push_mem, DumpList}, _From, State) -> {TableSize, Manifest, L0Pend} = case State#state.levelzero_pending of {true, Remainder, {StartKey, EndKey, Pid}} -> %% Need to handle not error scenarios? %% N.B. Sync call - so will be ready - ok = leveled_sft:sft_checkready(Pid), + {ok, SrcFN} = leveled_sft:sft_checkready(Pid), %% Reset ETS, but re-insert any remainder true = ets:delete_all_objects(State#state.memtable), true = ets:insert(State#state.memtable, Remainder), + ManifestEntry = #manifest_entry{start_key=StartKey, + end_key=EndKey, + owner=Pid, + filename=SrcFN}, {length(Remainder), lists:keystore(0, 1, State#state.manifest, - {0, [{StartKey, EndKey, Pid}]}), + {0, [ManifestEntry]}), ?L0PEND_RESET}; {false, _, _} -> {State#state.table_size, @@ -246,7 +254,9 @@ handle_call({push_mem, DumpList}, _From, State) -> Unexpected -> io:format("Unexpected value of ~w~n", [Unexpected]), error - end, + end, + %% Prompt clerk to ask about work - do this for every push_mem + ok = leveled_clerk:clerk_prompt(State#state.clerk, penciller), case do_push_to_mem(DumpList, TableSize, State#state.memtable) of {twist, ApproxTableSize} -> {reply, ok, State#state{table_size=ApproxTableSize, @@ -258,13 +268,14 @@ handle_call({push_mem, DumpList}, _From, State) -> L0SN = State#state.levelzero_sqn + 1, FileName = State#state.root_path ++ ?FILES_FP ++ "/" - ++ integer_to_list(L0SN), - SFT = leveled_sft:sft_new(FileName, - ets:tab2list(State#state.memtable), - [], - 0, - #sft_options{wait=false}), - {ok, L0Pid, Reply} = SFT, + ++ integer_to_list(L0SN) ++ "_0_0", + Dump = ets:tab2list(State#state.memtable), + L0_SFT = leveled_sft:sft_new(FileName, + Dump, + [], + 0, + #sft_options{wait=false}), + {ok, L0Pid, Reply} = L0_SFT, {{KL1Rem, []}, L0StartKey, L0EndKey} = Reply, {reply, ok, State#state{levelzero_pending={true, KL1Rem, @@ -285,10 +296,16 @@ handle_call({fetch, Key}, _From, State) -> {reply, fetch(Key, State#state.manifest, State#state.memtable), State}; handle_call(work_for_clerk, From, State) -> {UpdState, Work} = return_work(State, From), - {reply, Work, UpdState}. + {reply, Work, UpdState}; +handle_call({confirm_delete, FileName}, _From, State) -> + Reply = confirm_delete(FileName, + State#state.unreferenced_files, + State#state.registered_iterators), + {reply, Reply, State}. -handle_cast(_Msg, State) -> - {noreply, State}. +handle_cast({manifest_change, WI}, State) -> + {ok, UpdState} = commit_manifest_change(WI, State), + {noreply, UpdState}. handle_info(_Info, State) -> {noreply, State}. @@ -453,7 +470,7 @@ get_item(Index, List, Default) -> %% - the list of ongoing work needs to be cleared of this item -commit_manifest_change(ReturnedWorkItem, From, State) -> +commit_manifest_change(ReturnedWorkItem, State) -> NewMSN = State#state.manifest_sqn + 1, [SentWorkItem] = State#state.ongoing_work, RootPath = State#state.root_path, @@ -461,8 +478,8 @@ commit_manifest_change(ReturnedWorkItem, From, State) -> case {SentWorkItem#penciller_work.next_sqn, SentWorkItem#penciller_work.clerk} of - {NewMSN, From} -> - MTime = timer:diff_now(os:timestamp(), + {NewMSN, _From} -> + MTime = timer:now_diff(os:timestamp(), SentWorkItem#penciller_work.start_time), io:format("Merge to sqn ~w completed in ~w microseconds at Level ~w~n", @@ -477,14 +494,15 @@ commit_manifest_change(ReturnedWorkItem, From, State) -> io:format("Merge has been commmitted at sequence number ~w~n", [NewMSN]), NewManifest = ReturnedWorkItem#penciller_work.new_manifest, - {ok, State#state{ongoing_work=null, + %% io:format("Updated manifest is ~w~n", [NewManifest]), + {ok, State#state{ongoing_work=[], manifest_sqn=NewMSN, manifest=NewManifest, unreferenced_files=UnreferencedFilesUpd}}; - {MaybeWrongMSN, MaybeWrongClerk} -> - io:format("Merge commit from ~w at sqn ~w not matched to expected - clerk ~w or sqn ~w~n", - [From, NewMSN, MaybeWrongClerk, MaybeWrongMSN]), + {MaybeWrongMSN, From} -> + io:format("Merge commit at sqn ~w not matched to expected + sqn ~w from Clerk ~w~n", + [NewMSN, MaybeWrongMSN, From]), {error, State} end. @@ -505,9 +523,26 @@ filepath(RootPath, NewMSN, new_merge_files) -> update_deletions([], _NewMSN, UnreferencedFiles) -> UnreferencedFiles; update_deletions([ClearedFile|Tail], MSN, UnreferencedFiles) -> + io:format("Adding cleared file ~s to deletion list ~n", + [ClearedFile#manifest_entry.filename]), update_deletions(Tail, MSN, - lists:append(UnreferencedFiles, [{ClearedFile, MSN}])). + lists:append(UnreferencedFiles, + [{ClearedFile#manifest_entry.filename, MSN}])). + +confirm_delete(Filename, UnreferencedFiles, RegisteredIterators) -> + case lists:keyfind(Filename, 1, UnreferencedFiles) of + false -> + false; + {Filename, MSN} -> + LowSQN = lists:foldl(fun({_, SQN}, MinSQN) -> min(SQN, MinSQN) end, + infinity, + RegisteredIterators), + if + MSN >= LowSQN -> false; + true -> true + end + end. %%%============================================================================ %%% Test @@ -532,3 +567,16 @@ compaction_work_assessment_test() -> Manifest3 = [{0, []}, {1, L1Alt}], WorkQ3 = assess_workqueue([], 0, Manifest3), ?assertMatch(WorkQ3, [{1, Manifest3}]). + +confirm_delete_test() -> + Filename = 'test.sft', + UnreferencedFiles = [{'other.sft', 15}, {Filename, 10}], + RegisteredIterators1 = [{dummy_pid, 16}, {dummy_pid, 12}], + R1 = confirm_delete(Filename, UnreferencedFiles, RegisteredIterators1), + ?assertMatch(R1, true), + RegisteredIterators2 = [{dummy_pid, 10}, {dummy_pid, 12}], + R2 = confirm_delete(Filename, UnreferencedFiles, RegisteredIterators2), + ?assertMatch(R2, false), + RegisteredIterators3 = [{dummy_pid, 9}, {dummy_pid, 12}], + R3 = confirm_delete(Filename, UnreferencedFiles, RegisteredIterators3), + ?assertMatch(R3, false). \ No newline at end of file diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 2896796..cde8313 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -160,6 +160,7 @@ sft_clear/1, sft_checkready/1, sft_getfilename/1, + sft_setfordelete/2, strip_to_keyonly/1, generate_randomkeys/1]). @@ -182,6 +183,7 @@ -define(ITERATOR_SCANWIDTH, 1). -define(MERGE_SCANWIDTH, 8). -define(MAX_KEYS, ?SLOT_COUNT * ?BLOCK_COUNT * ?BLOCK_SIZE). +-define(DELETE_TIMEOUT, 60000). -record(state, {version = ?CURRENT_VERSION :: tuple(), @@ -199,7 +201,9 @@ filename :: string(), handle :: file:fd(), background_complete=false :: boolean(), - background_failure="Unknown" :: string()}). + background_failure="Unknown" :: string(), + ready_for_delete = false ::boolean(), + penciller :: pid()}). %%%============================================================================ @@ -223,7 +227,6 @@ sft_new(Filename, KL1, KL2, Level, Options) -> end, {ok, Pid, Reply}. - sft_open(Filename) -> {ok, Pid} = gen_server:start(?MODULE, [], []), case gen_server:call(Pid, {sft_open, Filename}, infinity) of @@ -233,6 +236,9 @@ sft_open(Filename) -> Error end. +sft_setfordelete(Pid, Penciller) -> + file_request(Pid, {set_for_delete, Penciller}). + sft_get(Pid, Key) -> file_request(Pid, {get_kv, Key}). @@ -266,18 +272,7 @@ sft_getfilename(Pid) -> file_request(Pid, Request) -> case check_pid(Pid) of ok -> - try - gen_server:call(Pid, Request, infinity) - catch - exit:{normal,_} when Request == file_close -> - %% Honest race condition in bitcask_eqc PULSE test. - ok; - exit:{noproc,_} when Request == file_close -> - %% Honest race condition in bitcask_eqc PULSE test. - ok; - X1:X2 -> - exit({file_request_error, self(), Request, X1, X2}) - end; + gen_server:call(Pid, Request, infinity); Error -> Error end. @@ -388,16 +383,42 @@ handle_call(clear, _From, State) -> handle_call(background_complete, _From, State) -> case State#state.background_complete of true -> - {reply, ok, State}; + {reply, {ok, State#state.filename}, State}; false -> {reply, {error, State#state.background_failure}, State} end; -handle_call(get_filename, _from, State) -> - {reply, State#state.filename, State}. +handle_call(get_filename, _From, State) -> + {reply, State#state.filename, State}; +handle_call({set_for_delete, Penciller}, _From, State) -> + {reply, + ok, + State#state{ready_for_delete=true, + penciller=Penciller}, + ?DELETE_TIMEOUT}. handle_cast(_Msg, State) -> {noreply, State}. +handle_info(timeout, State) -> + case State#state.ready_for_delete of + true -> + case leveled_penciller:pcl_confirmdelete(State#state.penciller, + State#state.filename) + of + true -> + io:format("Polled for deletion and now clearing ~s~n", + [State#state.filename]), + ok = file:close(State#state.handle), + ok = file:delete(State#state.filename), + {stop, shutdown, State}; + false -> + io:format("Polled for deletion but ~s not ready~n", + [State#state.filename]), + {noreply, State, ?DELETE_TIMEOUT} + end; + false -> + {noreply, State} + end; handle_info(_Info, State) -> {noreply, State}. @@ -583,7 +604,7 @@ fetch_range(Handle, FileMD, StartKey, EndKey, FunList, AccFun, ScanWidth, Acc) - LengthList, 0, PointerB + FileMD#state.slots_pointer, AccFun(null, Acc)); not_found -> - {complete, Acc} + {complete, AccFun(null, Acc)} end. fetch_range(Handle, FileMD, _StartKey, NearestKey, EndKey, FunList, AccFun, ScanWidth, @@ -989,7 +1010,12 @@ serialise_block(BlockKeyList) -> %% any lower sequence numbers should be compacted out of existence -key_dominates([H1|T1], [], Level) -> +key_dominates(KL1, KL2, Level) -> + key_dominates_expanded(maybe_expand_pointer(KL1), + maybe_expand_pointer(KL2), + Level). + +key_dominates_expanded([H1|T1], [], Level) -> {_, _, St1, _} = H1, case maybe_reap_expiredkey(St1, Level) of true -> @@ -997,7 +1023,7 @@ key_dominates([H1|T1], [], Level) -> false -> {{next_key, H1}, maybe_expand_pointer(T1), []} end; -key_dominates([], [H2|T2], Level) -> +key_dominates_expanded([], [H2|T2], Level) -> {_, _, St2, _} = H2, case maybe_reap_expiredkey(St2, Level) of true -> @@ -1005,7 +1031,7 @@ key_dominates([], [H2|T2], Level) -> false -> {{next_key, H2}, [], maybe_expand_pointer(T2)} end; -key_dominates([H1|T1], [H2|T2], Level) -> +key_dominates_expanded([H1|T1], [H2|T2], Level) -> {K1, Sq1, St1, _} = H1, {K2, Sq2, St2, _} = H2, case K1 of @@ -1051,7 +1077,7 @@ maybe_expand_pointer([]) -> maybe_expand_pointer([H|Tail]) -> case H of {next, SFTPid, StartKey} -> - io:format("Scanning further on PID ~w ~w~n", [SFTPid, StartKey]), + %% io:format("Scanning further on PID ~w ~w~n", [SFTPid, StartKey]), QResult = sft_getkvrange(SFTPid, StartKey, all, ?MERGE_SCANWIDTH), Acc = pointer_append_queryresults(QResult, SFTPid), lists:append(Acc, Tail); From 6e56b569b8852a528c0fd72aa4b15214ca818d61 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 12 Aug 2016 01:05:59 +0100 Subject: [PATCH 028/167] Auto-merge Allow for the clerk to merge continuously is no activity for the penciller to prompt. The penciller now must also correctly lock the manifest - to stop races between the creation of ne wL0 files and the completion of work by the clerk --- src/leveled_clerk.erl | 65 +++++++--- src/leveled_penciller.erl | 249 +++++++++++++++++++++++++------------- src/leveled_sft.erl | 8 +- 3 files changed, 221 insertions(+), 101 deletions(-) diff --git a/src/leveled_clerk.erl b/src/leveled_clerk.erl index bf5e252..1a181b2 100644 --- a/src/leveled_clerk.erl +++ b/src/leveled_clerk.erl @@ -20,7 +20,11 @@ -include_lib("eunit/include/eunit.hrl"). --record(state, {owner :: pid()}). +-define(INACTIVITY_TIMEOUT, 2000). +-define(HAPPYTIME_MULTIPLIER, 5). + +-record(state, {owner :: pid(), + in_backlog = false :: boolean()}). %%%============================================================================ %%% API @@ -47,20 +51,17 @@ handle_call({register, Owner}, _From, State) -> {reply, ok, State#state{owner=Owner}}. handle_cast(penciller_prompt, State) -> - case leveled_penciller:pcl_workforclerk(State#state.owner) of - none -> - io:format("Work prompted but none needed~n"), - {noreply, State}; - WI -> - {NewManifest, FilesToDelete} = merge(WI), - UpdWI = WI#penciller_work{new_manifest=NewManifest, - unreferenced_files=FilesToDelete}, - leveled_penciller:pcl_requestmanifestchange(State#state.owner, - UpdWI), - mark_for_delete(FilesToDelete, State#state.owner), - {noreply, State} - end. + Timeout = requestandhandle_work(State), + {noreply, State, Timeout}. +handle_info(timeout, State) -> + %% The pcl prompt will cause a penciller_prompt, to re-trigger timeout + case leveled_penciller:pcl_prompt(State#state.owner) of + ok -> + {noreply, State}; + pause -> + {noreply, State} + end; handle_info(_Info, State) -> {noreply, State}. @@ -75,6 +76,27 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================ +requestandhandle_work(State) -> + case leveled_penciller:pcl_workforclerk(State#state.owner) of + {none, Backlog} -> + io:format("Work prompted but none needed~n"), + case Backlog of + false -> + ?INACTIVITY_TIMEOUT * ?HAPPYTIME_MULTIPLIER; + _ -> + ?INACTIVITY_TIMEOUT + end; + {WI, _} -> + {NewManifest, FilesToDelete} = merge(WI), + UpdWI = WI#penciller_work{new_manifest=NewManifest, + unreferenced_files=FilesToDelete}, + leveled_penciller:pcl_requestmanifestchange(State#state.owner, + UpdWI), + mark_for_delete(FilesToDelete, State#state.owner), + ?INACTIVITY_TIMEOUT + end. + + merge(WI) -> SrcLevel = WI#penciller_work.src_level, {SrcF, UpdMFest1} = select_filetomerge(SrcLevel, @@ -106,6 +128,8 @@ merge(WI) -> %% %% TODO: need to think still about simply renaming when at %% lower level + io:format("File ~s to simply switch levels to level ~w~n", + [SrcF#manifest_entry.filename, SrcLevel + 1]), [SrcF]; _ -> perform_merge({SrcF#manifest_entry.owner, @@ -130,7 +154,13 @@ merge(WI) -> [binary, raw, write]), ok = file:write(Handle, term_to_binary(UpdMFest2)), ok = file:close(Handle), - {UpdMFest2, Candidates} + case lists:member(SrcF, MergedFiles) of + true -> + {UpdMFest2, Candidates}; + false -> + %% Can rub out src file as it is not part of output + {UpdMFest2, Candidates ++ [SrcF]} + end end. @@ -298,7 +328,10 @@ merge_file_test() -> {ok, PidL2_4, _} = leveled_sft:sft_new("../test/KL4_L2.sft", KL4_L2, [], 2), Result = perform_merge({PidL1_1, "../test/KL1_L1.sft"}, - [PidL2_1, PidL2_2, PidL2_3, PidL2_4], + [#manifest_entry{owner=PidL2_1}, + #manifest_entry{owner=PidL2_2}, + #manifest_entry{owner=PidL2_3}, + #manifest_entry{owner=PidL2_4}], 2, {"../test/", 99}), lists:foreach(fun(ManEntry) -> {o, B1, K1} = ManEntry#manifest_entry.start_key, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index fb7432d..44e70c0 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -159,7 +159,8 @@ pcl_fetch/2, pcl_workforclerk/1, pcl_requestmanifestchange/2, - pcl_confirmdelete/2]). + pcl_confirmdelete/2, + pcl_prompt/1]). -include_lib("eunit/include/eunit.hrl"). @@ -181,26 +182,26 @@ -record(state, {manifest = [] :: list(), ongoing_work = [] :: list(), manifest_sqn = 0 :: integer(), - levelzero_sqn = 0 :: integer(), registered_iterators = [] :: list(), unreferenced_files = [] :: list(), - root_path = "../test/" :: string(), + root_path = "../test" :: string(), table_size = 0 :: integer(), clerk :: pid(), levelzero_pending = {false, [], none} :: tuple(), - memtable}). + memtable, + backlog = false :: boolean()}). %%%============================================================================ %%% API %%%============================================================================ - + pcl_new() -> gen_server:start(?MODULE, [], []). -pcl_start(_RootDir) -> - %% TODO - %% Need to call startup to rebuild from disk +pcl_start(RootDir) -> + {ok, Pid} = gen_server:start(?MODULE, [], []), + gen_server:call(Pid, {load, RootDir}, infinity), ok. pcl_pushmem(Pid, DumpList) -> @@ -219,6 +220,9 @@ pcl_requestmanifestchange(Pid, WorkItem) -> pcl_confirmdelete(Pid, FileName) -> gen_server:call(Pid, {confirm_delete, FileName}). +pcl_prompt(Pid) -> + gen_server:call(Pid, prompt_compaction). + %%%============================================================================ %%% gen_server callbacks %%%============================================================================ @@ -229,79 +233,39 @@ init([]) -> {ok, #state{memtable=TID, clerk=Clerk}}. handle_call({push_mem, DumpList}, _From, State) -> - {TableSize, Manifest, L0Pend} = case State#state.levelzero_pending of - {true, Remainder, {StartKey, EndKey, Pid}} -> - %% Need to handle not error scenarios? - %% N.B. Sync call - so will be ready - {ok, SrcFN} = leveled_sft:sft_checkready(Pid), - %% Reset ETS, but re-insert any remainder - true = ets:delete_all_objects(State#state.memtable), - true = ets:insert(State#state.memtable, Remainder), - ManifestEntry = #manifest_entry{start_key=StartKey, - end_key=EndKey, - owner=Pid, - filename=SrcFN}, - {length(Remainder), - lists:keystore(0, - 1, - State#state.manifest, - {0, [ManifestEntry]}), - ?L0PEND_RESET}; - {false, _, _} -> - {State#state.table_size, - State#state.manifest, - State#state.levelzero_pending}; - Unexpected -> - io:format("Unexpected value of ~w~n", [Unexpected]), - error - end, - %% Prompt clerk to ask about work - do this for every push_mem - ok = leveled_clerk:clerk_prompt(State#state.clerk, penciller), - case do_push_to_mem(DumpList, TableSize, State#state.memtable) of - {twist, ApproxTableSize} -> - {reply, ok, State#state{table_size=ApproxTableSize, - manifest=Manifest, - levelzero_pending=L0Pend}}; - {roll, ApproxTableSize} -> - case {get_item(0, Manifest, []), L0Pend} of - {[], ?L0PEND_RESET} -> - L0SN = State#state.levelzero_sqn + 1, - FileName = State#state.root_path - ++ ?FILES_FP ++ "/" - ++ integer_to_list(L0SN) ++ "_0_0", - Dump = ets:tab2list(State#state.memtable), - L0_SFT = leveled_sft:sft_new(FileName, - Dump, - [], - 0, - #sft_options{wait=false}), - {ok, L0Pid, Reply} = L0_SFT, - {{KL1Rem, []}, L0StartKey, L0EndKey} = Reply, - {reply, ok, State#state{levelzero_pending={true, - KL1Rem, - {L0StartKey, - L0EndKey, - L0Pid}}, - table_size=ApproxTableSize, - levelzero_sqn=L0SN}}; - _ -> - io:format("Memory has exceeded limit but L0 file is still - awaiting compaction ~n"), - {reply, pause, State#state{table_size=ApproxTableSize, - manifest=Manifest, - levelzero_pending=L0Pend}} - end + case push_to_memory(DumpList, State) of + {ok, UpdState} -> + {reply, ok, UpdState}; + {{pause, Reason, Details}, UpdState} -> + io:format("Excess work due to - " ++ Reason, Details), + {reply, pause, UpdState#state{backlog=true}} end; handle_call({fetch, Key}, _From, State) -> {reply, fetch(Key, State#state.manifest, State#state.memtable), State}; handle_call(work_for_clerk, From, State) -> {UpdState, Work} = return_work(State, From), - {reply, Work, UpdState}; + {reply, {Work, UpdState#state.backlog}, UpdState}; handle_call({confirm_delete, FileName}, _From, State) -> Reply = confirm_delete(FileName, State#state.unreferenced_files, State#state.registered_iterators), - {reply, Reply, State}. + {reply, Reply, State}; +handle_call({load, RootDir}, _From, State) -> + {Manifest, ManifestSQN} = load_manifest(RootDir), + {UpdManifest, MaxSQN} = load_allsft(RootDir, Manifest), + {UpdMaxSQN} = load_levelzero(RootDir, MaxSQN), + {reply, UpdMaxSQN, State#state{root_path=RootDir, + manifest=UpdManifest, + manifest_sqn=ManifestSQN}}; +handle_call(prompt_compaction, _From, State) -> + case push_to_memory([], State) of + {ok, UpdState} -> + {reply, ok, UpdState#state{backlog=false}}; + {{pause, Reason, Details}, UpdState} -> + io:format("Excess work due to - " ++ Reason, Details), + {reply, pause, UpdState#state{backlog=true}} + end. + handle_cast({manifest_change, WI}, State) -> {ok, UpdState} = commit_manifest_change(WI, State), @@ -321,6 +285,81 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================ +push_to_memory(DumpList, State) -> + {TableSize, UpdState} = case State#state.levelzero_pending of + {true, Remainder, {StartKey, EndKey, Pid}} -> + %% Need to handle not error scenarios? + %% N.B. Sync call - so will be ready + {ok, SrcFN} = leveled_sft:sft_checkready(Pid), + %% Reset ETS, but re-insert any remainder + true = ets:delete_all_objects(State#state.memtable), + true = ets:insert(State#state.memtable, Remainder), + ManifestEntry = #manifest_entry{start_key=StartKey, + end_key=EndKey, + owner=Pid, + filename=SrcFN}, + {length(Remainder), + State#state{manifest=lists:keystore(0, + 1, + State#state.manifest, + {0, [ManifestEntry]}), + levelzero_pending=?L0PEND_RESET}}; + {false, _, _} -> + {State#state.table_size, State} + end, + + %% Prompt clerk to ask about work - do this for every push_mem + ok = leveled_clerk:clerk_prompt(UpdState#state.clerk, penciller), + + case do_push_to_mem(DumpList, TableSize, UpdState#state.memtable) of + {twist, ApproxTableSize} -> + {ok, UpdState#state{table_size=ApproxTableSize}}; + {roll, ApproxTableSize} -> + L0 = get_item(0, UpdState#state.manifest, []), + case {L0, manifest_locked(UpdState)} of + {[], false} -> + MSN = UpdState#state.manifest_sqn + 1, + FileName = UpdState#state.root_path + ++ "/" ++ ?FILES_FP ++ "/" + ++ integer_to_list(MSN) ++ "_0_0", + Dump = ets:tab2list(UpdState#state.memtable), + L0_SFT = leveled_sft:sft_new(FileName, + Dump, + [], + 0, + #sft_options{wait=false}), + {ok, L0Pid, Reply} = L0_SFT, + {{KL1Rem, []}, L0StartKey, L0EndKey} = Reply, + Backlog = length(KL1Rem), + Rsp = + if + Backlog > ?MAX_TABLESIZE -> + {pause, + "Backlog of ~w in memory table~n", + [Backlog]}; + true -> + ok + end, + {Rsp, + UpdState#state{levelzero_pending={true, + KL1Rem, + {L0StartKey, + L0EndKey, + L0Pid}}, + table_size=ApproxTableSize, + manifest_sqn=MSN}}; + {[], true} -> + {{pause, + "L0 file write blocked by change at sqn=~w~n", + [UpdState#state.manifest_sqn]}, + UpdState#state{table_size=ApproxTableSize}}; + _ -> + {{pause, + "L0 file write blocked by L0 file in manifest~n", + []}, + UpdState#state{table_size=ApproxTableSize}} + end + end. fetch(Key, Manifest, TID) -> case ets:lookup(TID, Key) of @@ -373,6 +412,22 @@ do_push_to_mem(DumpList, TableSize, MemTable) -> end. +%% Manifest lock - don't have two changes to the manifest happening +%% concurrently + +manifest_locked(State) -> + if + length(State#state.ongoing_work) > 0 -> + true; + true -> + case State#state.levelzero_pending of + {true, _, _} -> + true; + _ -> + false + end + end. + %% Work out what the current work queue should be %% @@ -387,11 +442,11 @@ return_work(State, From) -> case length(WorkQueue) of L when L > 0 -> [{SrcLevel, Manifest}|OtherWork] = WorkQueue, - io:format("Work at Level ~w to be scheduled for ~w - with ~w queue items outstanding~n", + io:format("Work at Level ~w to be scheduled for ~w with ~w " ++ + "queue items outstanding~n", [SrcLevel, From, length(OtherWork)]), - case State#state.ongoing_work of - [] -> + case {manifest_locked(State), State#state.ongoing_work} of + {false, _} -> %% No work currently outstanding %% Can allocate work NextSQN = State#state.manifest_sqn + 1, @@ -409,15 +464,20 @@ return_work(State, From) -> ledger_filepath = FP, manifest_file = ManFile}, {State#state{ongoing_work=[WI]}, WI}; - [OutstandingWork] -> + {true, [OutstandingWork]} -> %% Still awaiting a response - io:format("Ongoing work requested by ~w but work - outstanding from Level ~w and Clerk ~w - at sequence number ~w~n", + io:format("Ongoing work requested by ~w " ++ + "but work outstanding from Level ~w " ++ + "and Clerk ~w at sequence number ~w~n", [From, OutstandingWork#penciller_work.src_level, OutstandingWork#penciller_work.clerk, OutstandingWork#penciller_work.next_sqn]), + {State, none}; + {true, _} -> + %% Manifest locked + io:format("Manifest locked but no work outstanding " ++ + "with clerk~n"), {State, none} end; _ -> @@ -443,10 +503,8 @@ maybe_append_work(WorkQ, Level, Manifest, io:format("Outstanding compaction work items of ~w at level ~w~n", [FileCount - MaxFiles, Level]), lists:append(WorkQ, [{Level, Manifest}]); -maybe_append_work(WorkQ, Level, _Manifest, - _MaxFiles, FileCount) -> - io:format("No compaction work due to file count ~w at level ~w~n", - [FileCount, Level]), +maybe_append_work(WorkQ, _Level, _Manifest, + _MaxFiles, _FileCount) -> WorkQ. @@ -544,6 +602,29 @@ confirm_delete(Filename, UnreferencedFiles, RegisteredIterators) -> end end. + +%% load_manifest(RootDir), +%% {UpdManifest, MaxSQN} = load_allsft(RootDir, Manifest), +%% Level0SQN, UpdMaxSQN} = load_levelzero(RootDir, MaxSQN) + +load_manifest(_RootDir) -> + {{}, 0}. + +load_allsft(_RootDir, _Manifest) -> + %% Manifest has been persisted with PIDs that are no longer + %% valid, roll through each entry opening files and replacing Pids in + %% Manifest + {{}, 0}. + +load_levelzero(_RootDir, _MaxSQN) -> + %% When loading L0 manifest make sure that the lowest sequence number in + %% the L0 manifest is bigger than the highest in all levels below + %% - not True + %% ... what about key remainders? + %% ... need to rethink L0 + {0, 0}. + + %%%============================================================================ %%% Test %%%============================================================================ diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index cde8313..80d3fd4 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -161,6 +161,7 @@ sft_checkready/1, sft_getfilename/1, sft_setfordelete/2, + sft_getmaxsequencenumber/1, strip_to_keyonly/1, generate_randomkeys/1]). @@ -260,6 +261,9 @@ sft_checkready(Pid) -> sft_getfilename(Pid) -> gen_server:call(Pid, get_filename, infinty). +sft_getmaxsequencenumber(Pid) -> + gen_server:call(Pid, get_maxsqn, infinity). + %%%============================================================================ %%% API helper functions %%%============================================================================ @@ -394,7 +398,9 @@ handle_call({set_for_delete, Penciller}, _From, State) -> ok, State#state{ready_for_delete=true, penciller=Penciller}, - ?DELETE_TIMEOUT}. + ?DELETE_TIMEOUT}; +handle_call(get_maxsqn, _From, State) -> + {reply, State#state.highest_sqn, State}. handle_cast(_Msg, State) -> {noreply, State}. From 4586e2340a6c8dfe4371c8ebb5a5fa6e44e188c7 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 15 Aug 2016 16:43:39 +0100 Subject: [PATCH 029/167] Startup and Shutdown Support Added support for startup and shutdown of a Ledger. As aprt of this will now start tracking the highest sequence number. This also adds a safety check on pcl_pushmem to make sure that only keys with a higher sequenc enumber are being pushed in - and hence we can happily insert into the in-memory view without checking the sequence number. --- src/leveled_clerk.erl | 33 +++-- src/leveled_penciller.erl | 291 +++++++++++++++++++++++++++++--------- src/leveled_sft.erl | 172 +++++++++++----------- 3 files changed, 344 insertions(+), 152 deletions(-) diff --git a/src/leveled_clerk.erl b/src/leveled_clerk.erl index 1a181b2..107f12b 100644 --- a/src/leveled_clerk.erl +++ b/src/leveled_clerk.erl @@ -15,6 +15,7 @@ terminate/2, clerk_new/1, clerk_prompt/2, + clerk_stop/1, code_change/3, perform_merge/4]). @@ -23,8 +24,7 @@ -define(INACTIVITY_TIMEOUT, 2000). -define(HAPPYTIME_MULTIPLIER, 5). --record(state, {owner :: pid(), - in_backlog = false :: boolean()}). +-record(state, {owner :: pid()}). %%%============================================================================ %%% API @@ -35,11 +35,13 @@ clerk_new(Owner) -> ok = gen_server:call(Pid, {register, Owner}, infinity), {ok, Pid}. - clerk_prompt(Pid, penciller) -> gen_server:cast(Pid, penciller_prompt), ok. +clerk_stop(Pid) -> + gen_server:cast(Pid, stop). + %%%============================================================================ %%% gen_server callbacks %%%============================================================================ @@ -48,11 +50,13 @@ init([]) -> {ok, #state{}}. handle_call({register, Owner}, _From, State) -> - {reply, ok, State#state{owner=Owner}}. + {reply, ok, State#state{owner=Owner}, ?INACTIVITY_TIMEOUT}. handle_cast(penciller_prompt, State) -> Timeout = requestandhandle_work(State), - {noreply, State, Timeout}. + {noreply, State, Timeout}; +handle_cast(stop, State) -> + {stop, normal, State}. handle_info(timeout, State) -> %% The pcl prompt will cause a penciller_prompt, to re-trigger timeout @@ -90,10 +94,21 @@ requestandhandle_work(State) -> {NewManifest, FilesToDelete} = merge(WI), UpdWI = WI#penciller_work{new_manifest=NewManifest, unreferenced_files=FilesToDelete}, - leveled_penciller:pcl_requestmanifestchange(State#state.owner, - UpdWI), - mark_for_delete(FilesToDelete, State#state.owner), - ?INACTIVITY_TIMEOUT + R = leveled_penciller:pcl_requestmanifestchange(State#state.owner, + UpdWI), + case R of + ok -> + %% Request for manifest change must be a synchronous call + %% Otherwise cannot mark files for deletion (may erase + %% without manifest change on close) + mark_for_delete(FilesToDelete, State#state.owner), + ?INACTIVITY_TIMEOUT; + _ -> + %% New files will forever remain in an undetermined state + %% The disconnected files should be logged at start-up for + %% Manual clear-up + ?INACTIVITY_TIMEOUT + end end. diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 44e70c0..49de7c6 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -84,9 +84,8 @@ %% To initiate the Ledger the must consult the manifest, and then start a SFT %% management process for each file in the manifest. %% -%% The penciller should then try and read any persisted ETS table in the -%% on_shutdown folder. The Penciller must then discover the highest sequence -%% number in the ledger, and respond to the Bookie with that sequence number. +%% The penciller should then try and read any Level 0 file which has the +%% manifest sequence number one higher than the last store in the manifest. %% %% The Bookie will ask the Inker for any Keys seen beyond that sequence number %% before the startup of the overall store can be completed. @@ -153,14 +152,15 @@ handle_info/2, terminate/2, code_change/3, - pcl_new/0, pcl_start/1, pcl_pushmem/2, pcl_fetch/2, pcl_workforclerk/1, pcl_requestmanifestchange/2, pcl_confirmdelete/2, - pcl_prompt/1]). + pcl_prompt/1, + pcl_close/1, + pcl_getstartupsequencenumber/1]). -include_lib("eunit/include/eunit.hrl"). @@ -182,6 +182,7 @@ -record(state, {manifest = [] :: list(), ongoing_work = [] :: list(), manifest_sqn = 0 :: integer(), + ledger_sqn = 0 :: integer(), registered_iterators = [] :: list(), unreferenced_files = [] :: list(), root_path = "../test" :: string(), @@ -196,13 +197,8 @@ %%% API %%%============================================================================ -pcl_new() -> - gen_server:start(?MODULE, [], []). - pcl_start(RootDir) -> - {ok, Pid} = gen_server:start(?MODULE, [], []), - gen_server:call(Pid, {load, RootDir}, infinity), - ok. + gen_server:start(?MODULE, [RootDir], []). pcl_pushmem(Pid, DumpList) -> %% Bookie to dump memory onto penciller @@ -215,7 +211,7 @@ pcl_workforclerk(Pid) -> gen_server:call(Pid, work_for_clerk, infinity). pcl_requestmanifestchange(Pid, WorkItem) -> - gen_server:cast(Pid, {manifest_change, WorkItem}). + gen_server:call(Pid, {manifest_change, WorkItem}, infinity). pcl_confirmdelete(Pid, FileName) -> gen_server:call(Pid, {confirm_delete, FileName}). @@ -223,23 +219,116 @@ pcl_confirmdelete(Pid, FileName) -> pcl_prompt(Pid) -> gen_server:call(Pid, prompt_compaction). +pcl_getstartupsequencenumber(Pid) -> + gen_server:call(Pid, get_startup_sqn). + +pcl_close(Pid) -> + gen_server:call(Pid, close). + %%%============================================================================ %%% gen_server callbacks %%%============================================================================ -init([]) -> +init([RootPath]) -> TID = ets:new(?MEMTABLE, [ordered_set, private]), {ok, Clerk} = leveled_clerk:clerk_new(self()), - {ok, #state{memtable=TID, clerk=Clerk}}. + InitState = #state{memtable=TID, clerk=Clerk, root_path=RootPath}, + + %% Open manifest + ManifestPath = InitState#state.root_path ++ "/" ++ ?MANIFEST_FP ++ "/", + {ok, Filenames} = file:list_dir(ManifestPath), + CurrRegex = "nonzero_(?[0-9]+)\\." ++ ?CURRENT_FILEX, + ValidManSQNs = lists:foldl(fun(FN, Acc) -> + case re:run(FN, + CurrRegex, + [{capture, ['MSN'], list}]) of + nomatch -> + Acc; + {match, [Int]} when is_list(Int) -> + Acc ++ [list_to_integer(Int)]; + _ -> + Acc + end end, + [], + Filenames), + TopManSQN = lists:foldl(fun(X, MaxSQN) -> max(X, MaxSQN) end, + 0, + ValidManSQNs), + io:format("Store to be started based on " ++ + "manifest sequence number of ~w~n", [TopManSQN]), + case TopManSQN of + 0 -> + io:format("Seqence number of 0 indicates no valid manifest~n"), + {ok, InitState}; + _ -> + {ok, Bin} = file:read_file(filepath(InitState#state.root_path, + TopManSQN, + current_manifest)), + Manifest = binary_to_term(Bin), + {UpdManifest, MaxSQN} = open_all_filesinmanifest(Manifest), + io:format("Maximum sequence number of ~w " + ++ "found in nonzero levels~n", + [MaxSQN]), + + %% TODO + %% Find any L0 File left outstanding + L0FN = filepath(RootPath, + TopManSQN + 1, + new_merge_files) ++ "_0_0.sft", + case filelib:is_file(L0FN) of + true -> + io:format("L0 file found ~s~n", [L0FN]), + {ok, + L0Pid, + {L0StartKey, L0EndKey}} = leveled_sft:sft_open(L0FN), + L0SQN = leveled_sft:sft_getmaxsequencenumber(L0Pid), + ManifestEntry = #manifest_entry{start_key=L0StartKey, + end_key=L0EndKey, + owner=L0Pid, + filename=L0FN}, + UpdManifest2 = lists:keystore(0, + 1, + UpdManifest, + {0, [ManifestEntry]}), + io:format("L0 file had maximum sequence number of ~w~n", + [L0SQN]), + {ok, + InitState#state{manifest=UpdManifest2, + manifest_sqn=TopManSQN, + ledger_sqn=max(MaxSQN, L0SQN)}}; + false -> + io:format("No L0 file found~n"), + {ok, + InitState#state{manifest=UpdManifest, + manifest_sqn=TopManSQN, + ledger_sqn=MaxSQN}} + end + end. + handle_call({push_mem, DumpList}, _From, State) -> - case push_to_memory(DumpList, State) of - {ok, UpdState} -> - {reply, ok, UpdState}; - {{pause, Reason, Details}, UpdState} -> - io:format("Excess work due to - " ++ Reason, Details), - {reply, pause, UpdState#state{backlog=true}} - end; + StartWatch = os:timestamp(), + Response = case assess_sqn(DumpList) of + {MinSQN, MaxSQN} when MaxSQN > MinSQN, + MinSQN >= State#state.ledger_sqn -> + case push_to_memory(DumpList, State) of + {ok, UpdState} -> + {reply, ok, UpdState}; + {{pause, Reason, Details}, UpdState} -> + io:format("Excess work due to - " ++ Reason, Details), + {reply, pause, UpdState#state{backlog=true, + ledger_sqn=MaxSQN}} + end; + {MinSQN, MaxSQN} -> + io:format("Mismatch of sequence number expectations with push " + ++ "having sequence numbers between ~w and ~w " + ++ "but current sequence number is ~w~n", + [MinSQN, MaxSQN, State#state.ledger_sqn]), + {reply, refused, State} + end, + io:format("Push completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(),StartWatch)]), + Response; handle_call({fetch, Key}, _From, State) -> {reply, fetch(Key, State#state.manifest, State#state.memtable), State}; handle_call(work_for_clerk, From, State) -> @@ -249,14 +338,13 @@ handle_call({confirm_delete, FileName}, _From, State) -> Reply = confirm_delete(FileName, State#state.unreferenced_files, State#state.registered_iterators), - {reply, Reply, State}; -handle_call({load, RootDir}, _From, State) -> - {Manifest, ManifestSQN} = load_manifest(RootDir), - {UpdManifest, MaxSQN} = load_allsft(RootDir, Manifest), - {UpdMaxSQN} = load_levelzero(RootDir, MaxSQN), - {reply, UpdMaxSQN, State#state{root_path=RootDir, - manifest=UpdManifest, - manifest_sqn=ManifestSQN}}; + case Reply of + true -> + UF1 = lists:keydelete(FileName, 1, State#state.unreferenced_files), + {reply, true, State#state{unreferenced_files=UF1}}; + _ -> + {reply, Reply, State} + end; handle_call(prompt_compaction, _From, State) -> case push_to_memory([], State) of {ok, UpdState} -> @@ -264,17 +352,66 @@ handle_call(prompt_compaction, _From, State) -> {{pause, Reason, Details}, UpdState} -> io:format("Excess work due to - " ++ Reason, Details), {reply, pause, UpdState#state{backlog=true}} - end. - - -handle_cast({manifest_change, WI}, State) -> + end; +handle_call({manifest_change, WI}, _From, State) -> {ok, UpdState} = commit_manifest_change(WI, State), - {noreply, UpdState}. + {reply, ok, UpdState}; +handle_call(get_startup_sqn, _From, State) -> + {reply, State#state.ledger_sqn, State}; +handle_call(close, _From, State) -> + {stop, normal, ok, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. handle_info(_Info, State) -> {noreply, State}. -terminate(_Reason, _State) -> +terminate(_Reason, State) -> + %% When a Penciller shuts down it isn't safe to try an manage the safe + %% finishing of any outstanding work. The last commmitted manifest will + %% be used. + %% + %% Level 0 files lie outside of the manifest, and so if there is no L0 + %% file present it is safe to write the current contents of memory. If + %% there is a L0 file present - then the memory can be dropped (it is + %% recoverable from the ledger, and there should not be a lot to recover + %% as presumably the ETS file has been recently flushed, hence the presence + %% of a L0 file). + %% + %% The penciller should close each file in the unreferenced files, and + %% then each file in the manifest, and cast a close on the clerk. + %% The cast may not succeed as the clerk could be synchronously calling + %% the penciller looking for a manifest commit + %% + leveled_clerk:clerk_stop(State#state.clerk), + Dump = ets:tab2list(State#state.memtable), + case {State#state.levelzero_pending, + get_item(0, State#state.manifest, []), length(Dump)} of + {{false, _, _}, [], L} when L > 0 -> + MSN = State#state.manifest_sqn + 1, + FileName = State#state.root_path + ++ "/" ++ ?FILES_FP ++ "/" + ++ integer_to_list(MSN) ++ "_0_0", + {ok, + L0Pid, + {{KR1, _}, _SK, _HK}} = leveled_sft:sft_new(FileName ++ ".pnd", + Dump, + [], + 0), + io:format("Dump of memory on close to filename ~s with" + ++ " remainder ~w~n", [FileName, length(KR1)]), + leveled_sft:sft_close(L0Pid), + file:rename(FileName ++ ".pnd", FileName ++ ".sft"); + {{false, _, _}, [], L} when L == 0 -> + io:format("No keys to dump from memory when closing~n"); + _ -> + io:format("No opportunity to persist memory before closing " + ++ "with ~w keys discarded~n", [length(Dump)]) + end, + ok = close_files(0, State#state.manifest), + lists:foreach(fun({_FN, Pid, _SN}) -> leveled_sft:sft_close(Pid) end, + State#state.unreferenced_files), ok. code_change(_OldVsn, State, _Extra) -> @@ -288,7 +425,7 @@ code_change(_OldVsn, State, _Extra) -> push_to_memory(DumpList, State) -> {TableSize, UpdState} = case State#state.levelzero_pending of {true, Remainder, {StartKey, EndKey, Pid}} -> - %% Need to handle not error scenarios? + %% Need to handle error scenarios? %% N.B. Sync call - so will be ready {ok, SrcFN} = leveled_sft:sft_checkready(Pid), %% Reset ETS, but re-insert any remainder @@ -429,6 +566,7 @@ manifest_locked(State) -> end. + %% Work out what the current work queue should be %% %% The work queue should have a lower level work at the front, and no work @@ -485,8 +623,41 @@ return_work(State, From) -> end. +close_files(?MAX_LEVELS - 1, _Manifest) -> + ok; +close_files(Level, Manifest) -> + LevelList = get_item(Level, Manifest, []), + lists:foreach(fun(F) -> leveled_sft:sft_close(F#manifest_entry.owner) end, + LevelList), + close_files(Level + 1, Manifest). +open_all_filesinmanifest(Manifest) -> + open_all_filesinmanifest({Manifest, 0}, 0). + +open_all_filesinmanifest(Result, ?MAX_LEVELS - 1) -> + Result; +open_all_filesinmanifest({Manifest, TopSQN}, Level) -> + LevelList = get_item(Level, Manifest, []), + %% The Pids in the saved manifest related to now closed references + %% Need to roll over the manifest at this level starting new processes to + %5 replace them + LvlR = lists:foldl(fun(F, {FL, FL_SQN}) -> + FN = F#manifest_entry.filename, + {ok, P, _Keys} = leveled_sft:sft_open(FN), + F_SQN = leveled_sft:sft_getmaxsequencenumber(P), + {lists:append(FL, + [F#manifest_entry{owner = P}]), + max(FL_SQN, F_SQN)} + end, + {[], 0}, + LevelList), + %% Result is tuple of revised file list for this level in manifest, and + %% the maximum sequence number seen at this level + {LvlFL, LvlSQN} = LvlR, + UpdManifest = lists:keystore(Level, 1, Manifest, {Level, LvlFL}), + open_all_filesinmanifest({UpdManifest, max(TopSQN, LvlSQN)}, Level + 1). + assess_workqueue(WorkQ, ?MAX_LEVELS - 1, _Manifest) -> WorkQ; assess_workqueue(WorkQ, LevelToAssess, Manifest)-> @@ -539,8 +710,8 @@ commit_manifest_change(ReturnedWorkItem, State) -> {NewMSN, _From} -> MTime = timer:now_diff(os:timestamp(), SentWorkItem#penciller_work.start_time), - io:format("Merge to sqn ~w completed in ~w microseconds - at Level ~w~n", + io:format("Merge to sqn ~w completed in ~w microseconds " ++ + "at Level ~w~n", [SentWorkItem#penciller_work.next_sqn, MTime, SentWorkItem#penciller_work.src_level]), @@ -558,8 +729,8 @@ commit_manifest_change(ReturnedWorkItem, State) -> manifest=NewManifest, unreferenced_files=UnreferencedFilesUpd}}; {MaybeWrongMSN, From} -> - io:format("Merge commit at sqn ~w not matched to expected - sqn ~w from Clerk ~w~n", + io:format("Merge commit at sqn ~w not matched to expected" ++ + " sqn ~w from Clerk ~w~n", [NewMSN, MaybeWrongMSN, From]), {error, State} end. @@ -586,43 +757,36 @@ update_deletions([ClearedFile|Tail], MSN, UnreferencedFiles) -> update_deletions(Tail, MSN, lists:append(UnreferencedFiles, - [{ClearedFile#manifest_entry.filename, MSN}])). + [{ClearedFile#manifest_entry.filename, + ClearedFile#manifest_entry.owner, + MSN}])). confirm_delete(Filename, UnreferencedFiles, RegisteredIterators) -> case lists:keyfind(Filename, 1, UnreferencedFiles) of false -> false; - {Filename, MSN} -> + {Filename, _Pid, MSN} -> LowSQN = lists:foldl(fun({_, SQN}, MinSQN) -> min(SQN, MinSQN) end, infinity, RegisteredIterators), if - MSN >= LowSQN -> false; - true -> true + MSN >= LowSQN -> + false; + true -> + true end end. -%% load_manifest(RootDir), -%% {UpdManifest, MaxSQN} = load_allsft(RootDir, Manifest), -%% Level0SQN, UpdMaxSQN} = load_levelzero(RootDir, MaxSQN) -load_manifest(_RootDir) -> - {{}, 0}. +assess_sqn(DumpList) -> + assess_sqn(DumpList, infinity, 0). -load_allsft(_RootDir, _Manifest) -> - %% Manifest has been persisted with PIDs that are no longer - %% valid, roll through each entry opening files and replacing Pids in - %% Manifest - {{}, 0}. - -load_levelzero(_RootDir, _MaxSQN) -> - %% When loading L0 manifest make sure that the lowest sequence number in - %% the L0 manifest is bigger than the highest in all levels below - %% - not True - %% ... what about key remainders? - %% ... need to rethink L0 - {0, 0}. +assess_sqn([], MinSQN, MaxSQN) -> + {MinSQN, MaxSQN}; +assess_sqn([HeadKey|Tail], MinSQN, MaxSQN) -> + {_K, SQN} = leveled_sft:strip_to_key_seqn_only(HeadKey), + assess_sqn(Tail, min(MinSQN, SQN), max(MaxSQN, SQN)). %%%============================================================================ @@ -651,7 +815,8 @@ compaction_work_assessment_test() -> confirm_delete_test() -> Filename = 'test.sft', - UnreferencedFiles = [{'other.sft', 15}, {Filename, 10}], + UnreferencedFiles = [{'other.sft', dummy_owner, 15}, + {Filename, dummy_owner, 10}], RegisteredIterators1 = [{dummy_pid, 16}, {dummy_pid, 12}], R1 = confirm_delete(Filename, UnreferencedFiles, RegisteredIterators1), ?assertMatch(R1, true), diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 80d3fd4..fe4d331 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -163,6 +163,7 @@ sft_setfordelete/2, sft_getmaxsequencenumber/1, strip_to_keyonly/1, + strip_to_key_seqn_only/1, generate_randomkeys/1]). -include_lib("eunit/include/eunit.hrl"). @@ -231,29 +232,33 @@ sft_new(Filename, KL1, KL2, Level, Options) -> sft_open(Filename) -> {ok, Pid} = gen_server:start(?MODULE, [], []), case gen_server:call(Pid, {sft_open, Filename}, infinity) of - ok -> - {ok, Pid}; + {ok, {SK, EK}} -> + {ok, Pid, {SK, EK}}; Error -> Error end. sft_setfordelete(Pid, Penciller) -> - file_request(Pid, {set_for_delete, Penciller}). + gen_server:call(Pid, {set_for_delete, Penciller}, infinity). sft_get(Pid, Key) -> - file_request(Pid, {get_kv, Key}). + gen_server:call(Pid, {get_kv, Key}, infinity). sft_getkeyrange(Pid, StartKey, EndKey, ScanWidth) -> - file_request(Pid, {get_keyrange, StartKey, EndKey, ScanWidth}). + gen_server:call(Pid, + {get_keyrange, StartKey, EndKey, ScanWidth}, + infinity). sft_getkvrange(Pid, StartKey, EndKey, ScanWidth) -> - file_request(Pid, {get_kvrange, StartKey, EndKey, ScanWidth}). - -sft_close(Pid) -> - file_request(Pid, close). + gen_server:call(Pid, + {get_kvrange, StartKey, EndKey, ScanWidth}, + infinity). sft_clear(Pid) -> - file_request(Pid, clear). + gen_server:call(Pid, clear, infinity). + +sft_close(Pid) -> + gen_server:call(Pid, close, infinity). sft_checkready(Pid) -> gen_server:call(Pid, background_complete, infinity). @@ -264,36 +269,7 @@ sft_getfilename(Pid) -> sft_getmaxsequencenumber(Pid) -> gen_server:call(Pid, get_maxsqn, infinity). -%%%============================================================================ -%%% API helper functions -%%%============================================================================ -%% This saftey measure of checking the Pid is alive before perfoming any ops -%% is copied from the bitcask source code. -%% -%% It is not clear at present if this is necessary. - -file_request(Pid, Request) -> - case check_pid(Pid) of - ok -> - gen_server:call(Pid, Request, infinity); - Error -> - Error - end. - -check_pid(Pid) -> - IsPid = is_pid(Pid), - IsAlive = IsPid andalso is_process_alive(Pid), - case {IsAlive, IsPid} of - {true, _} -> - ok; - {false, true} -> - %% Same result as `file' module when accessing closed FD - {error, einval}; - _ -> - %% Same result as `file' module when providing wrong arg - {error, badarg} - end. %%%============================================================================ %%% gen_server callbacks @@ -363,8 +339,12 @@ handle_call({sft_new, Filename, KL1, KL2, Level}, _From, State) -> UpdFileMD#state{handle=ReadHandle, filename=Filename}} end; handle_call({sft_open, Filename}, _From, _State) -> - {_Handle, FileMD} = open_file(Filename), - {reply, {FileMD#state.smallest_key, FileMD#state.highest_key}, FileMD}; + {_Handle, FileMD} = open_file(#state{filename=Filename}), + io:format("Opened filename with name ~s~n", [Filename]), + {reply, + {ok, + {FileMD#state.smallest_key, FileMD#state.highest_key}}, + FileMD}; handle_call({get_kv, Key}, _From, State) -> Reply = fetch_keyvalue(State#state.handle, State, Key), {reply, Reply, State}; @@ -379,11 +359,9 @@ handle_call({get_kvrange, StartKey, EndKey, ScanWidth}, _From, State) -> ScanWidth), {reply, Reply, State}; handle_call(close, _From, State) -> - {reply, true, State}; + {stop, normal, ok, State}; handle_call(clear, _From, State) -> - ok = file:close(State#state.handle), - ok = file:delete(State#state.filename), - {reply, true, State}; + {stop, normal, ok, State#state{ready_for_delete=true}}; handle_call(background_complete, _From, State) -> case State#state.background_complete of true -> @@ -401,7 +379,7 @@ handle_call({set_for_delete, Penciller}, _From, State) -> ?DELETE_TIMEOUT}; handle_call(get_maxsqn, _From, State) -> {reply, State#state.highest_sqn, State}. - + handle_cast(_Msg, State) -> {noreply, State}. @@ -412,10 +390,6 @@ handle_info(timeout, State) -> State#state.filename) of true -> - io:format("Polled for deletion and now clearing ~s~n", - [State#state.filename]), - ok = file:close(State#state.handle), - ok = file:delete(State#state.filename), {stop, shutdown, State}; false -> io:format("Polled for deletion but ~s not ready~n", @@ -428,8 +402,18 @@ handle_info(timeout, State) -> handle_info(_Info, State) -> {noreply, State}. -terminate(_Reason, _State) -> - ok. +terminate(Reason, State) -> + io:format("Exit called for reason ~w on filename ~s~n", + [Reason, State#state.filename]), + case State#state.ready_for_delete of + true -> + io:format("Exit called and now clearing ~s~n", + [State#state.filename]), + ok = file:close(State#state.handle), + ok = file:delete(State#state.filename); + _ -> + ok = file:close(State#state.handle) + end. code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -453,7 +437,8 @@ create_file(FileName) when is_list(FileName) -> FileMD = #state{next_position=StartPos, filename=FileName}, {Handle, FileMD}; {error, Reason} -> - io:format("Error opening filename ~s with reason ~s", [FileName, Reason]), + io:format("Error opening filename ~s with reason ~s", + [FileName, Reason]), {error, Reason} end. @@ -484,7 +469,8 @@ open_file(FileMD) -> Ilen:32/integer, Flen:32/integer, Slen:32/integer>> = HeaderLengths, - {ok, SummaryBin} = file:pread(Handle, ?HEADER_LEN + Blen + Ilen + Flen, Slen), + {ok, SummaryBin} = file:pread(Handle, + ?HEADER_LEN + Blen + Ilen + Flen, Slen), {{LowSQN, HighSQN}, {LowKey, HighKey}} = binary_to_term(SummaryBin), {ok, SlotIndexBin} = file:pread(Handle, ?HEADER_LEN + Blen, Ilen), SlotIndex = binary_to_term(SlotIndexBin), @@ -552,14 +538,17 @@ fetch_keyvalue(Handle, FileMD, Key) -> %% Fetches a range of keys returning a list of {Key, SeqN} tuples fetch_range_keysonly(Handle, FileMD, StartKey, EndKey) -> - fetch_range(Handle, FileMD, StartKey, EndKey, [], fun acc_list_keysonly/2). + fetch_range(Handle, FileMD, StartKey, EndKey, [], + fun acc_list_keysonly/2). fetch_range_keysonly(Handle, FileMD, StartKey, EndKey, ScanWidth) -> - fetch_range(Handle, FileMD, StartKey, EndKey, [], fun acc_list_keysonly/2, ScanWidth). + fetch_range(Handle, FileMD, StartKey, EndKey, [], + fun acc_list_keysonly/2, ScanWidth). %% Fetches a range of keys returning the full tuple, including value fetch_range_kv(Handle, FileMD, StartKey, EndKey, ScanWidth) -> - fetch_range(Handle, FileMD, StartKey, EndKey, [], fun acc_list_kv/2, ScanWidth). + fetch_range(Handle, FileMD, StartKey, EndKey, [], + fun acc_list_kv/2, ScanWidth). acc_list_keysonly(null, empty) -> []; @@ -594,39 +583,60 @@ acc_list_kv(R, RList) -> %% used - e.g. counters, hash-lists to build bloom filters etc fetch_range(Handle, FileMD, StartKey, EndKey, FunList, AccFun) -> - fetch_range(Handle, FileMD, StartKey, EndKey, FunList, AccFun, ?ITERATOR_SCANWIDTH). + fetch_range(Handle, FileMD, StartKey, EndKey, FunList, + AccFun, ?ITERATOR_SCANWIDTH). fetch_range(Handle, FileMD, StartKey, EndKey, FunList, AccFun, ScanWidth) -> - fetch_range(Handle, FileMD, StartKey, EndKey, FunList, AccFun, ScanWidth, empty). + fetch_range(Handle, FileMD, StartKey, EndKey, FunList, + AccFun, ScanWidth, empty). -fetch_range(_Handle, _FileMD, StartKey, _EndKey, _FunList, _AccFun, 0, Acc) -> +fetch_range(_Handle, _FileMD, StartKey, _EndKey, _FunList, + _AccFun, 0, Acc) -> {partial, Acc, StartKey}; -fetch_range(Handle, FileMD, StartKey, EndKey, FunList, AccFun, ScanWidth, Acc) -> +fetch_range(Handle, FileMD, StartKey, EndKey, FunList, + AccFun, ScanWidth, Acc) -> %% get_nearestkey gets the last key in the index <= StartKey, or the next %% key along if {next, StartKey} is passed case get_nearestkey(FileMD#state.slot_index, StartKey) of {NearestKey, _Filter, {LengthList, PointerB}} -> - fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, FunList, AccFun, ScanWidth, - LengthList, 0, PointerB + FileMD#state.slots_pointer, + fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, FunList, + AccFun, ScanWidth, + LengthList, + 0, + PointerB + FileMD#state.slots_pointer, AccFun(null, Acc)); not_found -> {complete, AccFun(null, Acc)} end. -fetch_range(Handle, FileMD, _StartKey, NearestKey, EndKey, FunList, AccFun, ScanWidth, - LengthList, BlockNumber, _Pointer, Acc) +fetch_range(Handle, FileMD, _StartKey, NearestKey, EndKey, FunList, + AccFun, ScanWidth, + LengthList, + BlockNumber, + _Pointer, + Acc) when length(LengthList) == BlockNumber -> %% Reached the end of the slot. Move the start key on one to scan a new slot - fetch_range(Handle, FileMD, {next, NearestKey}, EndKey, FunList, AccFun, ScanWidth - 1, Acc); -fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, FunList, AccFun, ScanWidth, - LengthList, BlockNumber, Pointer, Acc) -> + fetch_range(Handle, FileMD, {next, NearestKey}, EndKey, FunList, + AccFun, ScanWidth - 1, + Acc); +fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, FunList, + AccFun, ScanWidth, + LengthList, + BlockNumber, + Pointer, + Acc) -> Block = fetch_block(Handle, LengthList, BlockNumber, Pointer), Results = scan_block(Block, StartKey, EndKey, FunList, AccFun, Acc), case Results of {partial, Acc1, StartKey} -> %% Move on to the next block - fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, FunList, AccFun, ScanWidth, - LengthList, BlockNumber + 1, Pointer, Acc1); + fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, FunList, + AccFun, ScanWidth, + LengthList, + BlockNumber + 1, + Pointer, + Acc1); {complete, Acc1} -> {complete, Acc1} end. @@ -665,8 +675,8 @@ applyfuns([HeadFun|OtherFuns], KV) -> fetch_keyvalue_fromblock([], _Key, _LengthList, _Handle, _StartOfSlot) -> not_present; -fetch_keyvalue_fromblock([BlockNumber|T], Key, LengthList, Handle, StartOfSlot) -> - BlockToCheck = fetch_block(Handle, LengthList, BlockNumber, StartOfSlot), +fetch_keyvalue_fromblock([BlockNmb|T], Key, LengthList, Handle, StartOfSlot) -> + BlockToCheck = fetch_block(Handle, LengthList, BlockNmb, StartOfSlot), Result = lists:keyfind(Key, 1, BlockToCheck), case Result of false -> @@ -675,9 +685,9 @@ fetch_keyvalue_fromblock([BlockNumber|T], Key, LengthList, Handle, StartOfSlot) KV end. -fetch_block(Handle, LengthList, BlockNumber, StartOfSlot) -> - Start = lists:sum(lists:sublist(LengthList, BlockNumber)), - Length = lists:nth(BlockNumber + 1, LengthList), +fetch_block(Handle, LengthList, BlockNmb, StartOfSlot) -> + Start = lists:sum(lists:sublist(LengthList, BlockNmb)), + Length = lists:nth(BlockNmb + 1, LengthList), {ok, BlockToCheckBin} = file:pread(Handle, Start + StartOfSlot, Length), binary_to_term(BlockToCheckBin). @@ -1384,18 +1394,20 @@ generate_randomsegfilter(BlockSize) -> Block4})). +generate_randomkeys({Count, StartSQN}) -> + generate_randomkeys(Count, StartSQN, []); generate_randomkeys(Count) -> - generate_randomkeys(Count, []). + generate_randomkeys(Count, 0, []). -generate_randomkeys(0, Acc) -> +generate_randomkeys(0, _SQN, Acc) -> Acc; -generate_randomkeys(Count, Acc) -> +generate_randomkeys(Count, SQN, Acc) -> RandKey = {{o, lists:concat(["Bucket", random:uniform(1024)]), lists:concat(["Key", random:uniform(1024)])}, - Count + 1, + SQN, {active, infinity}, null}, - generate_randomkeys(Count - 1, [RandKey|Acc]). + generate_randomkeys(Count - 1, SQN + 1, [RandKey|Acc]). generate_sequentialkeys(Count, Start) -> generate_sequentialkeys(Count + Start, Start, []). From ce0c55a2ec012b6ad179598fcafccbe56afd0475 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 16 Aug 2016 12:45:48 +0100 Subject: [PATCH 030/167] Resolve issue of Remainders Two issues looked at - There shouldn't be a remainder after writing the L0 file, as this could have overlapping sequence numbers which will be missed on restart - There should be a safety-check to stop the Clerk from doing a fake push too soon after a background L0 file ahs been written (as the fake push would lock the ledger waiting for the L0 file write to finish) --- src/leveled_clerk.erl | 4 +- src/leveled_penciller.erl | 172 ++++++++++++++++++++++---------------- src/leveled_sft.erl | 132 +++++++++++++++++------------ 3 files changed, 177 insertions(+), 131 deletions(-) diff --git a/src/leveled_clerk.erl b/src/leveled_clerk.erl index 107f12b..f423b59 100644 --- a/src/leveled_clerk.erl +++ b/src/leveled_clerk.erl @@ -251,7 +251,7 @@ do_merge(KL1, KL2, Level, {Filepath, MSN}, FileCounter, OutList) -> io:format("File to be created as part of MSN=~w Filename=~s~n", [MSN, FileName]), TS1 = os:timestamp(), - case leveled_sft:sft_new(FileName, KL1, KL2, Level) of + case leveled_sft:sft_new(FileName, KL1, KL2, Level + 1) of {ok, _Pid, {error, Reason}} -> io:format("Exiting due to error~w~n", [Reason]), error; @@ -263,7 +263,7 @@ do_merge(KL1, KL2, Level, {Filepath, MSN}, FileCounter, OutList) -> owner=Pid, filename=FileName}]), MTime = timer:now_diff(os:timestamp(), TS1), - io:format("file creation took ~w microseconds ~n", [MTime]), + io:format("File creation took ~w microseconds ~n", [MTime]), do_merge(KL1Rem, KL2Rem, Level, {Filepath, MSN}, FileCounter + 1, ExtMan) end. diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 49de7c6..c84bfe7 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -1,29 +1,35 @@ %% -------- PENCILLER --------- %% -%% The penciller is repsonsible for writing and re-writing the ledger - a +%% The penciller is responsible for writing and re-writing the ledger - a %% persisted, ordered view of non-recent Keys and Metadata which have been %% added to the store. %% - The penciller maintains a manifest of all the files within the current %% Ledger. -%% - The Penciller queues re-write (compaction) work up to be managed by Clerks +%% - The Penciller provides re-write (compaction) work up to be managed by +%% the Penciller's Clerk %% - The Penciller mainatins a register of iterators who have requested %% snapshots of the Ledger -%% - The accepts new dumps (in the form of immutable ets tables) from the -%% Bookie, and calls the Bookie once the process of pencilling this data in -%% the Ledger is complete - and the Bookie is free to forget about the data +%% - The accepts new dumps (in the form of lists of keys) from the Bookie, and +%% calls the Bookie once the process of pencilling this data in the Ledger is +%% complete - and the Bookie is free to forget about the data %% %% -------- LEDGER --------- %% %% The Ledger is divided into many levels -%% L0: ETS tables are received from the Bookie and merged into a single ETS +%% - L0: New keys are received from the Bookie and merged into a single ETS %% table, until that table is the size of a SFT file, and it is then persisted -%% as a SFT file at this level. Once the persistence is completed, the ETS -%% table can be dropped. There can be only one SFT file at Level 0, so -%% the work to merge that file to the lower level must be the highest priority, -%% as otherwise the database will stall. -%% L1 TO L7: May contain multiple non-overlapping PIDs managing sft files. -%% Compaction work should be sheduled if the number of files exceeds the target -%% size of the level, where the target size is 8 ^ n. +%% as a SFT file at this level. L0 SFT files can be larger than the normal +%% maximum size - so we don't have to consider problems of either having more +%% than one L0 file (and handling what happens on a crash between writing the +%% files when the second may have overlapping sequence numbers), or having a +%% remainder with overlapping in sequence numbers in memory after the file is +%% written. Once the persistence is completed, the ETS table can be erased. +%% There can be only one SFT file at Level 0, so the work to merge that file +%% to the lower level must be the highest priority, as otherwise writes to the +%% ledger will stall, when there is next a need to persist. +%% - L1 TO L7: May contain multiple processes managing non-overlapping sft +%% files. Compaction work should be sheduled if the number of files exceeds +%% the target size of the level, where the target size is 8 ^ n. %% %% The most recent revision of a Key can be found by checking each level until %% the key is found. To check a level the correct file must be sought from the @@ -33,28 +39,30 @@ %% If a compaction change takes the size of a level beyond the target size, %% then compaction work for that level + 1 should be added to the compaction %% work queue. -%% Compaction work is fetched by the Pencllier's Clerk because: +%% Compaction work is fetched by the Penciller's Clerk because: %% - it has timed out due to a period of inactivity %% - it has been triggered by the a cast to indicate the arrival of high %% priority compaction work %% The Penciller's Clerk (which performs compaction worker) will always call -%% the Penciller to find out the highest priority work currently in the queue +%% the Penciller to find out the highest priority work currently required %% whenever it has either completed work, or a timeout has occurred since it %% was informed there was no work to do. %% -%% When the clerk picks work off the queue it will take the current manifest -%% for the level and level - 1. The clerk will choose which file to compact -%% from level - 1, and once the compaction is complete will call to the -%% Penciller with the new version of the manifest to be written. +%% When the clerk picks work it will take the current manifest, and the +%% Penciller assumes the manifest sequence number is to be incremented. +%% When the clerk has completed the work it cna request that the manifest +%% change be committed by the Penciller. The commit is made through changing +%% the filename of the new manifest - so the Penciller is not held up by the +%% process of wiritng a file, just altering file system metadata. %% -%% Once the new version of the manifest had been persisted, the state of any -%% deleted files will be changed to pending deletion. In pending deletion they -%% will call the Penciller on a timeout to confirm that they are no longer in -%% use (by any iterators). +%% The manifest is locked by a clerk taking work, or by there being a need to +%% write a file to Level 0. If the manifest is locked, then new keys can still +%% be added in memory - however, the response to that push will be to "pause", +%% that is to say the Penciller will ask the Bookie to slowdown. %% %% ---------- PUSH ---------- %% -%% The Penciller must support the PUSH of an ETS table from the Bookie. The +%% The Penciller must support the PUSH of a dump of keys from the Bookie. The %% call to PUSH should be immediately acknowledged, and then work should be %% completed to merge the ETS table into the L0 ETS table. %% @@ -177,7 +185,8 @@ -define(ARCHIVE_FILEX, "arc"). -define(MEMTABLE, mem). -define(MAX_TABLESIZE, 32000). --define(L0PEND_RESET, {false, [], none}). +-define(PROMPT_WAIT_ONL0, 5). +-define(L0PEND_RESET, {false, null, null}). -record(state, {manifest = [] :: list(), ongoing_work = [] :: list(), @@ -188,7 +197,7 @@ root_path = "../test" :: string(), table_size = 0 :: integer(), clerk :: pid(), - levelzero_pending = {false, [], none} :: tuple(), + levelzero_pending = ?L0PEND_RESET :: tuple(), memtable, backlog = false :: boolean()}). @@ -230,13 +239,18 @@ pcl_close(Pid) -> %%%============================================================================ init([RootPath]) -> - TID = ets:new(?MEMTABLE, [ordered_set, private]), + TID = ets:new(?MEMTABLE, [ordered_set]), {ok, Clerk} = leveled_clerk:clerk_new(self()), InitState = #state{memtable=TID, clerk=Clerk, root_path=RootPath}, %% Open manifest ManifestPath = InitState#state.root_path ++ "/" ++ ?MANIFEST_FP ++ "/", - {ok, Filenames} = file:list_dir(ManifestPath), + {ok, Filenames} = case filelib:is_dir(ManifestPath) of + true -> + file:list_dir(ManifestPath); + false -> + {ok, []} + end, CurrRegex = "nonzero_(?[0-9]+)\\." ++ ?CURRENT_FILEX, ValidManSQNs = lists:foldl(fun(FN, Acc) -> case re:run(FN, @@ -270,8 +284,7 @@ init([RootPath]) -> ++ "found in nonzero levels~n", [MaxSQN]), - %% TODO - %% Find any L0 File left outstanding + %% Find any L0 files L0FN = filepath(RootPath, TopManSQN + 1, new_merge_files) ++ "_0_0.sft", @@ -311,6 +324,8 @@ handle_call({push_mem, DumpList}, _From, State) -> Response = case assess_sqn(DumpList) of {MinSQN, MaxSQN} when MaxSQN > MinSQN, MinSQN >= State#state.ledger_sqn -> + io:format("SQN check completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(),StartWatch)]), case push_to_memory(DumpList, State) of {ok, UpdState} -> {reply, ok, UpdState}; @@ -346,12 +361,31 @@ handle_call({confirm_delete, FileName}, _From, State) -> {reply, Reply, State} end; handle_call(prompt_compaction, _From, State) -> - case push_to_memory([], State) of - {ok, UpdState} -> - {reply, ok, UpdState#state{backlog=false}}; - {{pause, Reason, Details}, UpdState} -> - io:format("Excess work due to - " ++ Reason, Details), - {reply, pause, UpdState#state{backlog=true}} + %% If there is a prompt immediately after a L0 async write event then + %% there exists the potential for the prompt to stall the database. + %% Should only accept prompts if there has been a safe wait from the + %% last L0 write event. + Proceed = case State#state.levelzero_pending of + {true, _Pid, TS} -> + TD = timer:now_diff(os:timestamp(),TS), + if + TD < ?PROMPT_WAIT_ONL0 * 1000000 -> false; + true -> true + end; + ?L0PEND_RESET -> + true + end, + if + Proceed -> + case push_to_memory([], State) of + {ok, UpdState} -> + {reply, ok, UpdState#state{backlog=false}}; + {{pause, Reason, Details}, UpdState} -> + io:format("Excess work due to - " ++ Reason, Details), + {reply, pause, UpdState#state{backlog=true}} + end; + true -> + {reply, ok, State#state{backlog=false}} end; handle_call({manifest_change, WI}, _From, State) -> {ok, UpdState} = commit_manifest_change(WI, State), @@ -388,22 +422,21 @@ terminate(_Reason, State) -> Dump = ets:tab2list(State#state.memtable), case {State#state.levelzero_pending, get_item(0, State#state.manifest, []), length(Dump)} of - {{false, _, _}, [], L} when L > 0 -> + {?L0PEND_RESET, [], L} when L > 0 -> MSN = State#state.manifest_sqn + 1, FileName = State#state.root_path ++ "/" ++ ?FILES_FP ++ "/" ++ integer_to_list(MSN) ++ "_0_0", {ok, L0Pid, - {{KR1, _}, _SK, _HK}} = leveled_sft:sft_new(FileName ++ ".pnd", + {{[], []}, _SK, _HK}} = leveled_sft:sft_new(FileName ++ ".pnd", Dump, [], 0), - io:format("Dump of memory on close to filename ~s with" - ++ " remainder ~w~n", [FileName, length(KR1)]), + io:format("Dump of memory on close to filename ~s~n", [FileName]), leveled_sft:sft_close(L0Pid), file:rename(FileName ++ ".pnd", FileName ++ ".sft"); - {{false, _, _}, [], L} when L == 0 -> + {?L0PEND_RESET, [], L} when L == 0 -> io:format("No keys to dump from memory when closing~n"); _ -> io:format("No opportunity to persist memory before closing " @@ -424,31 +457,36 @@ code_change(_OldVsn, State, _Extra) -> push_to_memory(DumpList, State) -> {TableSize, UpdState} = case State#state.levelzero_pending of - {true, Remainder, {StartKey, EndKey, Pid}} -> + {true, Pid, _TS} -> %% Need to handle error scenarios? %% N.B. Sync call - so will be ready - {ok, SrcFN} = leveled_sft:sft_checkready(Pid), - %% Reset ETS, but re-insert any remainder + {ok, SrcFN, StartKey, EndKey} = leveled_sft:sft_checkready(Pid), true = ets:delete_all_objects(State#state.memtable), - true = ets:insert(State#state.memtable, Remainder), ManifestEntry = #manifest_entry{start_key=StartKey, end_key=EndKey, owner=Pid, filename=SrcFN}, - {length(Remainder), + {0, State#state{manifest=lists:keystore(0, 1, State#state.manifest, {0, [ManifestEntry]}), levelzero_pending=?L0PEND_RESET}}; - {false, _, _} -> + ?L0PEND_RESET -> {State#state.table_size, State} end, %% Prompt clerk to ask about work - do this for every push_mem ok = leveled_clerk:clerk_prompt(UpdState#state.clerk, penciller), - case do_push_to_mem(DumpList, TableSize, UpdState#state.memtable) of + SW2 = os:timestamp(), + MemoryInsertion = do_push_to_mem(DumpList, + TableSize, + UpdState#state.memtable), + io:format("Push into memory timed at ~w microseconds~n", + [timer:now_diff(os:timestamp(),SW2)]), + + case MemoryInsertion of {twist, ApproxTableSize} -> {ok, UpdState#state{table_size=ApproxTableSize}}; {roll, ApproxTableSize} -> @@ -459,30 +497,16 @@ push_to_memory(DumpList, State) -> FileName = UpdState#state.root_path ++ "/" ++ ?FILES_FP ++ "/" ++ integer_to_list(MSN) ++ "_0_0", - Dump = ets:tab2list(UpdState#state.memtable), - L0_SFT = leveled_sft:sft_new(FileName, - Dump, - [], - 0, - #sft_options{wait=false}), - {ok, L0Pid, Reply} = L0_SFT, - {{KL1Rem, []}, L0StartKey, L0EndKey} = Reply, - Backlog = length(KL1Rem), - Rsp = - if - Backlog > ?MAX_TABLESIZE -> - {pause, - "Backlog of ~w in memory table~n", - [Backlog]}; - true -> - ok - end, - {Rsp, + Opts = #sft_options{wait=false}, + {ok, L0Pid} = leveled_sft:sft_new(FileName, + UpdState#state.memtable, + [], + 0, + Opts), + {ok, UpdState#state{levelzero_pending={true, - KL1Rem, - {L0StartKey, - L0EndKey, - L0Pid}}, + L0Pid, + os:timestamp()}, table_size=ApproxTableSize, manifest_sqn=MSN}}; {[], true} -> @@ -558,20 +582,20 @@ manifest_locked(State) -> true; true -> case State#state.levelzero_pending of - {true, _, _} -> + {true, _Pid, _TS} -> true; _ -> false end end. - - %% Work out what the current work queue should be %% %% The work queue should have a lower level work at the front, and no work %% should be added to the queue if a compaction worker has already been asked %% to look at work at that level +%% +%% The full queue is calculated for logging purposes only return_work(State, From) -> WorkQueue = assess_workqueue([], diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index fe4d331..12e36e5 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -184,8 +184,8 @@ -define(HEADER_LEN, 56). -define(ITERATOR_SCANWIDTH, 1). -define(MERGE_SCANWIDTH, 8). --define(MAX_KEYS, ?SLOT_COUNT * ?BLOCK_COUNT * ?BLOCK_SIZE). -define(DELETE_TIMEOUT, 60000). +-define(MAX_KEYS, ?SLOT_COUNT * ?BLOCK_COUNT * ?BLOCK_SIZE). -record(state, {version = ?CURRENT_VERSION :: tuple(), @@ -202,8 +202,9 @@ summ_length :: integer(), filename :: string(), handle :: file:fd(), - background_complete=false :: boolean(), - background_failure="Unknown" :: string(), + background_complete = false :: boolean(), + background_failure = "Unknown" :: string(), + oversized_file = false :: boolean(), ready_for_delete = false ::boolean(), penciller :: pid()}). @@ -217,17 +218,17 @@ sft_new(Filename, KL1, KL2, Level) -> sft_new(Filename, KL1, KL2, Level, Options) -> {ok, Pid} = gen_server:start(?MODULE, [], []), - Reply = case Options#sft_options.wait of + case Options#sft_options.wait of true -> - gen_server:call(Pid, - {sft_new, Filename, KL1, KL2, Level}, - infinity); + Reply = gen_server:call(Pid, + {sft_new, Filename, KL1, KL2, Level}, + infinity), + {ok, Pid, Reply}; false -> - gen_server:call(Pid, - {sft_new, Filename, KL1, KL2, Level, background}, - infinity) - end, - {ok, Pid, Reply}. + gen_server:cast(Pid, + {sft_new, Filename, KL1, KL2, Level}), + {ok, Pid} + end. sft_open(Filename) -> {ok, Pid} = gen_server:start(?MODULE, [], []), @@ -278,47 +279,13 @@ sft_getmaxsequencenumber(Pid) -> init([]) -> {ok, #state{}}. -handle_call({sft_new, Filename, KL1, [], Level, background}, From, State) -> - {ListForFile, KL1Rem} = case length(KL1) of - L when L >= ?MAX_KEYS -> - lists:split(?MAX_KEYS, KL1); - _ -> - {KL1, []} - end, - StartKey = strip_to_keyonly(lists:nth(1, ListForFile)), - EndKey = strip_to_keyonly(lists:last(ListForFile)), - Ext = filename:extension(Filename), - Components = filename:split(Filename), - {TmpFilename, PrmFilename} = case Ext of - [] -> - {filename:join(Components) ++ ".pnd", filename:join(Components) ++ ".sft"}; - Ext -> - %% This seems unnecessarily hard - DN = filename:dirname(Filename), - FP = lists:last(Components), - FP_NOEXT = lists:sublist(FP, 1, 1 + length(FP) - length(Ext)), - {DN ++ "/" ++ FP_NOEXT ++ ".pnd", DN ++ "/" ++ FP_NOEXT ++ ".sft"} - end, - gen_server:reply(From, {{KL1Rem, []}, StartKey, EndKey}), - case create_file(TmpFilename) of - {error, Reason} -> - {noreply, State#state{background_complete=false, - background_failure=Reason}}; - {Handle, FileMD} -> - io:format("Creating file in background with input of size ~w~n", - [length(ListForFile)]), - % Key remainders must match to empty - Rename = {true, TmpFilename, PrmFilename}, - {ReadHandle, UpdFileMD, {[], []}} = complete_file(Handle, - FileMD, - ListForFile, - [], - Level, - Rename), - {noreply, UpdFileMD#state{handle=ReadHandle, - filename=PrmFilename, - background_complete=true}} - end; +handle_call({sft_new, Filename, KL1, [], 0}, _From, _State) -> + {ok, State} = create_levelzero(KL1, Filename), + {reply, + {{[], []}, + State#state.smallest_key, + State#state.highest_key}, + State}; handle_call({sft_new, Filename, KL1, KL2, Level}, _From, State) -> case create_file(Filename) of {error, Reason} -> @@ -365,7 +332,12 @@ handle_call(clear, _From, State) -> handle_call(background_complete, _From, State) -> case State#state.background_complete of true -> - {reply, {ok, State#state.filename}, State}; + {reply, + {ok, + State#state.filename, + State#state.smallest_key, + State#state.highest_key}, + State}; false -> {reply, {error, State#state.background_failure}, State} end; @@ -380,6 +352,9 @@ handle_call({set_for_delete, Penciller}, _From, State) -> handle_call(get_maxsqn, _From, State) -> {reply, State#state.highest_sqn, State}. +handle_cast({sft_new, Filename, Inp1, [], 0}, _State) -> + {ok, State} = create_levelzero(Inp1, Filename), + {noreply, State}; handle_cast(_Msg, State) -> {noreply, State}. @@ -423,6 +398,47 @@ code_change(_OldVsn, State, _Extra) -> %%%============================================================================ +create_levelzero(Inp1, Filename) -> + ListForFile = case is_list(Inp1) of + true -> + Inp1; + false -> + ets:tab2list(Inp1) + end, + Ext = filename:extension(Filename), + Components = filename:split(Filename), + {TmpFilename, PrmFilename} = case Ext of + [] -> + {filename:join(Components) ++ ".pnd", + filename:join(Components) ++ ".sft"}; + Ext -> + %% This seems unnecessarily hard + DN = filename:dirname(Filename), + FP = lists:last(Components), + FP_NOEXT = lists:sublist(FP, 1, 1 + length(FP) - length(Ext)), + {DN ++ "/" ++ FP_NOEXT ++ ".pnd", DN ++ "/" ++ FP_NOEXT ++ ".sft"} + end, + case create_file(TmpFilename) of + {error, Reason} -> + {error, + #state{background_complete=false, background_failure=Reason}}; + {Handle, FileMD} -> + InputSize = length(ListForFile), + io:format("Creating file with input of size ~w~n", [InputSize]), + Rename = {true, TmpFilename, PrmFilename}, + {ReadHandle, UpdFileMD, {[], []}} = complete_file(Handle, + FileMD, + ListForFile, + [], + 0, + Rename), + {ok, + UpdFileMD#state{handle=ReadHandle, + filename=PrmFilename, + background_complete=true, + oversized_file=InputSize>?MAX_KEYS}} + end. + %% Start a bare file with an initial header and no further details %% Return the {Handle, metadata record} create_file(FileName) when is_list(FileName) -> @@ -446,7 +462,9 @@ create_file(FileName) when is_list(FileName) -> create_header(initial) -> {Major, Minor} = ?CURRENT_VERSION, Version = <>, - Options = <<0:8>>, % Not thought of any options + %% Not thought of any options - options are ignored + Options = <<0:8>>, + %% Settings are currently ignored {BlSize, BlCount, SlCount} = {?BLOCK_COUNT, ?BLOCK_SIZE, ?SLOT_COUNT}, Settings = <>, {SpareO, SpareL} = {<<0:48>>, <<0:192>>}, @@ -871,6 +889,10 @@ sftwrite_function(finalise, SNExtremes, KeyExtremes}). +%% Level 0 files are of variable (infinite) size to avoid issues with having +%% any remainders when flushing from memory +maxslots_bylevel(_SlotTotal, 0) -> + continue; maxslots_bylevel(SlotTotal, _Level) -> case SlotTotal of ?SLOT_COUNT -> From 2a76eb364e731dbb122a024441c542f6d9df2803 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 5 Sep 2016 15:01:23 +0100 Subject: [PATCH 031/167] Inker - Initial Code An attempt to get a first inker that can build a ledger from a manifest as well as support simple get and put operations. Basic tests surround the building of manifests only at this stage - more work required for get and put. --- src/leveled_cdb.erl | 215 +++++++++++---- src/leveled_inker.erl | 555 +++++++++++++++++++++++++++++++++++++- src/leveled_penciller.erl | 41 +-- 3 files changed, 745 insertions(+), 66 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index e2c266f..3557609 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -16,9 +16,9 @@ %% %% This is to be used in eleveledb, and in this context: %% - Keys will be a Sequence Number -%% - Values will be a Checksum; Pointers (length * 3); Key; [Metadata]; [Value] -%% where the pointers can be used to extract just part of the value -%% (i.e. metadata only) +%% - Values will be a Checksum | Object | KeyAdditions +%% Where the KeyAdditions are all the Key changes required to be added to the +%% ledger to complete the changes (the addition of postings and tombstones). %% %% This module provides functions to create and query a CDB (constant database). %% A CDB implements a two-level hashtable which provides fast {key,value} @@ -58,7 +58,11 @@ cdb_open_reader/1, cdb_get/2, cdb_put/3, - cdb_close/1]). + cdb_lastkey/1, + cdb_filename/1, + cdb_keycheck/2, + cdb_close/1, + cdb_complete/1]). -include_lib("eunit/include/eunit.hrl"). @@ -70,8 +74,8 @@ -record(state, {hashtree, last_position :: integer(), - smallest_sqn :: integer(), - highest_sqn :: integer(), + last_key = empty, + hash_index = [] :: list(), filename :: string(), handle :: file:fd(), writer :: boolean}). @@ -108,6 +112,22 @@ cdb_put(Pid, Key, Value) -> cdb_close(Pid) -> gen_server:call(Pid, cdb_close, infinity). +cdb_complete(Pid) -> + gen_server:call(Pid, cdb_complete, infinity). + +%% Get the last key to be added to the file (which will have the highest +%% sequence number) +cdb_lastkey(Pid) -> + gen_server:call(Pid, cdb_lastkey, infinity). + +%% Get the filename of the database +cdb_filename(Pid) -> + gen_server:call(Pid, cdb_filename, infinity). + +%% Check to see if the key is probably present, will return either +%% probably or missing. Does not do a definitive check +cdb_keycheck(Pid, Key) -> + gen_server:call(Pid, {cdb_keycheck, Key}, infinity). %%%============================================================================ %%% gen_server callbacks @@ -118,29 +138,59 @@ init([]) -> handle_call({cdb_open_writer, Filename}, _From, State) -> io:format("Opening file for writing with filename ~s~n", [Filename]), - {LastPosition, HashTree} = open_active_file(Filename), + {LastPosition, HashTree, LastKey} = open_active_file(Filename), {ok, Handle} = file:open(Filename, [binary, raw, read, write, delayed_write]), {reply, ok, State#state{handle=Handle, last_position=LastPosition, + last_key=LastKey, filename=Filename, hashtree=HashTree, writer=true}}; handle_call({cdb_open_reader, Filename}, _From, State) -> io:format("Opening file for reading with filename ~s~n", [Filename]), {ok, Handle} = file:open(Filename, [binary, raw, read]), + Index = load_index(Handle), {reply, ok, State#state{handle=Handle, filename=Filename, - writer=false}}; + writer=false, + hash_index=Index}}; handle_call({cdb_get, Key}, _From, State) -> - case State#state.writer of - true -> + case {State#state.writer, State#state.hash_index} of + {true, _} -> {reply, get_mem(Key, State#state.handle, State#state.hashtree), State}; - false -> + {false, []} -> {reply, get(State#state.handle, Key), + State}; + {false, Cache} -> + {reply, + get_withcache(State#state.handle, Key, Cache), + State} + end; +handle_call({cdb_keycheck, Key}, _From, State) -> + case {State#state.writer, State#state.hash_index} of + {true, _} -> + {reply, + get_mem(Key, + State#state.handle, + State#state.hashtree, + loose_presence), + State}; + {false, []} -> + {reply, + get(State#state.handle, + Key, + loose_presence), + State}; + {false, Cache} -> + {reply, + get(State#state.handle, + Key, + loose_presence, + Cache), State} end; handle_call({cdb_put, Key, Value}, _From, State) -> @@ -151,10 +201,12 @@ handle_call({cdb_put, Key, Value}, _From, State) -> {State#state.last_position, State#state.hashtree}), case Result of roll -> + %% Key and value could not be written {reply, roll, State}; {UpdHandle, NewPosition, HashTree} -> {reply, ok, State#state{handle=UpdHandle, last_position=NewPosition, + last_key=Key, hashtree=HashTree}} end; false -> @@ -162,17 +214,31 @@ handle_call({cdb_put, Key, Value}, _From, State) -> {error, read_only}, State} end; +handle_call(cdb_lastkey, _From, State) -> + {reply, State#state.last_key, State}; +handle_call(cdb_filename, _From, State) -> + {reply, State#state.filename, State}; handle_call(cdb_close, _From, State) -> + ok = file:close(State#state.handle), + {stop, normal, ok, State}; +handle_call(cdb_complete, _From, State) -> case State#state.writer of true -> ok = close_file(State#state.handle, State#state.hashtree, - State#state.last_position); + State#state.last_position), + %% Rename file + NewName = filename:rootname(State#state.filename, ".pnd") + ++ ".cdb", + io:format("Renaming file from ~s to ~s~n", [State#state.filename, NewName]), + ok = file:rename(State#state.filename, NewName), + {stop, normal, {ok, NewName}, State}; false -> - ok = file:close(State#state.handle) - end, - {stop, normal, ok, State}. - + ok = file:close(State#state.handle), + {stop, normal, {ok, State#state.filename}, State} + end. + + handle_cast(_Msg, State) -> {noreply, State}. @@ -260,7 +326,7 @@ dump(FileName, CRCCheck) -> open_active_file(FileName) when is_list(FileName) -> {ok, Handle} = file:open(FileName, [binary, raw, read, write]), {ok, Position} = file:position(Handle, {bof, 256*?DWORD_SIZE}), - {LastPosition, HashTree} = scan_over_file(Handle, Position), + {LastPosition, HashTree, LastKey} = scan_over_file(Handle, Position), case file:position(Handle, eof) of {ok, LastPosition} -> ok = file:close(Handle); @@ -272,7 +338,7 @@ open_active_file(FileName) when is_list(FileName) -> ok = file:truncate(Handle), ok = file:close(Handle) end, - {LastPosition, HashTree}. + {LastPosition, HashTree, LastKey}. %% put(Handle, Key, Value, {LastPosition, HashDict}) -> {NewPosition, KeyDict} %% Append to an active file a new key/value pair returning an updated @@ -298,18 +364,22 @@ put(Handle, Key, Value, {LastPosition, HashTree}) -> %% get(FileName,Key) -> {key,value} %% Given a filename and a key, returns a key and value tuple. %% +get_withcache(Handle, Key, Cache) -> + get(Handle, Key, ?CRC_CHECK, Cache). + get(FileNameOrHandle, Key) -> get(FileNameOrHandle, Key, ?CRC_CHECK). -get(FileName, Key, CRCCheck) when is_list(FileName), is_list(Key) -> +get(FileNameOrHandle, Key, CRCCheck) -> + get(FileNameOrHandle, Key, CRCCheck, no_cache). + +get(FileName, Key, CRCCheck, Cache) when is_list(FileName), is_list(Key) -> {ok,Handle} = file:open(FileName,[binary, raw, read]), - get(Handle,Key, CRCCheck); -get(Handle, Key, CRCCheck) when is_tuple(Handle), is_list(Key) -> + get(Handle,Key, CRCCheck, Cache); +get(Handle, Key, CRCCheck, Cache) when is_tuple(Handle), is_list(Key) -> Hash = hash(Key), Index = hash_to_index(Hash), - {ok,_} = file:position(Handle, {bof, ?DWORD_SIZE * Index}), - % Get location of hashtable and number of entries in the hash - {HashTable, Count} = read_next_2_integers(Handle), + {HashTable, Count} = get_index(Handle, Index, Cache), % If the count is 0 for that index - key must be missing case Count of 0 -> @@ -326,14 +396,32 @@ get(Handle, Key, CRCCheck) when is_tuple(Handle), is_list(Key) -> search_hash_table(Handle, lists:append(L2, L1), Hash, Key, CRCCheck) end. +get_index(Handle, Index, no_cache) -> + {ok,_} = file:position(Handle, {bof, ?DWORD_SIZE * Index}), + % Get location of hashtable and number of entries in the hash + read_next_2_integers(Handle); +get_index(_Handle, Index, Cache) -> + lists:keyfind(Index, 1, Cache). + %% Get a Key/Value pair from an active CDB file (with no hash table written) %% This requires a key dictionary to be passed in (mapping keys to positions) %% Will return {Key, Value} or missing -get_mem(Key, Filename, HashTree) when is_list(Filename) -> +get_mem(Key, FNOrHandle, HashTree) -> + get_mem(Key, FNOrHandle, HashTree, ?CRC_CHECK). + +get_mem(Key, Filename, HashTree, CRCCheck) when is_list(Filename) -> {ok, Handle} = file:open(Filename, [binary, raw, read]), - get_mem(Key, Handle, HashTree); -get_mem(Key, Handle, HashTree) -> - extract_kvpair(Handle, get_hashtree(Key, HashTree), Key). + get_mem(Key, Handle, HashTree, CRCCheck); +get_mem(Key, Handle, HashTree, CRCCheck) -> + ListToCheck = get_hashtree(Key, HashTree), + case {CRCCheck, ListToCheck} of + {loose_presence, []} -> + missing; + {loose_presence, _L} -> + probably; + _ -> + extract_kvpair(Handle, ListToCheck, Key, CRCCheck) + end. %% Get the next key at a position in the file (or the first key if no position %% is passed). Will return both a key and the next position @@ -433,6 +521,15 @@ fold_keys(Handle, FoldFun, Acc0) -> %% Internal functions %%%%%%%%%%%%%%%%%%%% +load_index(Handle) -> + Index = lists:seq(0, 255), + lists:map(fun(X) -> + file:position(Handle, {bof, ?DWORD_SIZE * X}), + {HashTablePos, Count} = read_next_2_integers(Handle), + {X, {HashTablePos, Count}} end, + Index). + + %% Take an active file and write the hash details necessary to close that %% file and roll a new active file if requested. %% @@ -473,9 +570,6 @@ put_hashtree(Key, Position, HashTree) -> %% Function to extract a Key-Value pair given a file handle and a position %% Will confirm that the key matches and do a CRC check when requested -extract_kvpair(Handle, Positions, Key) -> - extract_kvpair(Handle, Positions, Key, ?CRC_CHECK). - extract_kvpair(_, [], _, _) -> missing; extract_kvpair(Handle, [Position|Rest], Key, Check) -> @@ -497,12 +591,12 @@ extract_kvpair(Handle, [Position|Rest], Key, Check) -> %% at that point return the position and the key dictionary scanned so far scan_over_file(Handle, Position) -> HashTree = array:new(256, {default, gb_trees:empty()}), - scan_over_file(Handle, Position, HashTree). + scan_over_file(Handle, Position, HashTree, empty). -scan_over_file(Handle, Position, HashTree) -> +scan_over_file(Handle, Position, HashTree, LastKey) -> case saferead_keyvalue(Handle) of false -> - {Position, HashTree}; + {Position, HashTree, LastKey}; {Key, ValueAsBin, KeyLength, ValueLength} -> case crccheck_value(ValueAsBin) of true -> @@ -510,14 +604,15 @@ scan_over_file(Handle, Position, HashTree) -> + ?DWORD_SIZE, scan_over_file(Handle, NewPosition, - put_hashtree(Key, Position, HashTree)); + put_hashtree(Key, Position, HashTree), + Key); false -> io:format("CRC check returned false on key of ~w ~n", [Key]), - {Position, HashTree} + {Position, HashTree, LastKey} end; eof -> - {Position, HashTree} + {Position, HashTree, LastKey} end. @@ -531,7 +626,6 @@ saferead_keyvalue(Handle) -> eof -> false; {KeyL, ValueL} -> - io:format("KeyL ~w ValueL ~w~n", [KeyL, ValueL]), case safe_read_next_term(Handle, KeyL) of {error, einval} -> false; @@ -540,7 +634,6 @@ saferead_keyvalue(Handle) -> false -> false; Key -> - io:format("Found Key of ~s~n", [Key]), case file:read(Handle, ValueL) of {error, einval} -> false; @@ -640,14 +733,25 @@ read_next_2_integers(Handle) -> %% Seach the hash table for the matching hash and key. Be prepared for %% multiple keys to have the same hash value. -search_hash_table(_Handle, [], _Hash, _Key, _CRCCHeck) -> +%% +%% There are three possible values of CRCCheck: +%% true - check the CRC before returning key & value +%% false - don't check the CRC before returning key & value +%% loose_presence - confirm that the hash of the key is present + +search_hash_table(_Handle, [], _Hash, _Key, _CRCCheck) -> missing; search_hash_table(Handle, [Entry|RestOfEntries], Hash, Key, CRCCheck) -> {ok, _} = file:position(Handle, Entry), {StoredHash, DataLoc} = read_next_2_integers(Handle), case StoredHash of Hash -> - KV = extract_kvpair(Handle, [DataLoc], Key, CRCCheck), + KV = case CRCCheck of + loose_presence -> + probably; + _ -> + extract_kvpair(Handle, [DataLoc], Key, CRCCheck) + end, case KV of missing -> search_hash_table(Handle, RestOfEntries, Hash, Key, CRCCheck); @@ -789,6 +893,13 @@ write_top_index_table(Handle, BasePos, List) -> lists:foldl(FnWriteIndex, BasePos, CompleteList), ok = file:advise(Handle, 0, ?DWORD_SIZE * 256, will_need). +%% To make this compatible with original Bernstein format this endian flip +%% and also the use of the standard hash function required. +%% +%% Hash function contains mysterious constants, some explanation here as to +%% what they are - +%% http://stackoverflow.com/ ++ +%% questions/10696223/reason-for-5381-number-in-djb-hash-function endian_flip(Int) -> <> = <>, @@ -962,12 +1073,24 @@ activewrite_singlewrite_test() -> InitialD1 = dict:store("0001", "Initial value", InitialD), ok = from_dict("../test/test_mem.cdb", InitialD1), io:format("New db file created ~n", []), - {LastPosition, KeyDict} = open_active_file("../test/test_mem.cdb"), + {LastPosition, KeyDict, _} = open_active_file("../test/test_mem.cdb"), io:format("File opened as new active file " "with LastPosition=~w ~n", [LastPosition]), - {_, _, UpdKeyDict} = put("../test/test_mem.cdb", Key, Value, {LastPosition, KeyDict}), + {_, _, UpdKeyDict} = put("../test/test_mem.cdb", + Key, Value, + {LastPosition, KeyDict}), io:format("New key and value added to active file ~n", []), - ?assertMatch({Key, Value}, get_mem(Key, "../test/test_mem.cdb", UpdKeyDict)), + ?assertMatch({Key, Value}, + get_mem(Key, "../test/test_mem.cdb", + UpdKeyDict)), + ?assertMatch(probably, + get_mem(Key, "../test/test_mem.cdb", + UpdKeyDict, + loose_presence)), + ?assertMatch(missing, + get_mem("not_present", "../test/test_mem.cdb", + UpdKeyDict, + loose_presence)), ok = file:delete("../test/test_mem.cdb"). search_hash_table_findinslot_test() -> @@ -992,6 +1115,8 @@ search_hash_table_findinslot_test() -> io:format("Slot 2 has Hash ~w Position ~w~n", [ReadH4, ReadP4]), ?assertMatch(0, ReadH4), ?assertMatch({"key1", "value1"}, get(Handle, Key1)), + ?assertMatch(probably, get(Handle, Key1, loose_presence)), + ?assertMatch(missing, get(Handle, "Key99", loose_presence)), {ok, _} = file:position(Handle, FirstHashPosition), FlipH3 = endian_flip(ReadH3), FlipP3 = endian_flip(ReadP3), @@ -1029,7 +1154,7 @@ getnextkey_inclemptyvalue_test() -> ok = file:delete("../test/hashtable2_test.cdb"). newactivefile_test() -> - {LastPosition, _} = open_active_file("../test/activefile_test.cdb"), + {LastPosition, _, _} = open_active_file("../test/activefile_test.cdb"), ?assertMatch(256 * ?DWORD_SIZE, LastPosition), Response = get_nextkey("../test/activefile_test.cdb"), ?assertMatch(nomorekeys, Response), diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 7564ed4..d19334b 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -1,11 +1,560 @@ %% -------- Inker --------- -%% %% +%% The Inker is responsible for managing access and updates to the Journal. +%% +%% The Inker maintains a manifest of what files make up the Journal, and which +%% file is the current append-only nursery log to accept new PUTs into the +%% Journal. The Inker also marshals GET requests to the appropriate database +%% file within the Journal (routed by sequence number). The Inker is also +%% responsible for scheduling compaction work to be carried out by the Inker's +%% clerk. +%% +%% -------- Journal --------- +%% +%% The Journal is a series of files originally named as _nursery.cdb +%% where the sequence number is the first object sequence number (key) within +%% the given database file. The files will be named *.cdb at the point they +%% have been made immutable (through a rename operation). Prior to this, they +%% will originally start out as a *.pnd file. +%% +%% At some stage in the future compacted versions of old journal cdb files may +%% be produced. These files will be named -.cdb, and once +%% the manifest is updated the original _nursery.cdb (or +%% _.cdb) files they replace will be erased. +%% +%% The current Journal is made up of a set of files referenced in the manifest, +%% combined with a set of files of the form _nursery.[cdb|pnd] with +%% a higher Sequence Number compared to the files in the manifest. +%% +%% The Journal is ordered by sequence number from front to back both within +%% and across files. +%% +%% On startup the Inker should open the manifest with the highest sequence +%% number, and this will contain the list of filenames that make up the +%% non-recent part of the Journal. The Manifest is completed by opening these +%% files plus any other files with a higher sequence number. The file with +%% the highest sequence number is assumed to to be the active writer. Any file +%% with a lower sequence number and a *.pnd extension should be re-rolled into +%% a *.cdb file. +%% +%% -------- Objects --------- +%% +%% From the perspective of the Inker, objects to store are made up of: +%% - A Primary Key (as an Erlang term) +%% - A sequence number (assigned by the Inker) +%% - An object (an Erlang term) +%% - A set of Key Deltas associated with the change +%% +%% -------- Manifest --------- +%% +%% The Journal has a manifest which is the current record of which cdb files +%% are currently active in the Journal (i.e. following compaction). The +%% manifest holds this information through two lists - a list of files which +%% are definitely in the current manifest, and a list of files which have been +%% removed, but may still be present on disk. The use of two lists is to +%% avoid any circumsatnces where a compaction event has led to the deletion of +%% a Journal file with a higher sequence number than any in the remaining +%% manifest. +%% +%% A new manifest file is saved for every compaction event. The manifest files +%% are saved using the filename .man once saved. The ManifestSQN +%% is incremented once for every compaction event. +%% +%% -------- Compaction --------- +%% +%% Compaction is a process whereby an Inker's clerk will: +%% - Request a snapshot of the Ledger, as well as the lowest sequence number +%% that is currently registerd by another snapshot owner +%% - Picks a Journal database file at random (not including the current +%% nursery log) +%% - Performs a random walk on keys and sequence numbers in the chosen CDB +%% file to extract a subset of 100 key and sequence number combinations from +%% the database +%% - Looks up the current sequence number for those keys in the Ledger +%% - If more than % (default n=20) of the keys are now at a higher sequence +%% number, then the database file is a candidate for compaction. In this case +%% each of the next 8 files in sequence should be checked until all those 8 +%% files have been checked or one of the files has been found to be below the +%% threshold. +%% - If a set of below-the-threshold files is found, the files are re-written +%% without any superceded values +%%- The clerk should then request that the Inker commit the manifest change +%% +%% -------- Inker's Clerk --------- +%% %% -%% -------- Ledger --------- %% %% - -module(leveled_inker). +-behaviour(gen_server). + +-include("../include/leveled.hrl"). + +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3, + ink_start/1, + ink_put/4, + ink_get/3, + ink_snap/1, + build_dummy_journal/0, + simple_manifest_reader/2]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(MANIFEST_FP, "journal_manifest"). +-define(FILES_FP, "journal_files"). +-define(JOURNAL_FILEX, "cdb"). +-define(MANIFEST_FILEX, "man"). +-define(PENDING_FILEX, "pnd"). + + +-record(state, {manifest = [] :: list(), + manifest_sqn = 0 :: integer(), + journal_sqn = 0 :: integer(), + active_journaldb :: pid(), + removed_journaldbs = [] :: list(), + root_path :: string()}). + + +%%%============================================================================ +%%% API +%%%============================================================================ + +ink_start(RootDir) -> + gen_server:start(?MODULE, [RootDir], []). + +ink_put(Pid, PrimaryKey, Object, KeyChanges) -> + gen_server:call(Pid, {put, PrimaryKey, Object, KeyChanges}, infinity). + +ink_get(Pid, PrimaryKey, SQN) -> + gen_server:call(Pid, {get, PrimaryKey, SQN}, infinity). + +ink_snap(Pid) -> + gen_server:call(Pid, snapshot, infinity). + +%%%============================================================================ +%%% gen_server callbacks +%%%============================================================================ + +init([RootPath]) -> + JournalFP = filepath(RootPath, journal_dir), + {ok, JournalFilenames} = case filelib:is_dir(JournalFP) of + true -> + file:list_dir(JournalFP); + false -> + filelib:ensure_dir(JournalFP), + {ok, []} + end, + ManifestFP = filepath(RootPath, manifest_dir), + {ok, ManifestFilenames} = case filelib:is_dir(ManifestFP) of + true -> + file:list_dir(ManifestFP); + false -> + filelib:ensure_dir(ManifestFP), + {ok, []} + end, + {Manifest, + ActiveJournal, + JournalSQN, + ManifestSQN} = build_manifest(ManifestFilenames, + JournalFilenames, + fun simple_manifest_reader/2, + RootPath), + {ok, #state{manifest = Manifest, + manifest_sqn = ManifestSQN, + journal_sqn = JournalSQN, + active_journaldb = ActiveJournal, + root_path = RootPath}}. + + +handle_call({put, Key, Object, KeyChanges}, From, State) -> + case put_object(Key, Object, KeyChanges, State) of + {ok, UpdState} -> + {reply, {ok, UpdState#state.journal_sqn}, UpdState}; + {rolling, UpdState} -> + gen_server:reply(From, {ok, UpdState#state.journal_sqn}), + {NewManifest, + NewManifestSQN} = roll_active_file(State#state.active_journaldb, + State#state.manifest, + State#state.manifest_sqn, + State#state.root_path), + {noreply, UpdState#state{manifest=NewManifest, + manifest_sqn=NewManifestSQN}}; + {blocked, UpdState} -> + {reply, blocked, UpdState} + end; +handle_call({get, Key, SQN}, _From, State) -> + {reply, get_object(Key, SQN, State#state.manifest), State}; +handle_call(snapshot, _From , State) -> + %% TODO: Not yet implemented registration of snapshot + %% Should return manifest and register the snapshot + {reply, State#state.manifest, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + + +%%%============================================================================ +%%% Internal functions +%%%============================================================================ + +put_object(PrimaryKey, Object, KeyChanges, State) -> + NewSQN = State#state.journal_sqn + 1, + Bin1 = term_to_binary({Object, KeyChanges}, [compressed]), + case leveled_cdb:cdb_put(State#state.active_journaldb, + {NewSQN, PrimaryKey}, + Bin1) of + ok -> + {ok, State#state{journal_sqn=NewSQN}}; + roll -> + FileName = filepath(State#state.root_path, NewSQN, new_journal), + {ok, NewJournalP} = leveled_cdb:cdb_open_writer(FileName), + case leveled_cdb:cdb_put(NewJournalP, + {NewSQN, PrimaryKey}, + Bin1) of + ok -> + {rolling, State#state{journal_sqn=NewSQN, + active_journaldb=NewJournalP}}; + roll -> + {blocked, State#state{journal_sqn=NewSQN, + active_journaldb=NewJournalP}} + end + end. + +roll_active_file(OldActiveJournal, Manifest, ManifestSQN, RootPath) -> + {ok, NewFilename} = leveled_cdb:cdb_complete(OldActiveJournal), + {ok, PidR} = leveled_cdb:cdb_open_reader(NewFilename), + JournalRegex2 = "nursery_(?[0-9]+)\\." ++ ?JOURNAL_FILEX, + [JournalSQN] = sequencenumbers_fromfilenames([NewFilename], + JournalRegex2, + 'SQN'), + NewManifest = lists:append(Manifest, {JournalSQN, NewFilename, PidR}), + NewManifestSQN = ManifestSQN + 1, + ok = simple_manifest_writer(NewManifest, NewManifestSQN, RootPath), + {NewManifest, NewManifestSQN}. + +get_object(PrimaryKey, SQN, Manifest) -> + JournalP = find_in_manifest(SQN, Manifest), + leveled_cdb:cdb_get(JournalP, {SQN, PrimaryKey}). + + +build_manifest(ManifestFilenames, + JournalFilenames, + ManifestRdrFun, + RootPath) -> + %% Setup root paths + JournalFP = filepath(RootPath, journal_dir), + %% Find the manifest with a highest Manifest sequence number + %% Open it and read it to get the current Confirmed Manifest + ManifestRegex = "(?[0-9]+)\\." ++ ?MANIFEST_FILEX, + ValidManSQNs = sequencenumbers_fromfilenames(ManifestFilenames, + ManifestRegex, + 'MSQN'), + {JournalSQN1, + ConfirmedManifest, + Removed, + ManifestSQN} = case length(ValidManSQNs) of + 0 -> + {0, [], [], 0}; + _ -> + PersistedManSQN = lists:max(ValidManSQNs), + {J1, M1, R1} = ManifestRdrFun(PersistedManSQN, + RootPath), + {J1, M1, R1, PersistedManSQN} + end, + + %% Find any more recent immutable files that have a higher sequence number + %% - the immutable files have already been rolled, and so have a completed + %% hashtree lookup + JournalRegex1 = "nursery_(?[0-9]+)\\." ++ ?JOURNAL_FILEX, + UnremovedJournalFiles = lists:foldl(fun(FN, Acc) -> + case lists:member(FN, Removed) of + true -> + Acc; + false -> + Acc ++ [FN] + end end, + [], + JournalFilenames), + OtherSQNs_imm = sequencenumbers_fromfilenames(UnremovedJournalFiles, + JournalRegex1, + 'SQN'), + Manifest1 = lists:foldl(fun(X, Acc) -> + if + X > JournalSQN1 + -> + FN = "nursery_" ++ + integer_to_list(X) + ++ "." ++ + ?JOURNAL_FILEX, + Acc ++ [{X, FN}]; + true + -> Acc + end end, + ConfirmedManifest, + lists:sort(OtherSQNs_imm)), + + %% Enrich the manifest so it contains the Pid of any of the immutable + %% entries + io:format("Manifest1 is ~w~n", [Manifest1]), + Manifest2 = lists:map(fun({X, Y}) -> + FN = filename:join(JournalFP, Y), + {ok, Pid} = leveled_cdb:cdb_open_reader(FN), + {X, Y, Pid} end, + Manifest1), + + %% Find any more recent mutable files that have a higher sequence number + %% Roll any mutable files which do not have the highest sequence number + %% to create the hashtree and complete the header entries + JournalRegex2 = "nursery_(?[0-9]+)\\." ++ ?PENDING_FILEX, + OtherSQNs_pnd = sequencenumbers_fromfilenames(JournalFilenames, + JournalRegex2, + 'SQN'), + + case length(OtherSQNs_pnd) of + 0 -> + %% Need to create a new active writer, but also find the highest + %% SQN from within the confirmed manifest + TopSQNInManifest = + case length(Manifest2) of + 0 -> + %% Manifest is empty and no active writers + %% can be found so database is empty + 0; + _ -> + TM = lists:last(lists:keysort(1,Manifest2)), + {_SQN, _FN, TMPid} = TM, + {HighSQN, _HighKey} = leveled_cdb:cdb_lastkey(TMPid), + HighSQN + end, + ActiveFN = filepath(RootPath, TopSQNInManifest + 1, new_journal), + {ok, ActiveJournal} = leveled_cdb:cdb_open_writer(ActiveFN), + {Manifest2, ActiveJournal, TopSQNInManifest, ManifestSQN}; + _ -> + + {ActiveJournalSQN, + Manifest3} = roll_pending_journals(lists:sort(OtherSQNs_pnd), + Manifest2, + RootPath), + %% Need to work out highest sequence number in tail file to feed + %% into opening of pending journal + ActiveFN = filepath(RootPath, ActiveJournalSQN, new_journal), + {ok, ActiveJournal} = leveled_cdb:cdb_open_writer(ActiveFN), + {HighestSQN, _HighestKey} = leveled_cdb:cdb_lastkey(ActiveJournal), + {Manifest3, ActiveJournal, HighestSQN, ManifestSQN} + end. + +close_allmanifest([], ActiveJournal) -> + leveled_cdb:cdb_close(ActiveJournal); +close_allmanifest([H|ManifestT], ActiveJournal) -> + {_, _, Pid} = H, + leveled_cdb:cdb_close(Pid), + close_allmanifest(ManifestT, ActiveJournal). + + +roll_pending_journals([TopJournalSQN], Manifest, _RootPath) + when is_integer(TopJournalSQN) -> + {TopJournalSQN, Manifest}; +roll_pending_journals([JournalSQN|T], Manifest, RootPath) -> + Filename = filepath(RootPath, JournalSQN, new_journal), + PidW = leveled_cdb:cdb_open_writer(Filename), + {ok, NewFilename} = leveled_cdb:cdb_complete(PidW), + {ok, PidR} = leveled_cdb:cdb_open_reader(NewFilename), + roll_pending_journals(T, + lists:append(Manifest, + {JournalSQN, NewFilename, PidR}), + RootPath). + + + +sequencenumbers_fromfilenames(Filenames, Regex, IntName) -> + lists:foldl(fun(FN, Acc) -> + case re:run(FN, + Regex, + [{capture, [IntName], list}]) of + nomatch -> + Acc; + {match, [Int]} when is_list(Int) -> + Acc ++ [list_to_integer(Int)]; + _ -> + Acc + end end, + [], + Filenames). + +find_in_manifest(_SQN, []) -> + error; +find_in_manifest(SQN, [{LowSQN, _FN, Pid}|_Tail]) when SQN >= LowSQN -> + Pid; +find_in_manifest(SQN, [_Head|Tail]) -> + find_in_manifest(SQN, Tail). + +filepath(RootPath, journal_dir) -> + RootPath ++ "/" ++ ?FILES_FP ++ "/"; +filepath(RootPath, manifest_dir) -> + RootPath ++ "/" ++ ?MANIFEST_FP ++ "/". + + +filepath(RootPath, NewSQN, new_journal) -> + filename:join(filepath(RootPath, journal_dir), + "nursery_" + ++ integer_to_list(NewSQN) + ++ "." ++ ?PENDING_FILEX). + + +simple_manifest_reader(SQN, RootPath) -> + ManifestPath = filepath(RootPath, manifest_dir), + {ok, MBin} = file:read_file(filename:join(ManifestPath, + integer_to_list(SQN) + ++ ".man")), + binary_to_term(MBin). + + +simple_manifest_writer(Manifest, ManSQN, RootPath) -> + ManPath = filepath(RootPath, manifest_dir), + NewFN = filename:join(ManPath, integer_to_list(ManSQN) ++ ?MANIFEST_FILEX), + TmpFN = filename:join(ManPath, integer_to_list(ManSQN) ++ ?PENDING_FILEX), + MBin = term_to_binary(Manifest), + case file:is_file(NewFN) of + true -> + io:format("Error - trying to write manifest for" + ++ " ManifestSQN=~w which already exists~n", [ManSQN]), + error; + false -> + io:format("Writing new version of manifest for " + ++ " manifestSQN=~w~n", [ManSQN]), + ok = file:write_file(TmpFN, MBin), + ok = file:rename(TmpFN, NewFN), + ok + end. + + + +%%%============================================================================ +%%% Test +%%%============================================================================ + +-ifdef(TEST). + +build_dummy_journal() -> + RootPath = "../test/inker", + JournalFP = filepath(RootPath, journal_dir), + ManifestFP = filepath(RootPath, manifest_dir), + ok = filelib:ensure_dir(RootPath), + ok = filelib:ensure_dir(JournalFP), + ok = filelib:ensure_dir(ManifestFP), + F1 = filename:join(JournalFP, "nursery_1.pnd"), + {ok, J1} = leveled_cdb:cdb_open_writer(F1), + {K1, V1} = {"Key1", "TestValue1"}, + {K2, V2} = {"Key2", "TestValue2"}, + ok = leveled_cdb:cdb_put(J1, {1, K1}, V1), + ok = leveled_cdb:cdb_put(J1, {2, K2}, V2), + {ok, _} = leveled_cdb:cdb_complete(J1), + F2 = filename:join(JournalFP, "nursery_3.pnd"), + {ok, J2} = leveled_cdb:cdb_open_writer(F2), + {K1, V3} = {"Key1", "TestValue3"}, + {K4, V4} = {"Key4", "TestValue4"}, + ok = leveled_cdb:cdb_put(J2, {3, K1}, V3), + ok = leveled_cdb:cdb_put(J2, {4, K4}, V4), + ok = leveled_cdb:cdb_close(J2), + Manifest = {2, [{1, "nursery_1.cdb"}], []}, + ManifestBin = term_to_binary(Manifest), + {ok, MF1} = file:open(filename:join(ManifestFP, "1.man"), + [binary, raw, read, write]), + ok = file:write(MF1, ManifestBin), + ok = file:close(MF1). + + +clean_testdir(RootPath) -> + clean_subdir(filepath(RootPath, journal_dir)), + clean_subdir(filepath(RootPath, manifest_dir)). + +clean_subdir(DirPath) -> + {ok, Files} = file:list_dir(DirPath), + lists:foreach(fun(FN) -> file:delete(filename:join(DirPath, FN)) end, + Files). + +simple_buildmanifest_test() -> + RootPath = "../test/inker", + build_dummy_journal(), + Res = build_manifest(["1.man"], + ["nursery_1.cdb", "nursery_3.pnd"], + fun simple_manifest_reader/2, + RootPath), + io:format("Build manifest output is ~w~n", [Res]), + {Man, ActJournal, HighSQN, ManSQN} = Res, + ?assertMatch(HighSQN, 4), + ?assertMatch(ManSQN, 1), + ?assertMatch([{1, "nursery_1.cdb", _}], Man), + {ActSQN, _ActK} = leveled_cdb:cdb_lastkey(ActJournal), + ?assertMatch(ActSQN, 4), + close_allmanifest(Man, ActJournal), + clean_testdir(RootPath). + +another_buildmanifest_test() -> + %% There is a rolled jounral file which is not yet in the manifest + RootPath = "../test/inker", + build_dummy_journal(), + FN = filepath(RootPath, 3, new_journal), + {ok, FileToRoll} = leveled_cdb:cdb_open_writer(FN), + {ok, _} = leveled_cdb:cdb_complete(FileToRoll), + FN2 = filepath(RootPath, 5, new_journal), + {ok, NewActiveJN} = leveled_cdb:cdb_open_writer(FN2), + {K5, V5} = {"Key5", "TestValue5"}, + {K6, V6} = {"Key6", "TestValue6"}, + ok = leveled_cdb:cdb_put(NewActiveJN, {5, K5}, V5), + ok = leveled_cdb:cdb_put(NewActiveJN, {6, K6}, V6), + ok = leveled_cdb:cdb_close(NewActiveJN), + %% Test setup - now build manifest + Res = build_manifest(["1.man"], + ["nursery_1.cdb", + "nursery_3.cdb", + "nursery_5.pnd"], + fun simple_manifest_reader/2, + RootPath), + io:format("Build manifest output is ~w~n", [Res]), + {Man, ActJournal, HighSQN, ManSQN} = Res, + ?assertMatch(HighSQN, 6), + ?assertMatch(ManSQN, 1), + ?assertMatch([{1, "nursery_1.cdb", _}, {3, "nursery_3.cdb", _}], Man), + {ActSQN, _ActK} = leveled_cdb:cdb_lastkey(ActJournal), + ?assertMatch(ActSQN, 6), + close_allmanifest(Man, ActJournal), + clean_testdir(RootPath). + + +empty_buildmanifest_test() -> + RootPath = "../test/inker/", + Res = build_manifest([], + [], + fun simple_manifest_reader/2, + RootPath), + io:format("Build manifest output is ~w~n", [Res]), + {Man, ActJournal, HighSQN, ManSQN} = Res, + ?assertMatch(Man, []), + ?assertMatch(ManSQN, 0), + ?assertMatch(HighSQN, 0), + empty = leveled_cdb:cdb_lastkey(ActJournal), + FN = leveled_cdb:cdb_filename(ActJournal), + %% The filename should be based on the next journal SQN (1) not 0 + ?assertMatch(FN, filepath(RootPath, 1, new_journal)), + close_allmanifest(Man, ActJournal), + clean_testdir(RootPath). + + +-endif. \ No newline at end of file diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index c84bfe7..6fe6eac 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -178,11 +178,8 @@ -define(MAX_WORK_WAIT, 300). -define(MANIFEST_FP, "ledger_manifest"). -define(FILES_FP, "ledger_files"). --define(SHUTDOWN_FP, "ledger_onshutdown"). -define(CURRENT_FILEX, "crr"). -define(PENDING_FILEX, "pnd"). --define(BACKUP_FILEX, "bak"). --define(ARCHIVE_FILEX, "arc"). -define(MEMTABLE, mem). -define(MAX_TABLESIZE, 32000). -define(PROMPT_WAIT_ONL0, 5). @@ -198,6 +195,7 @@ table_size = 0 :: integer(), clerk :: pid(), levelzero_pending = ?L0PEND_RESET :: tuple(), + levelzero_snapshot = [] :: list(), memtable, backlog = false :: boolean()}). @@ -471,7 +469,8 @@ push_to_memory(DumpList, State) -> 1, State#state.manifest, {0, [ManifestEntry]}), - levelzero_pending=?L0PEND_RESET}}; + levelzero_pending=?L0PEND_RESET, + levelzero_snapshot=[]}}; ?L0PEND_RESET -> {State#state.table_size, State} end, @@ -479,17 +478,16 @@ push_to_memory(DumpList, State) -> %% Prompt clerk to ask about work - do this for every push_mem ok = leveled_clerk:clerk_prompt(UpdState#state.clerk, penciller), - SW2 = os:timestamp(), MemoryInsertion = do_push_to_mem(DumpList, TableSize, - UpdState#state.memtable), - io:format("Push into memory timed at ~w microseconds~n", - [timer:now_diff(os:timestamp(),SW2)]), + UpdState#state.memtable, + UpdState#state.levelzero_snapshot), case MemoryInsertion of - {twist, ApproxTableSize} -> - {ok, UpdState#state{table_size=ApproxTableSize}}; - {roll, ApproxTableSize} -> + {twist, ApproxTableSize, UpdSnapshot} -> + {ok, UpdState#state{table_size=ApproxTableSize, + levelzero_snapshot=UpdSnapshot}}; + {roll, ApproxTableSize, UpdSnapshot} -> L0 = get_item(0, UpdState#state.manifest, []), case {L0, manifest_locked(UpdState)} of {[], false} -> @@ -508,17 +506,20 @@ push_to_memory(DumpList, State) -> L0Pid, os:timestamp()}, table_size=ApproxTableSize, - manifest_sqn=MSN}}; + manifest_sqn=MSN, + levelzero_snapshot=UpdSnapshot}}; {[], true} -> {{pause, "L0 file write blocked by change at sqn=~w~n", [UpdState#state.manifest_sqn]}, - UpdState#state{table_size=ApproxTableSize}}; + UpdState#state{table_size=ApproxTableSize, + levelzero_snapshot=UpdSnapshot}}; _ -> {{pause, "L0 file write blocked by L0 file in manifest~n", []}, - UpdState#state{table_size=ApproxTableSize}} + UpdState#state{table_size=ApproxTableSize, + levelzero_snapshot=UpdSnapshot}} end end. @@ -556,20 +557,24 @@ fetch(Key, Manifest, Level, FetchFun) -> end end. -do_push_to_mem(DumpList, TableSize, MemTable) -> +do_push_to_mem(DumpList, TableSize, MemTable, Snapshot) -> + SW = os:timestamp(), + UpdSnapshot = lists:append(Snapshot, DumpList), ets:insert(MemTable, DumpList), + io:format("Push into memory timed at ~w microseconds~n", + [timer:now_diff(os:timestamp(), SW)]), case TableSize + length(DumpList) of ApproxTableSize when ApproxTableSize > ?MAX_TABLESIZE -> case ets:info(MemTable, size) of ActTableSize when ActTableSize > ?MAX_TABLESIZE -> - {roll, ActTableSize}; + {roll, ActTableSize, UpdSnapshot}; ActTableSize -> io:format("Table size is actually ~w~n", [ActTableSize]), - {twist, ActTableSize} + {twist, ActTableSize, UpdSnapshot} end; ApproxTableSize -> io:format("Table size is approximately ~w~n", [ApproxTableSize]), - {twist, ApproxTableSize} + {twist, ApproxTableSize, UpdSnapshot} end. From f3a40e106db94ccc87275855dc5c5449c8726a24 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 5 Sep 2016 20:22:16 +0100 Subject: [PATCH 032/167] Inker improvements Resolve issue in CDB file when we have cached the index. Allow for Inker to find keys in the active journal --- src/leveled_cdb.erl | 13 ++++++----- src/leveled_inker.erl | 51 +++++++++++++++++++++++++++++++++---------- 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 3557609..88c8223 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -373,10 +373,10 @@ get(FileNameOrHandle, Key) -> get(FileNameOrHandle, Key, CRCCheck) -> get(FileNameOrHandle, Key, CRCCheck, no_cache). -get(FileName, Key, CRCCheck, Cache) when is_list(FileName), is_list(Key) -> - {ok,Handle} = file:open(FileName,[binary, raw, read]), - get(Handle,Key, CRCCheck, Cache); -get(Handle, Key, CRCCheck, Cache) when is_tuple(Handle), is_list(Key) -> +get(FileName, Key, CRCCheck, Cache) when is_list(FileName) -> + {ok, Handle} = file:open(FileName,[binary, raw, read]), + get(Handle, Key, CRCCheck, Cache); +get(Handle, Key, CRCCheck, Cache) when is_tuple(Handle) -> Hash = hash(Key), Index = hash_to_index(Hash), {HashTable, Count} = get_index(Handle, Index, Cache), @@ -401,7 +401,8 @@ get_index(Handle, Index, no_cache) -> % Get location of hashtable and number of entries in the hash read_next_2_integers(Handle); get_index(_Handle, Index, Cache) -> - lists:keyfind(Index, 1, Cache). + {_Pointer, Count} = lists:keyfind(Index, 1, Cache), + Count. %% Get a Key/Value pair from an active CDB file (with no hash table written) %% This requires a key dictionary to be passed in (mapping keys to positions) @@ -921,7 +922,7 @@ hash1(H, <>) -> hash_to_index(Hash) -> Hash band 255. -hash_to_slot(Hash,L) -> +hash_to_slot(Hash, L) -> (Hash bsr 8) rem L. %% Create a binary of the LengthKeyLengthValue, adding a CRC check diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index d19334b..b303f4e 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -118,6 +118,7 @@ manifest_sqn = 0 :: integer(), journal_sqn = 0 :: integer(), active_journaldb :: pid(), + active_journaldb_sqn :: integer(), removed_journaldbs = [] :: list(), root_path :: string()}). @@ -160,7 +161,7 @@ init([RootPath]) -> {ok, []} end, {Manifest, - ActiveJournal, + {ActiveJournal, LowActiveSQN}, JournalSQN, ManifestSQN} = build_manifest(ManifestFilenames, JournalFilenames, @@ -170,6 +171,7 @@ init([RootPath]) -> manifest_sqn = ManifestSQN, journal_sqn = JournalSQN, active_journaldb = ActiveJournal, + active_journaldb_sqn = LowActiveSQN, root_path = RootPath}}. @@ -190,7 +192,11 @@ handle_call({put, Key, Object, KeyChanges}, From, State) -> {reply, blocked, UpdState} end; handle_call({get, Key, SQN}, _From, State) -> - {reply, get_object(Key, SQN, State#state.manifest), State}; + {reply, get_object(Key, + SQN, + State#state.manifest, + State#state.active_journaldb, + State#state.active_journaldb_sqn), State}; handle_call(snapshot, _From , State) -> %% TODO: Not yet implemented registration of snapshot %% Should return manifest and register the snapshot @@ -248,9 +254,21 @@ roll_active_file(OldActiveJournal, Manifest, ManifestSQN, RootPath) -> ok = simple_manifest_writer(NewManifest, NewManifestSQN, RootPath), {NewManifest, NewManifestSQN}. -get_object(PrimaryKey, SQN, Manifest) -> - JournalP = find_in_manifest(SQN, Manifest), - leveled_cdb:cdb_get(JournalP, {SQN, PrimaryKey}). +get_object(PrimaryKey, SQN, Manifest, ActiveJournal, ActiveJournalSQN) -> + if + SQN < ActiveJournalSQN -> + JournalP = find_in_manifest(SQN, Manifest), + if + JournalP == error -> + io:format("Unable to find SQN~w in Manifest~w~n", + [SQN, Manifest]), + error; + true -> + leveled_cdb:cdb_get(JournalP, {SQN, PrimaryKey}) + end; + true -> + leveled_cdb:cdb_get(ActiveJournal, {SQN, PrimaryKey}) + end. build_manifest(ManifestFilenames, @@ -342,11 +360,14 @@ build_manifest(ManifestFilenames, {HighSQN, _HighKey} = leveled_cdb:cdb_lastkey(TMPid), HighSQN end, - ActiveFN = filepath(RootPath, TopSQNInManifest + 1, new_journal), + LowActiveSQN = TopSQNInManifest + 1, + ActiveFN = filepath(RootPath, LowActiveSQN, new_journal), {ok, ActiveJournal} = leveled_cdb:cdb_open_writer(ActiveFN), - {Manifest2, ActiveJournal, TopSQNInManifest, ManifestSQN}; + {Manifest2, + {ActiveJournal, LowActiveSQN}, + TopSQNInManifest, + ManifestSQN}; _ -> - {ActiveJournalSQN, Manifest3} = roll_pending_journals(lists:sort(OtherSQNs_pnd), Manifest2, @@ -356,7 +377,10 @@ build_manifest(ManifestFilenames, ActiveFN = filepath(RootPath, ActiveJournalSQN, new_journal), {ok, ActiveJournal} = leveled_cdb:cdb_open_writer(ActiveFN), {HighestSQN, _HighestKey} = leveled_cdb:cdb_lastkey(ActiveJournal), - {Manifest3, ActiveJournal, HighestSQN, ManifestSQN} + {Manifest3, + {ActiveJournal, ActiveJournalSQN}, + HighestSQN, + ManifestSQN} end. close_allmanifest([], ActiveJournal) -> @@ -497,12 +521,13 @@ simple_buildmanifest_test() -> fun simple_manifest_reader/2, RootPath), io:format("Build manifest output is ~w~n", [Res]), - {Man, ActJournal, HighSQN, ManSQN} = Res, + {Man, {ActJournal, ActJournalSQN}, HighSQN, ManSQN} = Res, ?assertMatch(HighSQN, 4), ?assertMatch(ManSQN, 1), ?assertMatch([{1, "nursery_1.cdb", _}], Man), {ActSQN, _ActK} = leveled_cdb:cdb_lastkey(ActJournal), ?assertMatch(ActSQN, 4), + ?assertMatch(ActJournalSQN, 3), close_allmanifest(Man, ActJournal), clean_testdir(RootPath). @@ -528,12 +553,13 @@ another_buildmanifest_test() -> fun simple_manifest_reader/2, RootPath), io:format("Build manifest output is ~w~n", [Res]), - {Man, ActJournal, HighSQN, ManSQN} = Res, + {Man, {ActJournal, ActJournalSQN}, HighSQN, ManSQN} = Res, ?assertMatch(HighSQN, 6), ?assertMatch(ManSQN, 1), ?assertMatch([{1, "nursery_1.cdb", _}, {3, "nursery_3.cdb", _}], Man), {ActSQN, _ActK} = leveled_cdb:cdb_lastkey(ActJournal), ?assertMatch(ActSQN, 6), + ?assertMatch(ActJournalSQN, 5), close_allmanifest(Man, ActJournal), clean_testdir(RootPath). @@ -545,10 +571,11 @@ empty_buildmanifest_test() -> fun simple_manifest_reader/2, RootPath), io:format("Build manifest output is ~w~n", [Res]), - {Man, ActJournal, HighSQN, ManSQN} = Res, + {Man, {ActJournal, ActJournalSQN}, HighSQN, ManSQN} = Res, ?assertMatch(Man, []), ?assertMatch(ManSQN, 0), ?assertMatch(HighSQN, 0), + ?assertMatch(ActJournalSQN, 1), empty = leveled_cdb:cdb_lastkey(ActJournal), FN = leveled_cdb:cdb_filename(ActJournal), %% The filename should be based on the next journal SQN (1) not 0 From f0e1c1d7eaa2512ddd91a08f39ece8b7e02ca332 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 6 Sep 2016 17:17:31 +0100 Subject: [PATCH 033/167] Basic GET/PUT and rolling in Inker Add support to roll file on PUT in the inker --- src/leveled_bookie.erl | 4 ++-- src/leveled_cdb.erl | 21 ++++++++++++--------- src/leveled_inker.erl | 36 ++++++++++++++++++++++++++---------- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 84f7b5a..0eab5bc 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -42,7 +42,7 @@ %% the request to the ledger. %% %% The inker will pass the request to the current (append only) CDB journal -%% fileto persist the change. The call should return either 'ok' or 'roll'. +%% file to persist the change. The call should return either 'ok' or 'roll'. %% 'roll' indicates that the CDB file has insufficient capacity for %% this write. @@ -59,7 +59,7 @@ %% %% The Bookie's memory consists of an in-memory ets table. Periodically, the %% current table is pushed to the Penciller for eventual persistence, and a -%% new tabble is started. +%% new table is started. %% %% This completes the non-deferrable work associated with a PUT %% diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 88c8223..2e63dbc 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -71,6 +71,7 @@ -define(CRC_CHECK, true). -define(MAX_FILE_SIZE, 3221225472). -define(BASE_POSITION, 2048). +-define(WRITE_OPS, [binary, raw, read, write]). -record(state, {hashtree, last_position :: integer(), @@ -139,8 +140,7 @@ init([]) -> handle_call({cdb_open_writer, Filename}, _From, State) -> io:format("Opening file for writing with filename ~s~n", [Filename]), {LastPosition, HashTree, LastKey} = open_active_file(Filename), - {ok, Handle} = file:open(Filename, [binary, raw, read, - write, delayed_write]), + {ok, Handle} = file:open(Filename, [sync | ?WRITE_OPS]), {reply, ok, State#state{handle=Handle, last_position=LastPosition, last_key=LastKey, @@ -220,7 +220,7 @@ handle_call(cdb_filename, _From, State) -> {reply, State#state.filename, State}; handle_call(cdb_close, _From, State) -> ok = file:close(State#state.handle), - {stop, normal, ok, State}; + {stop, normal, ok, State#state{handle=closed}}; handle_call(cdb_complete, _From, State) -> case State#state.writer of true -> @@ -247,7 +247,12 @@ handle_info(_Info, State) -> {noreply, State}. terminate(_Reason, State) -> - file:close(State#state.handle). + case State#state.handle of + closed -> + ok; + Handle -> + file:close(Handle) + end. code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -270,7 +275,7 @@ from_dict(FileName,Dict) -> %% this function creates a CDB %% create(FileName,KeyValueList) -> - {ok, Handle} = file:open(FileName, [binary, raw, read, write]), + {ok, Handle} = file:open(FileName, ?WRITE_OPS), {ok, _} = file:position(Handle, {bof, ?BASE_POSITION}), {BasePos, HashTree} = write_key_value_pairs(Handle, KeyValueList), close_file(Handle, HashTree, BasePos). @@ -324,7 +329,7 @@ dump(FileName, CRCCheck) -> %% tuples as the write_key_value_pairs function, and the current position, and %% the file handle open_active_file(FileName) when is_list(FileName) -> - {ok, Handle} = file:open(FileName, [binary, raw, read, write]), + {ok, Handle} = file:open(FileName, ?WRITE_OPS), {ok, Position} = file:position(Handle, {bof, 256*?DWORD_SIZE}), {LastPosition, HashTree, LastKey} = scan_over_file(Handle, Position), case file:position(Handle, eof) of @@ -345,14 +350,12 @@ open_active_file(FileName) when is_list(FileName) -> %% dictionary of Keys and positions. Returns an updated Position %% put(FileName, Key, Value, {LastPosition, HashTree}) when is_list(FileName) -> - {ok, Handle} = file:open(FileName, - [binary, raw, read, write, delayed_write]), + {ok, Handle} = file:open(FileName, ?WRITE_OPS), put(Handle, Key, Value, {LastPosition, HashTree}); put(Handle, Key, Value, {LastPosition, HashTree}) -> Bin = key_value_to_record({Key, Value}), PotentialNewSize = LastPosition + byte_size(Bin), if PotentialNewSize > ?MAX_FILE_SIZE -> - close_file(Handle, HashTree, LastPosition), roll; true -> ok = file:pwrite(Handle, LastPosition, Bin), diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index b303f4e..de75666 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -102,6 +102,7 @@ ink_put/4, ink_get/3, ink_snap/1, + ink_close/1, build_dummy_journal/0, simple_manifest_reader/2]). @@ -139,6 +140,9 @@ ink_get(Pid, PrimaryKey, SQN) -> ink_snap(Pid) -> gen_server:call(Pid, snapshot, infinity). +ink_close(Pid) -> + gen_server:call(Pid, close, infinity). + %%%============================================================================ %%% gen_server callbacks %%%============================================================================ @@ -200,7 +204,12 @@ handle_call({get, Key, SQN}, _From, State) -> handle_call(snapshot, _From , State) -> %% TODO: Not yet implemented registration of snapshot %% Should return manifest and register the snapshot - {reply, State#state.manifest, State}. + {reply, {State#state.manifest, + State#state.active_journaldb, + State#state.active_journaldb_sqn}, + State}; +handle_call(close, _From, State) -> + {stop, normal, ok, State}. handle_cast(_Msg, State) -> {noreply, State}. @@ -208,8 +217,12 @@ handle_cast(_Msg, State) -> handle_info(_Info, State) -> {noreply, State}. -terminate(_Reason, _State) -> - ok. +terminate(Reason, State) -> + io:format("Inker closing journal for reason ~w~n", [Reason]), + io:format("Close triggered with journal_sqn=~w and manifest_sqn=~w~n", + [State#state.journal_sqn, State#state.manifest_sqn]), + io:format("Manifest when closing is ~w~n", [State#state.manifest]), + close_allmanifest(State#state.manifest, State#state.active_journaldb). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -235,21 +248,24 @@ put_object(PrimaryKey, Object, KeyChanges, State) -> Bin1) of ok -> {rolling, State#state{journal_sqn=NewSQN, - active_journaldb=NewJournalP}}; + active_journaldb=NewJournalP, + active_journaldb_sqn=NewSQN}}; roll -> {blocked, State#state{journal_sqn=NewSQN, - active_journaldb=NewJournalP}} + active_journaldb=NewJournalP, + active_journaldb_sqn=NewSQN}} end end. roll_active_file(OldActiveJournal, Manifest, ManifestSQN, RootPath) -> + io:format("Rolling old journal ~w~n", [OldActiveJournal]), {ok, NewFilename} = leveled_cdb:cdb_complete(OldActiveJournal), {ok, PidR} = leveled_cdb:cdb_open_reader(NewFilename), JournalRegex2 = "nursery_(?[0-9]+)\\." ++ ?JOURNAL_FILEX, [JournalSQN] = sequencenumbers_fromfilenames([NewFilename], JournalRegex2, 'SQN'), - NewManifest = lists:append(Manifest, {JournalSQN, NewFilename, PidR}), + NewManifest = lists:append(Manifest, [{JournalSQN, NewFilename, PidR}]), NewManifestSQN = ManifestSQN + 1, ok = simple_manifest_writer(NewManifest, NewManifestSQN, RootPath), {NewManifest, NewManifestSQN}. @@ -387,7 +403,7 @@ close_allmanifest([], ActiveJournal) -> leveled_cdb:cdb_close(ActiveJournal); close_allmanifest([H|ManifestT], ActiveJournal) -> {_, _, Pid} = H, - leveled_cdb:cdb_close(Pid), + ok = leveled_cdb:cdb_close(Pid), close_allmanifest(ManifestT, ActiveJournal). @@ -396,12 +412,12 @@ roll_pending_journals([TopJournalSQN], Manifest, _RootPath) {TopJournalSQN, Manifest}; roll_pending_journals([JournalSQN|T], Manifest, RootPath) -> Filename = filepath(RootPath, JournalSQN, new_journal), - PidW = leveled_cdb:cdb_open_writer(Filename), + {ok, PidW} = leveled_cdb:cdb_open_writer(Filename), {ok, NewFilename} = leveled_cdb:cdb_complete(PidW), {ok, PidR} = leveled_cdb:cdb_open_reader(NewFilename), roll_pending_journals(T, lists:append(Manifest, - {JournalSQN, NewFilename, PidR}), + [{JournalSQN, NewFilename, PidR}]), RootPath). @@ -454,7 +470,7 @@ simple_manifest_writer(Manifest, ManSQN, RootPath) -> NewFN = filename:join(ManPath, integer_to_list(ManSQN) ++ ?MANIFEST_FILEX), TmpFN = filename:join(ManPath, integer_to_list(ManSQN) ++ ?PENDING_FILEX), MBin = term_to_binary(Manifest), - case file:is_file(NewFN) of + case filelib:is_file(NewFN) of true -> io:format("Error - trying to write manifest for" ++ " ManifestSQN=~w which already exists~n", [ManSQN]), From 0d905639be4707141ad145aac170d31f7d527298 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 7 Sep 2016 17:58:12 +0100 Subject: [PATCH 034/167] Testing of Inker rolling Journal Add test to show inker rolling journal. to achieve needs to make CDB size an option, and also alter the manifest sorting so that find_in_manifest actually works! --- include/leveled.hrl | 10 ++- src/leveled_cdb.erl | 115 +++++++++++++++++++++++--------- src/leveled_inker.erl | 152 ++++++++++++++++++++++++++++++++++-------- 3 files changed, 216 insertions(+), 61 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index fdf779c..f68a12d 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -18,4 +18,12 @@ {start_key :: tuple(), end_key :: tuple(), owner :: pid(), - filename :: string()}). \ No newline at end of file + filename :: string()}). + +-record(cdb_options, + {max_size :: integer()}). + +-record(inker_options, + {cdb_max_size :: integer(), + root_path :: string(), + cdb_options :: #cdb_options{}}). \ No newline at end of file diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 2e63dbc..513de01 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -47,6 +47,7 @@ -module(leveled_cdb). -behaviour(gen_server). +-include("../include/leveled.hrl"). -export([init/1, handle_call/3, @@ -55,6 +56,7 @@ terminate/2, code_change/3, cdb_open_writer/1, + cdb_open_writer/2, cdb_open_reader/1, cdb_get/2, cdb_put/3, @@ -79,7 +81,8 @@ hash_index = [] :: list(), filename :: string(), handle :: file:fd(), - writer :: boolean}). + writer :: boolean, + max_size :: integer()}). %%%============================================================================ @@ -87,7 +90,11 @@ %%%============================================================================ cdb_open_writer(Filename) -> - {ok, Pid} = gen_server:start(?MODULE, [], []), + %% No options passed + cdb_open_writer(Filename, #cdb_options{}). + +cdb_open_writer(Filename, Opts) -> + {ok, Pid} = gen_server:start(?MODULE, [Opts], []), case gen_server:call(Pid, {cdb_open_writer, Filename}, infinity) of ok -> {ok, Pid}; @@ -96,7 +103,7 @@ cdb_open_writer(Filename) -> end. cdb_open_reader(Filename) -> - {ok, Pid} = gen_server:start(?MODULE, [], []), + {ok, Pid} = gen_server:start(?MODULE, [#cdb_options{}], []), case gen_server:call(Pid, {cdb_open_reader, Filename}, infinity) of ok -> {ok, Pid}; @@ -134,27 +141,33 @@ cdb_keycheck(Pid, Key) -> %%% gen_server callbacks %%%============================================================================ -init([]) -> - {ok, #state{}}. +init([Opts]) -> + MaxSize = case Opts#cdb_options.max_size of + undefined -> + ?MAX_FILE_SIZE; + M -> + M + end, + {ok, #state{max_size=MaxSize}}. handle_call({cdb_open_writer, Filename}, _From, State) -> io:format("Opening file for writing with filename ~s~n", [Filename]), {LastPosition, HashTree, LastKey} = open_active_file(Filename), {ok, Handle} = file:open(Filename, [sync | ?WRITE_OPS]), {reply, ok, State#state{handle=Handle, - last_position=LastPosition, - last_key=LastKey, - filename=Filename, - hashtree=HashTree, - writer=true}}; + last_position=LastPosition, + last_key=LastKey, + filename=Filename, + hashtree=HashTree, + writer=true}}; handle_call({cdb_open_reader, Filename}, _From, State) -> io:format("Opening file for reading with filename ~s~n", [Filename]), {ok, Handle} = file:open(Filename, [binary, raw, read]), Index = load_index(Handle), {reply, ok, State#state{handle=Handle, - filename=Filename, - writer=false, - hash_index=Index}}; + filename=Filename, + writer=false, + hash_index=Index}}; handle_call({cdb_get, Key}, _From, State) -> case {State#state.writer, State#state.hash_index} of {true, _} -> @@ -198,7 +211,8 @@ handle_call({cdb_put, Key, Value}, _From, State) -> true -> Result = put(State#state.handle, Key, Value, - {State#state.last_position, State#state.hashtree}), + {State#state.last_position, State#state.hashtree}, + State#state.max_size), case Result of roll -> %% Key and value could not be written @@ -230,7 +244,8 @@ handle_call(cdb_complete, _From, State) -> %% Rename file NewName = filename:rootname(State#state.filename, ".pnd") ++ ".cdb", - io:format("Renaming file from ~s to ~s~n", [State#state.filename, NewName]), + io:format("Renaming file from ~s to ~s~n", + [State#state.filename, NewName]), ok = file:rename(State#state.filename, NewName), {stop, normal, {ok, NewName}, State}; false -> @@ -349,19 +364,24 @@ open_active_file(FileName) when is_list(FileName) -> %% Append to an active file a new key/value pair returning an updated %% dictionary of Keys and positions. Returns an updated Position %% -put(FileName, Key, Value, {LastPosition, HashTree}) when is_list(FileName) -> +put(FileName, Key, Value, {LastPosition, HashTree}, MaxSize) when is_list(FileName) -> {ok, Handle} = file:open(FileName, ?WRITE_OPS), - put(Handle, Key, Value, {LastPosition, HashTree}); -put(Handle, Key, Value, {LastPosition, HashTree}) -> + put(Handle, Key, Value, {LastPosition, HashTree}, MaxSize); +put(Handle, Key, Value, {LastPosition, HashTree}, MaxSize) -> Bin = key_value_to_record({Key, Value}), PotentialNewSize = LastPosition + byte_size(Bin), - if PotentialNewSize > ?MAX_FILE_SIZE -> + if PotentialNewSize > MaxSize -> roll; true -> ok = file:pwrite(Handle, LastPosition, Bin), {Handle, PotentialNewSize, put_hashtree(Key, LastPosition, HashTree)} end. +%% Should not be used for non-test PUTs by the inker - as the Max File Size +%% should be taken from the startup options not the default +put(FileName, Key, Value, {LastPosition, HashTree}) -> + put(FileName, Key, Value, {LastPosition, HashTree}, ?MAX_FILE_SIZE). + %% %% get(FileName,Key) -> {key,value} @@ -393,10 +413,14 @@ get(Handle, Key, CRCCheck, Cache) when is_tuple(Handle) -> Slot = hash_to_slot(Hash, Count), {ok, _} = file:position(Handle, {cur, Slot * ?DWORD_SIZE}), LastHashPosition = HashTable + ((Count-1) * ?DWORD_SIZE), - LocList = lists:seq(FirstHashPosition, LastHashPosition, ?DWORD_SIZE), + LocList = lists:seq(FirstHashPosition, + LastHashPosition, + ?DWORD_SIZE), % Split list around starting slot. {L1, L2} = lists:split(Slot, LocList), - search_hash_table(Handle, lists:append(L2, L1), Hash, Key, CRCCheck) + search_hash_table(Handle, + lists:append(L2, L1), + Hash, Key, CRCCheck) end. get_index(Handle, Index, no_cache) -> @@ -758,7 +782,11 @@ search_hash_table(Handle, [Entry|RestOfEntries], Hash, Key, CRCCheck) -> end, case KV of missing -> - search_hash_table(Handle, RestOfEntries, Hash, Key, CRCCheck); + search_hash_table(Handle, + RestOfEntries, + Hash, + Key, + CRCCheck); _ -> KV end; @@ -948,14 +976,24 @@ key_value_to_record({Key, Value}) -> write_key_value_pairs_1_test() -> {ok,Handle} = file:open("../test/test.cdb",write), - {_, HashTree} = write_key_value_pairs(Handle,[{"key1","value1"},{"key2","value2"}]), + {_, HashTree} = write_key_value_pairs(Handle, + [{"key1","value1"}, + {"key2","value2"}]), Hash1 = hash("key1"), Index1 = hash_to_index(Hash1), Hash2 = hash("key2"), Index2 = hash_to_index(Hash2), R0 = array:new(256, {default, gb_trees:empty()}), - R1 = array:set(Index1, gb_trees:insert(Hash1, [0], array:get(Index1, R0)), R0), - R2 = array:set(Index2, gb_trees:insert(Hash2, [30], array:get(Index2, R1)), R1), + R1 = array:set(Index1, + gb_trees:insert(Hash1, + [0], + array:get(Index1, R0)), + R0), + R2 = array:set(Index2, + gb_trees:insert(Hash2, + [30], + array:get(Index2, R1)), + R1), io:format("HashTree is ~w~n", [HashTree]), io:format("Expected HashTree is ~w~n", [R2]), ?assertMatch(R2, HashTree), @@ -965,8 +1003,16 @@ write_key_value_pairs_1_test() -> write_hash_tables_1_test() -> {ok, Handle} = file:open("../test/testx.cdb",write), R0 = array:new(256, {default, gb_trees:empty()}), - R1 = array:set(64, gb_trees:insert(6383014720, [18], array:get(64, R0)), R0), - R2 = array:set(67, gb_trees:insert(6383014723, [0], array:get(67, R1)), R1), + R1 = array:set(64, + gb_trees:insert(6383014720, + [18], + array:get(64, R0)), + R0), + R2 = array:set(67, + gb_trees:insert(6383014723, + [0], + array:get(67, R1)), + R1), Result = write_hash_tables(Handle, R2), io:format("write hash tables result of ~w ~n", [Result]), ?assertMatch(Result,[{67,16,2},{64,0,2}]), @@ -999,7 +1045,8 @@ find_open_slot_5_test() -> full_1_test() -> List1 = lists:sort([{"key1","value1"},{"key2","value2"}]), - create("../test/simple.cdb",lists:sort([{"key1","value1"},{"key2","value2"}])), + create("../test/simple.cdb", + lists:sort([{"key1","value1"},{"key2","value2"}])), List2 = lists:sort(dump("../test/simple.cdb")), ?assertMatch(List1,List2), ok = file:delete("../test/simple.cdb"). @@ -1103,7 +1150,8 @@ search_hash_table_findinslot_test() -> {"K4", "V4"}, {"K5", "V5"}, {"K6", "V6"}, {"K7", "V7"}, {"K8", "V8"}]), ok = from_dict("../test/hashtable1_test.cdb",D), - {ok, Handle} = file:open("../test/hashtable1_test.cdb", [binary, raw, read, write]), + {ok, Handle} = file:open("../test/hashtable1_test.cdb", + [binary, raw, read, write]), Hash = hash(Key1), Index = hash_to_index(Hash), {ok, _} = file:position(Handle, {bof, ?DWORD_SIZE*Index}), @@ -1124,12 +1172,17 @@ search_hash_table_findinslot_test() -> {ok, _} = file:position(Handle, FirstHashPosition), FlipH3 = endian_flip(ReadH3), FlipP3 = endian_flip(ReadP3), - RBin = <>, + RBin = <>, io:format("Replacement binary of ~w~n", [RBin]), {ok, OldBin} = file:pread(Handle, FirstHashPosition + (Slot -1) * ?DWORD_SIZE, 16), io:format("Bin to be replaced is ~w ~n", [OldBin]), - ok = file:pwrite(Handle, FirstHashPosition + (Slot -1) * ?DWORD_SIZE, RBin), + ok = file:pwrite(Handle, + FirstHashPosition + (Slot -1) * ?DWORD_SIZE, + RBin), ok = file:close(Handle), io:format("Find key following change to hash table~n"), ?assertMatch(missing, get("../test/hashtable1_test.cdb", Key1)), diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index de75666..6513e72 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -103,6 +103,7 @@ ink_get/3, ink_snap/1, ink_close/1, + ink_print_manifest/1, build_dummy_journal/0, simple_manifest_reader/2]). @@ -121,15 +122,16 @@ active_journaldb :: pid(), active_journaldb_sqn :: integer(), removed_journaldbs = [] :: list(), - root_path :: string()}). + root_path :: string(), + cdb_options :: #cdb_options{}}). %%%============================================================================ %%% API %%%============================================================================ -ink_start(RootDir) -> - gen_server:start(?MODULE, [RootDir], []). +ink_start(InkerOpts) -> + gen_server:start(?MODULE, [InkerOpts], []). ink_put(Pid, PrimaryKey, Object, KeyChanges) -> gen_server:call(Pid, {put, PrimaryKey, Object, KeyChanges}, infinity). @@ -143,11 +145,16 @@ ink_snap(Pid) -> ink_close(Pid) -> gen_server:call(Pid, close, infinity). +ink_print_manifest(Pid) -> + gen_server:call(Pid, print_manifest, infinity). + %%%============================================================================ %%% gen_server callbacks %%%============================================================================ -init([RootPath]) -> +init([InkerOpts]) -> + RootPath = InkerOpts#inker_options.root_path, + CDBopts = InkerOpts#inker_options.cdb_options, JournalFP = filepath(RootPath, journal_dir), {ok, JournalFilenames} = case filelib:is_dir(JournalFP) of true -> @@ -170,13 +177,15 @@ init([RootPath]) -> ManifestSQN} = build_manifest(ManifestFilenames, JournalFilenames, fun simple_manifest_reader/2, - RootPath), + RootPath, + CDBopts), {ok, #state{manifest = Manifest, manifest_sqn = ManifestSQN, journal_sqn = JournalSQN, active_journaldb = ActiveJournal, active_journaldb_sqn = LowActiveSQN, - root_path = RootPath}}. + root_path = RootPath, + cdb_options = CDBopts}}. handle_call({put, Key, Object, KeyChanges}, From, State) -> @@ -208,6 +217,9 @@ handle_call(snapshot, _From , State) -> State#state.active_journaldb, State#state.active_journaldb_sqn}, State}; +handle_call(print_manifest, _From, State) -> + manifest_printer(State#state.manifest), + {reply, ok, State}; handle_call(close, _From, State) -> {stop, normal, ok, State}. @@ -221,7 +233,8 @@ terminate(Reason, State) -> io:format("Inker closing journal for reason ~w~n", [Reason]), io:format("Close triggered with journal_sqn=~w and manifest_sqn=~w~n", [State#state.journal_sqn, State#state.manifest_sqn]), - io:format("Manifest when closing is ~w~n", [State#state.manifest]), + io:format("Manifest when closing is: ~n"), + manifest_printer(State#state.manifest), close_allmanifest(State#state.manifest, State#state.active_journaldb). code_change(_OldVsn, State, _Extra) -> @@ -242,7 +255,8 @@ put_object(PrimaryKey, Object, KeyChanges, State) -> {ok, State#state{journal_sqn=NewSQN}}; roll -> FileName = filepath(State#state.root_path, NewSQN, new_journal), - {ok, NewJournalP} = leveled_cdb:cdb_open_writer(FileName), + CDBopts = State#state.cdb_options, + {ok, NewJournalP} = leveled_cdb:cdb_open_writer(FileName, CDBopts), case leveled_cdb:cdb_put(NewJournalP, {NewSQN, PrimaryKey}, Bin1) of @@ -265,13 +279,13 @@ roll_active_file(OldActiveJournal, Manifest, ManifestSQN, RootPath) -> [JournalSQN] = sequencenumbers_fromfilenames([NewFilename], JournalRegex2, 'SQN'), - NewManifest = lists:append(Manifest, [{JournalSQN, NewFilename, PidR}]), + NewManifest = add_to_manifest(Manifest, {JournalSQN, NewFilename, PidR}), NewManifestSQN = ManifestSQN + 1, ok = simple_manifest_writer(NewManifest, NewManifestSQN, RootPath), {NewManifest, NewManifestSQN}. get_object(PrimaryKey, SQN, Manifest, ActiveJournal, ActiveJournalSQN) -> - if + Obj = if SQN < ActiveJournalSQN -> JournalP = find_in_manifest(SQN, Manifest), if @@ -284,13 +298,30 @@ get_object(PrimaryKey, SQN, Manifest, ActiveJournal, ActiveJournalSQN) -> end; true -> leveled_cdb:cdb_get(ActiveJournal, {SQN, PrimaryKey}) + end, + case Obj of + {{SQN, PK}, Bin} -> + {{SQN, PK}, binary_to_term(Bin)}; + _ -> + Obj end. - + build_manifest(ManifestFilenames, JournalFilenames, ManifestRdrFun, RootPath) -> + build_manifest(ManifestFilenames, + JournalFilenames, + ManifestRdrFun, + RootPath, + #cdb_options{}). + +build_manifest(ManifestFilenames, + JournalFilenames, + ManifestRdrFun, + RootPath, + CDBopts) -> %% Setup root paths JournalFP = filepath(RootPath, journal_dir), %% Find the manifest with a highest Manifest sequence number @@ -336,7 +367,7 @@ build_manifest(ManifestFilenames, integer_to_list(X) ++ "." ++ ?JOURNAL_FILEX, - Acc ++ [{X, FN}]; + add_to_manifest(Acc, {X, FN}); true -> Acc end end, @@ -345,11 +376,12 @@ build_manifest(ManifestFilenames, %% Enrich the manifest so it contains the Pid of any of the immutable %% entries - io:format("Manifest1 is ~w~n", [Manifest1]), - Manifest2 = lists:map(fun({X, Y}) -> - FN = filename:join(JournalFP, Y), - {ok, Pid} = leveled_cdb:cdb_open_reader(FN), - {X, Y, Pid} end, + io:format("Manifest on startup is: ~n"), + manifest_printer(Manifest1), + Manifest2 = lists:map(fun({LowSQN, FN}) -> + FP = filename:join(JournalFP, FN), + {ok, Pid} = leveled_cdb:cdb_open_reader(FP), + {LowSQN, FN, Pid} end, Manifest1), %% Find any more recent mutable files that have a higher sequence number @@ -378,7 +410,8 @@ build_manifest(ManifestFilenames, end, LowActiveSQN = TopSQNInManifest + 1, ActiveFN = filepath(RootPath, LowActiveSQN, new_journal), - {ok, ActiveJournal} = leveled_cdb:cdb_open_writer(ActiveFN), + {ok, ActiveJournal} = leveled_cdb:cdb_open_writer(ActiveFN, + CDBopts), {Manifest2, {ActiveJournal, LowActiveSQN}, TopSQNInManifest, @@ -391,7 +424,8 @@ build_manifest(ManifestFilenames, %% Need to work out highest sequence number in tail file to feed %% into opening of pending journal ActiveFN = filepath(RootPath, ActiveJournalSQN, new_journal), - {ok, ActiveJournal} = leveled_cdb:cdb_open_writer(ActiveFN), + {ok, ActiveJournal} = leveled_cdb:cdb_open_writer(ActiveFN, + CDBopts), {HighestSQN, _HighestKey} = leveled_cdb:cdb_lastkey(ActiveJournal), {Manifest3, {ActiveJournal, ActiveJournalSQN}, @@ -416,8 +450,8 @@ roll_pending_journals([JournalSQN|T], Manifest, RootPath) -> {ok, NewFilename} = leveled_cdb:cdb_complete(PidW), {ok, PidR} = leveled_cdb:cdb_open_reader(NewFilename), roll_pending_journals(T, - lists:append(Manifest, - [{JournalSQN, NewFilename, PidR}]), + add_to_manifest(Manifest, + {JournalSQN, NewFilename, PidR}), RootPath). @@ -437,6 +471,9 @@ sequencenumbers_fromfilenames(Filenames, Regex, IntName) -> [], Filenames). +add_to_manifest(Manifest, Entry) -> + lists:reverse(lists:sort([Entry|Manifest])). + find_in_manifest(_SQN, []) -> error; find_in_manifest(SQN, [{LowSQN, _FN, Pid}|_Tail]) when SQN >= LowSQN -> @@ -483,7 +520,17 @@ simple_manifest_writer(Manifest, ManSQN, RootPath) -> ok end. - +manifest_printer(Manifest) -> + lists:foreach(fun(X) -> + {SQN, FN} = case X of + {A, B, _PID} -> + {A, B}; + {A, B} -> + {A, B} + end, + io:format("At SQN=~w journal has filename ~s~n", + [SQN, FN]) end, + Manifest). %%%============================================================================ %%% Test @@ -502,15 +549,15 @@ build_dummy_journal() -> {ok, J1} = leveled_cdb:cdb_open_writer(F1), {K1, V1} = {"Key1", "TestValue1"}, {K2, V2} = {"Key2", "TestValue2"}, - ok = leveled_cdb:cdb_put(J1, {1, K1}, V1), - ok = leveled_cdb:cdb_put(J1, {2, K2}, V2), + ok = leveled_cdb:cdb_put(J1, {1, K1}, term_to_binary({V1, []})), + ok = leveled_cdb:cdb_put(J1, {2, K2}, term_to_binary({V2, []})), {ok, _} = leveled_cdb:cdb_complete(J1), F2 = filename:join(JournalFP, "nursery_3.pnd"), {ok, J2} = leveled_cdb:cdb_open_writer(F2), {K1, V3} = {"Key1", "TestValue3"}, {K4, V4} = {"Key4", "TestValue4"}, - ok = leveled_cdb:cdb_put(J2, {3, K1}, V3), - ok = leveled_cdb:cdb_put(J2, {4, K4}, V4), + ok = leveled_cdb:cdb_put(J2, {3, K1}, term_to_binary({V3, []})), + ok = leveled_cdb:cdb_put(J2, {4, K4}, term_to_binary({V4, []})), ok = leveled_cdb:cdb_close(J2), Manifest = {2, [{1, "nursery_1.cdb"}], []}, ManifestBin = term_to_binary(Manifest), @@ -558,8 +605,8 @@ another_buildmanifest_test() -> {ok, NewActiveJN} = leveled_cdb:cdb_open_writer(FN2), {K5, V5} = {"Key5", "TestValue5"}, {K6, V6} = {"Key6", "TestValue6"}, - ok = leveled_cdb:cdb_put(NewActiveJN, {5, K5}, V5), - ok = leveled_cdb:cdb_put(NewActiveJN, {6, K6}, V6), + ok = leveled_cdb:cdb_put(NewActiveJN, {5, K5}, term_to_binary({V5, []})), + ok = leveled_cdb:cdb_put(NewActiveJN, {6, K6}, term_to_binary({V6, []})), ok = leveled_cdb:cdb_close(NewActiveJN), %% Test setup - now build manifest Res = build_manifest(["1.man"], @@ -572,7 +619,7 @@ another_buildmanifest_test() -> {Man, {ActJournal, ActJournalSQN}, HighSQN, ManSQN} = Res, ?assertMatch(HighSQN, 6), ?assertMatch(ManSQN, 1), - ?assertMatch([{1, "nursery_1.cdb", _}, {3, "nursery_3.cdb", _}], Man), + ?assertMatch([{3, "nursery_3.cdb", _}, {1, "nursery_1.cdb", _}], Man), {ActSQN, _ActK} = leveled_cdb:cdb_lastkey(ActJournal), ?assertMatch(ActSQN, 6), ?assertMatch(ActJournalSQN, 5), @@ -599,5 +646,52 @@ empty_buildmanifest_test() -> close_allmanifest(Man, ActJournal), clean_testdir(RootPath). +simplejournal_test() -> + %% build up a database, and then open it through the gen_server wrap + %% Get and Put some keys + RootPath = "../test/inker", + build_dummy_journal(), + {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, + cdb_options=#cdb_options{}}), + R1 = ink_get(Ink1, "Key1", 1), + ?assertMatch(R1, {{1, "Key1"}, {"TestValue1", []}}), + R2 = ink_get(Ink1, "Key1", 3), + ?assertMatch(R2, {{3, "Key1"}, {"TestValue3", []}}), + {ok, NewSQN1} = ink_put(Ink1, "Key99", "TestValue99", []), + ?assertMatch(NewSQN1, 5), + R3 = ink_get(Ink1, "Key99", 5), + io:format("Result 3 is ~w~n", [R3]), + ?assertMatch(R3, {{5, "Key99"}, {"TestValue99", []}}), + ink_close(Ink1), + clean_testdir(RootPath). + +rollafile_simplejournal_test() -> + RootPath = "../test/inker", + build_dummy_journal(), + CDBopts = #cdb_options{max_size=300000}, + {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, + cdb_options=CDBopts}), + FunnyLoop = lists:seq(1, 48), + {ok, NewSQN1} = ink_put(Ink1, "KeyAA", "TestValueAA", []), + ?assertMatch(NewSQN1, 5), + ok = ink_print_manifest(Ink1), + R0 = ink_get(Ink1, "KeyAA", 5), + ?assertMatch(R0, {{5, "KeyAA"}, {"TestValueAA", []}}), + lists:foreach(fun(X) -> + {ok, _} = ink_put(Ink1, + "KeyZ" ++ integer_to_list(X), + crypto:rand_bytes(10000), + []) end, + FunnyLoop), + {ok, NewSQN2} = ink_put(Ink1, "KeyBB", "TestValueBB", []), + ?assertMatch(NewSQN2, 54), + ok = ink_print_manifest(Ink1), + R1 = ink_get(Ink1, "KeyAA", 5), + ?assertMatch(R1, {{5, "KeyAA"}, {"TestValueAA", []}}), + R2 = ink_get(Ink1, "KeyBB", 54), + ?assertMatch(R2, {{54, "KeyBB"}, {"TestValueBB", []}}), + ink_close(Ink1), + clean_testdir(RootPath). + -endif. \ No newline at end of file From edfe9e3bed1dd0c73e3e0c98cad3cd2392690f46 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 8 Sep 2016 14:21:30 +0100 Subject: [PATCH 035/167] Improved testing Improve testing of Penciller to show startup and shutdown with push, merging and fetch --- include/leveled.hrl | 6 +- src/leveled_bookie.erl | 74 +++++++++++++++++++++++++ src/leveled_inker.erl | 12 ++-- src/leveled_penciller.erl | 114 +++++++++++++++++++++++++++++++++----- src/leveled_sft.erl | 10 +++- 5 files changed, 193 insertions(+), 23 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index f68a12d..80cdc87 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -26,4 +26,8 @@ -record(inker_options, {cdb_max_size :: integer(), root_path :: string(), - cdb_options :: #cdb_options{}}). \ No newline at end of file + cdb_options :: #cdb_options{}}). + +-record(penciller_options, + {root_path :: string(), + max_inmemory_tablesize :: integer()}). \ No newline at end of file diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 0eab5bc..58d24b2 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -90,3 +90,77 @@ -module(leveled_bookie). +-behaviour(gen_server). + +-include("../include/leveled.hrl"). + +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-include_lib("eunit/include/eunit.hrl"). + + +-record(state, {inker :: pid(), + penciller :: pid()}). + + +%%%============================================================================ +%%% API +%%%============================================================================ + + + +%%%============================================================================ +%%% gen_server callbacks +%%%============================================================================ + +init([Opts]) -> + {InkerOpts, PencillerOpts} = set_options(Opts), + {Inker, Penciller} = startup(InkerOpts, PencillerOpts), + {ok, #state{inker=Inker, penciller=Penciller}}. + + +handle_call(_, _From, State) -> + {reply, ok, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + + +%%%============================================================================ +%%% Internal functions +%%%============================================================================ + +set_options(_Opts) -> + {#inker_options{}, #penciller_options{}}. + +startup(InkerOpts, PencillerOpts) -> + {ok, Inker} = leveled_inker:ink_start(InkerOpts), + {ok, Penciller} = leveled_penciller:pcl_start(PencillerOpts), + LedgerSQN = leveled_penciller:pcl_getstartupsequencenumber(Penciller), + KeyChanges = leveled_inker:ink_fetchkeychangesfrom(Inker, LedgerSQN), + ok = leveled_penciller:pcl_pushmem(Penciller, KeyChanges), + {Inker, Penciller}. + + + +%%%============================================================================ +%%% Test +%%%============================================================================ + +-ifdef(TEST). + +-endif. \ No newline at end of file diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 6513e72..25b839b 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -539,7 +539,7 @@ manifest_printer(Manifest) -> -ifdef(TEST). build_dummy_journal() -> - RootPath = "../test/inker", + RootPath = "../test/journal", JournalFP = filepath(RootPath, journal_dir), ManifestFP = filepath(RootPath, manifest_dir), ok = filelib:ensure_dir(RootPath), @@ -577,7 +577,7 @@ clean_subdir(DirPath) -> Files). simple_buildmanifest_test() -> - RootPath = "../test/inker", + RootPath = "../test/journal", build_dummy_journal(), Res = build_manifest(["1.man"], ["nursery_1.cdb", "nursery_3.pnd"], @@ -596,7 +596,7 @@ simple_buildmanifest_test() -> another_buildmanifest_test() -> %% There is a rolled jounral file which is not yet in the manifest - RootPath = "../test/inker", + RootPath = "../test/journal", build_dummy_journal(), FN = filepath(RootPath, 3, new_journal), {ok, FileToRoll} = leveled_cdb:cdb_open_writer(FN), @@ -628,7 +628,7 @@ another_buildmanifest_test() -> empty_buildmanifest_test() -> - RootPath = "../test/inker/", + RootPath = "../test/journal", Res = build_manifest([], [], fun simple_manifest_reader/2, @@ -649,7 +649,7 @@ empty_buildmanifest_test() -> simplejournal_test() -> %% build up a database, and then open it through the gen_server wrap %% Get and Put some keys - RootPath = "../test/inker", + RootPath = "../test/journal", build_dummy_journal(), {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, cdb_options=#cdb_options{}}), @@ -666,7 +666,7 @@ simplejournal_test() -> clean_testdir(RootPath). rollafile_simplejournal_test() -> - RootPath = "../test/inker", + RootPath = "../test/journal", build_dummy_journal(), CDBopts = #cdb_options{max_size=300000}, {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 6fe6eac..009cffb 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -197,15 +197,16 @@ levelzero_pending = ?L0PEND_RESET :: tuple(), levelzero_snapshot = [] :: list(), memtable, - backlog = false :: boolean()}). + backlog = false :: boolean(), + memtable_maxsize :: integer}). %%%============================================================================ %%% API %%%============================================================================ -pcl_start(RootDir) -> - gen_server:start(?MODULE, [RootDir], []). +pcl_start(PCLopts) -> + gen_server:start(?MODULE, [PCLopts], []). pcl_pushmem(Pid, DumpList) -> %% Bookie to dump memory onto penciller @@ -236,10 +237,20 @@ pcl_close(Pid) -> %%% gen_server callbacks %%%============================================================================ -init([RootPath]) -> +init([PCLopts]) -> + RootPath = PCLopts#penciller_options.root_path, + MaxTableSize = case PCLopts#penciller_options.max_inmemory_tablesize of + undefined -> + ?MAX_TABLESIZE; + M -> + M + end, TID = ets:new(?MEMTABLE, [ordered_set]), {ok, Clerk} = leveled_clerk:clerk_new(self()), - InitState = #state{memtable=TID, clerk=Clerk, root_path=RootPath}, + InitState = #state{memtable=TID, + clerk=Clerk, + root_path=RootPath, + memtable_maxsize=MaxTableSize}, %% Open manifest ManifestPath = InitState#state.root_path ++ "/" ++ ?MANIFEST_FP ++ "/", @@ -320,7 +331,7 @@ init([RootPath]) -> handle_call({push_mem, DumpList}, _From, State) -> StartWatch = os:timestamp(), Response = case assess_sqn(DumpList) of - {MinSQN, MaxSQN} when MaxSQN > MinSQN, + {MinSQN, MaxSQN} when MaxSQN >= MinSQN, MinSQN >= State#state.ledger_sqn -> io:format("SQN check completed in ~w microseconds~n", [timer:now_diff(os:timestamp(),StartWatch)]), @@ -436,6 +447,10 @@ terminate(_Reason, State) -> file:rename(FileName ++ ".pnd", FileName ++ ".sft"); {?L0PEND_RESET, [], L} when L == 0 -> io:format("No keys to dump from memory when closing~n"); + {{true, L0Pid, _TS}, _, _} -> + leveled_sft:sft_close(L0Pid), + io:format("No opportunity to persist memory before closing " + ++ "with ~w keys discarded~n", [length(Dump)]); _ -> io:format("No opportunity to persist memory before closing " ++ "with ~w keys discarded~n", [length(Dump)]) @@ -481,7 +496,8 @@ push_to_memory(DumpList, State) -> MemoryInsertion = do_push_to_mem(DumpList, TableSize, UpdState#state.memtable, - UpdState#state.levelzero_snapshot), + UpdState#state.levelzero_snapshot, + UpdState#state.memtable_maxsize), case MemoryInsertion of {twist, ApproxTableSize, UpdSnapshot} -> @@ -557,16 +573,16 @@ fetch(Key, Manifest, Level, FetchFun) -> end end. -do_push_to_mem(DumpList, TableSize, MemTable, Snapshot) -> +do_push_to_mem(DumpList, TableSize, MemTable, Snapshot, MaxSize) -> SW = os:timestamp(), UpdSnapshot = lists:append(Snapshot, DumpList), ets:insert(MemTable, DumpList), io:format("Push into memory timed at ~w microseconds~n", [timer:now_diff(os:timestamp(), SW)]), case TableSize + length(DumpList) of - ApproxTableSize when ApproxTableSize > ?MAX_TABLESIZE -> + ApproxTableSize when ApproxTableSize > MaxSize -> case ets:info(MemTable, size) of - ActTableSize when ActTableSize > ?MAX_TABLESIZE -> + ActTableSize when ActTableSize > MaxSize -> {roll, ActTableSize, UpdSnapshot}; ActTableSize -> io:format("Table size is actually ~w~n", [ActTableSize]), @@ -769,14 +785,19 @@ rename_manifest_files(RootPath, NewMSN) -> file:rename(filepath(RootPath, NewMSN, pending_manifest), filepath(RootPath, NewMSN, current_manifest)). +filepath(RootPath, manifest) -> + RootPath ++ "/" ++ ?MANIFEST_FP; +filepath(RootPath, files) -> + RootPath ++ "/" ++ ?FILES_FP. + filepath(RootPath, NewMSN, pending_manifest) -> - RootPath ++ "/" ++ ?MANIFEST_FP ++ "/" ++ "nonzero_" + filepath(RootPath, manifest) ++ "/" ++ "nonzero_" ++ integer_to_list(NewMSN) ++ "." ++ ?PENDING_FILEX; filepath(RootPath, NewMSN, current_manifest) -> - RootPath ++ "/" ++ ?MANIFEST_FP ++ "/" ++ "nonzero_" + filepath(RootPath, manifest) ++ "/" ++ "nonzero_" ++ integer_to_list(NewMSN) ++ "." ++ ?CURRENT_FILEX; filepath(RootPath, NewMSN, new_merge_files) -> - RootPath ++ "/" ++ ?FILES_FP ++ "/" ++ integer_to_list(NewMSN). + filepath(RootPath, files) ++ "/" ++ integer_to_list(NewMSN). update_deletions([], _NewMSN, UnreferencedFiles) -> UnreferencedFiles; @@ -822,6 +843,24 @@ assess_sqn([HeadKey|Tail], MinSQN, MaxSQN) -> %%% Test %%%============================================================================ +-ifdef(TEST). + +clean_testdir(RootPath) -> + clean_subdir(filepath(RootPath, manifest)), + clean_subdir(filepath(RootPath, files)). + +clean_subdir(DirPath) -> + case filelib:is_dir(DirPath) of + true -> + {ok, Files} = file:list_dir(DirPath), + lists:foreach(fun(FN) -> file:delete(filename:join(DirPath, FN)), + io:format("Delete file ~s/~s~n", + [DirPath, FN]) + end, + Files); + false -> + ok + end. compaction_work_assessment_test() -> L0 = [{{o, "B1", "K1"}, {o, "B3", "K3"}, dummy_pid}], @@ -854,4 +893,51 @@ confirm_delete_test() -> ?assertMatch(R2, false), RegisteredIterators3 = [{dummy_pid, 9}, {dummy_pid, 12}], R3 = confirm_delete(Filename, UnreferencedFiles, RegisteredIterators3), - ?assertMatch(R3, false). \ No newline at end of file + ?assertMatch(R3, false). + + +simple_server_test() -> + RootPath = "../test/ledger", + clean_testdir(RootPath), + {ok, PCL} = pcl_start(#penciller_options{root_path=RootPath, + max_inmemory_tablesize=1000}), + Key1 = {{o,"Bucket0001", "Key0001"},1, {active, infinity}, null}, + KL1 = lists:sort(leveled_sft:generate_randomkeys({1000, 2})), + Key2 = {{o,"Bucket0002", "Key0002"},1002, {active, infinity}, null}, + KL2 = lists:sort(leveled_sft:generate_randomkeys({1000, 1002})), + Key3 = {{o,"Bucket0003", "Key0003"},2002, {active, infinity}, null}, + ok = pcl_pushmem(PCL, [Key1]), + R1 = pcl_fetch(PCL, {o,"Bucket0001", "Key0001"}), + ?assertMatch(R1, Key1), + ok = pcl_pushmem(PCL, KL1), + R2 = pcl_fetch(PCL, {o,"Bucket0001", "Key0001"}), + ?assertMatch(R2, Key1), + S1 = pcl_pushmem(PCL, [Key2]), + if S1 == pause -> timer:sleep(2); true -> ok end, + R3 = pcl_fetch(PCL, {o,"Bucket0001", "Key0001"}), + R4 = pcl_fetch(PCL, {o,"Bucket0002", "Key0002"}), + ?assertMatch(R3, Key1), + ?assertMatch(R4, Key2), + S2 = pcl_pushmem(PCL, KL2), + if S2 == pause -> timer:sleep(2000); true -> ok end, + S3 = pcl_pushmem(PCL, [Key3]), + if S3 == pause -> timer:sleep(2000); true -> ok end, + R5 = pcl_fetch(PCL, {o,"Bucket0001", "Key0001"}), + R6 = pcl_fetch(PCL, {o,"Bucket0002", "Key0002"}), + R7 = pcl_fetch(PCL, {o,"Bucket0003", "Key0003"}), + ?assertMatch(R5, Key1), + ?assertMatch(R6, Key2), + ?assertMatch(R7, Key3), + ok = pcl_close(PCL), + {ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath, + max_inmemory_tablesize=1000}), + R8 = pcl_fetch(PCLr, {o,"Bucket0001", "Key0001"}), + R9 = pcl_fetch(PCLr, {o,"Bucket0002", "Key0002"}), + R10 = pcl_fetch(PCLr, {o,"Bucket0003", "Key0003"}), + ?assertMatch(R8, Key1), + ?assertMatch(R9, Key2), + ?assertMatch(R10, Key3), + ok = pcl_close(PCLr), + clean_testdir(RootPath). + +-endif. \ No newline at end of file diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 12e36e5..d420829 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -200,7 +200,7 @@ filter_pointer :: integer(), summ_pointer :: integer(), summ_length :: integer(), - filename :: string(), + filename = "not set" :: string(), handle :: file:fd(), background_complete = false :: boolean(), background_failure = "Unknown" :: string(), @@ -387,7 +387,12 @@ terminate(Reason, State) -> ok = file:close(State#state.handle), ok = file:delete(State#state.filename); _ -> - ok = file:close(State#state.handle) + case State#state.handle of + undefined -> + ok; + Handle -> + file:close(Handle) + end end. code_change(_OldVsn, State, _Extra) -> @@ -520,6 +525,7 @@ complete_file(Handle, FileMD, KL1, KL2, Level, Rename) -> false -> open_file(FileMD); {true, OldName, NewName} -> + io:format("Renaming file from ~s to ~s~n", [OldName, NewName]), ok = file:rename(OldName, NewName), open_file(FileMD#state{filename=NewName}) end, From 86666b1cb617d984eccdc2e258e35127353c8eed Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 9 Sep 2016 15:58:19 +0100 Subject: [PATCH 036/167] Scan over CDB file Make scanning over a CDB file generic rather than specific to read-in of active nursery log - open to be called as an external function to support other scanning behaviour. --- src/leveled_bookie.erl | 16 +++++++++-- src/leveled_cdb.erl | 58 +++++++++++++++++++++++++++------------ src/leveled_inker.erl | 19 +++++++++++++ src/leveled_penciller.erl | 38 +++++++++++++++++++++++-- 4 files changed, 108 insertions(+), 23 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 58d24b2..75e938c 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -53,9 +53,9 @@ %% - close the previously active journal file (writing the hashtree), and move %% it to the historic journal %% -%% Once the object has been persisted to the Journal, the Key and Metadata can -%% be added to the ledger. Initially this will be added to the Bookie's -%% in-memory view of recent changes only. +%% Once the object has been persisted to the Journal, the Key with Metadata +%% and the keychanges can be added to the ledger. Initially this will be +%% added to the Bookie'sin-memory view of recent changes only. %% %% The Bookie's memory consists of an in-memory ets table. Periodically, the %% current table is pushed to the Penciller for eventual persistence, and a @@ -107,6 +107,16 @@ -record(state, {inker :: pid(), penciller :: pid()}). +-record(item, {primary_key :: term(), + contents :: list(), + metadatas :: list(), + vclock, + hash :: integer(), + size :: integer(), + key_changes :: list()}) + + + %%%============================================================================ %%% API diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 513de01..f6444c7 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -63,6 +63,7 @@ cdb_lastkey/1, cdb_filename/1, cdb_keycheck/2, + cdb_scan/4, cdb_close/1, cdb_complete/1]). @@ -123,6 +124,11 @@ cdb_close(Pid) -> cdb_complete(Pid) -> gen_server:call(Pid, cdb_complete, infinity). +cdb_scan(Pid, StartPosition, FilterFun, InitAcc) -> + gen_server:call(Pid, + {cdb_scan, StartPosition, FilterFun, InitAcc}, + infinity). + %% Get the last key to be added to the file (which will have the highest %% sequence number) cdb_lastkey(Pid) -> @@ -232,6 +238,12 @@ handle_call(cdb_lastkey, _From, State) -> {reply, State#state.last_key, State}; handle_call(cdb_filename, _From, State) -> {reply, State#state.filename, State}; +handle_call({cdb_scan, StartPos, FilterFun, Acc}, _From, State) -> + {LastPosition, Acc2} = scan_over_file(State#state.handle, + StartPos, + FilterFun, + Acc), + {reply, {LastPosition, Acc2}, State}; handle_call(cdb_close, _From, State) -> ok = file:close(State#state.handle), {stop, normal, ok, State#state{handle=closed}}; @@ -346,7 +358,8 @@ dump(FileName, CRCCheck) -> open_active_file(FileName) when is_list(FileName) -> {ok, Handle} = file:open(FileName, ?WRITE_OPS), {ok, Position} = file:position(Handle, {bof, 256*?DWORD_SIZE}), - {LastPosition, HashTree, LastKey} = scan_over_file(Handle, Position), + {LastPosition, {HashTree, LastKey}} = startup_scan_over_file(Handle, + Position), case file:position(Handle, eof) of {ok, LastPosition} -> ok = file:close(Handle); @@ -617,32 +630,43 @@ extract_kvpair(Handle, [Position|Rest], Key, Check) -> %% Scan through the file until there is a failure to crc check an input, and %% at that point return the position and the key dictionary scanned so far -scan_over_file(Handle, Position) -> +startup_scan_over_file(Handle, Position) -> HashTree = array:new(256, {default, gb_trees:empty()}), - scan_over_file(Handle, Position, HashTree, empty). + scan_over_file(Handle, Position, fun startup_filter/4, {HashTree, empty}). -scan_over_file(Handle, Position, HashTree, LastKey) -> +%% Scan for key changes - scan over file returning applying FilterFun +%% The FilterFun should accept as input: +%% - Key, Value, Position, Accumulator, outputting a new Accumulator +%% and a loop|stop instruction as a tuple i.e. {loop, Acc} or {stop, Acc} + +scan_over_file(Handle, Position, FilterFun, Output) -> case saferead_keyvalue(Handle) of false -> - {Position, HashTree, LastKey}; + {Position, Output}; {Key, ValueAsBin, KeyLength, ValueLength} -> - case crccheck_value(ValueAsBin) of - true -> - NewPosition = Position + KeyLength + ValueLength + NewPosition = Position + KeyLength + ValueLength + ?DWORD_SIZE, - scan_over_file(Handle, - NewPosition, - put_hashtree(Key, Position, HashTree), - Key); - false -> - io:format("CRC check returned false on key of ~w ~n", - [Key]), - {Position, HashTree, LastKey} + case FilterFun(Key, ValueAsBin, Position, Output) of + {stop, UpdOutput} -> + {Position, UpdOutput}; + {loop, UpdOutput} -> + scan_over_file(Handle, NewPosition, FilterFun, UpdOutput) end; eof -> - {Position, HashTree, LastKey} + {Position, Output} end. +%% Specific filter to be used at startup to build a hashtree for an incomplete +%% cdb file, and returns at the end the hashtree and the final Key seen in the +%% journal + +startup_filter(Key, ValueAsBin, Position, {Hashtree, LastKey}) -> + case crccheck_value(ValueAsBin) of + true -> + {loop, {put_hashtree(Key, Position, Hashtree), Key}}; + false -> + {stop, {Hashtree, LastKey}} + end. %% Read the Key/Value at this point, returning {ok, Key, Value} %% catch expected exceptiosn associated with file corruption (or end) and diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 25b839b..56c982e 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -139,6 +139,9 @@ ink_put(Pid, PrimaryKey, Object, KeyChanges) -> ink_get(Pid, PrimaryKey, SQN) -> gen_server:call(Pid, {get, PrimaryKey, SQN}, infinity). +ink_fetchkeychanges(Pid, SQN) -> + gen_server:call(Pid, {fetch_keychanges, SQN}, infinity). + ink_snap(Pid) -> gen_server:call(Pid, snapshot, infinity). @@ -220,6 +223,12 @@ handle_call(snapshot, _From , State) -> handle_call(print_manifest, _From, State) -> manifest_printer(State#state.manifest), {reply, ok, State}; +handle_call({fetch_keychanges, SQN}, _From, State) -> + KeyChanges = fetch_key_changes(SQN, + State#state.manifest, + State#state.active_journaldb, + State#state.active_journaldb_sqn), + {reply, KeyChanges, State}; handle_call(close, _From, State) -> {stop, normal, ok, State}. @@ -455,6 +464,16 @@ roll_pending_journals([JournalSQN|T], Manifest, RootPath) -> RootPath). +fetch_key_changes(SQN, Manifest, ActiveJournal, ActiveSQN) -> + InitialChanges = case SQN of + SQN when SQN < ActiveSQN -> + fetch_key_changes(SQN, Manifest); + _ -> + [] + end, + RecentChanges = fetch_key_changes(SQN, ActiveJournal), + InitialChanges ++ RecentChanges. + sequencenumbers_fromfilenames(Filenames, Regex, IntName) -> lists:foldl(fun(FN, Acc) -> diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 009cffb..46fb8e9 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -901,11 +901,14 @@ simple_server_test() -> clean_testdir(RootPath), {ok, PCL} = pcl_start(#penciller_options{root_path=RootPath, max_inmemory_tablesize=1000}), - Key1 = {{o,"Bucket0001", "Key0001"},1, {active, infinity}, null}, + Key1 = {{o,"Bucket0001", "Key0001"}, 1, {active, infinity}, null}, KL1 = lists:sort(leveled_sft:generate_randomkeys({1000, 2})), - Key2 = {{o,"Bucket0002", "Key0002"},1002, {active, infinity}, null}, + Key2 = {{o,"Bucket0002", "Key0002"}, 1002, {active, infinity}, null}, KL2 = lists:sort(leveled_sft:generate_randomkeys({1000, 1002})), - Key3 = {{o,"Bucket0003", "Key0003"},2002, {active, infinity}, null}, + Key3 = {{o,"Bucket0003", "Key0003"}, 2002, {active, infinity}, null}, + KL3 = lists:sort(leveled_sft:generate_randomkeys({1000, 2002})), + Key4 = {{o,"Bucket0004", "Key0004"}, 3002, {active, infinity}, null}, + KL4 = lists:sort(leveled_sft:generate_randomkeys({1000, 3002})), ok = pcl_pushmem(PCL, [Key1]), R1 = pcl_fetch(PCL, {o,"Bucket0001", "Key0001"}), ?assertMatch(R1, Key1), @@ -931,12 +934,41 @@ simple_server_test() -> ok = pcl_close(PCL), {ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath, max_inmemory_tablesize=1000}), + TopSQN = pcl_getstartupsequencenumber(PCLr), + case TopSQN of + 2001 -> + %% Last push not persisted + S3a = pcl_pushmem(PCL, [Key3]), + if S3a == pause -> timer:sleep(2000); true -> ok end; + 2002 -> + %% everything got persisted + ok; + _ -> + io:format("Unexpected sequence number on restart ~w~n", [TopSQN]), + ok = pcl_close(PCLr), + clean_testdir(RootPath), + ?assertMatch(true, false) + end, R8 = pcl_fetch(PCLr, {o,"Bucket0001", "Key0001"}), R9 = pcl_fetch(PCLr, {o,"Bucket0002", "Key0002"}), R10 = pcl_fetch(PCLr, {o,"Bucket0003", "Key0003"}), ?assertMatch(R8, Key1), ?assertMatch(R9, Key2), ?assertMatch(R10, Key3), + S4 = pcl_pushmem(PCLr, KL3), + if S4 == pause -> timer:sleep(2000); true -> ok end, + S5 = pcl_pushmem(PCLr, [Key4]), + if S5 == pause -> timer:sleep(2000); true -> ok end, + S6 = pcl_pushmem(PCLr, KL4), + if S6 == pause -> timer:sleep(2000); true -> ok end, + R11 = pcl_fetch(PCLr, {o,"Bucket0001", "Key0001"}), + R12 = pcl_fetch(PCLr, {o,"Bucket0002", "Key0002"}), + R13 = pcl_fetch(PCLr, {o,"Bucket0003", "Key0003"}), + R14 = pcl_fetch(PCLr, {o,"Bucket0004", "Key0004"}), + ?assertMatch(R11, Key1), + ?assertMatch(R12, Key2), + ?assertMatch(R13, Key3), + ?assertMatch(R14, Key4), ok = pcl_close(PCLr), clean_testdir(RootPath). From e73a5bbf31cefdceb050c34b8823bb32f7837ca4 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 15 Sep 2016 10:53:24 +0100 Subject: [PATCH 037/167] WIP - First draft of Bookie code First draft of untested bookie code --- include/leveled.hrl | 24 ++- src/leveled_bookie.erl | 303 ++++++++++++++++++++++++++++++++++---- src/leveled_cdb.erl | 76 +++++++--- src/leveled_inker.erl | 124 +++++++++++----- src/leveled_penciller.erl | 13 +- src/leveled_sft.erl | 212 +++++++++++--------------- 6 files changed, 536 insertions(+), 216 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 80cdc87..030fdd4 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -30,4 +30,26 @@ -record(penciller_options, {root_path :: string(), - max_inmemory_tablesize :: integer()}). \ No newline at end of file + max_inmemory_tablesize :: integer()}). + +-record(bookie_options, + {root_path :: string(), + cache_size :: integer(), + metadata_extractor :: function(), + indexspec_converter :: function()}). + +%% Temp location for records related to riak + +-record(r_content, { + metadata, + value :: term() + }). + +-record(r_object, { + bucket, + key, + contents :: [#r_content{}], + vclock, + updatemetadata=dict:store(clean, true, dict:new()), + updatevalue :: term()}). + \ No newline at end of file diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 75e938c..1b68d85 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -33,19 +33,23 @@ %% -------- PUT -------- %% %% A PUT request consists of -%% - A primary Key -%% - Metadata associated with the primary key (2i, vector clock, object size) -%% - A value -%% - A set of secondary key changes which should be made as part of the commit +%% - A Primary Key and a Value +%% - IndexSpecs - a set of secondary key changes associated with the +%% transaction %% %% The Bookie takes the place request and passes it first to the Inker to add %% the request to the ledger. %% -%% The inker will pass the request to the current (append only) CDB journal -%% file to persist the change. The call should return either 'ok' or 'roll'. -%% 'roll' indicates that the CDB file has insufficient capacity for +%% The inker will pass the PK/Value/IndexSpecs to the current (append only) +%% CDB journal file to persist the change. The call should return either 'ok' +%% or 'roll'. -'roll' indicates that the CDB file has insufficient capacity for %% this write. - +%% +%% (Note that storing the IndexSpecs will create some duplication with the +%% Metadata wrapped up within the Object value. This Value and the IndexSpecs +%% are compressed before storage, so this should provide some mitigation for +%% the duplication). +%% %% In resonse to a 'roll', the inker should: %% - start a new active journal file with an open_write_request, and then; %% - call to PUT the object in this file; @@ -53,13 +57,31 @@ %% - close the previously active journal file (writing the hashtree), and move %% it to the historic journal %% -%% Once the object has been persisted to the Journal, the Key with Metadata -%% and the keychanges can be added to the ledger. Initially this will be -%% added to the Bookie'sin-memory view of recent changes only. +%% The inker will also return the SQN which the change has been made at, as +%% well as the object size on disk within the Journal. %% -%% The Bookie's memory consists of an in-memory ets table. Periodically, the -%% current table is pushed to the Penciller for eventual persistence, and a -%% new table is started. +%% Once the object has been persisted to the Journal, the Ledger can be updated. +%% The Ledger is updated by the Bookie applying a function (passed in at +%% startup) to the Value to return the Object Metadata, a function to generate +%% a hash of the Value and also taking the Primary Key, the IndexSpecs, the +%% Sequence Number in the Journal and the Object Size (returned from the +%% Inker). +%% +%% The Bookie should generate a series of ledger key changes from this +%% information, using a function passed in at startup. For Riak this will be +%% of the form: +%% {{o, Bucket, Key}, +%% SQN, +%% {Hash, Size, {Riak_Metadata}}, +%% {active, TS}|{tomb, TS}} or +%% {{i, Bucket, IndexTerm, IndexField, Key}, +%% SQN, +%% null, +%% {active, TS}|{tomb, TS}} +%% +%% Recent Ledger changes are retained initially in the Bookies' memory (in an +%% in-memory ets table). Periodically, the current table is pushed to the +%% Penciller for eventual persistence, and a new table is started. %% %% This completes the non-deferrable work associated with a PUT %% @@ -86,6 +108,18 @@ %% %% e.g. Get all for SegmentID/Partition %% +%% +%% +%% -------- On Startup -------- +%% +%% On startup the Bookie must restart both the Inker to load the Journal, and +%% the Penciller to load the Ledger. Once the Penciller has started, the +%% Bookie should request the highest sequence number in the Ledger, and then +%% and try and rebuild any missing information from the Journal +%% +%% To rebuild the Ledger it requests the Inker to scan over the files from +%% the sequence number and re-generate the Ledger changes. + -module(leveled_bookie). @@ -99,22 +133,28 @@ handle_cast/2, handle_info/2, terminate/2, - code_change/3]). + code_change/3, + book_start/1, + book_put/4, + book_get/2, + book_head/2, + strip_to_keyonly/1, + strip_to_keyseqonly/1, + strip_to_seqonly/1, + strip_to_statusonly/1, + strip_to_details/1]). -include_lib("eunit/include/eunit.hrl"). +-define(CACHE_SIZE, 1000). -record(state, {inker :: pid(), - penciller :: pid()}). - --record(item, {primary_key :: term(), - contents :: list(), - metadatas :: list(), - vclock, - hash :: integer(), - size :: integer(), - key_changes :: list()}) - + penciller :: pid(), + metadata_extractor :: function(), + indexspec_converter :: function(), + cache_size :: integer(), + back_pressure :: boolean(), + ledger_cache :: gb_trees:tree()}). @@ -122,7 +162,17 @@ %%% API %%%============================================================================ +book_start(Opts) -> + gen_server:start(?MODULE, [Opts], []). +book_put(Pid, PrimaryKey, Object, IndexSpecs) -> + gen_server:call(Pid, {put, PrimaryKey, Object, IndexSpecs}, infinity). + +book_get(Pid, PrimaryKey) -> + gen_server:call(Pid, {get, PrimaryKey}, infinity). + +book_head(Pid, PrimaryKey) -> + gen_server:call(Pid, {head, PrimaryKey}, infinity). %%%============================================================================ %%% gen_server callbacks @@ -131,11 +181,81 @@ init([Opts]) -> {InkerOpts, PencillerOpts} = set_options(Opts), {Inker, Penciller} = startup(InkerOpts, PencillerOpts), - {ok, #state{inker=Inker, penciller=Penciller}}. + Extractor = if + Opts#bookie_options.metadata_extractor == undefined -> + fun extract_metadata/2; + true -> + Opts#bookie_options.metadata_extractor + end, + Converter = if + Opts#bookie_options.indexspec_converter == undefined -> + fun convert_indexspecs/3; + true -> + Opts#bookie_options.indexspec_converter + end, + CacheSize = if + Opts#bookie_options.cache_size == undefined -> + ?CACHE_SIZE; + true -> + Opts#bookie_options.cache_size + end, + {ok, #state{inker=Inker, + penciller=Penciller, + metadata_extractor=Extractor, + indexspec_converter=Converter, + cache_size=CacheSize, + ledger_cache=gb_trees:empty()}}. -handle_call(_, _From, State) -> - {reply, ok, State}. +handle_call({put, PrimaryKey, Object, IndexSpecs}, From, State) -> + {ok, SQN, ObjSize} = leveled_inker:ink_put(PrimaryKey, Object, IndexSpecs), + Changes = preparefor_ledgercache(PrimaryKey, + SQN, + Object, + ObjSize, + IndexSpecs), + Cache0 = addto_ledgercache(Changes, State#state.ledger_cache), + gen_server:reply(From, ok), + case maybepush_ledgercache(State#state.cache_size, + Cache0, + State#state.penciller) of + {ok, NewCache} -> + {noreply, State#state{ledger_cache=NewCache, back_pressure=false}}; + {pause, NewCache} -> + {noreply, State#state{ledger_cache=NewCache, back_pressure=true}} + end; +handle_call({get, Key}, _From, State) -> + case fetch_head(Key, State#state.penciller, State#state.ledger_cache) of + not_present -> + {reply, not_found, State}; + Head -> + {Key, Seqn, Status} = strip_to_details(Head), + case Status of + {tomb, _} -> + {reply, not_found, State}; + {active, _} -> + case fetch_value(Key, Seqn, State#state.inker) of + not_present -> + {reply, not_found, State}; + Object -> + {reply, {ok, Object}, State} + end + end + end; +handle_call({head, Key}, _From, State) -> + case fetch_head(Key, State#state.penciller, State#state.ledger_cache) of + not_present -> + {reply, not_found, State}; + Head -> + {Key, _Seqn, Status} = strip_to_details(Head), + case Status of + {tomb, _} -> + {reply, not_found, State}; + {active, _} -> + MD = strip_to_mdonly(Head), + {reply, {ok, MD}, State} + end + end. handle_cast(_Msg, State) -> {noreply, State}. @@ -154,18 +274,137 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================ -set_options(_Opts) -> - {#inker_options{}, #penciller_options{}}. +set_options(Opts) -> + {#inker_options{root_path=Opts#bookie_options.root_path}, + #penciller_options{root_path=Opts#bookie_options.root_path}}. startup(InkerOpts, PencillerOpts) -> {ok, Inker} = leveled_inker:ink_start(InkerOpts), {ok, Penciller} = leveled_penciller:pcl_start(PencillerOpts), LedgerSQN = leveled_penciller:pcl_getstartupsequencenumber(Penciller), - KeyChanges = leveled_inker:ink_fetchkeychangesfrom(Inker, LedgerSQN), - ok = leveled_penciller:pcl_pushmem(Penciller, KeyChanges), + ok = leveled_inker:ink_loadpcl(LedgerSQN, fun load_fun/4, Penciller), {Inker, Penciller}. +fetch_head(Key, Penciller, Cache) -> + case gb_trees:lookup(Key, Cache) of + {value, Head} -> + Head; + none -> + case leveled_penciller:pcl_fetch(Penciller, Key) of + {Key, Head} -> + Head; + not_present -> + not_present + end + end. + +fetch_value(Key, SQN, Inker) -> + case leveled_inker:ink_fetch(Inker, Key, SQN) of + {ok, Value} -> + Value; + not_present -> + not_present + end. + +%% Format of a Key within the ledger is +%% {PrimaryKey, SQN, Metadata, Status} + +strip_to_keyonly({keyonly, K}) -> K; +strip_to_keyonly({K, _V}) -> K. + +strip_to_keyseqonly({K, {SeqN, _, _}}) -> {K, SeqN}. + +strip_to_statusonly({_, {_, St, _}}) -> St. + +strip_to_seqonly({_, {SeqN, _, _}}) -> SeqN. + +strip_to_details({K, {SeqN, St, _}}) -> {K, SeqN, St}. + +strip_to_mdonly({_, {_, _, MD}}) -> MD. + +get_metadatas(#r_object{contents=Contents}) -> + [Content#r_content.metadata || Content <- Contents]. + +set_vclock(Object=#r_object{}, VClock) -> Object#r_object{vclock=VClock}. + +vclock(#r_object{vclock=VClock}) -> VClock. + +to_binary(v0, Obj) -> + term_to_binary(Obj). + +hash(Obj=#r_object{}) -> + Vclock = vclock(Obj), + UpdObj = set_vclock(Obj, lists:sort(Vclock)), + erlang:phash2(to_binary(v0, UpdObj)). + +extract_metadata(Obj, Size) -> + {get_metadatas(Obj), vclock(Obj), hash(Obj), Size}. + +convert_indexspecs(IndexSpecs, SQN, PrimaryKey) -> + lists:map(fun({IndexOp, IndexField, IndexValue}) -> + Status = case IndexOp of + add -> + %% TODO: timestamp support + {active, infinity}; + remove -> + %% TODO: timestamps for delayed reaping + {tomb, infinity} + end, + {o, B, K} = PrimaryKey, + {{i, B, IndexField, IndexValue, K}, + {SQN, Status, null}} + end, + IndexSpecs). + + +preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs) -> + PrimaryChange = {PK, + {SQN, + {active, infinity}, + extract_metadata(Obj, Size)}}, + SecChanges = convert_indexspecs(IndexSpecs, SQN, PK), + [PrimaryChange] ++ SecChanges. + +addto_ledgercache(Changes, Cache) -> + lists:foldl(fun({{K, V}, Acc}) -> gb_trees:enter(K, V, Acc) end, + Cache, + Changes). + +maybepush_ledgercache(MaxCacheSize, Cache, Penciller) -> + CacheSize = gb_trees:size(Cache), + if + CacheSize > MaxCacheSize -> + case leveled_penciller:pcl_pushmem(Penciller, + gb_trees:to_list(Cache)) of + ok -> + {ok, gb_trees:empty()}; + pause -> + {pause, gb_trees:empty()}; + refused -> + {ok, Cache} + end; + true -> + {ok, Cache} + end. + +load_fun(KeyInLedger, ValueInLedger, _Position, Acc0) -> + {MinSQN, MaxSQN, Output} = Acc0, + {SQN, PK} = KeyInLedger, + {Obj, IndexSpecs} = ValueInLedger, + case SQN of + SQN when SQN < MinSQN -> + {loop, Acc0}; + SQN when SQN =< MaxSQN -> + %% TODO - get correct size in a more efficient manner + %% Need to have compressed size + Size = byte_size(term_to_binary(ValueInLedger, [compressed])), + Changes = preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs), + {loop, {MinSQN, MaxSQN, Output ++ Changes}}; + SQN when SQN > MaxSQN -> + {stop, Acc0} + end. + %%%============================================================================ %%% Test diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index f6444c7..cba127d 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -82,7 +82,7 @@ hash_index = [] :: list(), filename :: string(), handle :: file:fd(), - writer :: boolean, + writer :: boolean(), max_size :: integer()}). @@ -124,9 +124,16 @@ cdb_close(Pid) -> cdb_complete(Pid) -> gen_server:call(Pid, cdb_complete, infinity). -cdb_scan(Pid, StartPosition, FilterFun, InitAcc) -> +%% cdb_scan returns {LastPosition, Acc}. Use LastPosition as StartPosiiton to +%% continue from that point (calling function has to protect against) double +%% counting. +%% +%% LastPosition could be the atom complete when the last key processed was at +%% the end of the file. last_key must be defined in LoopState. + +cdb_scan(Pid, FilterFun, InitAcc, StartPosition) -> gen_server:call(Pid, - {cdb_scan, StartPosition, FilterFun, InitAcc}, + {cdb_scan, FilterFun, InitAcc, StartPosition}, infinity). %% Get the last key to be added to the file (which will have the highest @@ -238,15 +245,24 @@ handle_call(cdb_lastkey, _From, State) -> {reply, State#state.last_key, State}; handle_call(cdb_filename, _From, State) -> {reply, State#state.filename, State}; -handle_call({cdb_scan, StartPos, FilterFun, Acc}, _From, State) -> +handle_call({cdb_scan, FilterFun, Acc, StartPos}, _From, State) -> + {ok, StartPos0} = case StartPos of + undefined -> + file:position(State#state.handle, + ?BASE_POSITION); + StartPos -> + {ok, StartPos} + end, + ok = check_last_key(State#state.last_key), {LastPosition, Acc2} = scan_over_file(State#state.handle, - StartPos, + StartPos0, FilterFun, - Acc), + Acc, + State#state.last_key), {reply, {LastPosition, Acc2}, State}; handle_call(cdb_close, _From, State) -> ok = file:close(State#state.handle), - {stop, normal, ok, State#state{handle=closed}}; + {stop, normal, ok, State#state{handle=undefined}}; handle_call(cdb_complete, _From, State) -> case State#state.writer of true -> @@ -261,6 +277,9 @@ handle_call(cdb_complete, _From, State) -> ok = file:rename(State#state.filename, NewName), {stop, normal, {ok, NewName}, State}; false -> + ok = file:close(State#state.handle), + {stop, normal, {ok, State#state.filename}, State}; + undefined -> ok = file:close(State#state.handle), {stop, normal, {ok, State#state.filename}, State} end. @@ -275,7 +294,7 @@ handle_info(_Info, State) -> terminate(_Reason, State) -> case State#state.handle of - closed -> + undefined -> ok; Handle -> file:close(Handle) @@ -632,28 +651,36 @@ extract_kvpair(Handle, [Position|Rest], Key, Check) -> %% at that point return the position and the key dictionary scanned so far startup_scan_over_file(Handle, Position) -> HashTree = array:new(256, {default, gb_trees:empty()}), - scan_over_file(Handle, Position, fun startup_filter/4, {HashTree, empty}). + scan_over_file(Handle, + Position, + fun startup_filter/4, + {HashTree, empty}, + undefined). %% Scan for key changes - scan over file returning applying FilterFun %% The FilterFun should accept as input: %% - Key, Value, Position, Accumulator, outputting a new Accumulator %% and a loop|stop instruction as a tuple i.e. {loop, Acc} or {stop, Acc} -scan_over_file(Handle, Position, FilterFun, Output) -> +scan_over_file(Handle, Position, FilterFun, Output, LastKey) -> case saferead_keyvalue(Handle) of false -> {Position, Output}; {Key, ValueAsBin, KeyLength, ValueLength} -> NewPosition = Position + KeyLength + ValueLength + ?DWORD_SIZE, - case FilterFun(Key, ValueAsBin, Position, Output) of - {stop, UpdOutput} -> + case {FilterFun(Key, ValueAsBin, Position, Output), Key} of + {{stop, UpdOutput}, _} -> {Position, UpdOutput}; - {loop, UpdOutput} -> - scan_over_file(Handle, NewPosition, FilterFun, UpdOutput) - end; - eof -> - {Position, Output} + {{loop, UpdOutput}, LastKey} -> + {eof, UpdOutput}; + {{loop, UpdOutput}, _} -> + scan_over_file(Handle, + NewPosition, + FilterFun, + UpdOutput, + Key) + end end. %% Specific filter to be used at startup to build a hashtree for an incomplete @@ -668,6 +695,15 @@ startup_filter(Key, ValueAsBin, Position, {Hashtree, LastKey}) -> {stop, {Hashtree, LastKey}} end. +%% Confirm that the last key has been defined and set to a non-default value + +check_last_key(LastKey) -> + case LastKey of + undefined -> error; + empty -> error; + _ -> ok + end. + %% Read the Key/Value at this point, returning {ok, Key, Value} %% catch expected exceptiosn associated with file corruption (or end) and %% return eof @@ -765,7 +801,7 @@ read_next_term(Handle, Length, crc, Check) -> _ -> {false, binary_to_term(Bin)} end; - _ -> + false -> {ok, _} = file:position(Handle, {cur, 4}), {ok, Bin} = file:read(Handle, Length - 4), {unchecked, binary_to_term(Bin)} @@ -999,7 +1035,7 @@ key_value_to_record({Key, Value}) -> -ifdef(TEST). write_key_value_pairs_1_test() -> - {ok,Handle} = file:open("../test/test.cdb",write), + {ok,Handle} = file:open("../test/test.cdb",[write]), {_, HashTree} = write_key_value_pairs(Handle, [{"key1","value1"}, {"key2","value2"}]), @@ -1025,7 +1061,7 @@ write_key_value_pairs_1_test() -> write_hash_tables_1_test() -> - {ok, Handle} = file:open("../test/testx.cdb",write), + {ok, Handle} = file:open("../test/testx.cdb", [write]), R0 = array:new(256, {default, gb_trees:empty()}), R1 = array:set(64, gb_trees:insert(6383014720, diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 56c982e..e5302c0 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -101,11 +101,14 @@ ink_start/1, ink_put/4, ink_get/3, + ink_fetch/3, + ink_loadpcl/4, ink_snap/1, ink_close/1, ink_print_manifest/1, build_dummy_journal/0, - simple_manifest_reader/2]). + simple_manifest_reader/2, + clean_testdir/1]). -include_lib("eunit/include/eunit.hrl"). @@ -114,7 +117,8 @@ -define(JOURNAL_FILEX, "cdb"). -define(MANIFEST_FILEX, "man"). -define(PENDING_FILEX, "pnd"). - +-define(LOADING_PAUSE, 5000). +-define(LOADING_BATCH, 1000). -record(state, {manifest = [] :: list(), manifest_sqn = 0 :: integer(), @@ -139,8 +143,8 @@ ink_put(Pid, PrimaryKey, Object, KeyChanges) -> ink_get(Pid, PrimaryKey, SQN) -> gen_server:call(Pid, {get, PrimaryKey, SQN}, infinity). -ink_fetchkeychanges(Pid, SQN) -> - gen_server:call(Pid, {fetch_keychanges, SQN}, infinity). +ink_fetch(Pid, PrimaryKey, SQN) -> + gen_server:call(Pid, {fetch, PrimaryKey, SQN}, infinity). ink_snap(Pid) -> gen_server:call(Pid, snapshot, infinity). @@ -148,6 +152,9 @@ ink_snap(Pid) -> ink_close(Pid) -> gen_server:call(Pid, close, infinity). +ink_loadpcl(Pid, MinSQN, FilterFun, Penciller) -> + gen_server:call(Pid, {load_pcl, MinSQN, FilterFun, Penciller}, infinity). + ink_print_manifest(Pid) -> gen_server:call(Pid, print_manifest, infinity). @@ -193,10 +200,10 @@ init([InkerOpts]) -> handle_call({put, Key, Object, KeyChanges}, From, State) -> case put_object(Key, Object, KeyChanges, State) of - {ok, UpdState} -> - {reply, {ok, UpdState#state.journal_sqn}, UpdState}; - {rolling, UpdState} -> - gen_server:reply(From, {ok, UpdState#state.journal_sqn}), + {ok, UpdState, ObjSize} -> + {reply, {ok, UpdState#state.journal_sqn, ObjSize}, UpdState}; + {rolling, UpdState, ObjSize} -> + gen_server:reply(From, {ok, UpdState#state.journal_sqn, ObjSize}), {NewManifest, NewManifestSQN} = roll_active_file(State#state.active_journaldb, State#state.manifest, @@ -207,12 +214,31 @@ handle_call({put, Key, Object, KeyChanges}, From, State) -> {blocked, UpdState} -> {reply, blocked, UpdState} end; +handle_call({fetch, Key, SQN}, _From, State) -> + case get_object(Key, + SQN, + State#state.manifest, + State#state.active_journaldb, + State#state.active_journaldb_sqn) of + {{SQN, Key}, {Value, _IndexSpecs}} -> + {reply, {ok, Value}, State}; + Other -> + io:format("Unexpected failure to fetch value for" ++ + "Key=~s SQN=~w with reason ~w", [Key, SQN, Other]), + {reply, not_present, State} + end; handle_call({get, Key, SQN}, _From, State) -> {reply, get_object(Key, SQN, State#state.manifest, State#state.active_journaldb, State#state.active_journaldb_sqn), State}; +handle_call({load_pcl, StartSQN, FilterFun, Penciller}, _From, State) -> + Manifest = State#state.manifest ++ [{State#state.active_journaldb_sqn, + dummy, + State#state.active_journaldb}], + Reply = load_from_sequence(StartSQN, FilterFun, Penciller, Manifest), + {reply, Reply, State}; handle_call(snapshot, _From , State) -> %% TODO: Not yet implemented registration of snapshot %% Should return manifest and register the snapshot @@ -223,12 +249,6 @@ handle_call(snapshot, _From , State) -> handle_call(print_manifest, _From, State) -> manifest_printer(State#state.manifest), {reply, ok, State}; -handle_call({fetch_keychanges, SQN}, _From, State) -> - KeyChanges = fetch_key_changes(SQN, - State#state.manifest, - State#state.active_journaldb, - State#state.active_journaldb_sqn), - {reply, KeyChanges, State}; handle_call(close, _From, State) -> {stop, normal, ok, State}. @@ -257,11 +277,12 @@ code_change(_OldVsn, State, _Extra) -> put_object(PrimaryKey, Object, KeyChanges, State) -> NewSQN = State#state.journal_sqn + 1, Bin1 = term_to_binary({Object, KeyChanges}, [compressed]), + ObjSize = byte_size(Bin1), case leveled_cdb:cdb_put(State#state.active_journaldb, {NewSQN, PrimaryKey}, Bin1) of ok -> - {ok, State#state{journal_sqn=NewSQN}}; + {ok, State#state{journal_sqn=NewSQN}, ObjSize}; roll -> FileName = filepath(State#state.root_path, NewSQN, new_journal), CDBopts = State#state.cdb_options, @@ -270,9 +291,11 @@ put_object(PrimaryKey, Object, KeyChanges, State) -> {NewSQN, PrimaryKey}, Bin1) of ok -> - {rolling, State#state{journal_sqn=NewSQN, - active_journaldb=NewJournalP, - active_journaldb_sqn=NewSQN}}; + {rolling, + State#state{journal_sqn=NewSQN, + active_journaldb=NewJournalP, + active_journaldb_sqn=NewSQN}, + ObjSize}; roll -> {blocked, State#state{journal_sqn=NewSQN, active_journaldb=NewJournalP, @@ -464,16 +487,49 @@ roll_pending_journals([JournalSQN|T], Manifest, RootPath) -> RootPath). -fetch_key_changes(SQN, Manifest, ActiveJournal, ActiveSQN) -> - InitialChanges = case SQN of - SQN when SQN < ActiveSQN -> - fetch_key_changes(SQN, Manifest); - _ -> - [] - end, - RecentChanges = fetch_key_changes(SQN, ActiveJournal), - InitialChanges ++ RecentChanges. +%% Scan between sequence numbers applying FilterFun to each entry where +%% FilterFun{K, V, Acc} -> Penciller Key List +%% Load the output for the CDB file into the Penciller. +load_from_sequence(_MinSQN, _FilterFun, _Penciller, []) -> + ok; +load_from_sequence(MinSQN, FilterFun, Penciller, [{LowSQN, _FN, Pid}|ManTail]) + when MinSQN >= LowSQN -> + ok = load_between_sequence(MinSQN, + MinSQN + ?LOADING_BATCH, + FilterFun, + Penciller, + Pid, + undefined), + load_from_sequence(MinSQN, FilterFun, Penciller, ManTail); +load_from_sequence(MinSQN, FilterFun, Penciller, [_H|ManTail]) -> + load_from_sequence(MinSQN, FilterFun, Penciller, ManTail). + +load_between_sequence(MinSQN, MaxSQN, FilterFun, Penciller, CDBpid, StartPos) -> + InitAcc = {MinSQN, MaxSQN, []}, + case leveled_cdb:cdb_scan(CDBpid, FilterFun, InitAcc, StartPos) of + {eof, Acc} -> + ok = push_to_penciller(Penciller, Acc), + ok; + {LastPosition, Acc} -> + ok = push_to_penciller(Penciller, Acc), + load_between_sequence(MaxSQN + 1, + MaxSQN + 1 + ?LOADING_BATCH, + FilterFun, + Penciller, + CDBpid, + LastPosition) + end. + +push_to_penciller(Penciller, KeyList) -> + R = leveled_penciler:pcl_pushmem(Penciller, KeyList), + if + R == pause -> + timer:sleep(?LOADING_PAUSE); + true -> + ok + end. + sequencenumbers_fromfilenames(Filenames, Regex, IntName) -> lists:foldl(fun(FN, Acc) -> @@ -676,7 +732,7 @@ simplejournal_test() -> ?assertMatch(R1, {{1, "Key1"}, {"TestValue1", []}}), R2 = ink_get(Ink1, "Key1", 3), ?assertMatch(R2, {{3, "Key1"}, {"TestValue3", []}}), - {ok, NewSQN1} = ink_put(Ink1, "Key99", "TestValue99", []), + {ok, NewSQN1, _ObjSize} = ink_put(Ink1, "Key99", "TestValue99", []), ?assertMatch(NewSQN1, 5), R3 = ink_get(Ink1, "Key99", 5), io:format("Result 3 is ~w~n", [R3]), @@ -691,18 +747,18 @@ rollafile_simplejournal_test() -> {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, cdb_options=CDBopts}), FunnyLoop = lists:seq(1, 48), - {ok, NewSQN1} = ink_put(Ink1, "KeyAA", "TestValueAA", []), + {ok, NewSQN1, _ObjSize} = ink_put(Ink1, "KeyAA", "TestValueAA", []), ?assertMatch(NewSQN1, 5), ok = ink_print_manifest(Ink1), R0 = ink_get(Ink1, "KeyAA", 5), ?assertMatch(R0, {{5, "KeyAA"}, {"TestValueAA", []}}), lists:foreach(fun(X) -> - {ok, _} = ink_put(Ink1, - "KeyZ" ++ integer_to_list(X), - crypto:rand_bytes(10000), - []) end, + {ok, _, _} = ink_put(Ink1, + "KeyZ" ++ integer_to_list(X), + crypto:rand_bytes(10000), + []) end, FunnyLoop), - {ok, NewSQN2} = ink_put(Ink1, "KeyBB", "TestValueBB", []), + {ok, NewSQN2, _ObjSize} = ink_put(Ink1, "KeyBB", "TestValueBB", []), ?assertMatch(NewSQN2, 54), ok = ink_print_manifest(Ink1), R1 = ink_get(Ink1, "KeyAA", 5), diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 46fb8e9..09babae 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -168,7 +168,8 @@ pcl_confirmdelete/2, pcl_prompt/1, pcl_close/1, - pcl_getstartupsequencenumber/1]). + pcl_getstartupsequencenumber/1, + clean_testdir/1]). -include_lib("eunit/include/eunit.hrl"). @@ -835,7 +836,7 @@ assess_sqn(DumpList) -> assess_sqn([], MinSQN, MaxSQN) -> {MinSQN, MaxSQN}; assess_sqn([HeadKey|Tail], MinSQN, MaxSQN) -> - {_K, SQN} = leveled_sft:strip_to_key_seqn_only(HeadKey), + {_K, SQN} = leveled_bookie:strip_to_keyseqonly(HeadKey), assess_sqn(Tail, min(MinSQN, SQN), max(MaxSQN, SQN)). @@ -901,13 +902,13 @@ simple_server_test() -> clean_testdir(RootPath), {ok, PCL} = pcl_start(#penciller_options{root_path=RootPath, max_inmemory_tablesize=1000}), - Key1 = {{o,"Bucket0001", "Key0001"}, 1, {active, infinity}, null}, + Key1 = {{o,"Bucket0001", "Key0001"}, {1, {active, infinity}, null}}, KL1 = lists:sort(leveled_sft:generate_randomkeys({1000, 2})), - Key2 = {{o,"Bucket0002", "Key0002"}, 1002, {active, infinity}, null}, + Key2 = {{o,"Bucket0002", "Key0002"}, {1002, {active, infinity}, null}}, KL2 = lists:sort(leveled_sft:generate_randomkeys({1000, 1002})), - Key3 = {{o,"Bucket0003", "Key0003"}, 2002, {active, infinity}, null}, + Key3 = {{o,"Bucket0003", "Key0003"}, {2002, {active, infinity}, null}}, KL3 = lists:sort(leveled_sft:generate_randomkeys({1000, 2002})), - Key4 = {{o,"Bucket0004", "Key0004"}, 3002, {active, infinity}, null}, + Key4 = {{o,"Bucket0004", "Key0004"}, {3002, {active, infinity}, null}}, KL4 = lists:sort(leveled_sft:generate_randomkeys({1000, 3002})), ok = pcl_pushmem(PCL, [Key1]), R1 = pcl_fetch(PCL, {o,"Bucket0001", "Key0001"}), diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index d420829..d4b81fb 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -162,8 +162,6 @@ sft_getfilename/1, sft_setfordelete/2, sft_getmaxsequencenumber/1, - strip_to_keyonly/1, - strip_to_key_seqn_only/1, generate_randomkeys/1]). -include_lib("eunit/include/eunit.hrl"). @@ -203,7 +201,7 @@ filename = "not set" :: string(), handle :: file:fd(), background_complete = false :: boolean(), - background_failure = "Unknown" :: string(), + background_failure :: tuple(), oversized_file = false :: boolean(), ready_for_delete = false ::boolean(), penciller :: pid()}). @@ -458,7 +456,7 @@ create_file(FileName) when is_list(FileName) -> FileMD = #state{next_position=StartPos, filename=FileName}, {Handle, FileMD}; {error, Reason} -> - io:format("Error opening filename ~s with reason ~s", + io:format("Error opening filename ~s with reason ~w", [FileName, Reason]), {error, Reason} end. @@ -579,7 +577,7 @@ acc_list_keysonly(null, empty) -> acc_list_keysonly(null, RList) -> RList; acc_list_keysonly(R, RList) -> - lists:append(RList, [strip_to_key_seqn_only(R)]). + lists:append(RList, [leveled_bookie:strip_to_keyseqonly(R)]). acc_list_kv(null, empty) -> []; @@ -668,7 +666,7 @@ fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, FunList, scan_block([], StartKey, _EndKey, _FunList, _AccFun, Acc) -> {partial, Acc, StartKey}; scan_block([HeadKV|T], StartKey, EndKey, FunList, AccFun, Acc) -> - K = strip_to_keyonly(HeadKV), + K = leveled_bookie:strip_to_keyonly(HeadKV), case K of K when K < StartKey, StartKey /= all -> scan_block(T, StartKey, EndKey, FunList, AccFun, Acc); @@ -676,11 +674,11 @@ scan_block([HeadKV|T], StartKey, EndKey, FunList, AccFun, Acc) -> {complete, Acc}; _ -> case applyfuns(FunList, HeadKV) of - include -> + true -> %% Add result to the accumulator scan_block(T, StartKey, EndKey, FunList, AccFun, AccFun(HeadKV, Acc)); - skip -> + false -> scan_block(T, StartKey, EndKey, FunList, AccFun, Acc) end @@ -688,13 +686,13 @@ scan_block([HeadKV|T], StartKey, EndKey, FunList, AccFun, Acc) -> applyfuns([], _KV) -> - include; + true; applyfuns([HeadFun|OtherFuns], KV) -> case HeadFun(KV) of true -> applyfuns(OtherFuns, KV); false -> - skip + false end. fetch_keyvalue_fromblock([], _Key, _LengthList, _Handle, _StartOfSlot) -> @@ -1005,20 +1003,20 @@ create_slot(KL1, KL2, Level, BlockCount, SegLists, SerialisedSlot, LengthList, {BlockKeyList, Status, {LSNb, HSNb}, SegmentList, KL1b, KL2b} = create_block(KL1, KL2, Level), - case LowKey of + TrackingMetadata = case LowKey of null -> [NewLowKeyV|_] = BlockKeyList, - TrackingMetadata = {strip_to_keyonly(NewLowKeyV), - min(LSN, LSNb), max(HSN, HSNb), - strip_to_keyonly(last(BlockKeyList, - {last, LastKey})), - Status}; + {leveled_bookie:strip_to_keyonly(NewLowKeyV), + min(LSN, LSNb), max(HSN, HSNb), + leveled_bookie:strip_to_keyonly(last(BlockKeyList, + {last, LastKey})), + Status}; _ -> - TrackingMetadata = {LowKey, - min(LSN, LSNb), max(HSN, HSNb), - strip_to_keyonly(last(BlockKeyList, - {last, LastKey})), - Status} + {LowKey, + min(LSN, LSNb), max(HSN, HSNb), + leveled_bookie:strip_to_keyonly(last(BlockKeyList, + {last, LastKey})), + Status} end, SerialisedBlock = serialise_block(BlockKeyList), BlockLength = byte_size(SerialisedBlock), @@ -1034,11 +1032,6 @@ last([E|Es], PrevLast) -> last(E, Es, PrevLast). last(_, [E|Es], PrevLast) -> last(E, Es, PrevLast); last(E, [], _) -> E. -strip_to_keyonly({keyonly, K}) -> K; -strip_to_keyonly({K, _, _, _}) -> K. - -strip_to_key_seqn_only({K, SeqN, _, _}) -> {K, SeqN}. - serialise_block(BlockKeyList) -> term_to_binary(BlockKeyList, [{compressed, ?COMPRESSION_LEVEL}]). @@ -1060,7 +1053,7 @@ key_dominates(KL1, KL2, Level) -> Level). key_dominates_expanded([H1|T1], [], Level) -> - {_, _, St1, _} = H1, + St1 = leveled_bookie:strip_to_statusonly(H1), case maybe_reap_expiredkey(St1, Level) of true -> {skipped_key, maybe_expand_pointer(T1), []}; @@ -1068,7 +1061,7 @@ key_dominates_expanded([H1|T1], [], Level) -> {{next_key, H1}, maybe_expand_pointer(T1), []} end; key_dominates_expanded([], [H2|T2], Level) -> - {_, _, St2, _} = H2, + St2 = leveled_bookie:strip_to_statusonly(H2), case maybe_reap_expiredkey(St2, Level) of true -> {skipped_key, [], maybe_expand_pointer(T2)}; @@ -1076,8 +1069,8 @@ key_dominates_expanded([], [H2|T2], Level) -> {{next_key, H2}, [], maybe_expand_pointer(T2)} end; key_dominates_expanded([H1|T1], [H2|T2], Level) -> - {K1, Sq1, St1, _} = H1, - {K2, Sq2, St2, _} = H2, + {K1, Sq1, St1} = leveled_bookie:strip_to_details(H1), + {K2, Sq2, St2} = leveled_bookie:strip_to_details(H2), case K1 of K2 -> case Sq1 > Sq2 of @@ -1139,14 +1132,16 @@ pointer_append_queryresults(Results, QueryPid) -> end. -%% Update the sequence numbers -update_sequencenumbers({_, SN, _, _}, 0, 0) -> +%% Update the sequence numbers +update_sequencenumbers(Item, LSN, HSN) when is_tuple(Item) -> + update_sequencenumbers(leveled_bookie:strip_to_seqonly(Item), LSN, HSN); +update_sequencenumbers(SN, 0, 0) -> {SN, SN}; -update_sequencenumbers({_, SN, _, _}, LSN, HSN) when SN < LSN -> +update_sequencenumbers(SN, LSN, HSN) when SN < LSN -> {SN, HSN}; -update_sequencenumbers({_, SN, _, _}, LSN, HSN) when SN > HSN -> +update_sequencenumbers(SN, LSN, HSN) when SN > HSN -> {LSN, SN}; -update_sequencenumbers({_, _, _, _}, LSN, HSN) -> +update_sequencenumbers(_SN, LSN, HSN) -> {LSN, HSN}. @@ -1250,7 +1245,7 @@ merge_seglists({SegList1, SegList2, SegList3, SegList4}) -> lists:sort(Stage4). hash_for_segmentid(KV) -> - erlang:phash2(strip_to_keyonly(KV), ?MAX_SEG_HASH). + erlang:phash2(leveled_bookie:strip_to_keyonly(KV), ?MAX_SEG_HASH). %% Check for a given list of segments in the filter, returning in normal @@ -1396,30 +1391,6 @@ findremainder(BitStr, Factor) -> %%% Test %%%============================================================================ -speedtest_check_forsegment(_, 0, _, _) -> - true; -speedtest_check_forsegment(SegFilter, LoopCount, CRCCheck, IDsToCheck) -> - check_for_segments(SegFilter, gensegmentids(IDsToCheck), CRCCheck), - speedtest_check_forsegment(SegFilter, LoopCount - 1, CRCCheck, IDsToCheck). - -gensegmentids(Count) -> - gensegmentids([], Count). - -gensegmentids(GeneratedIDs, 0) -> - lists:sort(GeneratedIDs); -gensegmentids(GeneratedIDs, Count) -> - gensegmentids([random:uniform(1024*1024)|GeneratedIDs], Count - 1). - - -generate_randomsegfilter(BlockSize) -> - Block1 = gensegmentids(BlockSize), - Block2 = gensegmentids(BlockSize), - Block3 = gensegmentids(BlockSize), - Block4 = gensegmentids(BlockSize), - serialise_segment_filter(generate_segment_filter({Block1, - Block2, - Block3, - Block4})). generate_randomkeys({Count, StartSQN}) -> @@ -1433,8 +1404,8 @@ generate_randomkeys(Count, SQN, Acc) -> RandKey = {{o, lists:concat(["Bucket", random:uniform(1024)]), lists:concat(["Key", random:uniform(1024)])}, - SQN, - {active, infinity}, null}, + {SQN, + {active, infinity}, null}}, generate_randomkeys(Count - 1, SQN + 1, [RandKey|Acc]). generate_sequentialkeys(Count, Start) -> @@ -1447,76 +1418,71 @@ generate_sequentialkeys(Target, Incr, Acc) -> NextKey = {{o, "BucketSeq", lists:concat(["Key", KeyStr])}, - 5, - {active, infinity}, null}, + {5, + {active, infinity}, null}}, generate_sequentialkeys(Target, Incr + 1, [NextKey|Acc]). -dummy_test() -> - R = speedtest_check_forsegment(a, 0, b, c), - ?assertMatch(R, true), - _ = generate_randomsegfilter(8). - simple_create_block_test() -> - KeyList1 = [{{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}, - {{o, "Bucket1", "Key3"}, 2, {active, infinity}, null}], - KeyList2 = [{{o, "Bucket1", "Key2"}, 3, {active, infinity}, null}], + KeyList1 = [{{o, "Bucket1", "Key1"}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key3"}, {2, {active, infinity}, null}}], + KeyList2 = [{{o, "Bucket1", "Key2"}, {3, {active, infinity}, null}}], {MergedKeyList, ListStatus, SN, _, _, _} = create_block(KeyList1, KeyList2, 1), ?assertMatch(partial, ListStatus), [H1|T1] = MergedKeyList, - ?assertMatch(H1, {{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}), + ?assertMatch(H1, {{o, "Bucket1", "Key1"}, {1, {active, infinity}, null}}), [H2|T2] = T1, - ?assertMatch(H2, {{o, "Bucket1", "Key2"}, 3, {active, infinity}, null}), - ?assertMatch(T2, [{{o, "Bucket1", "Key3"}, 2, {active, infinity}, null}]), + ?assertMatch(H2, {{o, "Bucket1", "Key2"}, {3, {active, infinity}, null}}), + ?assertMatch(T2, [{{o, "Bucket1", "Key3"}, {2, {active, infinity}, null}}]), ?assertMatch(SN, {1,3}). dominate_create_block_test() -> - KeyList1 = [{{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}, - {{o, "Bucket1", "Key2"}, 2, {active, infinity}, null}], - KeyList2 = [{{o, "Bucket1", "Key2"}, 3, {tomb, infinity}, null}], + KeyList1 = [{{o, "Bucket1", "Key1"}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key2"}, {2, {active, infinity}, null}}], + KeyList2 = [{{o, "Bucket1", "Key2"}, {3, {tomb, infinity}, null}}], {MergedKeyList, ListStatus, SN, _, _, _} = create_block(KeyList1, KeyList2, 1), ?assertMatch(partial, ListStatus), [K1, K2] = MergedKeyList, - ?assertMatch(K1, {{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}), - ?assertMatch(K2, {{o, "Bucket1", "Key2"}, 3, {tomb, infinity}, null}), + ?assertMatch(K1, {{o, "Bucket1", "Key1"}, {1, {active, infinity}, null}}), + ?assertMatch(K2, {{o, "Bucket1", "Key2"}, {3, {tomb, infinity}, null}}), ?assertMatch(SN, {1,3}). sample_keylist() -> - KeyList1 = [{{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}, - {{o, "Bucket1", "Key3"}, 1, {active, infinity}, null}, - {{o, "Bucket1", "Key5"}, 1, {active, infinity}, null}, - {{o, "Bucket1", "Key7"}, 1, {active, infinity}, null}, - {{o, "Bucket1", "Key9"}, 1, {active, infinity}, null}, - {{o, "Bucket2", "Key1"}, 1, {active, infinity}, null}, - {{o, "Bucket2", "Key3"}, 1, {active, infinity}, null}, - {{o, "Bucket2", "Key5"}, 1, {active, infinity}, null}, - {{o, "Bucket2", "Key7"}, 1, {active, infinity}, null}, - {{o, "Bucket2", "Key9"}, 1, {active, infinity}, null}, - {{o, "Bucket3", "Key1"}, 1, {active, infinity}, null}, - {{o, "Bucket3", "Key3"}, 1, {active, infinity}, null}, - {{o, "Bucket3", "Key5"}, 1, {active, infinity}, null}, - {{o, "Bucket3", "Key7"}, 1, {active, infinity}, null}, - {{o, "Bucket3", "Key9"}, 1, {active, infinity}, null}, - {{o, "Bucket4", "Key1"}, 1, {active, infinity}, null}], - KeyList2 = [{{o, "Bucket1", "Key2"}, 1, {active, infinity}, null}, - {{o, "Bucket1", "Key4"}, 1, {active, infinity}, null}, - {{o, "Bucket1", "Key6"}, 1, {active, infinity}, null}, - {{o, "Bucket1", "Key8"}, 1, {active, infinity}, null}, - {{o, "Bucket1", "Key9a"}, 1, {active, infinity}, null}, - {{o, "Bucket1", "Key9b"}, 1, {active, infinity}, null}, - {{o, "Bucket1", "Key9c"}, 1, {active, infinity}, null}, - {{o, "Bucket1", "Key9d"}, 1, {active, infinity}, null}, - {{o, "Bucket2", "Key2"}, 1, {active, infinity}, null}, - {{o, "Bucket2", "Key4"}, 1, {active, infinity}, null}, - {{o, "Bucket2", "Key6"}, 1, {active, infinity}, null}, - {{o, "Bucket2", "Key8"}, 1, {active, infinity}, null}, - {{o, "Bucket3", "Key2"}, 1, {active, infinity}, null}, - {{o, "Bucket3", "Key4"}, 3, {active, infinity}, null}, - {{o, "Bucket3", "Key6"}, 2, {active, infinity}, null}, - {{o, "Bucket3", "Key8"}, 1, {active, infinity}, null}], + KeyList1 = [{{o, "Bucket1", "Key1"}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key3"}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key5"}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key7"}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key9"}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key1"}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key3"}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key5"}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key7"}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key9"}, {1, {active, infinity}, null}}, + {{o, "Bucket3", "Key1"}, {1, {active, infinity}, null}}, + {{o, "Bucket3", "Key3"}, {1, {active, infinity}, null}}, + {{o, "Bucket3", "Key5"}, {1, {active, infinity}, null}}, + {{o, "Bucket3", "Key7"}, {1, {active, infinity}, null}}, + {{o, "Bucket3", "Key9"}, {1, {active, infinity}, null}}, + {{o, "Bucket4", "Key1"}, {1, {active, infinity}, null}}], + KeyList2 = [{{o, "Bucket1", "Key2"}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key4"}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key6"}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key8"}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key9a"}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key9b"}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key9c"}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key9d"}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key2"}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key4"}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key6"}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key8"}, {1, {active, infinity}, null}}, + {{o, "Bucket3", "Key2"}, {1, {active, infinity}, null}}, + {{o, "Bucket3", "Key4"}, {3, {active, infinity}, null}}, + {{o, "Bucket3", "Key6"}, {2, {active, infinity}, null}}, + {{o, "Bucket3", "Key8"}, {1, {active, infinity}, null}}], {KeyList1, KeyList2}. alternating_create_block_test() -> @@ -1528,12 +1494,12 @@ alternating_create_block_test() -> ?assertMatch(BlockSize, 32), ?assertMatch(ListStatus, complete), K1 = lists:nth(1, MergedKeyList), - ?assertMatch(K1, {{o, "Bucket1", "Key1"}, 1, {active, infinity}, null}), + ?assertMatch(K1, {{o, "Bucket1", "Key1"}, {1, {active, infinity}, null}}), K11 = lists:nth(11, MergedKeyList), - ?assertMatch(K11, {{o, "Bucket1", "Key9b"}, 1, {active, infinity}, null}), + ?assertMatch(K11, {{o, "Bucket1", "Key9b"}, {1, {active, infinity}, null}}), K32 = lists:nth(32, MergedKeyList), - ?assertMatch(K32, {{o, "Bucket4", "Key1"}, 1, {active, infinity}, null}), - HKey = {{o, "Bucket1", "Key0"}, 1, {active, infinity}, null}, + ?assertMatch(K32, {{o, "Bucket4", "Key1"}, {1, {active, infinity}, null}}), + HKey = {{o, "Bucket1", "Key0"}, {1, {active, infinity}, null}}, {_, ListStatus2, _, _, _, _} = create_block([HKey|KeyList1], KeyList2, 1), ?assertMatch(ListStatus2, full). @@ -1690,7 +1656,7 @@ initial_create_file_test() -> Result1 = fetch_keyvalue(UpdHandle, UpdFileMD, {o, "Bucket1", "Key8"}), io:format("Result is ~w~n", [Result1]), ?assertMatch(Result1, {{o, "Bucket1", "Key8"}, - 1, {active, infinity}, null}), + {1, {active, infinity}, null}}), Result2 = fetch_keyvalue(UpdHandle, UpdFileMD, {o, "Bucket1", "Key88"}), io:format("Result is ~w~n", [Result2]), ?assertMatch(Result2, not_present), @@ -1705,18 +1671,18 @@ big_create_file_test() -> {Handle, FileMD, {_KL1Rem, _KL2Rem}} = complete_file(InitHandle, InitFileMD, KL1, KL2, 1), - [{K1, Sq1, St1, V1}|_] = KL1, - [{K2, Sq2, St2, V2}|_] = KL2, + [{K1, {Sq1, St1, V1}}|_] = KL1, + [{K2, {Sq2, St2, V2}}|_] = KL2, Result1 = fetch_keyvalue(Handle, FileMD, K1), Result2 = fetch_keyvalue(Handle, FileMD, K2), - ?assertMatch(Result1, {K1, Sq1, St1, V1}), - ?assertMatch(Result2, {K2, Sq2, St2, V2}), + ?assertMatch(Result1, {K1, {Sq1, St1, V1}}), + ?assertMatch(Result2, {K2, {Sq2, St2, V2}}), SubList = lists:sublist(KL2, 1000), FailedFinds = lists:foldl(fun(K, Acc) -> - {Kn, _, _, _} = K, + {Kn, {_, _, _}} = K, Rn = fetch_keyvalue(Handle, FileMD, Kn), case Rn of - {Kn, _, _, _} -> + {Kn, {_, _, _}} -> Acc; _ -> Acc + 1 From 5127119669785f2f79243072b35c87476610472a Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 15 Sep 2016 15:14:49 +0100 Subject: [PATCH 038/167] First draft of bookie Now with the most simple of tests --- src/leveled_bookie.erl | 93 +++++++++++++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 23 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 1b68d85..25326f2 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -135,18 +135,20 @@ terminate/2, code_change/3, book_start/1, - book_put/4, - book_get/2, - book_head/2, + book_riakput/3, + book_riakget/3, + book_riakhead/3, + book_close/1, strip_to_keyonly/1, strip_to_keyseqonly/1, strip_to_seqonly/1, - strip_to_statusonly/1, - strip_to_details/1]). + strip_to_statusonly/1]). -include_lib("eunit/include/eunit.hrl"). -define(CACHE_SIZE, 1000). +-define(JOURNAL_FP, "journal"). +-define(LEDGER_FP, "ledger"). -record(state, {inker :: pid(), penciller :: pid(), @@ -165,15 +167,21 @@ book_start(Opts) -> gen_server:start(?MODULE, [Opts], []). -book_put(Pid, PrimaryKey, Object, IndexSpecs) -> +book_riakput(Pid, Object, IndexSpecs) -> + PrimaryKey = {o, Object#r_object.bucket, Object#r_object.key}, gen_server:call(Pid, {put, PrimaryKey, Object, IndexSpecs}, infinity). -book_get(Pid, PrimaryKey) -> +book_riakget(Pid, Bucket, Key) -> + PrimaryKey = {o, Bucket, Key}, gen_server:call(Pid, {get, PrimaryKey}, infinity). -book_head(Pid, PrimaryKey) -> +book_riakhead(Pid, Bucket, Key) -> + PrimaryKey = {o, Bucket, Key}, gen_server:call(Pid, {head, PrimaryKey}, infinity). +book_close(Pid) -> + gen_server:call(Pid, close, infinity). + %%%============================================================================ %%% gen_server callbacks %%%============================================================================ @@ -208,7 +216,10 @@ init([Opts]) -> handle_call({put, PrimaryKey, Object, IndexSpecs}, From, State) -> - {ok, SQN, ObjSize} = leveled_inker:ink_put(PrimaryKey, Object, IndexSpecs), + {ok, SQN, ObjSize} = leveled_inker:ink_put(State#state.inker, + PrimaryKey, + Object, + IndexSpecs), Changes = preparefor_ledgercache(PrimaryKey, SQN, Object, @@ -229,7 +240,7 @@ handle_call({get, Key}, _From, State) -> not_present -> {reply, not_found, State}; Head -> - {Key, Seqn, Status} = strip_to_details(Head), + {Seqn, Status, _MD} = striphead_to_details(Head), case Status of {tomb, _} -> {reply, not_found, State}; @@ -247,15 +258,17 @@ handle_call({head, Key}, _From, State) -> not_present -> {reply, not_found, State}; Head -> - {Key, _Seqn, Status} = strip_to_details(Head), + {_Seqn, Status, MD} = striphead_to_details(Head), case Status of {tomb, _} -> {reply, not_found, State}; {active, _} -> - MD = strip_to_mdonly(Head), - {reply, {ok, MD}, State} + OMD = build_metadata_object(Key, MD), + {reply, {ok, OMD}, State} end - end. + end; +handle_call(close, _From, State) -> + {stop, normal, ok, State}. handle_cast(_Msg, State) -> {noreply, State}. @@ -263,8 +276,10 @@ handle_cast(_Msg, State) -> handle_info(_Info, State) -> {noreply, State}. -terminate(_Reason, _State) -> - ok. +terminate(Reason, State) -> + io:format("Bookie closing for reason ~w~n", [Reason]), + ok = leveled_inker:ink_close(State#state.inker), + ok = leveled_penciller:pcl_close(State#state.penciller). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -275,14 +290,18 @@ code_change(_OldVsn, State, _Extra) -> %%%============================================================================ set_options(Opts) -> - {#inker_options{root_path=Opts#bookie_options.root_path}, - #penciller_options{root_path=Opts#bookie_options.root_path}}. + %% TODO: Change the max size default, and allow setting through options + {#inker_options{root_path = Opts#bookie_options.root_path ++ + "/" ++ ?JOURNAL_FP, + cdb_options = #cdb_options{max_size=30000}}, + #penciller_options{root_path=Opts#bookie_options.root_path ++ + "/" ++ ?LEDGER_FP}}. startup(InkerOpts, PencillerOpts) -> {ok, Inker} = leveled_inker:ink_start(InkerOpts), {ok, Penciller} = leveled_penciller:pcl_start(PencillerOpts), LedgerSQN = leveled_penciller:pcl_getstartupsequencenumber(Penciller), - ok = leveled_inker:ink_loadpcl(LedgerSQN, fun load_fun/4, Penciller), + ok = leveled_inker:ink_loadpcl(Inker, LedgerSQN, fun load_fun/4, Penciller), {Inker, Penciller}. @@ -319,9 +338,7 @@ strip_to_statusonly({_, {_, St, _}}) -> St. strip_to_seqonly({_, {SeqN, _, _}}) -> SeqN. -strip_to_details({K, {SeqN, St, _}}) -> {K, SeqN, St}. - -strip_to_mdonly({_, {_, _, MD}}) -> MD. +striphead_to_details({SeqN, St, MD}) -> {SeqN, St, MD}. get_metadatas(#r_object{contents=Contents}) -> [Content#r_content.metadata || Content <- Contents]. @@ -341,6 +358,14 @@ hash(Obj=#r_object{}) -> extract_metadata(Obj, Size) -> {get_metadatas(Obj), vclock(Obj), hash(Obj), Size}. +build_metadata_object(PrimaryKey, Head) -> + {o, Bucket, Key} = PrimaryKey, + {MD, VC, _, _} = Head, + Contents = lists:foldl(fun(X, Acc) -> Acc ++ [#r_content{metadata=X}] end, + [], + MD), + #r_object{contents=Contents, bucket=Bucket, key=Key, vclock=VC}. + convert_indexspecs(IndexSpecs, SQN, PrimaryKey) -> lists:map(fun({IndexOp, IndexField, IndexValue}) -> Status = case IndexOp of @@ -367,7 +392,7 @@ preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs) -> [PrimaryChange] ++ SecChanges. addto_ledgercache(Changes, Cache) -> - lists:foldl(fun({{K, V}, Acc}) -> gb_trees:enter(K, V, Acc) end, + lists:foldl(fun({K, V}, Acc) -> gb_trees:enter(K, V, Acc) end, Cache, Changes). @@ -412,4 +437,26 @@ load_fun(KeyInLedger, ValueInLedger, _Position, Acc0) -> -ifdef(TEST). +reset_filestructure() -> + RootPath = "../test", + leveled_inker:clean_testdir(RootPath ++ "/" ++ ?JOURNAL_FP), + leveled_penciller:clean_testdir(RootPath ++ "/" ++ ?LEDGER_FP), + RootPath. + +single_key_test() -> + RootPath = reset_filestructure(), + {ok, Bookie} = book_start(#bookie_options{root_path=RootPath}), + {B1, K1, V1, Spec1, MD} = {"Bucket1", + "Key1", + "Value1", + [], + {"MDK1", "MDV1"}}, + Content = #r_content{metadata=MD, value=V1}, + Object = #r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]}, + ok = book_riakput(Bookie, Object, Spec1), + {ok, F1} = book_riakget(Bookie, B1, K1), + ?assertMatch(F1, Object), + ok = book_close(Bookie), + reset_filestructure(). + -endif. \ No newline at end of file From b452fbe27c97f93f42c349d8efe00f0ab7917b94 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 15 Sep 2016 18:38:23 +0100 Subject: [PATCH 039/167] End-to-end test Changes to ensure working of first end-to-end test (with a single Key and Value) --- src/leveled_bookie.erl | 26 +++++++++++++------ src/leveled_cdb.erl | 53 ++++++++++++++++++++++++++------------- src/leveled_inker.erl | 17 ++++++++----- src/leveled_penciller.erl | 11 +++++++- src/leveled_sft.erl | 5 ++-- 5 files changed, 77 insertions(+), 35 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 25326f2..e95aa80 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -142,7 +142,8 @@ strip_to_keyonly/1, strip_to_keyseqonly/1, strip_to_seqonly/1, - strip_to_statusonly/1]). + strip_to_statusonly/1, + striphead_to_details/1]). -include_lib("eunit/include/eunit.hrl"). @@ -301,7 +302,11 @@ startup(InkerOpts, PencillerOpts) -> {ok, Inker} = leveled_inker:ink_start(InkerOpts), {ok, Penciller} = leveled_penciller:pcl_start(PencillerOpts), LedgerSQN = leveled_penciller:pcl_getstartupsequencenumber(Penciller), - ok = leveled_inker:ink_loadpcl(Inker, LedgerSQN, fun load_fun/4, Penciller), + io:format("LedgerSQN=~w at startup~n", [LedgerSQN]), + ok = leveled_inker:ink_loadpcl(Inker, + LedgerSQN + 1, + fun load_fun/5, + Penciller), {Inker, Penciller}. @@ -413,10 +418,11 @@ maybepush_ledgercache(MaxCacheSize, Cache, Penciller) -> {ok, Cache} end. -load_fun(KeyInLedger, ValueInLedger, _Position, Acc0) -> +load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> {MinSQN, MaxSQN, Output} = Acc0, {SQN, PK} = KeyInLedger, - {Obj, IndexSpecs} = ValueInLedger, + io:format("Reloading changes with SQN=~w PK=~w~n", [SQN, PK]), + {Obj, IndexSpecs} = binary_to_term(ExtractFun(ValueInLedger)), case SQN of SQN when SQN < MinSQN -> {loop, Acc0}; @@ -445,7 +451,7 @@ reset_filestructure() -> single_key_test() -> RootPath = reset_filestructure(), - {ok, Bookie} = book_start(#bookie_options{root_path=RootPath}), + {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath}), {B1, K1, V1, Spec1, MD} = {"Bucket1", "Key1", "Value1", @@ -453,10 +459,14 @@ single_key_test() -> {"MDK1", "MDV1"}}, Content = #r_content{metadata=MD, value=V1}, Object = #r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]}, - ok = book_riakput(Bookie, Object, Spec1), - {ok, F1} = book_riakget(Bookie, B1, K1), + ok = book_riakput(Bookie1, Object, Spec1), + {ok, F1} = book_riakget(Bookie1, B1, K1), ?assertMatch(F1, Object), - ok = book_close(Bookie), + ok = book_close(Bookie1), + {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath}), + {ok, F2} = book_riakget(Bookie2, B1, K1), + ?assertMatch(F2, Object), + ok = book_close(Bookie2), reset_filestructure(). -endif. \ No newline at end of file diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index cba127d..24ffc71 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -253,13 +253,17 @@ handle_call({cdb_scan, FilterFun, Acc, StartPos}, _From, State) -> StartPos -> {ok, StartPos} end, - ok = check_last_key(State#state.last_key), - {LastPosition, Acc2} = scan_over_file(State#state.handle, - StartPos0, - FilterFun, - Acc, - State#state.last_key), - {reply, {LastPosition, Acc2}, State}; + case check_last_key(State#state.last_key) of + ok -> + {LastPosition, Acc2} = scan_over_file(State#state.handle, + StartPos0, + FilterFun, + Acc, + State#state.last_key), + {reply, {LastPosition, Acc2}, State}; + empty -> + {reply, {eof, Acc}, State} + end; handle_call(cdb_close, _From, State) -> ok = file:close(State#state.handle), {stop, normal, ok, State#state{handle=undefined}}; @@ -382,11 +386,11 @@ open_active_file(FileName) when is_list(FileName) -> case file:position(Handle, eof) of {ok, LastPosition} -> ok = file:close(Handle); - {ok, _} -> - LogDetails = [LastPosition, file:position(Handle, eof)], + {ok, EndPosition} -> + LogDetails = [LastPosition, EndPosition], io:format("File to be truncated at last position of ~w " "with end of file at ~w~n", LogDetails), - {ok, LastPosition} = file:position(Handle, LastPosition), + {ok, _LastPosition} = file:position(Handle, LastPosition), ok = file:truncate(Handle), ok = file:close(Handle) end, @@ -653,25 +657,33 @@ startup_scan_over_file(Handle, Position) -> HashTree = array:new(256, {default, gb_trees:empty()}), scan_over_file(Handle, Position, - fun startup_filter/4, + fun startup_filter/5, {HashTree, empty}, - undefined). + empty). %% Scan for key changes - scan over file returning applying FilterFun %% The FilterFun should accept as input: -%% - Key, Value, Position, Accumulator, outputting a new Accumulator -%% and a loop|stop instruction as a tuple i.e. {loop, Acc} or {stop, Acc} +%% - Key, ValueBin, Position, Accumulator, Fun (to extract values from Binary) +%% -> outputting a new Accumulator and a loop|stop instruction as a tuple +%% i.e. {loop, Acc} or {stop, Acc} scan_over_file(Handle, Position, FilterFun, Output, LastKey) -> case saferead_keyvalue(Handle) of false -> + io:format("Failure to read Key/Value at Position ~w" + ++ " in scan~n", [Position]), {Position, Output}; {Key, ValueAsBin, KeyLength, ValueLength} -> NewPosition = Position + KeyLength + ValueLength + ?DWORD_SIZE, - case {FilterFun(Key, ValueAsBin, Position, Output), Key} of + case {FilterFun(Key, + ValueAsBin, + Position, + Output, + fun extract_value/1), + Key} of {{stop, UpdOutput}, _} -> - {Position, UpdOutput}; + {NewPosition, UpdOutput}; {{loop, UpdOutput}, LastKey} -> {eof, UpdOutput}; {{loop, UpdOutput}, _} -> @@ -687,7 +699,7 @@ scan_over_file(Handle, Position, FilterFun, Output, LastKey) -> %% cdb file, and returns at the end the hashtree and the final Key seen in the %% journal -startup_filter(Key, ValueAsBin, Position, {Hashtree, LastKey}) -> +startup_filter(Key, ValueAsBin, Position, {Hashtree, LastKey}, _ExtractFun) -> case crccheck_value(ValueAsBin) of true -> {loop, {put_hashtree(Key, Position, Hashtree), Key}}; @@ -700,7 +712,7 @@ startup_filter(Key, ValueAsBin, Position, {Hashtree, LastKey}) -> check_last_key(LastKey) -> case LastKey of undefined -> error; - empty -> error; + empty -> empty; _ -> ok end. @@ -807,6 +819,11 @@ read_next_term(Handle, Length, crc, Check) -> {unchecked, binary_to_term(Bin)} end. +%% Extract value from binary containing CRC +extract_value(ValueAsBin) -> + <<_CRC:32/integer, Bin/binary>> = ValueAsBin, + binary_to_term(Bin). + %% Used for reading lengths %% Note that the endian_flip is required to make the file format compatible diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index e5302c0..0c0529c 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -276,6 +276,10 @@ code_change(_OldVsn, State, _Extra) -> put_object(PrimaryKey, Object, KeyChanges, State) -> NewSQN = State#state.journal_sqn + 1, + %% TODO: The term goes through a double binary_to_term conversion + %% as the CDB will also do the same conversion + %% Perhaps have CDB started up in apure binary mode, when it doesn't + %5 receive terms? Bin1 = term_to_binary({Object, KeyChanges}, [compressed]), ObjSize = byte_size(Bin1), case leveled_cdb:cdb_put(State#state.active_journaldb, @@ -493,8 +497,9 @@ roll_pending_journals([JournalSQN|T], Manifest, RootPath) -> load_from_sequence(_MinSQN, _FilterFun, _Penciller, []) -> ok; -load_from_sequence(MinSQN, FilterFun, Penciller, [{LowSQN, _FN, Pid}|ManTail]) +load_from_sequence(MinSQN, FilterFun, Penciller, [{LowSQN, FN, Pid}|ManTail]) when MinSQN >= LowSQN -> + io:format("Loading from filename ~s from SQN ~w~n", [FN, MinSQN]), ok = load_between_sequence(MinSQN, MinSQN + ?LOADING_BATCH, FilterFun, @@ -508,11 +513,11 @@ load_from_sequence(MinSQN, FilterFun, Penciller, [_H|ManTail]) -> load_between_sequence(MinSQN, MaxSQN, FilterFun, Penciller, CDBpid, StartPos) -> InitAcc = {MinSQN, MaxSQN, []}, case leveled_cdb:cdb_scan(CDBpid, FilterFun, InitAcc, StartPos) of - {eof, Acc} -> - ok = push_to_penciller(Penciller, Acc), + {eof, {_AccMinSQN, _AccMaxSQN, AccKL}} -> + ok = push_to_penciller(Penciller, AccKL), ok; - {LastPosition, Acc} -> - ok = push_to_penciller(Penciller, Acc), + {LastPosition, {_AccMinSQN, _AccMaxSQN, AccKL}} -> + ok = push_to_penciller(Penciller, AccKL), load_between_sequence(MaxSQN + 1, MaxSQN + 1 + ?LOADING_BATCH, FilterFun, @@ -522,7 +527,7 @@ load_between_sequence(MinSQN, MaxSQN, FilterFun, Penciller, CDBpid, StartPos) -> end. push_to_penciller(Penciller, KeyList) -> - R = leveled_penciler:pcl_pushmem(Penciller, KeyList), + R = leveled_penciller:pcl_pushmem(Penciller, KeyList), if R == pause -> timer:sleep(?LOADING_PAUSE); diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 09babae..777baba 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -161,6 +161,7 @@ terminate/2, code_change/3, pcl_start/1, + pcl_quickstart/1, pcl_pushmem/2, pcl_fetch/2, pcl_workforclerk/1, @@ -205,6 +206,9 @@ %%%============================================================================ %%% API %%%============================================================================ + +pcl_quickstart(RootPath) -> + pcl_start(#penciller_options{root_path=RootPath}). pcl_start(PCLopts) -> gen_server:start(?MODULE, [PCLopts], []). @@ -349,7 +353,10 @@ handle_call({push_mem, DumpList}, _From, State) -> ++ "having sequence numbers between ~w and ~w " ++ "but current sequence number is ~w~n", [MinSQN, MaxSQN, State#state.ledger_sqn]), - {reply, refused, State} + {reply, refused, State}; + empty -> + io:format("Empty request pushed to Penciller~n"), + {reply, ok, State} end, io:format("Push completed in ~w microseconds~n", [timer:now_diff(os:timestamp(),StartWatch)]), @@ -830,6 +837,8 @@ confirm_delete(Filename, UnreferencedFiles, RegisteredIterators) -> +assess_sqn([]) -> + empty; assess_sqn(DumpList) -> assess_sqn(DumpList, infinity, 0). diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index d4b81fb..793edc5 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -1069,8 +1069,9 @@ key_dominates_expanded([], [H2|T2], Level) -> {{next_key, H2}, [], maybe_expand_pointer(T2)} end; key_dominates_expanded([H1|T1], [H2|T2], Level) -> - {K1, Sq1, St1} = leveled_bookie:strip_to_details(H1), - {K2, Sq2, St2} = leveled_bookie:strip_to_details(H2), + {{K1, V1}, {K2, V2}} = {H1, H2}, + {Sq1, St1, _MD1} = leveled_bookie:striphead_to_details(V1), + {Sq2, St2, _MD2} = leveled_bookie:striphead_to_details(V2), case K1 of K2 -> case Sq1 > Sq2 of From 7c28ffbd96b0854f4bb6e23587a70e5a89a27519 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 19 Sep 2016 15:31:26 +0100 Subject: [PATCH 040/167] Further bookie test - CDB optimisation and Inker manifest correction Additional bookie test revealed that the persisting/reading of inker manifests was inconsistent and buggy. Also, the CDB files were inffeciently writing the top index table - needed to be improved as this is blokicng on a roll --- src/leveled_bookie.erl | 75 ++++++++++++++++++++++ src/leveled_cdb.erl | 137 +++++++++++++++++++++++++++++------------ src/leveled_inker.erl | 36 +++++++---- 3 files changed, 198 insertions(+), 50 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index e95aa80..0658767 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -448,7 +448,25 @@ reset_filestructure() -> leveled_inker:clean_testdir(RootPath ++ "/" ++ ?JOURNAL_FP), leveled_penciller:clean_testdir(RootPath ++ "/" ++ ?LEDGER_FP), RootPath. + +generate_multiple_objects(Count, KeyNumber) -> + generate_multiple_objects(Count, KeyNumber, []). +generate_multiple_objects(0, _KeyNumber, ObjL) -> + ObjL; +generate_multiple_objects(Count, KeyNumber, ObjL) -> + Obj = {"Bucket", + "Key" ++ integer_to_list(KeyNumber), + crypto:rand_bytes(128), + [], + [{"MDK", "MDV" ++ integer_to_list(KeyNumber)}, + {"MDK2", "MDV" ++ integer_to_list(KeyNumber)}]}, + {B1, K1, V1, Spec1, MD} = Obj, + Content = #r_content{metadata=MD, value=V1}, + Obj1 = #r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]}, + generate_multiple_objects(Count - 1, KeyNumber + 1, ObjL ++ [{Obj1, Spec1}]). + + single_key_test() -> RootPath = reset_filestructure(), {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath}), @@ -469,4 +487,61 @@ single_key_test() -> ok = book_close(Bookie2), reset_filestructure(). +multi_key_test() -> + RootPath = reset_filestructure(), + {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath}), + {B1, K1, V1, Spec1, MD1} = {"Bucket", + "Key1", + "Value1", + [], + {"MDK1", "MDV1"}}, + C1 = #r_content{metadata=MD1, value=V1}, + Obj1 = #r_object{bucket=B1, key=K1, contents=[C1], vclock=[{'a',1}]}, + {B2, K2, V2, Spec2, MD2} = {"Bucket", + "Key2", + "Value2", + [], + {"MDK2", "MDV2"}}, + C2 = #r_content{metadata=MD2, value=V2}, + Obj2 = #r_object{bucket=B2, key=K2, contents=[C2], vclock=[{'a',1}]}, + ok = book_riakput(Bookie1, Obj1, Spec1), + ObjL1 = generate_multiple_objects(100, 3), + SW1 = os:timestamp(), + lists:foreach(fun({O, S}) -> ok = book_riakput(Bookie1, O, S) end, ObjL1), + io:format("PUT of 100 objects completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(),SW1)]), + ok = book_riakput(Bookie1, Obj2, Spec2), + {ok, F1A} = book_riakget(Bookie1, B1, K1), + ?assertMatch(F1A, Obj1), + {ok, F2A} = book_riakget(Bookie1, B2, K2), + ?assertMatch(F2A, Obj2), + ObjL2 = generate_multiple_objects(100, 103), + SW2 = os:timestamp(), + lists:foreach(fun({O, S}) -> ok = book_riakput(Bookie1, O, S) end, ObjL2), + io:format("PUT of 100 objects completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(),SW2)]), + {ok, F1B} = book_riakget(Bookie1, B1, K1), + ?assertMatch(F1B, Obj1), + {ok, F2B} = book_riakget(Bookie1, B2, K2), + ?assertMatch(F2B, Obj2), + ok = book_close(Bookie1), + %% Now reopen the file, and confirm that a fetch is still possible + {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath}), + {ok, F1C} = book_riakget(Bookie2, B1, K1), + ?assertMatch(F1C, Obj1), + {ok, F2C} = book_riakget(Bookie2, B2, K2), + ?assertMatch(F2C, Obj2), + ObjL3 = generate_multiple_objects(100, 203), + SW3 = os:timestamp(), + lists:foreach(fun({O, S}) -> ok = book_riakput(Bookie2, O, S) end, ObjL3), + io:format("PUT of 100 objects completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(),SW3)]), + {ok, F1D} = book_riakget(Bookie2, B1, K1), + ?assertMatch(F1D, Obj1), + {ok, F2D} = book_riakget(Bookie2, B2, K2), + ?assertMatch(F2D, Obj2), + ok = book_close(Bookie2), + reset_filestructure(), + ?assertMatch(true, false). + -endif. \ No newline at end of file diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 24ffc71..ec807ea 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -177,7 +177,9 @@ handle_call({cdb_open_reader, Filename}, _From, State) -> io:format("Opening file for reading with filename ~s~n", [Filename]), {ok, Handle} = file:open(Filename, [binary, raw, read]), Index = load_index(Handle), + LastKey = find_lastkey(Handle, Index), {reply, ok, State#state{handle=Handle, + last_key=LastKey, filename=Filename, writer=false, hash_index=Index}}; @@ -464,8 +466,8 @@ get_index(Handle, Index, no_cache) -> % Get location of hashtable and number of entries in the hash read_next_2_integers(Handle); get_index(_Handle, Index, Cache) -> - {_Pointer, Count} = lists:keyfind(Index, 1, Cache), - Count. + {Index, {Pointer, Count}} = lists:keyfind(Index, 1, Cache), + {Pointer, Count}. %% Get a Key/Value pair from an active CDB file (with no hash table written) %% This requires a key dictionary to be passed in (mapping keys to positions) @@ -593,6 +595,31 @@ load_index(Handle) -> {X, {HashTablePos, Count}} end, Index). +%% Function to find the LastKey in the file +find_lastkey(Handle, IndexCache) -> + LastPosition = scan_index(Handle, IndexCache), + {ok, _} = file:position(Handle, LastPosition), + {KeyLength, _ValueLength} = read_next_2_integers(Handle), + read_next_term(Handle, KeyLength). + +scan_index(Handle, IndexCache) -> + lists:foldl(fun({_X, {Pos, Count}}, LastPosition) -> + scan_index(Handle, Pos, 0, Count, LastPosition) end, + 0, + IndexCache). + +scan_index(_Handle, _Position, Count, Checks, LastPosition) + when Count == Checks -> + LastPosition; +scan_index(Handle, Position, Count, Checks, LastPosition) -> + {ok, _} = file:position(Handle, Position + ?DWORD_SIZE * Count), + {_Hash, HPosition} = read_next_2_integers(Handle), + scan_index(Handle, + Position, + Count + 1 , + Checks, + max(LastPosition, HPosition)). + %% Take an active file and write the hash details necessary to close that %% file and roll a new active file if requested. @@ -601,8 +628,13 @@ load_index(Handle) -> %% the hash tables close_file(Handle, HashTree, BasePos) -> {ok, BasePos} = file:position(Handle, BasePos), + SW = os:timestamp(), L2 = write_hash_tables(Handle, HashTree), + io:format("Hash Table write took ~w microseconds~n", + [timer:now_diff(os:timestamp(),SW)]), write_top_index_table(Handle, BasePos, L2), + io:format("Top Index Table write took ~w microseconds~n", + [timer:now_diff(os:timestamp(),SW)]), file:close(Handle). @@ -661,6 +693,19 @@ startup_scan_over_file(Handle, Position) -> {HashTree, empty}, empty). +%% Specific filter to be used at startup to build a hashtree for an incomplete +%% cdb file, and returns at the end the hashtree and the final Key seen in the +%% journal + +startup_filter(Key, ValueAsBin, Position, {Hashtree, LastKey}, _ExtractFun) -> + case crccheck_value(ValueAsBin) of + true -> + {loop, {put_hashtree(Key, Position, Hashtree), Key}}; + false -> + {stop, {Hashtree, LastKey}} + end. + + %% Scan for key changes - scan over file returning applying FilterFun %% The FilterFun should accept as input: %% - Key, ValueBin, Position, Accumulator, Fun (to extract values from Binary) @@ -674,39 +719,34 @@ scan_over_file(Handle, Position, FilterFun, Output, LastKey) -> ++ " in scan~n", [Position]), {Position, Output}; {Key, ValueAsBin, KeyLength, ValueLength} -> - NewPosition = Position + KeyLength + ValueLength - + ?DWORD_SIZE, - case {FilterFun(Key, + NewPosition = case Key of + LastKey -> + eof; + _ -> + Position + KeyLength + ValueLength + + ?DWORD_SIZE + end, + case FilterFun(Key, ValueAsBin, Position, Output, - fun extract_value/1), - Key} of - {{stop, UpdOutput}, _} -> + fun extract_value/1) of + {stop, UpdOutput} -> {NewPosition, UpdOutput}; - {{loop, UpdOutput}, LastKey} -> - {eof, UpdOutput}; - {{loop, UpdOutput}, _} -> - scan_over_file(Handle, - NewPosition, - FilterFun, - UpdOutput, - Key) + {loop, UpdOutput} -> + case NewPosition of + eof -> + {eof, UpdOutput}; + _ -> + scan_over_file(Handle, + NewPosition, + FilterFun, + UpdOutput, + LastKey) + end end end. -%% Specific filter to be used at startup to build a hashtree for an incomplete -%% cdb file, and returns at the end the hashtree and the final Key seen in the -%% journal - -startup_filter(Key, ValueAsBin, Position, {Hashtree, LastKey}, _ExtractFun) -> - case crccheck_value(ValueAsBin) of - true -> - {loop, {put_hashtree(Key, Position, Hashtree), Key}}; - false -> - {stop, {Hashtree, LastKey}} - end. - %% Confirm that the last key has been defined and set to a non-default value check_last_key(LastKey) -> @@ -972,17 +1012,16 @@ find_open_slot1([_|RestOfSlots], [_|RestOfEntries]) -> write_top_index_table(Handle, BasePos, List) -> % fold function to find any missing index tuples, and add one a replacement % in this case with a count of 0. Also orders the list by index - FnMakeIndex = fun(I, Acc) -> + FnMakeIndex = fun(I) -> case lists:keysearch(I, 1, List) of {value, Tuple} -> - [Tuple|Acc]; + Tuple; false -> - [{I, BasePos, 0}|Acc] + {I, BasePos, 0} end end, % Fold function to write the index entries - FnWriteIndex = fun({Index, Pos, Count}, CurrPos) -> - {ok, _} = file:position(Handle, ?DWORD_SIZE * Index), + FnWriteIndex = fun({_Index, Pos, Count}, {AccBin, CurrPos}) -> case Count == 0 of true -> PosLE = endian_flip(CurrPos), @@ -992,14 +1031,16 @@ write_top_index_table(Handle, BasePos, List) -> NextPos = Pos + (Count * ?DWORD_SIZE) end, CountLE = endian_flip(Count), - Bin = <>, - file:write(Handle, Bin), - NextPos + {<>, NextPos} end, Seq = lists:seq(0, 255), - CompleteList = lists:keysort(1, lists:foldl(FnMakeIndex, [], Seq)), - lists:foldl(FnWriteIndex, BasePos, CompleteList), + CompleteList = lists:keysort(1, lists:map(FnMakeIndex, Seq)), + {IndexBin, _Pos} = lists:foldl(FnWriteIndex, + {<<>>, BasePos}, + CompleteList), + {ok, _} = file:position(Handle, 0), + ok = file:write(Handle, IndexBin), ok = file:advise(Handle, 0, ?DWORD_SIZE * 256, will_need). %% To make this compatible with original Bernstein format this endian flip @@ -1131,7 +1172,7 @@ full_1_test() -> full_2_test() -> List1 = lists:sort([{lists:flatten(io_lib:format("~s~p",[Prefix,Plug])), lists:flatten(io_lib:format("value~p",[Plug]))} - || Plug <- lists:seq(1,2000), + || Plug <- lists:seq(1,200), Prefix <- ["dsd","so39ds","oe9%#*(","020dkslsldclsldowlslf%$#", "tiep4||","qweq"]]), create("../test/full.cdb",List1), @@ -1394,4 +1435,22 @@ fold2_test() -> ?assertMatch(RD2, Result), ok = file:delete("../test/fold2_test.cdb"). +find_lastkey_test() -> + {ok, P1} = cdb_open_writer("../test/lastkey.pnd"), + ok = cdb_put(P1, "Key1", "Value1"), + ok = cdb_put(P1, "Key3", "Value3"), + ok = cdb_put(P1, "Key2", "Value2"), + R1 = cdb_lastkey(P1), + ?assertMatch(R1, "Key2"), + ok = cdb_close(P1), + {ok, P2} = cdb_open_writer("../test/lastkey.pnd"), + R2 = cdb_lastkey(P2), + ?assertMatch(R2, "Key2"), + {ok, F2} = cdb_complete(P2), + {ok, P3} = cdb_open_reader(F2), + R3 = cdb_lastkey(P3), + ?assertMatch(R3, "Key2"), + ok = cdb_close(P3), + ok = file:delete("../test/lastkey.cdb"). + -endif. diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 0c0529c..b816e5f 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -308,16 +308,25 @@ put_object(PrimaryKey, Object, KeyChanges, State) -> end. roll_active_file(OldActiveJournal, Manifest, ManifestSQN, RootPath) -> + SW = os:timestamp(), io:format("Rolling old journal ~w~n", [OldActiveJournal]), {ok, NewFilename} = leveled_cdb:cdb_complete(OldActiveJournal), + io:format("Rolling old journal S1 completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(),SW)]), {ok, PidR} = leveled_cdb:cdb_open_reader(NewFilename), + io:format("Rolling old journal S2 completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(),SW)]), JournalRegex2 = "nursery_(?[0-9]+)\\." ++ ?JOURNAL_FILEX, [JournalSQN] = sequencenumbers_fromfilenames([NewFilename], JournalRegex2, 'SQN'), NewManifest = add_to_manifest(Manifest, {JournalSQN, NewFilename, PidR}), + io:format("Rolling old journal S3 completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(),SW)]), NewManifestSQN = ManifestSQN + 1, ok = simple_manifest_writer(NewManifest, NewManifestSQN, RootPath), + io:format("Rolling old journal S4 completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(),SW)]), {NewManifest, NewManifestSQN}. get_object(PrimaryKey, SQN, Manifest, ActiveJournal, ActiveJournalSQN) -> @@ -358,8 +367,6 @@ build_manifest(ManifestFilenames, ManifestRdrFun, RootPath, CDBopts) -> - %% Setup root paths - JournalFP = filepath(RootPath, journal_dir), %% Find the manifest with a highest Manifest sequence number %% Open it and read it to get the current Confirmed Manifest ManifestRegex = "(?[0-9]+)\\." ++ ?MANIFEST_FILEX, @@ -374,9 +381,12 @@ build_manifest(ManifestFilenames, {0, [], [], 0}; _ -> PersistedManSQN = lists:max(ValidManSQNs), - {J1, M1, R1} = ManifestRdrFun(PersistedManSQN, - RootPath), - {J1, M1, R1, PersistedManSQN} + M1 = ManifestRdrFun(PersistedManSQN, RootPath), + J1 = lists:foldl(fun({JSQN, _FN}, Acc) -> + max(JSQN, Acc) end, + 0, + M1), + {J1, M1, [], PersistedManSQN} end, %% Find any more recent immutable files that have a higher sequence number @@ -415,8 +425,7 @@ build_manifest(ManifestFilenames, io:format("Manifest on startup is: ~n"), manifest_printer(Manifest1), Manifest2 = lists:map(fun({LowSQN, FN}) -> - FP = filename:join(JournalFP, FN), - {ok, Pid} = leveled_cdb:cdb_open_reader(FP), + {ok, Pid} = leveled_cdb:cdb_open_reader(FN), {LowSQN, FN, Pid} end, Manifest1), @@ -498,7 +507,7 @@ roll_pending_journals([JournalSQN|T], Manifest, RootPath) -> load_from_sequence(_MinSQN, _FilterFun, _Penciller, []) -> ok; load_from_sequence(MinSQN, FilterFun, Penciller, [{LowSQN, FN, Pid}|ManTail]) - when MinSQN >= LowSQN -> + when LowSQN >= MinSQN -> io:format("Loading from filename ~s from SQN ~w~n", [FN, MinSQN]), ok = load_between_sequence(MinSQN, MinSQN + ?LOADING_BATCH, @@ -576,6 +585,8 @@ filepath(RootPath, NewSQN, new_journal) -> simple_manifest_reader(SQN, RootPath) -> ManifestPath = filepath(RootPath, manifest_dir), + io:format("Opening manifest file at ~s with SQN ~w~n", + [ManifestPath, SQN]), {ok, MBin} = file:read_file(filename:join(ManifestPath, integer_to_list(SQN) ++ ".man")), @@ -584,9 +595,12 @@ simple_manifest_reader(SQN, RootPath) -> simple_manifest_writer(Manifest, ManSQN, RootPath) -> ManPath = filepath(RootPath, manifest_dir), - NewFN = filename:join(ManPath, integer_to_list(ManSQN) ++ ?MANIFEST_FILEX), - TmpFN = filename:join(ManPath, integer_to_list(ManSQN) ++ ?PENDING_FILEX), - MBin = term_to_binary(Manifest), + NewFN = filename:join(ManPath, + integer_to_list(ManSQN) ++ "." ++ ?MANIFEST_FILEX), + TmpFN = filename:join(ManPath, + integer_to_list(ManSQN) ++ "." ++ ?PENDING_FILEX), + MBin = term_to_binary(lists:map(fun({SQN, FN, _PID}) -> {SQN, FN} end, + Manifest)), case filelib:is_file(NewFN) of true -> io:format("Error - trying to write manifest for" From a1c970a66acd23e283a6bdf443074869f5b46ac6 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 19 Sep 2016 15:56:35 +0100 Subject: [PATCH 041/167] Manifest ordering Be more explicit about manifest ordering to stop keys being laoded in incorrect order --- src/leveled_bookie.erl | 5 ++--- src/leveled_inker.erl | 19 +++++++------------ 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 0658767..0ae31e8 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -457,7 +457,7 @@ generate_multiple_objects(0, _KeyNumber, ObjL) -> generate_multiple_objects(Count, KeyNumber, ObjL) -> Obj = {"Bucket", "Key" ++ integer_to_list(KeyNumber), - crypto:rand_bytes(128), + crypto:rand_bytes(1024), [], [{"MDK", "MDV" ++ integer_to_list(KeyNumber)}, {"MDK2", "MDV" ++ integer_to_list(KeyNumber)}]}, @@ -541,7 +541,6 @@ multi_key_test() -> {ok, F2D} = book_riakget(Bookie2, B2, K2), ?assertMatch(F2D, Obj2), ok = book_close(Bookie2), - reset_filestructure(), - ?assertMatch(true, false). + reset_filestructure(). -endif. \ No newline at end of file diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index b816e5f..9b0c7c4 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -189,7 +189,7 @@ init([InkerOpts]) -> fun simple_manifest_reader/2, RootPath, CDBopts), - {ok, #state{manifest = Manifest, + {ok, #state{manifest = lists:reverse(lists:keysort(1, Manifest)), manifest_sqn = ManifestSQN, journal_sqn = JournalSQN, active_journaldb = ActiveJournal, @@ -224,7 +224,7 @@ handle_call({fetch, Key, SQN}, _From, State) -> {reply, {ok, Value}, State}; Other -> io:format("Unexpected failure to fetch value for" ++ - "Key=~s SQN=~w with reason ~w", [Key, SQN, Other]), + "Key=~w SQN=~w with reason ~w", [Key, SQN, Other]), {reply, not_present, State} end; handle_call({get, Key, SQN}, _From, State) -> @@ -234,9 +234,10 @@ handle_call({get, Key, SQN}, _From, State) -> State#state.active_journaldb, State#state.active_journaldb_sqn), State}; handle_call({load_pcl, StartSQN, FilterFun, Penciller}, _From, State) -> - Manifest = State#state.manifest ++ [{State#state.active_journaldb_sqn, - dummy, - State#state.active_journaldb}], + Manifest = lists:reverse(State#state.manifest) + ++ [{State#state.active_journaldb_sqn, + dummy, + State#state.active_journaldb}], Reply = load_from_sequence(StartSQN, FilterFun, Penciller, Manifest), {reply, Reply, State}; handle_call(snapshot, _From , State) -> @@ -311,21 +312,15 @@ roll_active_file(OldActiveJournal, Manifest, ManifestSQN, RootPath) -> SW = os:timestamp(), io:format("Rolling old journal ~w~n", [OldActiveJournal]), {ok, NewFilename} = leveled_cdb:cdb_complete(OldActiveJournal), - io:format("Rolling old journal S1 completed in ~w microseconds~n", - [timer:now_diff(os:timestamp(),SW)]), {ok, PidR} = leveled_cdb:cdb_open_reader(NewFilename), - io:format("Rolling old journal S2 completed in ~w microseconds~n", - [timer:now_diff(os:timestamp(),SW)]), JournalRegex2 = "nursery_(?[0-9]+)\\." ++ ?JOURNAL_FILEX, [JournalSQN] = sequencenumbers_fromfilenames([NewFilename], JournalRegex2, 'SQN'), NewManifest = add_to_manifest(Manifest, {JournalSQN, NewFilename, PidR}), - io:format("Rolling old journal S3 completed in ~w microseconds~n", - [timer:now_diff(os:timestamp(),SW)]), NewManifestSQN = ManifestSQN + 1, ok = simple_manifest_writer(NewManifest, NewManifestSQN, RootPath), - io:format("Rolling old journal S4 completed in ~w microseconds~n", + io:format("Rolling old journal completed in ~w microseconds~n", [timer:now_diff(os:timestamp(),SW)]), {NewManifest, NewManifestSQN}. From 4e28e4173cfe7fea25638b1d43112e6a9a6efc51 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 19 Sep 2016 18:50:11 +0100 Subject: [PATCH 042/167] Rebar and eunit changes Initial rebar compile - which exposed eunit tets failures associated with changes to file structures and filename references --- ebin/eleveleddb.app | 11 ++++++++++ src/eleveleddb.app.src | 12 +++++++++++ src/eleveleddb_app.erl | 16 +++++++++++++++ src/eleveleddb_sup.erl | 27 +++++++++++++++++++++++++ src/leveled_clerk.erl | 10 ++++----- src/leveled_inker.erl | 46 +++++++++++++++++++++++------------------- 6 files changed, 96 insertions(+), 26 deletions(-) create mode 100644 ebin/eleveleddb.app create mode 100644 src/eleveleddb.app.src create mode 100644 src/eleveleddb_app.erl create mode 100644 src/eleveleddb_sup.erl diff --git a/ebin/eleveleddb.app b/ebin/eleveleddb.app new file mode 100644 index 0000000..11d7f25 --- /dev/null +++ b/ebin/eleveleddb.app @@ -0,0 +1,11 @@ +{application,eleveleddb, + [{description,[]}, + {vsn,"1"}, + {registered,[]}, + {applications,[kernel,stdlib]}, + {mod,{eleveleddb_app,[]}}, + {env,[]}, + {modules,[eleveleddb_app,eleveleddb_sup,leveled_bookie, + leveled_cdb,leveled_clerk,leveled_inker, + leveled_iterator,leveled_penciller,leveled_rice, + leveled_sft]}]}. diff --git a/src/eleveleddb.app.src b/src/eleveleddb.app.src new file mode 100644 index 0000000..6d0069e --- /dev/null +++ b/src/eleveleddb.app.src @@ -0,0 +1,12 @@ +{application, eleveleddb, + [ + {description, ""}, + {vsn, "1"}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]}, + {mod, { eleveleddb_app, []}}, + {env, []} + ]}. diff --git a/src/eleveleddb_app.erl b/src/eleveleddb_app.erl new file mode 100644 index 0000000..18f7546 --- /dev/null +++ b/src/eleveleddb_app.erl @@ -0,0 +1,16 @@ +-module(eleveleddb_app). + +-behaviour(application). + +%% Application callbacks +-export([start/2, stop/1]). + +%% =================================================================== +%% Application callbacks +%% =================================================================== + +start(_StartType, _StartArgs) -> + eleveleddb_sup:start_link(). + +stop(_State) -> + ok. diff --git a/src/eleveleddb_sup.erl b/src/eleveleddb_sup.erl new file mode 100644 index 0000000..37ef58b --- /dev/null +++ b/src/eleveleddb_sup.erl @@ -0,0 +1,27 @@ +-module(eleveleddb_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callbacks +-export([init/1]). + +%% Helper macro for declaring children of supervisor +-define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). + +%% =================================================================== +%% API functions +%% =================================================================== + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%% =================================================================== +%% Supervisor callbacks +%% =================================================================== + +init([]) -> + {ok, { {one_for_one, 5, 10}, []} }. + diff --git a/src/leveled_clerk.erl b/src/leveled_clerk.erl index f423b59..717455f 100644 --- a/src/leveled_clerk.erl +++ b/src/leveled_clerk.erl @@ -295,8 +295,8 @@ generate_randomkeys(Count, Acc, BucketLow, BRange) -> RandKey = {{o, "Bucket" ++ BNumber, "Key" ++ KNumber}, - Count + 1, - {active, infinity}, null}, + {Count + 1, + {active, infinity}, null}}, generate_randomkeys(Count - 1, [RandKey|Acc], BucketLow, BRange). choose_pid_toquery([ManEntry|_T], Key) when @@ -310,8 +310,8 @@ choose_pid_toquery([_H|T], Key) -> find_randomkeys(_FList, 0, _Source) -> ok; find_randomkeys(FList, Count, Source) -> - K1 = leveled_sft:strip_to_keyonly(lists:nth(random:uniform(length(Source)), - Source)), + KV1 = lists:nth(random:uniform(length(Source)), Source), + K1 = leveled_bookie:strip_to_keyonly(KV1), P1 = choose_pid_toquery(FList, K1), FoundKV = leveled_sft:sft_get(P1, K1), case FoundKV of @@ -319,7 +319,7 @@ find_randomkeys(FList, Count, Source) -> io:format("Failed to find ~w in ~w~n", [K1, P1]), ?assertMatch(true, false); _ -> - Found = leveled_sft:strip_to_keyonly(FoundKV), + Found = leveled_bookie:strip_to_keyonly(FoundKV), io:format("success finding ~w in ~w~n", [K1, P1]), ?assertMatch(K1, Found) end, diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 9b0c7c4..9d150f4 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -400,20 +400,22 @@ build_manifest(ManifestFilenames, OtherSQNs_imm = sequencenumbers_fromfilenames(UnremovedJournalFiles, JournalRegex1, 'SQN'), - Manifest1 = lists:foldl(fun(X, Acc) -> - if - X > JournalSQN1 - -> - FN = "nursery_" ++ - integer_to_list(X) - ++ "." ++ - ?JOURNAL_FILEX, - add_to_manifest(Acc, {X, FN}); - true - -> Acc - end end, - ConfirmedManifest, - lists:sort(OtherSQNs_imm)), + ExtendManifestFun = fun(X, Acc) -> + if + X > JournalSQN1 + -> + FN = filepath(RootPath, journal_dir) + ++ "nursery_" ++ + integer_to_list(X) + ++ "." ++ + ?JOURNAL_FILEX, + add_to_manifest(Acc, {X, FN}); + true + -> Acc + end end, + Manifest1 = lists:foldl(ExtendManifestFun, + ConfirmedManifest, + lists:sort(OtherSQNs_imm)), %% Enrich the manifest so it contains the Pid of any of the immutable %% entries @@ -648,7 +650,7 @@ build_dummy_journal() -> ok = leveled_cdb:cdb_put(J2, {3, K1}, term_to_binary({V3, []})), ok = leveled_cdb:cdb_put(J2, {4, K4}, term_to_binary({V4, []})), ok = leveled_cdb:cdb_close(J2), - Manifest = {2, [{1, "nursery_1.cdb"}], []}, + Manifest = [{1, "../test/journal/journal_files/nursery_1.cdb"}], ManifestBin = term_to_binary(Manifest), {ok, MF1} = file:open(filename:join(ManifestFP, "1.man"), [binary, raw, read, write]), @@ -669,14 +671,15 @@ simple_buildmanifest_test() -> RootPath = "../test/journal", build_dummy_journal(), Res = build_manifest(["1.man"], - ["nursery_1.cdb", "nursery_3.pnd"], + ["../test/journal/journal_files/nursery_1.cdb", + "../test/journal/journal_files/nursery_3.pnd"], fun simple_manifest_reader/2, RootPath), io:format("Build manifest output is ~w~n", [Res]), {Man, {ActJournal, ActJournalSQN}, HighSQN, ManSQN} = Res, ?assertMatch(HighSQN, 4), ?assertMatch(ManSQN, 1), - ?assertMatch([{1, "nursery_1.cdb", _}], Man), + ?assertMatch([{1, "../test/journal/journal_files/nursery_1.cdb", _}], Man), {ActSQN, _ActK} = leveled_cdb:cdb_lastkey(ActJournal), ?assertMatch(ActSQN, 4), ?assertMatch(ActJournalSQN, 3), @@ -699,16 +702,17 @@ another_buildmanifest_test() -> ok = leveled_cdb:cdb_close(NewActiveJN), %% Test setup - now build manifest Res = build_manifest(["1.man"], - ["nursery_1.cdb", - "nursery_3.cdb", - "nursery_5.pnd"], + ["../test/journal/journal_files/nursery_1.cdb", + "../test/journal/journal_files/nursery_3.cdb", + "../test/journal/journal_files/nursery_5.pnd"], fun simple_manifest_reader/2, RootPath), io:format("Build manifest output is ~w~n", [Res]), {Man, {ActJournal, ActJournalSQN}, HighSQN, ManSQN} = Res, ?assertMatch(HighSQN, 6), ?assertMatch(ManSQN, 1), - ?assertMatch([{3, "nursery_3.cdb", _}, {1, "nursery_1.cdb", _}], Man), + ?assertMatch([{3, "../test/journal/journal_files/nursery_3.cdb", _}, + {1, "../test/journal/journal_files/nursery_1.cdb", _}], Man), {ActSQN, _ActK} = leveled_cdb:cdb_lastkey(ActJournal), ?assertMatch(ActSQN, 6), ?assertMatch(ActJournalSQN, 5), From c10eaa75cb3d61657e8a7f0c53bf23b1a3e66ddd Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 20 Sep 2016 10:17:24 +0100 Subject: [PATCH 043/167] Dialyzer changes Some chnages to improve dialyzer pass rate --- .gitignore | 4 +++- .rebar/erlcinfo | Bin 0 -> 531 bytes ebin/eleveleddb.app | 11 ----------- rebar.lock | 1 + src/leveled_cdb.erl | 3 +-- src/leveled_clerk.erl | 20 +++++++++++--------- src/leveled_penciller.erl | 32 +++++++++++++++++--------------- 7 files changed, 33 insertions(+), 38 deletions(-) create mode 100644 .rebar/erlcinfo delete mode 100644 ebin/eleveleddb.app create mode 100644 rebar.lock diff --git a/.gitignore b/.gitignore index 1ef2775..26dc633 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -*.beam \ No newline at end of file +*.beam +/.eunit +/_build \ No newline at end of file diff --git a/.rebar/erlcinfo b/.rebar/erlcinfo new file mode 100644 index 0000000000000000000000000000000000000000..f883b7cc3e21883e73352197d3098883ab2bc41c GIT binary patch literal 531 zcmV+u0_^>RPyhf5b9i2@Rm)D}FcdWnuhJJy3j!ow5({QQA}XPZnOVeZCCn$Dd9pcqo}XLiYYa7lxq)$MUg}^_Q77 z%>qd-X{T#WNK-1Za?%cTk@P4PzRT$=?h!NwArkTr=Bw8+J_^$HrKu@B3@CX*7&ye47)`mZg zL0>25Io}It79tqURlXWgN0d|lBe{kP>435C>w6z0Y%xXvI*wX zlA;w$ocD|&s@)uVjuUkhZsX5ZKw38xY(rDk9K5-5P)A#QW80}D-H7NbT!Ck+FsSRv ztH9CIbJW^bs^h*K;j;QRTAQKX%7*Ds#ae}J@Y&^!YCSZ!K!d5*Zu3U{jB2vgyT{b} zjmx)rO@-tuuu1ZK{toabJM*7Y3@ZQt literal 0 HcmV?d00001 diff --git a/ebin/eleveleddb.app b/ebin/eleveleddb.app deleted file mode 100644 index 11d7f25..0000000 --- a/ebin/eleveleddb.app +++ /dev/null @@ -1,11 +0,0 @@ -{application,eleveleddb, - [{description,[]}, - {vsn,"1"}, - {registered,[]}, - {applications,[kernel,stdlib]}, - {mod,{eleveleddb_app,[]}}, - {env,[]}, - {modules,[eleveleddb_app,eleveleddb_sup,leveled_bookie, - leveled_cdb,leveled_clerk,leveled_inker, - leveled_iterator,leveled_penciller,leveled_rice, - leveled_sft]}]}. diff --git a/rebar.lock b/rebar.lock new file mode 100644 index 0000000..57afcca --- /dev/null +++ b/rebar.lock @@ -0,0 +1 @@ +[]. diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index ec807ea..3355153 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -1323,8 +1323,7 @@ getnextkey_inclemptyvalue_test() -> {_, Handle, P6} = get_nextkey(Handle, P5), {_, Handle, P7} = get_nextkey(Handle, P6), {_, Handle, P8} = get_nextkey(Handle, P7), - {LastKey, Info} = get_nextkey(Handle, P8), - ?assertMatch(nomorekeys, Info), + {LastKey, nomorekeys} = get_nextkey(Handle, P8), ?assertMatch("K1", LastKey), ok = file:delete("../test/hashtable2_test.cdb"). diff --git a/src/leveled_clerk.erl b/src/leveled_clerk.erl index 717455f..0197f7b 100644 --- a/src/leveled_clerk.erl +++ b/src/leveled_clerk.erl @@ -314,15 +314,17 @@ find_randomkeys(FList, Count, Source) -> K1 = leveled_bookie:strip_to_keyonly(KV1), P1 = choose_pid_toquery(FList, K1), FoundKV = leveled_sft:sft_get(P1, K1), - case FoundKV of - not_present -> - io:format("Failed to find ~w in ~w~n", [K1, P1]), - ?assertMatch(true, false); - _ -> - Found = leveled_bookie:strip_to_keyonly(FoundKV), - io:format("success finding ~w in ~w~n", [K1, P1]), - ?assertMatch(K1, Found) - end, + Check = case FoundKV of + not_present -> + io:format("Failed to find ~w in ~w~n", [K1, P1]), + error; + _ -> + Found = leveled_bookie:strip_to_keyonly(FoundKV), + io:format("success finding ~w in ~w~n", [K1, P1]), + ?assertMatch(K1, Found), + ok + end, + ?assertMatch(Check, ok), find_randomkeys(FList, Count - 1, Source). diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 777baba..a3feeef 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -200,7 +200,7 @@ levelzero_snapshot = [] :: list(), memtable, backlog = false :: boolean(), - memtable_maxsize :: integer}). + memtable_maxsize :: integer()}). %%%============================================================================ @@ -945,20 +945,22 @@ simple_server_test() -> {ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath, max_inmemory_tablesize=1000}), TopSQN = pcl_getstartupsequencenumber(PCLr), - case TopSQN of - 2001 -> - %% Last push not persisted - S3a = pcl_pushmem(PCL, [Key3]), - if S3a == pause -> timer:sleep(2000); true -> ok end; - 2002 -> - %% everything got persisted - ok; - _ -> - io:format("Unexpected sequence number on restart ~w~n", [TopSQN]), - ok = pcl_close(PCLr), - clean_testdir(RootPath), - ?assertMatch(true, false) - end, + Check = case TopSQN of + 2001 -> + %% Last push not persisted + S3a = pcl_pushmem(PCL, [Key3]), + if S3a == pause -> timer:sleep(2000); true -> ok end, + ok; + 2002 -> + %% everything got persisted + ok; + _ -> + io:format("Unexpected sequence number on restart ~w~n", [TopSQN]), + ok = pcl_close(PCLr), + clean_testdir(RootPath), + error + end, + ?assertMatch(Check, ok), R8 = pcl_fetch(PCLr, {o,"Bucket0001", "Key0001"}), R9 = pcl_fetch(PCLr, {o,"Bucket0002", "Key0002"}), R10 = pcl_fetch(PCLr, {o,"Bucket0003", "Key0003"}), From aa7d235c4d257a938c64effa86a7cf34ea04138a Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 20 Sep 2016 16:13:36 +0100 Subject: [PATCH 044/167] Rename clerk and CDB Speed-Up CDB did many "bitty" reads/writes when scanning or writing hash tables - change these to bult reads and writes to speed up. CDB also added capabilities to fetch positions and get keys by position to help with iclerk role. --- src/leveled_cdb.erl | 171 ++++++++++++++---- src/leveled_iclerk.erl | 90 +++++++++ src/{leveled_clerk.erl => leveled_pclerk.erl} | 2 +- src/leveled_penciller.erl | 6 +- 4 files changed, 225 insertions(+), 44 deletions(-) create mode 100644 src/leveled_iclerk.erl rename src/{leveled_clerk.erl => leveled_pclerk.erl} (99%) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 3355153..b3672f9 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -60,6 +60,8 @@ cdb_open_reader/1, cdb_get/2, cdb_put/3, + cdb_getpositions/1, + cdb_getkey/2, cdb_lastkey/1, cdb_filename/1, cdb_keycheck/2, @@ -96,7 +98,7 @@ cdb_open_writer(Filename) -> cdb_open_writer(Filename, Opts) -> {ok, Pid} = gen_server:start(?MODULE, [Opts], []), - case gen_server:call(Pid, {cdb_open_writer, Filename}, infinity) of + case gen_server:call(Pid, {open_writer, Filename}, infinity) of ok -> {ok, Pid}; Error -> @@ -105,7 +107,7 @@ cdb_open_writer(Filename, Opts) -> cdb_open_reader(Filename) -> {ok, Pid} = gen_server:start(?MODULE, [#cdb_options{}], []), - case gen_server:call(Pid, {cdb_open_reader, Filename}, infinity) of + case gen_server:call(Pid, {open_reader, Filename}, infinity) of ok -> {ok, Pid}; Error -> @@ -113,10 +115,16 @@ cdb_open_reader(Filename) -> end. cdb_get(Pid, Key) -> - gen_server:call(Pid, {cdb_get, Key}, infinity). + gen_server:call(Pid, {get_kv, Key}, infinity). cdb_put(Pid, Key, Value) -> - gen_server:call(Pid, {cdb_put, Key, Value}, infinity). + gen_server:call(Pid, {put_kv, Key, Value}, infinity). + +cdb_getpositions(Pid) -> + gen_server:call(Pid, get_positions, infinity). + +cdb_getkey(Pid, Position) -> + gen_server:call(Pid, {get_key, Position}, infinity). cdb_close(Pid) -> gen_server:call(Pid, cdb_close, infinity). @@ -148,7 +156,7 @@ cdb_filename(Pid) -> %% Check to see if the key is probably present, will return either %% probably or missing. Does not do a definitive check cdb_keycheck(Pid, Key) -> - gen_server:call(Pid, {cdb_keycheck, Key}, infinity). + gen_server:call(Pid, {key_check, Key}, infinity). %%%============================================================================ %%% gen_server callbacks @@ -163,7 +171,7 @@ init([Opts]) -> end, {ok, #state{max_size=MaxSize}}. -handle_call({cdb_open_writer, Filename}, _From, State) -> +handle_call({open_writer, Filename}, _From, State) -> io:format("Opening file for writing with filename ~s~n", [Filename]), {LastPosition, HashTree, LastKey} = open_active_file(Filename), {ok, Handle} = file:open(Filename, [sync | ?WRITE_OPS]), @@ -173,7 +181,7 @@ handle_call({cdb_open_writer, Filename}, _From, State) -> filename=Filename, hashtree=HashTree, writer=true}}; -handle_call({cdb_open_reader, Filename}, _From, State) -> +handle_call({open_reader, Filename}, _From, State) -> io:format("Opening file for reading with filename ~s~n", [Filename]), {ok, Handle} = file:open(Filename, [binary, raw, read]), Index = load_index(Handle), @@ -183,7 +191,7 @@ handle_call({cdb_open_reader, Filename}, _From, State) -> filename=Filename, writer=false, hash_index=Index}}; -handle_call({cdb_get, Key}, _From, State) -> +handle_call({get_kv, Key}, _From, State) -> case {State#state.writer, State#state.hash_index} of {true, _} -> {reply, @@ -198,7 +206,7 @@ handle_call({cdb_get, Key}, _From, State) -> get_withcache(State#state.handle, Key, Cache), State} end; -handle_call({cdb_keycheck, Key}, _From, State) -> +handle_call({key_check, Key}, _From, State) -> case {State#state.writer, State#state.hash_index} of {true, _} -> {reply, @@ -221,7 +229,7 @@ handle_call({cdb_keycheck, Key}, _From, State) -> Cache), State} end; -handle_call({cdb_put, Key, Value}, _From, State) -> +handle_call({put_kv, Key, Value}, _From, State) -> case State#state.writer of true -> Result = put(State#state.handle, @@ -247,6 +255,13 @@ handle_call(cdb_lastkey, _From, State) -> {reply, State#state.last_key, State}; handle_call(cdb_filename, _From, State) -> {reply, State#state.filename, State}; +handle_call(get_positions, _From, State) -> + {reply, scan_index(State#state.handle, + State#state.hash_index, + {fun scan_index_returnpositions/4, []}), + State}; +handle_call({get_key, Position}, _From, State) -> + {reply, extract_key(State#state.handle, Position), State}; handle_call({cdb_scan, FilterFun, Acc, StartPos}, _From, State) -> {ok, StartPos0} = case StartPos of undefined -> @@ -353,7 +368,6 @@ dump(FileName, CRCCheck) -> Fn1 = fun(_I,Acc) -> {KL,VL} = read_next_2_integers(Handle), Key = read_next_term(Handle, KL), - io:format("Key read of ~w~n", [Key]), case read_next_term(Handle, VL, crc, CRCCheck) of {false, _} -> {ok, CurrLoc} = file:position(Handle, cur), @@ -597,28 +611,33 @@ load_index(Handle) -> %% Function to find the LastKey in the file find_lastkey(Handle, IndexCache) -> - LastPosition = scan_index(Handle, IndexCache), + LastPosition = scan_index(Handle, IndexCache, {fun scan_index_findlast/4, 0}), {ok, _} = file:position(Handle, LastPosition), {KeyLength, _ValueLength} = read_next_2_integers(Handle), read_next_term(Handle, KeyLength). -scan_index(Handle, IndexCache) -> - lists:foldl(fun({_X, {Pos, Count}}, LastPosition) -> - scan_index(Handle, Pos, 0, Count, LastPosition) end, - 0, +scan_index(Handle, IndexCache, {ScanFun, InitAcc}) -> + lists:foldl(fun({_X, {Pos, Count}}, Acc) -> + ScanFun(Handle, Pos, Count, Acc) + end, + InitAcc, IndexCache). -scan_index(_Handle, _Position, Count, Checks, LastPosition) - when Count == Checks -> - LastPosition; -scan_index(Handle, Position, Count, Checks, LastPosition) -> - {ok, _} = file:position(Handle, Position + ?DWORD_SIZE * Count), - {_Hash, HPosition} = read_next_2_integers(Handle), - scan_index(Handle, - Position, - Count + 1 , - Checks, - max(LastPosition, HPosition)). +scan_index_findlast(Handle, Position, Count, LastPosition) -> + {ok, _} = file:position(Handle, Position), + lists:foldl(fun({_Hash, HPos}, MaxPos) -> max(HPos, MaxPos) end, + LastPosition, + read_next_n_integerpairs(Handle, Count)). + +scan_index_returnpositions(Handle, Position, Count, PosList0) -> + {ok, _} = file:position(Handle, Position), + lists:foldl(fun({Hash, HPosition}, PosList) -> + case Hash of + 0 -> PosList; + _ -> PosList ++ [HPosition] + end end, + PosList0, + read_next_n_integerpairs(Handle, Count)). %% Take an active file and write the hash details necessary to close that @@ -628,13 +647,14 @@ scan_index(Handle, Position, Count, Checks, LastPosition) -> %% the hash tables close_file(Handle, HashTree, BasePos) -> {ok, BasePos} = file:position(Handle, BasePos), - SW = os:timestamp(), + SW1 = os:timestamp(), L2 = write_hash_tables(Handle, HashTree), + SW2 = os:timestamp(), io:format("Hash Table write took ~w microseconds~n", - [timer:now_diff(os:timestamp(),SW)]), + [timer:now_diff(SW2, SW1)]), write_top_index_table(Handle, BasePos, L2), io:format("Top Index Table write took ~w microseconds~n", - [timer:now_diff(os:timestamp(),SW)]), + [timer:now_diff(os:timestamp(),SW2)]), file:close(Handle). @@ -683,6 +703,11 @@ extract_kvpair(Handle, [Position|Rest], Key, Check) -> extract_kvpair(Handle, Rest, Key, Check) end. +extract_key(Handle, Position) -> + {ok, _} = file:position(Handle, Position), + {KeyLength, _ValueLength} = read_next_2_integers(Handle), + read_next_term(Handle, KeyLength). + %% Scan through the file until there is a failure to crc check an input, and %% at that point return the position and the key dictionary scanned so far startup_scan_over_file(Handle, Position) -> @@ -876,6 +901,17 @@ read_next_2_integers(Handle) -> ReadError end. +read_next_n_integerpairs(Handle, NumberOfPairs) -> + {ok, Block} = file:read(Handle, ?DWORD_SIZE * NumberOfPairs), + read_integerpairs(Block, []). + +read_integerpairs(<<>>, Pairs) -> + Pairs; +read_integerpairs(<>, Pairs) -> + read_integerpairs(<>, + Pairs ++ [{endian_flip(Int1), + endian_flip(Int2)}]). + %% Seach the hash table for the matching hash and key. Be prepared for %% multiple keys to have the same hash value. %% @@ -941,17 +977,19 @@ write_key_value_pairs(Handle, [HeadPair|TailList], Acc) -> write_hash_tables(Handle, HashTree) -> Seq = lists:seq(0, 255), {ok, StartPos} = file:position(Handle, cur), - write_hash_tables(Seq, Handle, HashTree, StartPos, []). - -write_hash_tables([], Handle, _, StartPos, IndexList) -> + {IndexList, HashTreeBin} = write_hash_tables(Seq, HashTree, StartPos, [], <<>>), + ok = file:write(Handle, HashTreeBin), {ok, EndPos} = file:position(Handle, cur), ok = file:advise(Handle, StartPos, EndPos - StartPos, will_need), - IndexList; -write_hash_tables([Index|Rest], Handle, HashTree, StartPos, IndexList) -> + IndexList. + +write_hash_tables([], _HashTree, _CurrPos, IndexList, HashTreeBin) -> + {IndexList, HashTreeBin}; +write_hash_tables([Index|Rest], HashTree, CurrPos, IndexList, HashTreeBin) -> Tree = array:get(Index, HashTree), case gb_trees:keys(Tree) of [] -> - write_hash_tables(Rest, Handle, HashTree, StartPos, IndexList); + write_hash_tables(Rest, HashTree, CurrPos, IndexList, HashTreeBin); _ -> HashList = gb_trees:to_list(Tree), BinList = build_binaryhashlist(HashList, []), @@ -965,10 +1003,14 @@ write_hash_tables([Index|Rest], Handle, HashTree, StartPos, IndexList) -> end, NewSlotList = lists:foldl(Fn, SlotList, BinList), - {ok, CurrPos} = file:position(Handle, cur), - file:write(Handle, NewSlotList), - write_hash_tables(Rest, Handle, HashTree, StartPos, - [{Index, CurrPos, IndexLength}|IndexList]) + NewSlotBin = lists:foldl(fun(X, Acc) -> <> end, + HashTreeBin, + NewSlotList), + write_hash_tables(Rest, + HashTree, + CurrPos + length(NewSlotList) * ?DWORD_SIZE, + [{Index, CurrPos, IndexLength}|IndexList], + NewSlotBin) end. %% The list created from the original HashTree may have duplicate positions @@ -1452,4 +1494,53 @@ find_lastkey_test() -> ok = cdb_close(P3), ok = file:delete("../test/lastkey.cdb"). +get_keys_byposition_simple_test() -> + {ok, P1} = cdb_open_writer("../test/poskey.pnd"), + ok = cdb_put(P1, "Key1", "Value1"), + ok = cdb_put(P1, "Key3", "Value3"), + ok = cdb_put(P1, "Key2", "Value2"), + KeyList = ["Key1", "Key2", "Key3"], + {ok, F2} = cdb_complete(P1), + {ok, P2} = cdb_open_reader(F2), + PositionList = cdb_getpositions(P2), + io:format("Position list of ~w~n", [PositionList]), + L1 = length(PositionList), + ?assertMatch(L1, 3), + lists:foreach(fun(Pos) -> + Key = cdb_getkey(P2, Pos), + Check = lists:member(Key, KeyList), + ?assertMatch(Check, true) end, + PositionList), + ok = cdb_close(P2), + ok = file:delete(F2). + +generate_sequentialkeys(0, KVList) -> + KVList; +generate_sequentialkeys(Count, KVList) -> + KV = {"Key" ++ integer_to_list(Count), "Value" ++ integer_to_list(Count)}, + generate_sequentialkeys(Count - 1, KVList ++ [KV]). + +get_keys_byposition_manykeys_test() -> + KeyCount = 1024, + {ok, P1} = cdb_open_writer("../test/poskeymany.pnd"), + KVList = generate_sequentialkeys(KeyCount, []), + lists:foreach(fun({K, V}) -> cdb_put(P1, K, V) end, KVList), + SW1 = os:timestamp(), + {ok, F2} = cdb_complete(P1), + SW2 = os:timestamp(), + io:format("CDB completed in ~w microseconds~n", + [timer:now_diff(SW2, SW1)]), + {ok, P2} = cdb_open_reader(F2), + SW3 = os:timestamp(), + io:format("CDB opened for read in ~w microseconds~n", + [timer:now_diff(SW3, SW2)]), + PositionList = cdb_getpositions(P2), + io:format("Positions fetched in ~w microseconds~n", + [timer:now_diff(os:timestamp(), SW3)]), + L1 = length(PositionList), + ?assertMatch(L1, KeyCount), + ok = cdb_close(P2), + ok = file:delete(F2). + + -endif. diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl new file mode 100644 index 0000000..3f95547 --- /dev/null +++ b/src/leveled_iclerk.erl @@ -0,0 +1,90 @@ + + +-module(leveled_iclerk). + +-behaviour(gen_server). + +-include("../include/leveled.hrl"). + +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + clerk_new/1, + clerk_compact/3, + clerk_remove/2, + clerk_stop/1, + code_change/3]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(KEYS_TO_CHECK, 100). + +-record(state, {owner :: pid()}). + +%%%============================================================================ +%%% API +%%%============================================================================ + +clerk_new(Owner) -> + {ok, Pid} = gen_server:start(?MODULE, [], []), + ok = gen_server:call(Pid, {register, Owner}, infinity), + {ok, Pid}. + +clerk_compact(Pid, InkerManifest, Penciller) -> + gen_server:cast(Pid, {compact, InkerManifest, Penciller}), + ok. + +clerk_remove(Pid, Removals) -> + gen_server:cast(Pid, {remove, Removals}), + ok. + +clerk_stop(Pid) -> + gen_server:cast(Pid, stop). + +%%%============================================================================ +%%% gen_server callbacks +%%%============================================================================ + +init([]) -> + {ok, #state{}}. + +handle_call({register, Owner}, _From, State) -> + {reply, ok, State#state{owner=Owner}}. + +handle_cast({compact, InkerManifest, Penciller, Timeout}, State) -> + ok = journal_compact(InkerManifest, Penciller, Timeout, State#state.owner), + {noreply, State}; +handle_cast({remove, _Removals}, State) -> + {noreply, State}; +handle_cast(stop, State) -> + {stop, normal, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + + +%%%============================================================================ +%%% Internal functions +%%%============================================================================ + +journal_compact(_InkerManifest, _Penciller, _Timeout, _Owner) -> + ok. + +check_all_files(_InkerManifest) -> + ok. + +window_closed(_Timeout) -> + true. + + +%%%============================================================================ +%%% Test +%%%============================================================================ diff --git a/src/leveled_clerk.erl b/src/leveled_pclerk.erl similarity index 99% rename from src/leveled_clerk.erl rename to src/leveled_pclerk.erl index 0197f7b..d3334da 100644 --- a/src/leveled_clerk.erl +++ b/src/leveled_pclerk.erl @@ -2,7 +2,7 @@ %% level and cleaning out of old files across a level --module(leveled_clerk). +-module(leveled_pclerk). -behaviour(gen_server). diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index a3feeef..8ec90c5 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -251,7 +251,7 @@ init([PCLopts]) -> M end, TID = ets:new(?MEMTABLE, [ordered_set]), - {ok, Clerk} = leveled_clerk:clerk_new(self()), + {ok, Clerk} = leveled_pclerk:clerk_new(self()), InitState = #state{memtable=TID, clerk=Clerk, root_path=RootPath, @@ -435,7 +435,7 @@ terminate(_Reason, State) -> %% The cast may not succeed as the clerk could be synchronously calling %% the penciller looking for a manifest commit %% - leveled_clerk:clerk_stop(State#state.clerk), + leveled_pclerk:clerk_stop(State#state.clerk), Dump = ets:tab2list(State#state.memtable), case {State#state.levelzero_pending, get_item(0, State#state.manifest, []), length(Dump)} of @@ -499,7 +499,7 @@ push_to_memory(DumpList, State) -> end, %% Prompt clerk to ask about work - do this for every push_mem - ok = leveled_clerk:clerk_prompt(UpdState#state.clerk, penciller), + ok = leveled_pclerk:clerk_prompt(UpdState#state.clerk, penciller), MemoryInsertion = do_push_to_mem(DumpList, TableSize, From 66d6db4e1194e73d2f4c682cdba6b3009b248bed Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 20 Sep 2016 18:24:05 +0100 Subject: [PATCH 045/167] Support for random sampling Makes the ability to get positions and the fetch directly by position more generic - supporting the fetch of different flavours of combinations, and requesting a sample of positions not just all --- src/leveled_cdb.erl | 160 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 126 insertions(+), 34 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index b3672f9..40a4aaf 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -6,18 +6,15 @@ %% The primary differences are: %% - Support for incrementally writing a CDB file while keeping the hash table %% in memory -%% - Support for merging of multiple CDB files with a key-checking function to -%% allow for compaction -%% - Automatic adding of a helper object that will keep a small proportion of -%% keys to be used when checking to see if the cdb file is a candidate for -%% compaction %% - The ability to scan a database and accumulate all the Key, Values to %% rebuild in-memory tables on startup +%% - The ability to scan a database in blocks of sequence numbers %% %% This is to be used in eleveledb, and in this context: -%% - Keys will be a Sequence Number -%% - Values will be a Checksum | Object | KeyAdditions -%% Where the KeyAdditions are all the Key changes required to be added to the +%% - Keys will be a combinatio of the PrimaryKey and the Sequence Number +%% - Values will be a serialised version on the whole object, and the +%% IndexChanges associated with the transaction +%% Where the IndexChanges are all the Key changes required to be added to the %% ledger to complete the changes (the addition of postings and tombstones). %% %% This module provides functions to create and query a CDB (constant database). @@ -60,8 +57,8 @@ cdb_open_reader/1, cdb_get/2, cdb_put/3, - cdb_getpositions/1, - cdb_getkey/2, + cdb_getpositions/2, + cdb_directfetch/3, cdb_lastkey/1, cdb_filename/1, cdb_keycheck/2, @@ -120,11 +117,15 @@ cdb_get(Pid, Key) -> cdb_put(Pid, Key, Value) -> gen_server:call(Pid, {put_kv, Key, Value}, infinity). -cdb_getpositions(Pid) -> - gen_server:call(Pid, get_positions, infinity). +%% SampleSize can be an integer or the atom all +cdb_getpositions(Pid, SampleSize) -> + gen_server:call(Pid, {get_positions, SampleSize}, infinity). -cdb_getkey(Pid, Position) -> - gen_server:call(Pid, {get_key, Position}, infinity). +%% Info can be key_only, key_size (size being the size of the value) or +%% key_value_check (with the check part indicating if the CRC is correct for +%% the value) +cdb_directfetch(Pid, PositionList, Info) -> + gen_server:call(Pid, {direct_fetch, PositionList, Info}, infinity). cdb_close(Pid) -> gen_server:call(Pid, cdb_close, infinity). @@ -255,13 +256,45 @@ handle_call(cdb_lastkey, _From, State) -> {reply, State#state.last_key, State}; handle_call(cdb_filename, _From, State) -> {reply, State#state.filename, State}; -handle_call(get_positions, _From, State) -> - {reply, scan_index(State#state.handle, - State#state.hash_index, - {fun scan_index_returnpositions/4, []}), - State}; -handle_call({get_key, Position}, _From, State) -> - {reply, extract_key(State#state.handle, Position), State}; +handle_call({get_positions, SampleSize}, _From, State) -> + case SampleSize of + all -> + {reply, scan_index(State#state.handle, + State#state.hash_index, + {fun scan_index_returnpositions/4, []}), + State}; + _ -> + SeededL = lists:map(fun(X) -> {random:uniform(), X} end, + State#state.hash_index), + SortedL = lists:keysort(1, SeededL), + RandomisedHashIndex = lists:map(fun({_R, X}) -> X end, SortedL), + {reply, + scan_index_forsample(State#state.handle, + RandomisedHashIndex, + fun scan_index_returnpositions/4, + [], + SampleSize), + State} + end; +handle_call({direct_fetch, PositionList, Info}, _From, State) -> + H = State#state.handle, + case Info of + key_only -> + KeyList = lists:map(fun(P) -> + extract_key(H, P) end, + PositionList), + {reply, KeyList, State}; + key_size -> + KeySizeList = lists:map(fun(P) -> + extract_key_size(H, P) end, + PositionList), + {reply, KeySizeList, State}; + key_value_check -> + KVCList = lists:map(fun(P) -> + extract_key_value_check(H, P) end, + PositionList), + {reply, KVCList, State} + end; handle_call({cdb_scan, FilterFun, Acc, StartPos}, _From, State) -> {ok, StartPos0} = case StartPos of undefined -> @@ -383,7 +416,7 @@ dump(FileName, CRCCheck) -> {ok, _} = file:position(Handle, CurrLoc), [Return | Acc] end, - lists:foldr(Fn1,[],lists:seq(0,NumberOfPairs-1)). + lists:foldr(Fn1, [], lists:seq(0, NumberOfPairs-1)). %% Open an active file - one for which it is assumed the hash tables have not %% yet been written @@ -611,7 +644,9 @@ load_index(Handle) -> %% Function to find the LastKey in the file find_lastkey(Handle, IndexCache) -> - LastPosition = scan_index(Handle, IndexCache, {fun scan_index_findlast/4, 0}), + LastPosition = scan_index(Handle, + IndexCache, + {fun scan_index_findlast/4, 0}), {ok, _} = file:position(Handle, LastPosition), {KeyLength, _ValueLength} = read_next_2_integers(Handle), read_next_term(Handle, KeyLength). @@ -623,6 +658,22 @@ scan_index(Handle, IndexCache, {ScanFun, InitAcc}) -> InitAcc, IndexCache). +scan_index_forsample(_Handle, [], _ScanFun, Acc, SampleSize) -> + lists:sublist(Acc, SampleSize); +scan_index_forsample(Handle, [CacheEntry|Tail], ScanFun, Acc, SampleSize) -> + case length(Acc) of + L when L >= SampleSize -> + lists:sublist(Acc, SampleSize); + _ -> + {_X, {Pos, Count}} = CacheEntry, + scan_index_forsample(Handle, + Tail, + ScanFun, + ScanFun(Handle, Pos, Count, Acc), + SampleSize) + end. + + scan_index_findlast(Handle, Position, Count, LastPosition) -> {ok, _} = file:position(Handle, Position), lists:foldl(fun({_Hash, HPos}, MaxPos) -> max(HPos, MaxPos) end, @@ -708,6 +759,18 @@ extract_key(Handle, Position) -> {KeyLength, _ValueLength} = read_next_2_integers(Handle), read_next_term(Handle, KeyLength). +extract_key_size(Handle, Position) -> + {ok, _} = file:position(Handle, Position), + {KeyLength, ValueLength} = read_next_2_integers(Handle), + {read_next_term(Handle, KeyLength), ValueLength}. + +extract_key_value_check(Handle, Position) -> + {ok, _} = file:position(Handle, Position), + {KeyLength, ValueLength} = read_next_2_integers(Handle), + K = read_next_term(Handle, KeyLength), + {Check, V} = read_next_term(Handle, ValueLength, crc, true), + {K, V, Check}. + %% Scan through the file until there is a failure to crc check an input, and %% at that point return the position and the key dictionary scanned so far startup_scan_over_file(Handle, Position) -> @@ -977,7 +1040,11 @@ write_key_value_pairs(Handle, [HeadPair|TailList], Acc) -> write_hash_tables(Handle, HashTree) -> Seq = lists:seq(0, 255), {ok, StartPos} = file:position(Handle, cur), - {IndexList, HashTreeBin} = write_hash_tables(Seq, HashTree, StartPos, [], <<>>), + {IndexList, HashTreeBin} = write_hash_tables(Seq, + HashTree, + StartPos, + [], + <<>>), ok = file:write(Handle, HashTreeBin), {ok, EndPos} = file:position(Handle, cur), ok = file:advise(Handle, StartPos, EndPos - StartPos, will_need), @@ -1003,7 +1070,8 @@ write_hash_tables([Index|Rest], HashTree, CurrPos, IndexList, HashTreeBin) -> end, NewSlotList = lists:foldl(Fn, SlotList, BinList), - NewSlotBin = lists:foldl(fun(X, Acc) -> <> end, + NewSlotBin = lists:foldl(fun(X, Acc) -> + <> end, HashTreeBin, NewSlotList), write_hash_tables(Rest, @@ -1027,9 +1095,11 @@ build_binaryhashlist([{Hash, [Position|TailP]}|TailKV], BinList) -> NewBin = <>, case TailP of [] -> - build_binaryhashlist(TailKV, [{Hash, NewBin}|BinList]); + build_binaryhashlist(TailKV, + [{Hash, NewBin}|BinList]); _ -> - build_binaryhashlist([{Hash, TailP}|TailKV], [{Hash, NewBin}|BinList]) + build_binaryhashlist([{Hash, TailP}|TailKV], + [{Hash, NewBin}|BinList]) end. %% Slot is zero based because it comes from a REM @@ -1502,15 +1572,29 @@ get_keys_byposition_simple_test() -> KeyList = ["Key1", "Key2", "Key3"], {ok, F2} = cdb_complete(P1), {ok, P2} = cdb_open_reader(F2), - PositionList = cdb_getpositions(P2), + PositionList = cdb_getpositions(P2, all), io:format("Position list of ~w~n", [PositionList]), - L1 = length(PositionList), - ?assertMatch(L1, 3), - lists:foreach(fun(Pos) -> - Key = cdb_getkey(P2, Pos), + ?assertMatch(3, length(PositionList)), + R1 = cdb_directfetch(P2, PositionList, key_only), + ?assertMatch(3, length(R1)), + lists:foreach(fun(Key) -> Check = lists:member(Key, KeyList), ?assertMatch(Check, true) end, - PositionList), + R1), + R2 = cdb_directfetch(P2, PositionList, key_size), + ?assertMatch(3, length(R2)), + lists:foreach(fun({Key, _Size}) -> + Check = lists:member(Key, KeyList), + ?assertMatch(Check, true) end, + R2), + R3 = cdb_directfetch(P2, PositionList, key_value_check), + ?assertMatch(3, length(R3)), + lists:foreach(fun({Key, Value, Check}) -> + ?assertMatch(Check, true), + {K, V} = cdb_get(P2, Key), + ?assertMatch(K, Key), + ?assertMatch(V, Value) end, + R3), ok = cdb_close(P2), ok = file:delete(F2). @@ -1534,11 +1618,19 @@ get_keys_byposition_manykeys_test() -> SW3 = os:timestamp(), io:format("CDB opened for read in ~w microseconds~n", [timer:now_diff(SW3, SW2)]), - PositionList = cdb_getpositions(P2), + PositionList = cdb_getpositions(P2, all), io:format("Positions fetched in ~w microseconds~n", [timer:now_diff(os:timestamp(), SW3)]), L1 = length(PositionList), ?assertMatch(L1, KeyCount), + + SampleList1 = cdb_getpositions(P2, 10), + ?assertMatch(10, length(SampleList1)), + SampleList2 = cdb_getpositions(P2, KeyCount), + ?assertMatch(KeyCount, length(SampleList2)), + SampleList3 = cdb_getpositions(P2, KeyCount + 1), + ?assertMatch(KeyCount, length(SampleList3)), + ok = cdb_close(P2), ok = file:delete(F2). From d3e985ed803f046f961873c063a393ec6b360d6e Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 21 Sep 2016 18:31:42 +0100 Subject: [PATCH 046/167] Refactor Penciller Push Two aspects of pushing to the penciller have been refactored: 1 - Allow the penciller to respond before the ETS table has been updated to unlock the Bookie sooner. 2 - Change the way the copy of the memtable is stored to work more effectively with snapshots wihtout locking the Penciller any further on a snapshot or push request --- .rebar/erlcinfo | Bin 531 -> 0 bytes include/leveled.hrl | 1 + src/leveled_iclerk.erl | 24 +- src/leveled_pclerk.erl | 6 +- src/leveled_penciller.erl | 613 +++++++++++++++++++++++++------------- 5 files changed, 433 insertions(+), 211 deletions(-) delete mode 100644 .rebar/erlcinfo diff --git a/.rebar/erlcinfo b/.rebar/erlcinfo deleted file mode 100644 index f883b7cc3e21883e73352197d3098883ab2bc41c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 531 zcmV+u0_^>RPyhf5b9i2@Rm)D}FcdWnuhJJy3j!ow5({QQA}XPZnOVeZCCn$Dd9pcqo}XLiYYa7lxq)$MUg}^_Q77 z%>qd-X{T#WNK-1Za?%cTk@P4PzRT$=?h!NwArkTr=Bw8+J_^$HrKu@B3@CX*7&ye47)`mZg zL0>25Io}It79tqURlXWgN0d|lBe{kP>435C>w6z0Y%xXvI*wX zlA;w$ocD|&s@)uVjuUkhZsX5ZKw38xY(rDk9K5-5P)A#QW80}D-H7NbT!Ck+FsSRv ztH9CIbJW^bs^h*K;j;QRTAQKX%7*Ds#ae}J@Y&^!YCSZ!K!d5*Zu3U{jB2vgyT{b} zjmx)rO@-tuuu1ZK{toabJM*7Y3@ZQt diff --git a/include/leveled.hrl b/include/leveled.hrl index 030fdd4..0f929cc 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -30,6 +30,7 @@ -record(penciller_options, {root_path :: string(), + penciller :: pid(), max_inmemory_tablesize :: integer()}). -record(bookie_options, diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 3f95547..335a649 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -19,7 +19,8 @@ -include_lib("eunit/include/eunit.hrl"). --define(KEYS_TO_CHECK, 100). +-define(BATCH_SIZE, 16) +-define(BATCHES_TO_CHECK, 8). -record(state, {owner :: pid()}). @@ -81,6 +82,27 @@ journal_compact(_InkerManifest, _Penciller, _Timeout, _Owner) -> check_all_files(_InkerManifest) -> ok. +check_single_file(CDB, _PencilSnapshot, SampleSize, BatchSize) -> + PositionList = leveled_cdb:cdb_getpositions(CDB, SampleSize), + KeySizeList = fetch_inbatches(PositionList, BatchSize, CDB, []), + KeySizeList. + %% TODO: + %% Need to check the penciller snapshot to see if these keys are at the + %% right sequence number + %% + %% The calculate the proportion (by value size) of the CDB which is at the + %% wrong sequence number to help determine eligibility for compaction + %% + %% BIG TODO: + %% Need to snapshot a penciller + +fetch_inbatches([], _BatchSize, _CDB, CheckedList) -> + CheckedList; +fetch_inbatches(PositionList, BatchSize, CDB, CheckedList) -> + {Batch, Tail} = lists:split(BatchSize, PositionList), + KL_List = leveled_cdb:direct_fetch(CDB, Batch, key_size), + fetch_inbatches(Tail, BatchSize, CDB, CheckedList ++ KL_List). + window_closed(_Timeout) -> true. diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index d3334da..c54ba01 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -59,12 +59,12 @@ handle_cast(stop, State) -> {stop, normal, State}. handle_info(timeout, State) -> - %% The pcl prompt will cause a penciller_prompt, to re-trigger timeout case leveled_penciller:pcl_prompt(State#state.owner) of ok -> - {noreply, State}; + Timeout = requestandhandle_work(State), + {noreply, State, Timeout}; pause -> - {noreply, State} + {noreply, State, ?INACTIVITY_TIMEOUT} end; handle_info(_Info, State) -> {noreply, State}. diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 8ec90c5..225431f 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -86,6 +86,10 @@ %% files valid at the point of the snapshot until either the iterator is %% completed or has timed out. %% +%% Snapshot requests may request a filtered view of the ETS table (whihc may +%% be quicker than requesting the full table), or requets a snapshot of only +%% the persisted part of the Ledger +%% %% ---------- ON STARTUP ---------- %% %% On Startup the Bookie with ask the Penciller to initiate the Ledger first. @@ -146,6 +150,66 @@ %% table and build a new table starting with the remainder, and the keys from %% the latest push. %% +%% ---------- NOTES ON THE USE OF ETS ---------- +%% +%% Insertion into ETS is very fast, and so using ETS does not slow the PUT +%% path. However, an ETS table is mutable, so it does complicate the +%% snapshotting of the Ledger. +%% +%% Some alternatives have been considered: +%% +%% A1 - Use gb_trees not ETS table +%% * Speed of inserts are too slow especially as the Bookie is blocked until +%% the insert is complete. Inserting 32K very simple keys takes 250ms. Only +%% the naive commands can be used, as Keys may be present - so not easy to +%% optimise. There is a lack of bulk operations +%% +%% A2 - Use some other structure other than gb_trees or ETS tables +%% * There is nothing else that will support iterators, so snapshots would +%% either need to do a conversion when they request the snapshot if +%% they need to iterate, or iterate through map functions scanning all the +%% keys. The conversion may not be expensive, as we know loading into an ETS +%% table is fast - but there may be some hidden overheads with creating and +%5 destroying many ETS tables. +%% +%% A3 - keep a parallel list of lists of things that have gone in the ETS +%% table in the format they arrived in +%% * There is doubling up of memory, and the snapshot must do some work to +%% make use of these lists. This combines the continued use of fast ETS +%% with the solution of A2 at a memory cost. +%% +%% A4 - Try and cache the conversion to be shared between snapshots registered +%% at the same Ledger SQN +%% * This is a rif on A2/A3, but if generally there is o(10) or o(100) seconds +%% between memory pushes, but much more frequent snapshots this may be more +%% efficient +%% +%% A5 - Produce a specific snapshot of the ETS table via an iterator on demand +%% for each snapshot +%% * So if a snapshot was required for na iterator, the Penciller would block +%% whilst it iterated over the ETS table first to produce a snapshot-specific +%% immutbale view. If the snapshot was required for a long-lived complete view +%% of the database the Penciller would block for a tab2list. +%% +%% A6 - Have snapshots incrementally create and share immutable trees, from a +%% parallel cache of changes +%% * This is a variance on A3. As changes are pushed to the Penciller in the +%% form of lists the Penciller updates a cache of the lists that are contained +%% in the current ETS table. These lists are returned to the snapshot when +%% the snapshot is registered. All snapshots it is assumed will convert these +%% lists into a gb_tree to use, but following that conversion they can cast +%% to the Penciller to refine the cache, so that the cache will become a +%% gb_tree up the ledger SQN at which the snapshot is registered, and now only +%% store new lists for subsequent updates. Future snapshot requests (before +%% the ets table is flushed) will now receive the array (if no changes have) +%% been made, or the array and only the lists needed to incrementally change +%% the array. If changes are infrequent, each snapshot request will pay the +%% full 20ms to 250ms cost of producing the array (although perhaps the clerk +%% could also update periodiclaly to avoid this). If changes are frequent, +%% the snapshot will generally not require to do a conversion, or will only +%% be required to do a small conversion +%% +%% A6 is the preferred option -module(leveled_penciller). @@ -169,7 +233,10 @@ pcl_confirmdelete/2, pcl_prompt/1, pcl_close/1, + pcl_registersnapshot/2, + pcl_updatesnapshotcache/3, pcl_getstartupsequencenumber/1, + roll_new_tree/3, clean_testdir/1]). -include_lib("eunit/include/eunit.hrl"). @@ -187,21 +254,26 @@ -define(PROMPT_WAIT_ONL0, 5). -define(L0PEND_RESET, {false, null, null}). +-record(l0snapshot, {increments = [] :: list, + tree = gb_trees:empty() :: gb_trees:tree(), + ledger_sqn = 0 :: integer()}). + -record(state, {manifest = [] :: list(), ongoing_work = [] :: list(), manifest_sqn = 0 :: integer(), ledger_sqn = 0 :: integer(), - registered_iterators = [] :: list(), + registered_snapshots = [] :: list(), unreferenced_files = [] :: list(), root_path = "../test" :: string(), table_size = 0 :: integer(), clerk :: pid(), levelzero_pending = ?L0PEND_RESET :: tuple(), - levelzero_snapshot = [] :: list(), + memtable_copy = #l0snapshot{} :: #l0snapshot{}, memtable, backlog = false :: boolean(), memtable_maxsize :: integer()}). + %%%============================================================================ %%% API @@ -227,22 +299,265 @@ pcl_requestmanifestchange(Pid, WorkItem) -> gen_server:call(Pid, {manifest_change, WorkItem}, infinity). pcl_confirmdelete(Pid, FileName) -> - gen_server:call(Pid, {confirm_delete, FileName}). + gen_server:call(Pid, {confirm_delete, FileName}, infinity). pcl_prompt(Pid) -> - gen_server:call(Pid, prompt_compaction). + gen_server:call(Pid, prompt_compaction, infinity). pcl_getstartupsequencenumber(Pid) -> - gen_server:call(Pid, get_startup_sqn). + gen_server:call(Pid, get_startup_sqn, infinity). + +pcl_registersnapshot(Pid, Snapshot) -> + gen_server:call(Pid, {register_snapshot, Snapshot}, infinity). + +pcl_updatesnapshotcache(Pid, Tree, SQN) -> + gen_server:cast(Pid, {update_snapshotcache, Tree, SQN}). pcl_close(Pid) -> gen_server:call(Pid, close). + %%%============================================================================ %%% gen_server callbacks %%%============================================================================ init([PCLopts]) -> + case PCLopts#penciller_options.root_path of + undefined -> + {ok, #state{}}; + _RootPath -> + start_from_file(PCLopts) + end. + + +handle_call({push_mem, DumpList}, From, State0) -> + % The process for pushing to memory is as follows + % - Check that the inbound list does not contain any Keys with a lower + % sequence number than any existing keys (assess_sqn/1) + % - Check that any file that had been sent to be written to L0 previously + % is now completed. If it is wipe out the in-memory view as this is now + % safely persisted. This will block waiting for this to complete if it + % hasn't (checkready_pushmem/1). + % - Quick check to see if there is a need to write a L0 file + % (quickcheck_pushmem/3). If there clearly isn't, then we can reply, and + % then add to memory in the background before updating the loop state + % - Push the update into memory (do_pushtomem/3) + % - If we haven't got through quickcheck now need to check if there is a + % definite need to write a new L0 file (roll_memory/2). If all clear this + % will write the file in the background and allow a response to the user. + % If not the change has still been made but the the L0 file will not have + % been prompted - so the reply does not indicate failure but returns the + % atom 'pause' to signal a loose desire for back-pressure to be applied. + % The only reason in this case why there should be a pause is if the + % manifest is locked pending completion of a manifest change - so reacting + % to the pause signal may not be sensible + StartWatch = os:timestamp(), + case assess_sqn(DumpList) of + {MinSQN, MaxSQN} when MaxSQN >= MinSQN, + MinSQN >= State0#state.ledger_sqn -> + MaxTableSize = State0#state.memtable_maxsize, + {TableSize0, State1} = checkready_pushtomem(State0), + case quickcheck_pushtomem(DumpList, + TableSize0, + MaxTableSize) of + {twist, TableSize1} -> + gen_server:reply(From, ok), + io:format("Reply made on push in ~w microseconds~n", + [timer:now_diff(os:timestamp(), StartWatch)]), + L0Snap = do_pushtomem(DumpList, + State1#state.memtable, + State1#state.memtable_copy, + MaxSQN), + io:format("Push completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(), StartWatch)]), + {noreply, + State1#state{memtable_copy=L0Snap, + table_size=TableSize1, + ledger_sqn=MaxSQN}}; + {maybe_roll, TableSize1} -> + L0Snap = do_pushtomem(DumpList, + State1#state.memtable, + State1#state.memtable_copy, + MaxSQN), + + case roll_memory(State1, MaxTableSize) of + {ok, L0Pend, ManSN, TableSize2} -> + io:format("Push completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(), StartWatch)]), + {reply, + ok, + State1#state{levelzero_pending=L0Pend, + table_size=TableSize2, + manifest_sqn=ManSN, + memtable_copy=L0Snap, + ledger_sqn=MaxSQN, + backlog=false}}; + {pause, Reason, Details} -> + io:format("Excess work due to - " ++ Reason, + Details), + {reply, + pause, + State1#state{backlog=true, + memtable_copy=L0Snap, + table_size=TableSize1, + ledger_sqn=MaxSQN}} + end + end; + {MinSQN, MaxSQN} -> + io:format("Mismatch of sequence number expectations with push " + ++ "having sequence numbers between ~w and ~w " + ++ "but current sequence number is ~w~n", + [MinSQN, MaxSQN, State0#state.ledger_sqn]), + {reply, refused, State0}; + empty -> + io:format("Empty request pushed to Penciller~n"), + {reply, ok, State0} + end; +handle_call({fetch, Key}, _From, State) -> + {reply, fetch(Key, State#state.manifest, State#state.memtable), State}; +handle_call(work_for_clerk, From, State) -> + {UpdState, Work} = return_work(State, From), + {reply, {Work, UpdState#state.backlog}, UpdState}; +handle_call({confirm_delete, FileName}, _From, State) -> + Reply = confirm_delete(FileName, + State#state.unreferenced_files, + State#state.registered_snapshots), + case Reply of + true -> + UF1 = lists:keydelete(FileName, 1, State#state.unreferenced_files), + {reply, true, State#state{unreferenced_files=UF1}}; + _ -> + {reply, Reply, State} + end; +handle_call(prompt_compaction, _From, State) -> + %% If there is a prompt immediately after a L0 async write event then + %% there exists the potential for the prompt to stall the database. + %% Should only accept prompts if there has been a safe wait from the + %% last L0 write event. + Proceed = case State#state.levelzero_pending of + {true, _Pid, TS} -> + TD = timer:now_diff(os:timestamp(),TS), + if + TD < ?PROMPT_WAIT_ONL0 * 1000000 -> false; + true -> true + end; + ?L0PEND_RESET -> + true + end, + if + Proceed -> + {_TableSize, State1} = checkready_pushtomem(State), + case roll_memory(State1, State1#state.memtable_maxsize) of + {ok, L0Pend, MSN, TableSize} -> + io:format("Prompted push completed~n"), + {reply, ok, State1#state{levelzero_pending=L0Pend, + table_size=TableSize, + manifest_sqn=MSN, + backlog=false}}; + {pause, Reason, Details} -> + io:format("Excess work due to - " ++ Reason, Details), + {reply, pause, State1#state{backlog=true}} + end; + true -> + {reply, ok, State#state{backlog=false}} + end; +handle_call({manifest_change, WI}, _From, State) -> + {ok, UpdState} = commit_manifest_change(WI, State), + {reply, ok, UpdState}; +handle_call(get_startup_sqn, _From, State) -> + {reply, State#state.ledger_sqn, State}; +handle_call({register_snapshot, Snapshot}, _From, State) -> + Rs = [{Snapshot, State#state.ledger_sqn}|State#state.registered_snapshots], + {reply, + {ok, + State#state.ledger_sqn, + State#state.manifest, + State#state.memtable_copy}, + State#state{registered_snapshots = Rs}}; +handle_call(close, _From, State) -> + {stop, normal, ok, State}. + +handle_cast({update_snapshotcache, Tree, SQN}, State) -> + MemTableC = cache_tree_in_memcopy(State#state.memtable_copy, Tree, SQN), + {noreply, State#state{memtable_copy=MemTableC}}; +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, State) -> + %% When a Penciller shuts down it isn't safe to try an manage the safe + %% finishing of any outstanding work. The last commmitted manifest will + %% be used. + %% + %% Level 0 files lie outside of the manifest, and so if there is no L0 + %% file present it is safe to write the current contents of memory. If + %% there is a L0 file present - then the memory can be dropped (it is + %% recoverable from the ledger, and there should not be a lot to recover + %% as presumably the ETS file has been recently flushed, hence the presence + %% of a L0 file). + %% + %% The penciller should close each file in the unreferenced files, and + %% then each file in the manifest, and cast a close on the clerk. + %% The cast may not succeed as the clerk could be synchronously calling + %% the penciller looking for a manifest commit + %% + leveled_pclerk:clerk_stop(State#state.clerk), + Dump = ets:tab2list(State#state.memtable), + case {State#state.levelzero_pending, + get_item(0, State#state.manifest, []), length(Dump)} of + {?L0PEND_RESET, [], L} when L > 0 -> + MSN = State#state.manifest_sqn + 1, + FileName = State#state.root_path + ++ "/" ++ ?FILES_FP ++ "/" + ++ integer_to_list(MSN) ++ "_0_0", + {ok, + L0Pid, + {{[], []}, _SK, _HK}} = leveled_sft:sft_new(FileName ++ ".pnd", + Dump, + [], + 0), + io:format("Dump of memory on close to filename ~s~n", [FileName]), + leveled_sft:sft_close(L0Pid), + file:rename(FileName ++ ".pnd", FileName ++ ".sft"); + {?L0PEND_RESET, [], L} when L == 0 -> + io:format("No keys to dump from memory when closing~n"); + {{true, L0Pid, _TS}, _, _} -> + leveled_sft:sft_close(L0Pid), + io:format("No opportunity to persist memory before closing " + ++ "with ~w keys discarded~n", [length(Dump)]); + _ -> + io:format("No opportunity to persist memory before closing " + ++ "with ~w keys discarded~n", [length(Dump)]) + end, + ok = close_files(0, State#state.manifest), + lists:foreach(fun({_FN, Pid, _SN}) -> leveled_sft:sft_close(Pid) end, + State#state.unreferenced_files), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%============================================================================ +%%% External functions +%%%============================================================================ + +roll_new_tree(Tree, [], HighSQN) -> + {Tree, HighSQN}; +roll_new_tree(Tree, [{SQN, KVList}|TailIncs], HighSQN) when SQN >= HighSQN -> + UpdTree = lists:foldl(fun({K, V}, TreeAcc) -> + gb_trees:enter(K, V, TreeAcc) end, + Tree, + KVList), + roll_new_tree(UpdTree, TailIncs, SQN). + + +%%%============================================================================ +%%% Internal functions +%%%============================================================================ + +start_from_file(PCLopts) -> RootPath = PCLopts#penciller_options.root_path, MaxTableSize = case PCLopts#penciller_options.max_inmemory_tablesize of undefined -> @@ -331,152 +646,9 @@ init([PCLopts]) -> ledger_sqn=MaxSQN}} end end. - - -handle_call({push_mem, DumpList}, _From, State) -> - StartWatch = os:timestamp(), - Response = case assess_sqn(DumpList) of - {MinSQN, MaxSQN} when MaxSQN >= MinSQN, - MinSQN >= State#state.ledger_sqn -> - io:format("SQN check completed in ~w microseconds~n", - [timer:now_diff(os:timestamp(),StartWatch)]), - case push_to_memory(DumpList, State) of - {ok, UpdState} -> - {reply, ok, UpdState}; - {{pause, Reason, Details}, UpdState} -> - io:format("Excess work due to - " ++ Reason, Details), - {reply, pause, UpdState#state{backlog=true, - ledger_sqn=MaxSQN}} - end; - {MinSQN, MaxSQN} -> - io:format("Mismatch of sequence number expectations with push " - ++ "having sequence numbers between ~w and ~w " - ++ "but current sequence number is ~w~n", - [MinSQN, MaxSQN, State#state.ledger_sqn]), - {reply, refused, State}; - empty -> - io:format("Empty request pushed to Penciller~n"), - {reply, ok, State} - end, - io:format("Push completed in ~w microseconds~n", - [timer:now_diff(os:timestamp(),StartWatch)]), - Response; -handle_call({fetch, Key}, _From, State) -> - {reply, fetch(Key, State#state.manifest, State#state.memtable), State}; -handle_call(work_for_clerk, From, State) -> - {UpdState, Work} = return_work(State, From), - {reply, {Work, UpdState#state.backlog}, UpdState}; -handle_call({confirm_delete, FileName}, _From, State) -> - Reply = confirm_delete(FileName, - State#state.unreferenced_files, - State#state.registered_iterators), - case Reply of - true -> - UF1 = lists:keydelete(FileName, 1, State#state.unreferenced_files), - {reply, true, State#state{unreferenced_files=UF1}}; - _ -> - {reply, Reply, State} - end; -handle_call(prompt_compaction, _From, State) -> - %% If there is a prompt immediately after a L0 async write event then - %% there exists the potential for the prompt to stall the database. - %% Should only accept prompts if there has been a safe wait from the - %% last L0 write event. - Proceed = case State#state.levelzero_pending of - {true, _Pid, TS} -> - TD = timer:now_diff(os:timestamp(),TS), - if - TD < ?PROMPT_WAIT_ONL0 * 1000000 -> false; - true -> true - end; - ?L0PEND_RESET -> - true - end, - if - Proceed -> - case push_to_memory([], State) of - {ok, UpdState} -> - {reply, ok, UpdState#state{backlog=false}}; - {{pause, Reason, Details}, UpdState} -> - io:format("Excess work due to - " ++ Reason, Details), - {reply, pause, UpdState#state{backlog=true}} - end; - true -> - {reply, ok, State#state{backlog=false}} - end; -handle_call({manifest_change, WI}, _From, State) -> - {ok, UpdState} = commit_manifest_change(WI, State), - {reply, ok, UpdState}; -handle_call(get_startup_sqn, _From, State) -> - {reply, State#state.ledger_sqn, State}; -handle_call(close, _From, State) -> - {stop, normal, ok, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, State) -> - %% When a Penciller shuts down it isn't safe to try an manage the safe - %% finishing of any outstanding work. The last commmitted manifest will - %% be used. - %% - %% Level 0 files lie outside of the manifest, and so if there is no L0 - %% file present it is safe to write the current contents of memory. If - %% there is a L0 file present - then the memory can be dropped (it is - %% recoverable from the ledger, and there should not be a lot to recover - %% as presumably the ETS file has been recently flushed, hence the presence - %% of a L0 file). - %% - %% The penciller should close each file in the unreferenced files, and - %% then each file in the manifest, and cast a close on the clerk. - %% The cast may not succeed as the clerk could be synchronously calling - %% the penciller looking for a manifest commit - %% - leveled_pclerk:clerk_stop(State#state.clerk), - Dump = ets:tab2list(State#state.memtable), - case {State#state.levelzero_pending, - get_item(0, State#state.manifest, []), length(Dump)} of - {?L0PEND_RESET, [], L} when L > 0 -> - MSN = State#state.manifest_sqn + 1, - FileName = State#state.root_path - ++ "/" ++ ?FILES_FP ++ "/" - ++ integer_to_list(MSN) ++ "_0_0", - {ok, - L0Pid, - {{[], []}, _SK, _HK}} = leveled_sft:sft_new(FileName ++ ".pnd", - Dump, - [], - 0), - io:format("Dump of memory on close to filename ~s~n", [FileName]), - leveled_sft:sft_close(L0Pid), - file:rename(FileName ++ ".pnd", FileName ++ ".sft"); - {?L0PEND_RESET, [], L} when L == 0 -> - io:format("No keys to dump from memory when closing~n"); - {{true, L0Pid, _TS}, _, _} -> - leveled_sft:sft_close(L0Pid), - io:format("No opportunity to persist memory before closing " - ++ "with ~w keys discarded~n", [length(Dump)]); - _ -> - io:format("No opportunity to persist memory before closing " - ++ "with ~w keys discarded~n", [length(Dump)]) - end, - ok = close_files(0, State#state.manifest), - lists:foreach(fun({_FN, Pid, _SN}) -> leveled_sft:sft_close(Pid) end, - State#state.unreferenced_files), - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. -%%%============================================================================ -%%% Internal functions -%%%============================================================================ - -push_to_memory(DumpList, State) -> +checkready_pushtomem(State) -> {TableSize, UpdState} = case State#state.levelzero_pending of {true, Pid, _TS} -> %% Need to handle error scenarios? @@ -493,60 +665,63 @@ push_to_memory(DumpList, State) -> State#state.manifest, {0, [ManifestEntry]}), levelzero_pending=?L0PEND_RESET, - levelzero_snapshot=[]}}; + memtable_copy=#l0snapshot{}}}; ?L0PEND_RESET -> {State#state.table_size, State} end, %% Prompt clerk to ask about work - do this for every push_mem ok = leveled_pclerk:clerk_prompt(UpdState#state.clerk, penciller), - - MemoryInsertion = do_push_to_mem(DumpList, - TableSize, - UpdState#state.memtable, - UpdState#state.levelzero_snapshot, - UpdState#state.memtable_maxsize), - - case MemoryInsertion of - {twist, ApproxTableSize, UpdSnapshot} -> - {ok, UpdState#state{table_size=ApproxTableSize, - levelzero_snapshot=UpdSnapshot}}; - {roll, ApproxTableSize, UpdSnapshot} -> - L0 = get_item(0, UpdState#state.manifest, []), - case {L0, manifest_locked(UpdState)} of + {TableSize, UpdState}. + +quickcheck_pushtomem(DumpList, TableSize, MaxSize) -> + case TableSize + length(DumpList) of + ApproxTableSize when ApproxTableSize > MaxSize -> + {maybe_roll, ApproxTableSize}; + ApproxTableSize -> + io:format("Table size is approximately ~w~n", [ApproxTableSize]), + {twist, ApproxTableSize} + end. + +do_pushtomem(DumpList, MemTable, Snapshot, MaxSQN) -> + SW = os:timestamp(), + UpdSnapshot = add_increment_to_memcopy(Snapshot, MaxSQN, DumpList), + ets:insert(MemTable, DumpList), + io:format("Push into memory timed at ~w microseconds~n", + [timer:now_diff(os:timestamp(), SW)]), + UpdSnapshot. + +roll_memory(State, MaxSize) -> + case ets:info(State#state.memtable, size) of + Size when Size > MaxSize -> + L0 = get_item(0, State#state.manifest, []), + case {L0, manifest_locked(State)} of {[], false} -> - MSN = UpdState#state.manifest_sqn + 1, - FileName = UpdState#state.root_path + MSN = State#state.manifest_sqn + 1, + FileName = State#state.root_path ++ "/" ++ ?FILES_FP ++ "/" ++ integer_to_list(MSN) ++ "_0_0", Opts = #sft_options{wait=false}, {ok, L0Pid} = leveled_sft:sft_new(FileName, - UpdState#state.memtable, + State#state.memtable, [], 0, Opts), - {ok, - UpdState#state{levelzero_pending={true, - L0Pid, - os:timestamp()}, - table_size=ApproxTableSize, - manifest_sqn=MSN, - levelzero_snapshot=UpdSnapshot}}; + {ok, {true, L0Pid, os:timestamp()}, MSN, Size}; {[], true} -> - {{pause, - "L0 file write blocked by change at sqn=~w~n", - [UpdState#state.manifest_sqn]}, - UpdState#state{table_size=ApproxTableSize, - levelzero_snapshot=UpdSnapshot}}; + {pause, + "L0 file write blocked by change at sqn=~w~n", + [State#state.manifest_sqn]}; _ -> - {{pause, + {pause, "L0 file write blocked by L0 file in manifest~n", - []}, - UpdState#state{table_size=ApproxTableSize, - levelzero_snapshot=UpdSnapshot}} - end + []} + end; + Size -> + {ok, ?L0PEND_RESET, State#state.manifest_sqn, Size} end. + fetch(Key, Manifest, TID) -> case ets:lookup(TID, Key) of [Object] -> @@ -581,27 +756,6 @@ fetch(Key, Manifest, Level, FetchFun) -> end end. -do_push_to_mem(DumpList, TableSize, MemTable, Snapshot, MaxSize) -> - SW = os:timestamp(), - UpdSnapshot = lists:append(Snapshot, DumpList), - ets:insert(MemTable, DumpList), - io:format("Push into memory timed at ~w microseconds~n", - [timer:now_diff(os:timestamp(), SW)]), - case TableSize + length(DumpList) of - ApproxTableSize when ApproxTableSize > MaxSize -> - case ets:info(MemTable, size) of - ActTableSize when ActTableSize > MaxSize -> - {roll, ActTableSize, UpdSnapshot}; - ActTableSize -> - io:format("Table size is actually ~w~n", [ActTableSize]), - {twist, ActTableSize, UpdSnapshot} - end; - ApproxTableSize -> - io:format("Table size is approximately ~w~n", [ApproxTableSize]), - {twist, ApproxTableSize, UpdSnapshot} - end. - - %% Manifest lock - don't have two changes to the manifest happening %% concurrently @@ -675,6 +829,32 @@ return_work(State, From) -> {State, none} end. +%% Update the memtable copy if the tree created advances the SQN +cache_tree_in_memcopy(MemCopy, Tree, SQN) -> + case MemCopy#l0snapshot.ledger_sqn of + CurrentSQN when SQN > CurrentSQN -> + % Discard any merged increments + io:format("Updating cache with new tree at SQN=~w~n", [SQN]), + Incs = lists:foldl(fun({PushSQN, PushL}, Acc) -> + if + SQN >= PushSQN -> + Acc; + true -> + Acc ++ {PushSQN, PushL} + end end, + [], + MemCopy#l0snapshot.increments), + #l0snapshot{ledger_sqn = SQN, + increments = Incs, + tree = Tree}; + _ -> + MemCopy + end. + +add_increment_to_memcopy(MemCopy, SQN, KVList) -> + Incs = MemCopy#l0snapshot.increments ++ [{SQN, KVList}], + MemCopy#l0snapshot{increments=Incs}. + close_files(?MAX_LEVELS - 1, _Manifest) -> ok; @@ -819,14 +999,14 @@ update_deletions([ClearedFile|Tail], MSN, UnreferencedFiles) -> ClearedFile#manifest_entry.owner, MSN}])). -confirm_delete(Filename, UnreferencedFiles, RegisteredIterators) -> +confirm_delete(Filename, UnreferencedFiles, RegisteredSnapshots) -> case lists:keyfind(Filename, 1, UnreferencedFiles) of false -> false; {Filename, _Pid, MSN} -> LowSQN = lists:foldl(fun({_, SQN}, MinSQN) -> min(SQN, MinSQN) end, infinity, - RegisteredIterators), + RegisteredSnapshots), if MSN >= LowSQN -> false; @@ -984,4 +1164,23 @@ simple_server_test() -> ok = pcl_close(PCLr), clean_testdir(RootPath). +memcopy_test() -> + KVL1 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), + "Value" ++ integer_to_list(X) ++ "A"} end, + lists:seq(1, 1000)), + KVL2 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), + "Value" ++ integer_to_list(X) ++ "B"} end, + lists:seq(1001, 2000)), + KVL3 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), + "Value" ++ integer_to_list(X) ++ "C"} end, + lists:seq(1, 1000)), + MemCopy0 = #l0snapshot{}, + MemCopy1 = add_increment_to_memcopy(MemCopy0, 1000, KVL1), + MemCopy2 = add_increment_to_memcopy(MemCopy1, 2000, KVL2), + MemCopy3 = add_increment_to_memcopy(MemCopy2, 3000, KVL3), + {Tree1, HighSQN1} = roll_new_tree(gb_trees:empty(), MemCopy3#l0snapshot.increments, 0), + Size1 = gb_trees:size(Tree1), + ?assertMatch(2000, Size1), + ?assertMatch(3000, HighSQN1). + -endif. \ No newline at end of file From c64d67d9fbcc07063257492983d851adc5423a3d Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 23 Sep 2016 18:50:29 +0100 Subject: [PATCH 047/167] Snapshot Work - Interim Commit Some initial work to get snapshots going. Changes required, as need to snapshot through the Bookie to ensure that there is no race between extracting the Bookie's in-memory view and the Penciller's view if a push_to_mem has occurred inbetween. A lot still outstanding, especially around Inker snapshots, and handling timeouts --- include/leveled.hrl | 11 +++-- src/leveled_bookie.erl | 21 +++++++++ src/leveled_iclerk.erl | 13 ------ src/leveled_inker.erl | 94 ++++++++++++++++++++++---------------- src/leveled_penciller.erl | 96 +++++++++++++++++++++++++++++++-------- 5 files changed, 160 insertions(+), 75 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 0f929cc..232aed7 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -26,12 +26,17 @@ -record(inker_options, {cdb_max_size :: integer(), root_path :: string(), - cdb_options :: #cdb_options{}}). + cdb_options :: #cdb_options{}, + start_snapshot = false :: boolean, + source_inker :: pid(), + requestor :: pid()}). -record(penciller_options, {root_path :: string(), - penciller :: pid(), - max_inmemory_tablesize :: integer()}). + max_inmemory_tablesize :: integer(), + start_snapshot = false :: boolean(), + source_penciller :: pid(), + requestor :: pid()}). -record(bookie_options, {root_path :: string(), diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 0ae31e8..948ea66 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -138,6 +138,8 @@ book_riakput/3, book_riakget/3, book_riakhead/3, + book_snapshotstore/3, + book_snapshotledger/3, book_close/1, strip_to_keyonly/1, strip_to_keyseqonly/1, @@ -180,6 +182,12 @@ book_riakhead(Pid, Bucket, Key) -> PrimaryKey = {o, Bucket, Key}, gen_server:call(Pid, {head, PrimaryKey}, infinity). +book_snapshotstore(Pid, Requestor, Timeout) -> + gen_server:call(Pid, {snapshot, Requestor, store, Timeout}, infinity). + +book_snapshotledger(Pid, Requestor, Timeout) -> + gen_server:call(Pid, {snapshot, Requestor, ledger, Timeout}, infinity). + book_close(Pid) -> gen_server:call(Pid, close, infinity). @@ -268,6 +276,19 @@ handle_call({head, Key}, _From, State) -> {reply, {ok, OMD}, State} end end; +handle_call({snapshot, Requestor, SnapType, _Timeout}, _From, State) -> + PCLopts = #penciller_options{start_snapshot=true, + source_penciller=State#state.penciller, + requestor=Requestor}, + {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), + case SnapType of + store -> + InkerOpts = #inker_options{}, + {ok, JournalSnapshot} = leveled_inker:ink_start(InkerOpts), + {reply, {ok, LedgerSnapshot, JournalSnapshot}, State}; + ledger -> + {reply, {ok, LedgerSnapshot, null}, State} + end; handle_call(close, _From, State) -> {stop, normal, ok, State}. diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 335a649..1fe049d 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -12,7 +12,6 @@ handle_info/2, terminate/2, clerk_new/1, - clerk_compact/3, clerk_remove/2, clerk_stop/1, code_change/3]). @@ -33,10 +32,6 @@ clerk_new(Owner) -> ok = gen_server:call(Pid, {register, Owner}, infinity), {ok, Pid}. -clerk_compact(Pid, InkerManifest, Penciller) -> - gen_server:cast(Pid, {compact, InkerManifest, Penciller}), - ok. - clerk_remove(Pid, Removals) -> gen_server:cast(Pid, {remove, Removals}), ok. @@ -54,9 +49,6 @@ init([]) -> handle_call({register, Owner}, _From, State) -> {reply, ok, State#state{owner=Owner}}. -handle_cast({compact, InkerManifest, Penciller, Timeout}, State) -> - ok = journal_compact(InkerManifest, Penciller, Timeout, State#state.owner), - {noreply, State}; handle_cast({remove, _Removals}, State) -> {noreply, State}; handle_cast(stop, State) -> @@ -76,11 +68,6 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================ -journal_compact(_InkerManifest, _Penciller, _Timeout, _Owner) -> - ok. - -check_all_files(_InkerManifest) -> - ok. check_single_file(CDB, _PencilSnapshot, SampleSize, BatchSize) -> PositionList = leveled_cdb:cdb_getpositions(CDB, SampleSize), diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 9d150f4..c5ddf94 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -103,7 +103,7 @@ ink_get/3, ink_fetch/3, ink_loadpcl/4, - ink_snap/1, + ink_registersnapshot/2, ink_close/1, ink_print_manifest/1, build_dummy_journal/0, @@ -146,8 +146,8 @@ ink_get(Pid, PrimaryKey, SQN) -> ink_fetch(Pid, PrimaryKey, SQN) -> gen_server:call(Pid, {fetch, PrimaryKey, SQN}, infinity). -ink_snap(Pid) -> - gen_server:call(Pid, snapshot, infinity). +ink_registersnapshot(Pid, Requestor) -> + gen_server:call(Pid, {snapshot, Requestor}, infinity). ink_close(Pid) -> gen_server:call(Pid, close, infinity). @@ -163,39 +163,20 @@ ink_print_manifest(Pid) -> %%%============================================================================ init([InkerOpts]) -> - RootPath = InkerOpts#inker_options.root_path, - CDBopts = InkerOpts#inker_options.cdb_options, - JournalFP = filepath(RootPath, journal_dir), - {ok, JournalFilenames} = case filelib:is_dir(JournalFP) of - true -> - file:list_dir(JournalFP); - false -> - filelib:ensure_dir(JournalFP), - {ok, []} - end, - ManifestFP = filepath(RootPath, manifest_dir), - {ok, ManifestFilenames} = case filelib:is_dir(ManifestFP) of - true -> - file:list_dir(ManifestFP); - false -> - filelib:ensure_dir(ManifestFP), - {ok, []} - end, - {Manifest, - {ActiveJournal, LowActiveSQN}, - JournalSQN, - ManifestSQN} = build_manifest(ManifestFilenames, - JournalFilenames, - fun simple_manifest_reader/2, - RootPath, - CDBopts), - {ok, #state{manifest = lists:reverse(lists:keysort(1, Manifest)), - manifest_sqn = ManifestSQN, - journal_sqn = JournalSQN, - active_journaldb = ActiveJournal, - active_journaldb_sqn = LowActiveSQN, - root_path = RootPath, - cdb_options = CDBopts}}. + case {InkerOpts#inker_options.root_path, + InkerOpts#inker_options.start_snapshot} of + {undefined, true} -> + SrcInker = InkerOpts#inker_options.source_inker, + Requestor = InkerOpts#inker_options.requestor, + {ok, + {ActiveJournalDB, + Manifest}} = ink_registersnapshot(SrcInker, Requestor), + {ok, #state{manifest=Manifest, + active_journaldb=ActiveJournalDB}}; + %% Need to do something about timeout + {_RootPath, false} -> + start_from_file(InkerOpts) + end. handle_call({put, Key, Object, KeyChanges}, From, State) -> @@ -240,12 +221,11 @@ handle_call({load_pcl, StartSQN, FilterFun, Penciller}, _From, State) -> State#state.active_journaldb}], Reply = load_from_sequence(StartSQN, FilterFun, Penciller, Manifest), {reply, Reply, State}; -handle_call(snapshot, _From , State) -> +handle_call({register_snapshot, _Requestor}, _From , State) -> %% TODO: Not yet implemented registration of snapshot %% Should return manifest and register the snapshot {reply, {State#state.manifest, - State#state.active_journaldb, - State#state.active_journaldb_sqn}, + State#state.active_journaldb}, State}; handle_call(print_manifest, _From, State) -> manifest_printer(State#state.manifest), @@ -275,6 +255,42 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================ +start_from_file(InkerOpts) -> + RootPath = InkerOpts#inker_options.root_path, + CDBopts = InkerOpts#inker_options.cdb_options, + JournalFP = filepath(RootPath, journal_dir), + {ok, JournalFilenames} = case filelib:is_dir(JournalFP) of + true -> + file:list_dir(JournalFP); + false -> + filelib:ensure_dir(JournalFP), + {ok, []} + end, + ManifestFP = filepath(RootPath, manifest_dir), + {ok, ManifestFilenames} = case filelib:is_dir(ManifestFP) of + true -> + file:list_dir(ManifestFP); + false -> + filelib:ensure_dir(ManifestFP), + {ok, []} + end, + {Manifest, + {ActiveJournal, LowActiveSQN}, + JournalSQN, + ManifestSQN} = build_manifest(ManifestFilenames, + JournalFilenames, + fun simple_manifest_reader/2, + RootPath, + CDBopts), + {ok, #state{manifest = lists:reverse(lists:keysort(1, Manifest)), + manifest_sqn = ManifestSQN, + journal_sqn = JournalSQN, + active_journaldb = ActiveJournal, + active_journaldb_sqn = LowActiveSQN, + root_path = RootPath, + cdb_options = CDBopts}}. + + put_object(PrimaryKey, Object, KeyChanges, State) -> NewSQN = State#state.journal_sqn + 1, %% TODO: The term goes through a double binary_to_term conversion diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 225431f..53c48fe 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -235,6 +235,7 @@ pcl_close/1, pcl_registersnapshot/2, pcl_updatesnapshotcache/3, + pcl_loadsnapshot/2, pcl_getstartupsequencenumber/1, roll_new_tree/3, clean_testdir/1]). @@ -254,7 +255,7 @@ -define(PROMPT_WAIT_ONL0, 5). -define(L0PEND_RESET, {false, null, null}). --record(l0snapshot, {increments = [] :: list, +-record(l0snapshot, {increments = [] :: list(), tree = gb_trees:empty() :: gb_trees:tree(), ledger_sqn = 0 :: integer()}). @@ -269,9 +270,13 @@ clerk :: pid(), levelzero_pending = ?L0PEND_RESET :: tuple(), memtable_copy = #l0snapshot{} :: #l0snapshot{}, + levelzero_snapshot = gb_trees:empty() :: gb_trees:tree(), memtable, backlog = false :: boolean(), - memtable_maxsize :: integer()}). + memtable_maxsize :: integer(), + is_snapshot = false :: boolean(), + snapshot_fully_loaded = false :: boolean(), + source_penciller :: pid()}). @@ -313,6 +318,9 @@ pcl_registersnapshot(Pid, Snapshot) -> pcl_updatesnapshotcache(Pid, Tree, SQN) -> gen_server:cast(Pid, {update_snapshotcache, Tree, SQN}). +pcl_loadsnapshot(Pid, Increment) -> + gen_server:call(Pid, {load_snapshot, Increment}, infinity). + pcl_close(Pid) -> gen_server:call(Pid, close). @@ -322,10 +330,21 @@ pcl_close(Pid) -> %%%============================================================================ init([PCLopts]) -> - case PCLopts#penciller_options.root_path of - undefined -> - {ok, #state{}}; - _RootPath -> + case {PCLopts#penciller_options.root_path, + PCLopts#penciller_options.start_snapshot} of + {undefined, true} -> + SrcPenciller = PCLopts#penciller_options.source_penciller, + {ok, {LedgerSQN, + MemTableCopy, + Manifest}} = pcl_registersnapshot(SrcPenciller, self()), + + {ok, #state{memtable_copy=MemTableCopy, + is_snapshot=true, + source_penciller=SrcPenciller, + manifest=Manifest, + ledger_sqn=LedgerSQN}}; + %% Need to do something about timeout + {_RootPath, false} -> start_from_file(PCLopts) end. @@ -474,6 +493,22 @@ handle_call({register_snapshot, Snapshot}, _From, State) -> State#state.manifest, State#state.memtable_copy}, State#state{registered_snapshots = Rs}}; +handle_call({load_snapshot, Increment}, _From, State) -> + MemTableCopy = State#state.memtable_copy, + {Tree0, TreeSQN0} = roll_new_tree(MemTableCopy#l0snapshot.tree, + MemTableCopy#l0snapshot.increments, + MemTableCopy#l0snapshot.ledger_sqn), + if + TreeSQN0 > MemTableCopy#l0snapshot.ledger_sqn -> + pcl_updatesnapshotcache(State#state.source_penciller, + Tree0, + TreeSQN0) + end, + {Tree1, TreeSQN1} = roll_new_tree(Tree0, [Increment], TreeSQN0), + io:format("Snapshot loaded to start at SQN~w~n", [TreeSQN1]), + {reply, ok, State#state{levelzero_snapshot=Tree1, + ledger_sqn=TreeSQN1, + snapshot_fully_loaded=true}}; handle_call(close, _From, State) -> {stop, normal, ok, State}. @@ -539,19 +574,6 @@ terminate(_Reason, State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -%%%============================================================================ -%%% External functions -%%%============================================================================ - -roll_new_tree(Tree, [], HighSQN) -> - {Tree, HighSQN}; -roll_new_tree(Tree, [{SQN, KVList}|TailIncs], HighSQN) when SQN >= HighSQN -> - UpdTree = lists:foldl(fun({K, V}, TreeAcc) -> - gb_trees:enter(K, V, TreeAcc) end, - Tree, - KVList), - roll_new_tree(UpdTree, TailIncs, SQN). - %%%============================================================================ %%% Internal functions @@ -829,6 +851,19 @@ return_work(State, From) -> {State, none} end. + +%% This takes the three parts of a memtable copy - the increments, the tree +%% and the SQN at which the tree was formed, and outputs a new tree + +roll_new_tree(Tree, [], HighSQN) -> + {Tree, HighSQN}; +roll_new_tree(Tree, [{SQN, KVList}|TailIncs], HighSQN) when SQN >= HighSQN -> + UpdTree = lists:foldl(fun({K, V}, TreeAcc) -> + gb_trees:enter(K, V, TreeAcc) end, + Tree, + KVList), + roll_new_tree(UpdTree, TailIncs, SQN). + %% Update the memtable copy if the tree created advances the SQN cache_tree_in_memcopy(MemCopy, Tree, SQN) -> case MemCopy#l0snapshot.ledger_sqn of @@ -855,7 +890,6 @@ add_increment_to_memcopy(MemCopy, SQN, KVList) -> Incs = MemCopy#l0snapshot.increments ++ [{SQN, KVList}], MemCopy#l0snapshot{increments=Incs}. - close_files(?MAX_LEVELS - 1, _Manifest) -> ok; close_files(Level, Manifest) -> @@ -1182,5 +1216,27 @@ memcopy_test() -> Size1 = gb_trees:size(Tree1), ?assertMatch(2000, Size1), ?assertMatch(3000, HighSQN1). + +memcopy_updatecache_test() -> + KVL1 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), + "Value" ++ integer_to_list(X) ++ "A"} end, + lists:seq(1, 1000)), + KVL2 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), + "Value" ++ integer_to_list(X) ++ "B"} end, + lists:seq(1001, 2000)), + KVL3 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), + "Value" ++ integer_to_list(X) ++ "C"} end, + lists:seq(1, 1000)), + MemCopy0 = #l0snapshot{}, + MemCopy1 = add_increment_to_memcopy(MemCopy0, 1000, KVL1), + MemCopy2 = add_increment_to_memcopy(MemCopy1, 2000, KVL2), + MemCopy3 = add_increment_to_memcopy(MemCopy2, 3000, KVL3), + ?assertMatch(0, MemCopy3#l0snapshot.ledger_sqn), + {Tree1, HighSQN1} = roll_new_tree(gb_trees:empty(), MemCopy3#l0snapshot.increments, 0), + MemCopy4 = cache_tree_in_memcopy(MemCopy3, Tree1, HighSQN1), + ?assertMatch(0, length(MemCopy4#l0snapshot.increments)), + Size2 = gb_trees:size(MemCopy4#l0snapshot.tree), + ?assertMatch(2000, Size2), + ?assertMatch(3000, MemCopy4#l0snapshot.ledger_sqn). -endif. \ No newline at end of file From e2bb09b873da1c720f3c7df4ef1d5bee5913a546 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 26 Sep 2016 10:55:08 +0100 Subject: [PATCH 048/167] Snapshot testing Work to test the checking of sequence numbers in snapshots as required by the inkers clerk to calculate the percentage of a file which is compactable --- src/leveled_iclerk.erl | 65 +++++-- src/leveled_inker.erl | 23 ++- src/leveled_penciller.erl | 354 ++++++++++++++++++++++++++------------ 3 files changed, 311 insertions(+), 131 deletions(-) diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 1fe049d..fee2417 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -12,16 +12,19 @@ handle_info/2, terminate/2, clerk_new/1, + clerk_compact/5, clerk_remove/2, clerk_stop/1, code_change/3]). -include_lib("eunit/include/eunit.hrl"). --define(BATCH_SIZE, 16) +-define(SAMPLE_SIZE, 200). +-define(BATCH_SIZE, 16). -define(BATCHES_TO_CHECK, 8). --record(state, {owner :: pid()}). +-record(state, {owner :: pid(), + penciller_snapshot :: pid()}). %%%============================================================================ %%% API @@ -36,6 +39,9 @@ clerk_remove(Pid, Removals) -> gen_server:cast(Pid, {remove, Removals}), ok. +clerk_compact(Pid, Manifest, ManifestSQN, Penciller, Timeout) -> + gen_server:cast(Pid, {compact, Manifest, ManifestSQN, Penciller, Timeout}). + clerk_stop(Pid) -> gen_server:cast(Pid, stop). @@ -49,6 +55,15 @@ init([]) -> handle_call({register, Owner}, _From, State) -> {reply, ok, State#state{owner=Owner}}. +handle_cast({compact, Manifest, _ManifestSQN, Penciller, _Timeout}, State) -> + PclOpts = #penciller_options{start_snapshot = true, + source_penciller = Penciller, + requestor = self()}, + PclSnap = leveled_penciller:pcl_start(PclOpts), + ok = leveled_penciller:pcl_loadsnapshot(PclSnap, []), + _CandidateList = scan_all_files(Manifest, PclSnap), + %% TODO - Lots + {noreply, State}; handle_cast({remove, _Removals}, State) -> {noreply, State}; handle_cast(stop, State) -> @@ -69,20 +84,38 @@ code_change(_OldVsn, State, _Extra) -> %%%============================================================================ -check_single_file(CDB, _PencilSnapshot, SampleSize, BatchSize) -> +check_single_file(CDB, PclSnap, SampleSize, BatchSize) -> PositionList = leveled_cdb:cdb_getpositions(CDB, SampleSize), KeySizeList = fetch_inbatches(PositionList, BatchSize, CDB, []), - KeySizeList. - %% TODO: - %% Need to check the penciller snapshot to see if these keys are at the - %% right sequence number - %% - %% The calculate the proportion (by value size) of the CDB which is at the - %% wrong sequence number to help determine eligibility for compaction - %% - %% BIG TODO: - %% Need to snapshot a penciller - + R0 = lists:foldl(fun(KS, {ActSize, RplSize}) -> + {{PK, SQN}, Size} = KS, + Chk = leveled_pcl:pcl_checksequencenumber(PclSnap, + PK, + SQN), + case Chk of + true -> + {ActSize + Size, RplSize}; + false -> + {ActSize, RplSize + Size} + end end, + {0, 0}, + KeySizeList), + {ActiveSize, ReplacedSize} = R0, + 100 * (ActiveSize / (ActiveSize + ReplacedSize)). + +scan_all_files(Manifest, Penciller) -> + scan_all_files(Manifest, Penciller, []). + +scan_all_files([], _Penciller, CandidateList) -> + CandidateList; +scan_all_files([{LowSQN, FN, JournalP}|Tail], Penciller, CandidateList) -> + CompactPerc = check_single_file(JournalP, + Penciller, + ?SAMPLE_SIZE, + ?BATCH_SIZE), + scan_all_files(Tail, Penciller, CandidateList ++ + [{LowSQN, FN, JournalP, CompactPerc}]). + fetch_inbatches([], _BatchSize, _CDB, CheckedList) -> CheckedList; fetch_inbatches(PositionList, BatchSize, CDB, CheckedList) -> @@ -90,8 +123,8 @@ fetch_inbatches(PositionList, BatchSize, CDB, CheckedList) -> KL_List = leveled_cdb:direct_fetch(CDB, Batch, key_size), fetch_inbatches(Tail, BatchSize, CDB, CheckedList ++ KL_List). -window_closed(_Timeout) -> - true. + + %%%============================================================================ diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index c5ddf94..b7a54a6 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -104,6 +104,7 @@ ink_fetch/3, ink_loadpcl/4, ink_registersnapshot/2, + ink_compactjournal/3, ink_close/1, ink_print_manifest/1, build_dummy_journal/0, @@ -126,8 +127,10 @@ active_journaldb :: pid(), active_journaldb_sqn :: integer(), removed_journaldbs = [] :: list(), + registered_snapshots = [] :: list(), root_path :: string(), - cdb_options :: #cdb_options{}}). + cdb_options :: #cdb_options{}, + clerk :: pid()}). %%%============================================================================ @@ -155,6 +158,9 @@ ink_close(Pid) -> ink_loadpcl(Pid, MinSQN, FilterFun, Penciller) -> gen_server:call(Pid, {load_pcl, MinSQN, FilterFun, Penciller}, infinity). +ink_compactjournal(Pid, Penciller, Timeout) -> + gen_server:call(Pid, {compact_journal, Penciller, Timeout}, infinty). + ink_print_manifest(Pid) -> gen_server:call(Pid, print_manifest, infinity). @@ -221,15 +227,18 @@ handle_call({load_pcl, StartSQN, FilterFun, Penciller}, _From, State) -> State#state.active_journaldb}], Reply = load_from_sequence(StartSQN, FilterFun, Penciller, Manifest), {reply, Reply, State}; -handle_call({register_snapshot, _Requestor}, _From , State) -> - %% TODO: Not yet implemented registration of snapshot - %% Should return manifest and register the snapshot +handle_call({register_snapshot, Requestor}, _From , State) -> + Rs = [{Requestor, + State#state.manifest_sqn}|State#state.registered_snapshots], {reply, {State#state.manifest, State#state.active_journaldb}, - State}; + State#state{registered_snapshots=Rs}}; handle_call(print_manifest, _From, State) -> manifest_printer(State#state.manifest), {reply, ok, State}; +handle_call({compact_journal, Penciller, Timeout}, _From, State) -> + leveled_iclerk:clerk_compact(Penciller, Timeout), + {reply, ok, State}; handle_call(close, _From, State) -> {stop, normal, ok, State}. @@ -256,6 +265,7 @@ code_change(_OldVsn, State, _Extra) -> %%%============================================================================ start_from_file(InkerOpts) -> + {ok, Clerk} = leveled_iclerk:clerk_new(self()), RootPath = InkerOpts#inker_options.root_path, CDBopts = InkerOpts#inker_options.cdb_options, JournalFP = filepath(RootPath, journal_dir), @@ -288,7 +298,8 @@ start_from_file(InkerOpts) -> active_journaldb = ActiveJournal, active_journaldb_sqn = LowActiveSQN, root_path = RootPath, - cdb_options = CDBopts}}. + cdb_options = CDBopts, + clerk = Clerk}}. put_object(PrimaryKey, Object, KeyChanges, State) -> diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 53c48fe..9747b9a 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -228,6 +228,7 @@ pcl_quickstart/1, pcl_pushmem/2, pcl_fetch/2, + pcl_checksequencenumber/3, pcl_workforclerk/1, pcl_requestmanifestchange/2, pcl_confirmdelete/2, @@ -297,6 +298,9 @@ pcl_pushmem(Pid, DumpList) -> pcl_fetch(Pid, Key) -> gen_server:call(Pid, {fetch, Key}, infinity). +pcl_checksequencenumber(Pid, Key, SQN) -> + gen_server:call(Pid, {check_sqn, Key, SQN}, infinity). + pcl_workforclerk(Pid) -> gen_server:call(Pid, work_for_clerk, infinity). @@ -334,9 +338,10 @@ init([PCLopts]) -> PCLopts#penciller_options.start_snapshot} of {undefined, true} -> SrcPenciller = PCLopts#penciller_options.source_penciller, - {ok, {LedgerSQN, - MemTableCopy, - Manifest}} = pcl_registersnapshot(SrcPenciller, self()), + {ok, + LedgerSQN, + Manifest, + MemTableCopy} = pcl_registersnapshot(SrcPenciller, self()), {ok, #state{memtable_copy=MemTableCopy, is_snapshot=true, @@ -349,7 +354,175 @@ init([PCLopts]) -> end. -handle_call({push_mem, DumpList}, From, State0) -> +handle_call({push_mem, DumpList}, From, State) -> + if + State#state.is_snapshot == true -> + {reply, bad_request, State}; + true -> + writer_call({push_mem, DumpList}, From, State) + end; +handle_call({confirm_delete, FileName}, _From, State) -> + if + State#state.is_snapshot == true -> + {reply, bad_request, State}; + true -> + writer_call({confirm_delete, FileName}, _From, State) + end; +handle_call(prompt_compaction, _From, State) -> + if + State#state.is_snapshot == true -> + {reply, bad_request, State}; + true -> + writer_call(prompt_compaction, _From, State) + end; +handle_call({manifest_change, WI}, _From, State) -> + if + State#state.is_snapshot == true -> + {reply, bad_request, State}; + true -> + writer_call({manifest_change, WI}, _From, State) + end; +handle_call({check_sqn, Key, SQN}, _From, State) -> + Obj = if + State#state.is_snapshot == true -> + fetch_snap(Key, + State#state.manifest, + State#state.levelzero_snapshot); + true -> + fetch(Key, + State#state.manifest, + State#state.memtable) + end, + Reply = case Obj of + not_present -> + false; + Obj -> + SQNToCompare = leveled_bookie:strip_to_seqonly(Obj), + if + SQNToCompare > SQN -> + false; + true -> + true + end + end, + {reply, Reply, State}; +handle_call({fetch, Key}, _From, State) -> + Reply = if + State#state.is_snapshot == true -> + fetch_snap(Key, + State#state.manifest, + State#state.levelzero_snapshot); + true -> + fetch(Key, + State#state.manifest, + State#state.memtable) + end, + {reply, Reply, State}; +handle_call(work_for_clerk, From, State) -> + {UpdState, Work} = return_work(State, From), + {reply, {Work, UpdState#state.backlog}, UpdState}; +handle_call(get_startup_sqn, _From, State) -> + {reply, State#state.ledger_sqn, State}; +handle_call({register_snapshot, Snapshot}, _From, State) -> + Rs = [{Snapshot, State#state.ledger_sqn}|State#state.registered_snapshots], + {reply, + {ok, + State#state.ledger_sqn, + State#state.manifest, + State#state.memtable_copy}, + State#state{registered_snapshots = Rs}}; +handle_call({load_snapshot, Increment}, _From, State) -> + MemTableCopy = State#state.memtable_copy, + {Tree0, TreeSQN0} = roll_new_tree(MemTableCopy#l0snapshot.tree, + MemTableCopy#l0snapshot.increments, + MemTableCopy#l0snapshot.ledger_sqn), + if + TreeSQN0 > MemTableCopy#l0snapshot.ledger_sqn -> + pcl_updatesnapshotcache(State#state.source_penciller, + Tree0, + TreeSQN0) + end, + {Tree1, TreeSQN1} = roll_new_tree(Tree0, [Increment], TreeSQN0), + io:format("Snapshot loaded to start at SQN~w~n", [TreeSQN1]), + {reply, ok, State#state{levelzero_snapshot=Tree1, + ledger_sqn=TreeSQN1, + snapshot_fully_loaded=true}}; +handle_call(close, _From, State) -> + {stop, normal, ok, State}. + +handle_cast({update_snapshotcache, Tree, SQN}, State) -> + MemTableC = cache_tree_in_memcopy(State#state.memtable_copy, Tree, SQN), + {noreply, State#state{memtable_copy=MemTableC}}; +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, State) -> + %% When a Penciller shuts down it isn't safe to try an manage the safe + %% finishing of any outstanding work. The last commmitted manifest will + %% be used. + %% + %% Level 0 files lie outside of the manifest, and so if there is no L0 + %% file present it is safe to write the current contents of memory. If + %% there is a L0 file present - then the memory can be dropped (it is + %% recoverable from the ledger, and there should not be a lot to recover + %% as presumably the ETS file has been recently flushed, hence the presence + %% of a L0 file). + %% + %% The penciller should close each file in the unreferenced files, and + %% then each file in the manifest, and cast a close on the clerk. + %% The cast may not succeed as the clerk could be synchronously calling + %% the penciller looking for a manifest commit + %% + if + State#state.is_snapshot == true -> + ok; + true -> + leveled_pclerk:clerk_stop(State#state.clerk), + Dump = ets:tab2list(State#state.memtable), + case {State#state.levelzero_pending, + get_item(0, State#state.manifest, []), length(Dump)} of + {?L0PEND_RESET, [], L} when L > 0 -> + MSN = State#state.manifest_sqn + 1, + FileName = State#state.root_path + ++ "/" ++ ?FILES_FP ++ "/" + ++ integer_to_list(MSN) ++ "_0_0", + NewSFT = leveled_sft:sft_new(FileName ++ ".pnd", + Dump, + [], + 0), + {ok, L0Pid, {{[], []}, _SK, _HK}} = NewSFT, + io:format("Dump of memory on close to filename ~s~n", + [FileName]), + leveled_sft:sft_close(L0Pid), + file:rename(FileName ++ ".pnd", FileName ++ ".sft"); + {?L0PEND_RESET, [], L} when L == 0 -> + io:format("No keys to dump from memory when closing~n"); + {{true, L0Pid, _TS}, _, _} -> + leveled_sft:sft_close(L0Pid), + io:format("No opportunity to persist memory before closing" + ++ " with ~w keys discarded~n", + [length(Dump)]); + _ -> + io:format("No opportunity to persist memory before closing" + ++ " with ~w keys discarded~n", + [length(Dump)]) + end, + ok = close_files(0, State#state.manifest), + lists:foreach(fun({_FN, Pid, _SN}) -> + leveled_sft:sft_close(Pid) end, + State#state.unreferenced_files), + ok + end. + + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + + +writer_call({push_mem, DumpList}, From, State0) -> % The process for pushing to memory is as follows % - Check that the inbound list does not contain any Keys with a lower % sequence number than any existing keys (assess_sqn/1) @@ -432,12 +605,7 @@ handle_call({push_mem, DumpList}, From, State0) -> io:format("Empty request pushed to Penciller~n"), {reply, ok, State0} end; -handle_call({fetch, Key}, _From, State) -> - {reply, fetch(Key, State#state.manifest, State#state.memtable), State}; -handle_call(work_for_clerk, From, State) -> - {UpdState, Work} = return_work(State, From), - {reply, {Work, UpdState#state.backlog}, UpdState}; -handle_call({confirm_delete, FileName}, _From, State) -> +writer_call({confirm_delete, FileName}, _From, State) -> Reply = confirm_delete(FileName, State#state.unreferenced_files, State#state.registered_snapshots), @@ -448,7 +616,7 @@ handle_call({confirm_delete, FileName}, _From, State) -> _ -> {reply, Reply, State} end; -handle_call(prompt_compaction, _From, State) -> +writer_call(prompt_compaction, _From, State) -> %% If there is a prompt immediately after a L0 async write event then %% there exists the potential for the prompt to stall the database. %% Should only accept prompts if there has been a safe wait from the @@ -480,99 +648,10 @@ handle_call(prompt_compaction, _From, State) -> true -> {reply, ok, State#state{backlog=false}} end; -handle_call({manifest_change, WI}, _From, State) -> +writer_call({manifest_change, WI}, _From, State) -> {ok, UpdState} = commit_manifest_change(WI, State), - {reply, ok, UpdState}; -handle_call(get_startup_sqn, _From, State) -> - {reply, State#state.ledger_sqn, State}; -handle_call({register_snapshot, Snapshot}, _From, State) -> - Rs = [{Snapshot, State#state.ledger_sqn}|State#state.registered_snapshots], - {reply, - {ok, - State#state.ledger_sqn, - State#state.manifest, - State#state.memtable_copy}, - State#state{registered_snapshots = Rs}}; -handle_call({load_snapshot, Increment}, _From, State) -> - MemTableCopy = State#state.memtable_copy, - {Tree0, TreeSQN0} = roll_new_tree(MemTableCopy#l0snapshot.tree, - MemTableCopy#l0snapshot.increments, - MemTableCopy#l0snapshot.ledger_sqn), - if - TreeSQN0 > MemTableCopy#l0snapshot.ledger_sqn -> - pcl_updatesnapshotcache(State#state.source_penciller, - Tree0, - TreeSQN0) - end, - {Tree1, TreeSQN1} = roll_new_tree(Tree0, [Increment], TreeSQN0), - io:format("Snapshot loaded to start at SQN~w~n", [TreeSQN1]), - {reply, ok, State#state{levelzero_snapshot=Tree1, - ledger_sqn=TreeSQN1, - snapshot_fully_loaded=true}}; -handle_call(close, _From, State) -> - {stop, normal, ok, State}. + {reply, ok, UpdState}. -handle_cast({update_snapshotcache, Tree, SQN}, State) -> - MemTableC = cache_tree_in_memcopy(State#state.memtable_copy, Tree, SQN), - {noreply, State#state{memtable_copy=MemTableC}}; -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, State) -> - %% When a Penciller shuts down it isn't safe to try an manage the safe - %% finishing of any outstanding work. The last commmitted manifest will - %% be used. - %% - %% Level 0 files lie outside of the manifest, and so if there is no L0 - %% file present it is safe to write the current contents of memory. If - %% there is a L0 file present - then the memory can be dropped (it is - %% recoverable from the ledger, and there should not be a lot to recover - %% as presumably the ETS file has been recently flushed, hence the presence - %% of a L0 file). - %% - %% The penciller should close each file in the unreferenced files, and - %% then each file in the manifest, and cast a close on the clerk. - %% The cast may not succeed as the clerk could be synchronously calling - %% the penciller looking for a manifest commit - %% - leveled_pclerk:clerk_stop(State#state.clerk), - Dump = ets:tab2list(State#state.memtable), - case {State#state.levelzero_pending, - get_item(0, State#state.manifest, []), length(Dump)} of - {?L0PEND_RESET, [], L} when L > 0 -> - MSN = State#state.manifest_sqn + 1, - FileName = State#state.root_path - ++ "/" ++ ?FILES_FP ++ "/" - ++ integer_to_list(MSN) ++ "_0_0", - {ok, - L0Pid, - {{[], []}, _SK, _HK}} = leveled_sft:sft_new(FileName ++ ".pnd", - Dump, - [], - 0), - io:format("Dump of memory on close to filename ~s~n", [FileName]), - leveled_sft:sft_close(L0Pid), - file:rename(FileName ++ ".pnd", FileName ++ ".sft"); - {?L0PEND_RESET, [], L} when L == 0 -> - io:format("No keys to dump from memory when closing~n"); - {{true, L0Pid, _TS}, _, _} -> - leveled_sft:sft_close(L0Pid), - io:format("No opportunity to persist memory before closing " - ++ "with ~w keys discarded~n", [length(Dump)]); - _ -> - io:format("No opportunity to persist memory before closing " - ++ "with ~w keys discarded~n", [length(Dump)]) - end, - ok = close_files(0, State#state.manifest), - lists:foreach(fun({_FN, Pid, _SN}) -> leveled_sft:sft_close(Pid) end, - State#state.unreferenced_files), - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. %%%============================================================================ @@ -744,6 +823,14 @@ roll_memory(State, MaxSize) -> end. +fetch_snap(Key, Manifest, Tree) -> + case gb_trees:lookup(Key, Tree) of + {value, Value} -> + {Key, Value}; + none -> + fetch(Key, Manifest, 0, fun leveled_sft:sft_get/2) + end. + fetch(Key, Manifest, TID) -> case ets:lookup(TID, Key) of [Object] -> @@ -777,6 +864,7 @@ fetch(Key, Manifest, Level, FetchFun) -> ObjectFound end end. + %% Manifest lock - don't have two changes to the manifest happening %% concurrently @@ -862,7 +950,9 @@ roll_new_tree(Tree, [{SQN, KVList}|TailIncs], HighSQN) when SQN >= HighSQN -> gb_trees:enter(K, V, TreeAcc) end, Tree, KVList), - roll_new_tree(UpdTree, TailIncs, SQN). + roll_new_tree(UpdTree, TailIncs, SQN); +roll_new_tree(Tree, [_H|TailIncs], HighSQN) -> + roll_new_tree(Tree, TailIncs, HighSQN). %% Update the memtable copy if the tree created advances the SQN cache_tree_in_memcopy(MemCopy, Tree, SQN) -> @@ -1146,9 +1236,9 @@ simple_server_test() -> ?assertMatch(R3, Key1), ?assertMatch(R4, Key2), S2 = pcl_pushmem(PCL, KL2), - if S2 == pause -> timer:sleep(2000); true -> ok end, + if S2 == pause -> timer:sleep(1000); true -> ok end, S3 = pcl_pushmem(PCL, [Key3]), - if S3 == pause -> timer:sleep(2000); true -> ok end, + if S3 == pause -> timer:sleep(1000); true -> ok end, R5 = pcl_fetch(PCL, {o,"Bucket0001", "Key0001"}), R6 = pcl_fetch(PCL, {o,"Bucket0002", "Key0002"}), R7 = pcl_fetch(PCL, {o,"Bucket0003", "Key0003"}), @@ -1163,7 +1253,7 @@ simple_server_test() -> 2001 -> %% Last push not persisted S3a = pcl_pushmem(PCL, [Key3]), - if S3a == pause -> timer:sleep(2000); true -> ok end, + if S3a == pause -> timer:sleep(1000); true -> ok end, ok; 2002 -> %% everything got persisted @@ -1182,11 +1272,11 @@ simple_server_test() -> ?assertMatch(R9, Key2), ?assertMatch(R10, Key3), S4 = pcl_pushmem(PCLr, KL3), - if S4 == pause -> timer:sleep(2000); true -> ok end, + if S4 == pause -> timer:sleep(1000); true -> ok end, S5 = pcl_pushmem(PCLr, [Key4]), - if S5 == pause -> timer:sleep(2000); true -> ok end, + if S5 == pause -> timer:sleep(1000); true -> ok end, S6 = pcl_pushmem(PCLr, KL4), - if S6 == pause -> timer:sleep(2000); true -> ok end, + if S6 == pause -> timer:sleep(1000); true -> ok end, R11 = pcl_fetch(PCLr, {o,"Bucket0001", "Key0001"}), R12 = pcl_fetch(PCLr, {o,"Bucket0002", "Key0002"}), R13 = pcl_fetch(PCLr, {o,"Bucket0003", "Key0003"}), @@ -1195,6 +1285,52 @@ simple_server_test() -> ?assertMatch(R12, Key2), ?assertMatch(R13, Key3), ?assertMatch(R14, Key4), + SnapOpts = #penciller_options{start_snapshot = true, + source_penciller = PCLr, + requestor = self()}, + {ok, PclSnap} = pcl_start(SnapOpts), + ok = pcl_loadsnapshot(PclSnap, []), + ?assertMatch(Key1, pcl_fetch(PclSnap, {o,"Bucket0001", "Key0001"})), + ?assertMatch(Key2, pcl_fetch(PclSnap, {o,"Bucket0002", "Key0002"})), + ?assertMatch(Key3, pcl_fetch(PclSnap, {o,"Bucket0003", "Key0003"})), + ?assertMatch(Key4, pcl_fetch(PclSnap, {o,"Bucket0004", "Key0004"})), + ?assertMatch(true, pcl_checksequencenumber(PclSnap, + {o,"Bucket0001", "Key0001"}, + 1)), + ?assertMatch(true, pcl_checksequencenumber(PclSnap, + {o,"Bucket0002", "Key0002"}, + 1002)), + ?assertMatch(true, pcl_checksequencenumber(PclSnap, + {o,"Bucket0003", "Key0003"}, + 2002)), + ?assertMatch(true, pcl_checksequencenumber(PclSnap, + {o,"Bucket0004", "Key0004"}, + 3002)), + % Add some more keys and confirm that chekc sequence number still + % sees the old version in the previous snapshot, but will see the new version + % in a new snapshot + Key1A = {{o,"Bucket0001", "Key0001"}, {4002, {active, infinity}, null}}, + KL1A = lists:sort(leveled_sft:generate_randomkeys({4002, 2})), + S7 = pcl_pushmem(PCLr, [Key1A]), + if S7 == pause -> timer:sleep(1000); true -> ok end, + S8 = pcl_pushmem(PCLr, KL1A), + if S8 == pause -> timer:sleep(1000); true -> ok end, + ?assertMatch(true, pcl_checksequencenumber(PclSnap, + {o,"Bucket0001", "Key0001"}, + 1)), + ok = pcl_close(PclSnap), + {ok, PclSnap2} = pcl_start(SnapOpts), + ok = pcl_loadsnapshot(PclSnap2, []), + ?assertMatch(false, pcl_checksequencenumber(PclSnap2, + {o,"Bucket0001", "Key0001"}, + 1)), + ?assertMatch(true, pcl_checksequencenumber(PclSnap2, + {o,"Bucket0001", "Key0001"}, + 4002)), + ?assertMatch(true, pcl_checksequencenumber(PclSnap2, + {o,"Bucket0002", "Key0002"}, + 1002)), + ok = pcl_close(PclSnap2), ok = pcl_close(PCLr), clean_testdir(RootPath). From d24b100aa6509e76016c207bc436d61679dd13df Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 27 Sep 2016 14:58:26 +0100 Subject: [PATCH 049/167] Initial work on Journal Compaction Largely untested work at this stage to allow for the Inker to request the Inker's clerk to perform a single round of compact based on the best run of files it can find. --- include/leveled.hrl | 10 +- src/leveled_cdb.erl | 27 ++- src/leveled_iclerk.erl | 438 +++++++++++++++++++++++++++++++++++++---- src/leveled_inker.erl | 71 ++++++- src/leveled_sft.erl | 4 +- 5 files changed, 494 insertions(+), 56 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 232aed7..321ff7c 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -21,13 +21,14 @@ filename :: string()}). -record(cdb_options, - {max_size :: integer()}). + {max_size :: integer(), + file_path :: string()}). -record(inker_options, {cdb_max_size :: integer(), root_path :: string(), cdb_options :: #cdb_options{}, - start_snapshot = false :: boolean, + start_snapshot = false :: boolean(), source_inker :: pid(), requestor :: pid()}). @@ -44,6 +45,11 @@ metadata_extractor :: function(), indexspec_converter :: function()}). +-record(iclerk_options, + {inker :: pid(), + max_run_length :: integer(), + cdb_options :: #cdb_options{}}). + %% Temp location for records related to riak -record(r_content, { diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 40a4aaf..b1eb11a 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -60,11 +60,13 @@ cdb_getpositions/2, cdb_directfetch/3, cdb_lastkey/1, + cdb_firstkey/1, cdb_filename/1, cdb_keycheck/2, cdb_scan/4, cdb_close/1, - cdb_complete/1]). + cdb_complete/1, + cdb_destroy/1]). -include_lib("eunit/include/eunit.hrl"). @@ -133,6 +135,9 @@ cdb_close(Pid) -> cdb_complete(Pid) -> gen_server:call(Pid, cdb_complete, infinity). +cdb_destroy(Pid) -> + gen_server:cast(Pid, destroy). + %% cdb_scan returns {LastPosition, Acc}. Use LastPosition as StartPosiiton to %% continue from that point (calling function has to protect against) double %% counting. @@ -150,6 +155,9 @@ cdb_scan(Pid, FilterFun, InitAcc, StartPosition) -> cdb_lastkey(Pid) -> gen_server:call(Pid, cdb_lastkey, infinity). +cdb_firstkey(Pid) -> + gen_server:call(Pid, cdb_firstkey, infinity). + %% Get the filename of the database cdb_filename(Pid) -> gen_server:call(Pid, cdb_filename, infinity). @@ -254,6 +262,8 @@ handle_call({put_kv, Key, Value}, _From, State) -> end; handle_call(cdb_lastkey, _From, State) -> {reply, State#state.last_key, State}; +handle_call(cdb_firstkey, _From, State) -> + {reply, extract_key(State#state.handle, ?BASE_POSITION), State}; handle_call(cdb_filename, _From, State) -> {reply, State#state.filename, State}; handle_call({get_positions, SampleSize}, _From, State) -> @@ -339,7 +349,10 @@ handle_call(cdb_complete, _From, State) -> end. - +handle_cast(destroy, State) -> + ok = file:close(State#state.handle), + ok = file:delete(State#state.filename), + {noreply, State}; handle_cast(_Msg, State) -> {noreply, State}. @@ -1551,16 +1564,14 @@ find_lastkey_test() -> ok = cdb_put(P1, "Key1", "Value1"), ok = cdb_put(P1, "Key3", "Value3"), ok = cdb_put(P1, "Key2", "Value2"), - R1 = cdb_lastkey(P1), - ?assertMatch(R1, "Key2"), + ?assertMatch("Key2", cdb_lastkey(P1)), + ?assertMatch("Key1", cdb_firstkey(P1)), ok = cdb_close(P1), {ok, P2} = cdb_open_writer("../test/lastkey.pnd"), - R2 = cdb_lastkey(P2), - ?assertMatch(R2, "Key2"), + ?assertMatch("Key2", cdb_lastkey(P2)), {ok, F2} = cdb_complete(P2), {ok, P3} = cdb_open_reader(F2), - R3 = cdb_lastkey(P3), - ?assertMatch(R3, "Key2"), + ?assertMatch("Key2", cdb_lastkey(P3)), ok = cdb_close(P3), ok = file:delete("../test/lastkey.cdb"). diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index fee2417..2ead967 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -12,35 +12,47 @@ handle_info/2, terminate/2, clerk_new/1, - clerk_compact/5, + clerk_compact/4, clerk_remove/2, clerk_stop/1, code_change/3]). -include_lib("eunit/include/eunit.hrl"). +-define(JOURNAL_FILEX, "cdb"). +-define(PENDING_FILEX, "pnd"). -define(SAMPLE_SIZE, 200). -define(BATCH_SIZE, 16). -define(BATCHES_TO_CHECK, 8). +%% How many consecutive files to compact in one run +-define(MAX_COMPACTION_RUN, 4). +%% Sliding scale to allow preference of longer runs up to maximum +-define(SINGLEFILE_COMPACTION_TARGET, 60.0). +-define(MAXRUN_COMPACTION_TARGET, 80.0). + +-record(state, {inker :: pid(), + max_run_length :: integer(), + cdb_options}). + +-record(candidate, {low_sqn :: integer(), + filename :: string(), + journal :: pid(), + compaction_perc :: float()}). --record(state, {owner :: pid(), - penciller_snapshot :: pid()}). %%%============================================================================ %%% API %%%============================================================================ -clerk_new(Owner) -> - {ok, Pid} = gen_server:start(?MODULE, [], []), - ok = gen_server:call(Pid, {register, Owner}, infinity), - {ok, Pid}. +clerk_new(InkerClerkOpts) -> + gen_server:start(?MODULE, [InkerClerkOpts], []). clerk_remove(Pid, Removals) -> gen_server:cast(Pid, {remove, Removals}), ok. -clerk_compact(Pid, Manifest, ManifestSQN, Penciller, Timeout) -> - gen_server:cast(Pid, {compact, Manifest, ManifestSQN, Penciller, Timeout}). +clerk_compact(Pid, Penciller, Inker, Timeout) -> + clerk_compact(Pid, Penciller, Inker, Timeout). clerk_stop(Pid) -> gen_server:cast(Pid, stop). @@ -49,21 +61,60 @@ clerk_stop(Pid) -> %%% gen_server callbacks %%%============================================================================ -init([]) -> - {ok, #state{}}. +init([IClerkOpts]) -> + case IClerkOpts#iclerk_options.max_run_length of + undefined -> + {ok, #state{max_run_length = ?MAX_COMPACTION_RUN, + inker = IClerkOpts#iclerk_options.inker, + cdb_options = IClerkOpts#iclerk_options.cdb_options}}; + MRL -> + {ok, #state{max_run_length = MRL, + inker = IClerkOpts#iclerk_options.inker, + cdb_options = IClerkOpts#iclerk_options.cdb_options}} + end. -handle_call({register, Owner}, _From, State) -> - {reply, ok, State#state{owner=Owner}}. +handle_call(_Msg, _From, State) -> + {reply, not_supprted, State}. -handle_cast({compact, Manifest, _ManifestSQN, Penciller, _Timeout}, State) -> +handle_cast({compact, Penciller, Inker, _Timeout}, State) -> + % Need to fetch manifest at start rather than have it be passed in + % Don't want to process a queued call waiting on an old manifest + Manifest = leveled_inker:ink_getmanifest(Inker), + MaxRunLength = State#state.max_run_length, PclOpts = #penciller_options{start_snapshot = true, source_penciller = Penciller, requestor = self()}, - PclSnap = leveled_penciller:pcl_start(PclOpts), - ok = leveled_penciller:pcl_loadsnapshot(PclSnap, []), - _CandidateList = scan_all_files(Manifest, PclSnap), - %% TODO - Lots - {noreply, State}; + FilterFun = fun leveled_penciller:pcl_checksequencenumber/3, + FilterServer = leveled_penciller:pcl_start(PclOpts), + ok = leveled_penciller:pcl_loadsnapshot(FilterServer, []), + Candidates = scan_all_files(Manifest, + FilterFun, + FilterServer), + BestRun = assess_candidates(Candidates, MaxRunLength), + case score_run(BestRun, MaxRunLength) of + Score when Score > 0 -> + print_compaction_run(BestRun, MaxRunLength), + CDBopts = State#state.cdb_options, + {ManifestSlice, + PromptDelete} = compact_files(BestRun, + CDBopts, + FilterFun, + FilterServer), + FilesToDelete = lists:map(fun(C) -> + {C#candidate.low_sqn, + C#candidate.filename, + C#candidate.journal} + end, + BestRun), + ok = leveled_inker:ink_updatemanifest(Inker, + ManifestSlice, + PromptDelete, + FilesToDelete), + {noreply, State}; + Score -> + io:format("No compaction run as highest score=~w~n", [Score]), + {noreply, State} + end; handle_cast({remove, _Removals}, State) -> {noreply, State}; handle_cast(stop, State) -> @@ -84,15 +135,13 @@ code_change(_OldVsn, State, _Extra) -> %%%============================================================================ -check_single_file(CDB, PclSnap, SampleSize, BatchSize) -> +check_single_file(CDB, FilterFun, FilterServer, SampleSize, BatchSize) -> PositionList = leveled_cdb:cdb_getpositions(CDB, SampleSize), KeySizeList = fetch_inbatches(PositionList, BatchSize, CDB, []), R0 = lists:foldl(fun(KS, {ActSize, RplSize}) -> - {{PK, SQN}, Size} = KS, - Chk = leveled_pcl:pcl_checksequencenumber(PclSnap, - PK, - SQN), - case Chk of + {{SQN, PK}, Size} = KS, + Check = FilterFun(FilterServer, PK, SQN), + case Check of true -> {ActSize + Size, RplSize}; false -> @@ -101,32 +150,345 @@ check_single_file(CDB, PclSnap, SampleSize, BatchSize) -> {0, 0}, KeySizeList), {ActiveSize, ReplacedSize} = R0, - 100 * (ActiveSize / (ActiveSize + ReplacedSize)). + 100 * ActiveSize / (ActiveSize + ReplacedSize). -scan_all_files(Manifest, Penciller) -> - scan_all_files(Manifest, Penciller, []). +scan_all_files(Manifest, FilterFun, FilterServer) -> + scan_all_files(Manifest, FilterFun, FilterServer, []). -scan_all_files([], _Penciller, CandidateList) -> +scan_all_files([], _FilterFun, _FilterServer, CandidateList) -> CandidateList; -scan_all_files([{LowSQN, FN, JournalP}|Tail], Penciller, CandidateList) -> - CompactPerc = check_single_file(JournalP, - Penciller, - ?SAMPLE_SIZE, - ?BATCH_SIZE), - scan_all_files(Tail, Penciller, CandidateList ++ - [{LowSQN, FN, JournalP, CompactPerc}]). +scan_all_files([Entry|Tail], FilterFun, FilterServer, CandidateList) -> + {LowSQN, FN, JournalP} = Entry, + CpctPerc = check_single_file(JournalP, + FilterFun, + FilterServer, + ?SAMPLE_SIZE, + ?BATCH_SIZE), + scan_all_files(Tail, + FilterFun, + FilterServer, + CandidateList ++ + [#candidate{low_sqn = LowSQN, + filename = FN, + journal = JournalP, + compaction_perc = CpctPerc}]). fetch_inbatches([], _BatchSize, _CDB, CheckedList) -> CheckedList; fetch_inbatches(PositionList, BatchSize, CDB, CheckedList) -> - {Batch, Tail} = lists:split(BatchSize, PositionList), - KL_List = leveled_cdb:direct_fetch(CDB, Batch, key_size), + {Batch, Tail} = if + length(PositionList) >= BatchSize -> + lists:split(BatchSize, PositionList); + true -> + {PositionList, []} + end, + KL_List = leveled_cdb:cdb_directfetch(CDB, Batch, key_size), fetch_inbatches(Tail, BatchSize, CDB, CheckedList ++ KL_List). +assess_candidates(AllCandidates, MaxRunLength) -> + NaiveBestRun = assess_candidates(AllCandidates, MaxRunLength, [], []), + case length(AllCandidates) of + L when L > MaxRunLength, MaxRunLength > 1 -> + %% Assess with different offsets from the start + SqL = lists:seq(1, MaxRunLength - 1), + lists:foldl(fun(Counter, BestRun) -> + SubList = lists:nthtail(Counter, + AllCandidates), + assess_candidates(SubList, + MaxRunLength, + [], + BestRun) + end, + NaiveBestRun, + SqL); + _ -> + NaiveBestRun + end. + +assess_candidates([], _MaxRunLength, _CurrentRun0, BestAssessment) -> + BestAssessment; +assess_candidates([HeadC|Tail], MaxRunLength, CurrentRun0, BestAssessment) -> + CurrentRun1 = choose_best_assessment(CurrentRun0 ++ [HeadC], + [HeadC], + MaxRunLength), + assess_candidates(Tail, + MaxRunLength, + CurrentRun1, + choose_best_assessment(CurrentRun1, + BestAssessment, + MaxRunLength)). +choose_best_assessment(RunToAssess, BestRun, MaxRunLength) -> + case length(RunToAssess) of + LR1 when LR1 > MaxRunLength -> + BestRun; + _ -> + AssessScore = score_run(RunToAssess, MaxRunLength), + BestScore = score_run(BestRun, MaxRunLength), + if + AssessScore > BestScore -> + RunToAssess; + true -> + BestRun + end + end. + +score_run([], _MaxRunLength) -> + 0.0; +score_run(Run, MaxRunLength) -> + TargetIncr = case MaxRunLength of + 1 -> + 0.0; + MaxRunSize -> + (?MAXRUN_COMPACTION_TARGET + - ?SINGLEFILE_COMPACTION_TARGET) + / (MaxRunSize - 1) + end, + Target = ?SINGLEFILE_COMPACTION_TARGET + TargetIncr * (length(Run) - 1), + RunTotal = lists:foldl(fun(Cand, Acc) -> + Acc + Cand#candidate.compaction_perc end, + 0.0, + Run), + Target - RunTotal / length(Run). +print_compaction_run(BestRun, MaxRunLength) -> + io:format("Compaction to be performed on ~w files with score of ~w~n", + [length(BestRun), score_run(BestRun, MaxRunLength)]), + lists:foreach(fun(File) -> + io:format("Filename ~s is part of compaction run~n", + [File#candidate.filename]) + end, + BestRun). + +compact_files([], _CDBopts, _FilterFun, _FilterServer) -> + {[], 0}; +compact_files(BestRun, CDBopts, FilterFun, FilterServer) -> + BatchesOfPositions = get_all_positions(BestRun, []), + compact_files(BatchesOfPositions, + CDBopts, + null, + FilterFun, + FilterServer, + [], + true). + +compact_files([], _CDBopts, _ActiveJournal0, _FilterFun, _FilterServer, + ManSlice0, PromptDelete0) -> + %% Need to close the active file + {ManSlice0, PromptDelete0}; +compact_files([Batch|T], CDBopts, ActiveJournal0, FilterFun, FilterServer, + ManSlice0, PromptDelete0) -> + {SrcJournal, PositionList} = Batch, + KVCs0 = leveled_cdb:cdb_directfetch(SrcJournal, + PositionList, + key_value_check), + R0 = filter_output(KVCs0, + FilterFun, + FilterServer), + {KVCs1, PromptDelete1} = R0, + PromptDelete2 = case {PromptDelete0, PromptDelete1} of + {true, true} -> + true; + _ -> + false + end, + {ActiveJournal1, ManSlice1} = write_values(KVCs1, + CDBopts, + ActiveJournal0, + ManSlice0), + compact_files(T, CDBopts, ActiveJournal1, FilterFun, FilterServer, + ManSlice1, PromptDelete2). + +get_all_positions([], PositionBatches) -> + PositionBatches; +get_all_positions([HeadRef|RestOfBest], PositionBatches) -> + SrcJournal = HeadRef#candidate.journal, + Positions = leveled_cdb:cdb_getpositions(SrcJournal, all), + Batches = split_positions_into_batches(Positions, SrcJournal, []), + get_all_positions(RestOfBest, PositionBatches ++ Batches). + +split_positions_into_batches([], _Journal, Batches) -> + Batches; +split_positions_into_batches(Positions, Journal, Batches) -> + {ThisBatch, Tail} = lists:split(?BATCH_SIZE, Positions), + split_positions_into_batches(Tail, + Journal, + Batches ++ [{Journal, ThisBatch}]). + + +filter_output(KVCs, FilterFun, FilterServer) -> + lists:foldl(fun(KVC, {Acc, PromptDelete}) -> + {{SQN, PK}, _V, CrcCheck} = KVC, + KeyValid = FilterFun(FilterServer, PK, SQN), + case {KeyValid, CrcCheck} of + {true, true} -> + {Acc ++ [KVC], PromptDelete}; + {false, _} -> + {Acc, PromptDelete}; + {_, false} -> + io:format("Corrupted value found for " ++ " + Key ~w at SQN ~w~n", [PK, SQN]), + {Acc, false} + end + end, + {[], true}, + KVCs). + + +write_values([KVC|Rest], CDBopts, ActiveJournal0, ManSlice0) -> + {{SQN, PK}, V, _CrcCheck} = KVC, + {ok, ActiveJournal1} = case ActiveJournal0 of + null -> + FP = CDBopts#cdb_options.file_path, + FN = leveled_inker:filepath(FP, + SQN, + new_journal), + leveled_cdb:cdb_open_writer(FN, + CDBopts); + _ -> + {ok, ActiveJournal0} + end, + R = leveled_cdb:cdb_put(ActiveJournal1, {SQN, PK}, V), + case R of + ok -> + write_values(Rest, CDBopts, ActiveJournal1, ManSlice0); + roll -> + {ok, NewFN} = leveled_cdb:cdb_complete(ActiveJournal1), + {ok, PidR} = leveled_cdb:cdb_open_reader(NewFN), + {StartSQN, _PK} = leveled_cdb:cdb_firstkey(PidR), + ManSlice1 = ManSlice0 ++ [{StartSQN, NewFN, PidR}], + write_values(Rest, CDBopts, null, ManSlice1) + end. + + + + + + + + + %%%============================================================================ %%% Test %%%============================================================================ + + +-ifdef(TEST). + +simple_score_test() -> + Run1 = [#candidate{compaction_perc = 75.0}, + #candidate{compaction_perc = 75.0}, + #candidate{compaction_perc = 76.0}, + #candidate{compaction_perc = 70.0}], + ?assertMatch(6.0, score_run(Run1, 4)), + Run2 = [#candidate{compaction_perc = 75.0}], + ?assertMatch(-15.0, score_run(Run2, 4)), + ?assertMatch(0.0, score_run([], 4)). + +score_compare_test() -> + Run1 = [#candidate{compaction_perc = 75.0}, + #candidate{compaction_perc = 75.0}, + #candidate{compaction_perc = 76.0}, + #candidate{compaction_perc = 70.0}], + ?assertMatch(6.0, score_run(Run1, 4)), + Run2 = [#candidate{compaction_perc = 75.0}], + ?assertMatch(Run1, choose_best_assessment(Run1, Run2, 4)), + ?assertMatch(Run2, choose_best_assessment(Run1 ++ Run2, Run2, 4)). + +find_bestrun_test() -> +%% Tests dependent on these defaults +%% -define(MAX_COMPACTION_RUN, 4). +%% -define(SINGLEFILE_COMPACTION_TARGET, 60.0). +%% -define(MAXRUN_COMPACTION_TARGET, 80.0). +%% Tested first with blocks significant as no back-tracking + Block1 = [#candidate{compaction_perc = 75.0}, + #candidate{compaction_perc = 85.0}, + #candidate{compaction_perc = 62.0}, + #candidate{compaction_perc = 70.0}], + Block2 = [#candidate{compaction_perc = 58.0}, + #candidate{compaction_perc = 95.0}, + #candidate{compaction_perc = 95.0}, + #candidate{compaction_perc = 65.0}], + Block3 = [#candidate{compaction_perc = 90.0}, + #candidate{compaction_perc = 100.0}, + #candidate{compaction_perc = 100.0}, + #candidate{compaction_perc = 100.0}], + Block4 = [#candidate{compaction_perc = 75.0}, + #candidate{compaction_perc = 76.0}, + #candidate{compaction_perc = 76.0}, + #candidate{compaction_perc = 60.0}], + Block5 = [#candidate{compaction_perc = 80.0}, + #candidate{compaction_perc = 80.0}], + CList0 = Block1 ++ Block2 ++ Block3 ++ Block4 ++ Block5, + ?assertMatch(Block4, assess_candidates(CList0, 4, [], [])), + CList1 = CList0 ++ [#candidate{compaction_perc = 20.0}], + ?assertMatch([#candidate{compaction_perc = 20.0}], + assess_candidates(CList1, 4, [], [])), + CList2 = Block4 ++ Block3 ++ Block2 ++ Block1 ++ Block5, + ?assertMatch(Block4, assess_candidates(CList2, 4, [], [])), + CList3 = Block5 ++ Block1 ++ Block2 ++ Block3 ++ Block4, + ?assertMatch([#candidate{compaction_perc = 62.0}, + #candidate{compaction_perc = 70.0}, + #candidate{compaction_perc = 58.0}], + assess_candidates(CList3, 4, [], [])), + %% Now do some back-tracking to get a genuinely optimal solution without + %% needing to re-order + ?assertMatch([#candidate{compaction_perc = 62.0}, + #candidate{compaction_perc = 70.0}, + #candidate{compaction_perc = 58.0}], + assess_candidates(CList0, 4)), + ?assertMatch([#candidate{compaction_perc = 62.0}, + #candidate{compaction_perc = 70.0}, + #candidate{compaction_perc = 58.0}], + assess_candidates(CList0, 5)), + ?assertMatch([#candidate{compaction_perc = 62.0}, + #candidate{compaction_perc = 70.0}, + #candidate{compaction_perc = 58.0}, + #candidate{compaction_perc = 95.0}, + #candidate{compaction_perc = 95.0}, + #candidate{compaction_perc = 65.0}], + assess_candidates(CList0, 6)). + +check_single_file_test() -> + RP = "../test/journal", + FN1 = leveled_inker:filepath(RP, 1, new_journal), + {ok, CDB1} = leveled_cdb:cdb_open_writer(FN1, #cdb_options{}), + {K1, V1} = {{1, "Key1"}, term_to_binary("Value1")}, + {K2, V2} = {{2, "Key2"}, term_to_binary("Value2")}, + {K3, V3} = {{3, "Key3"}, term_to_binary("Value3")}, + {K4, V4} = {{4, "Key1"}, term_to_binary("Value4")}, + {K5, V5} = {{5, "Key1"}, term_to_binary("Value5")}, + {K6, V6} = {{6, "Key1"}, term_to_binary("Value6")}, + {K7, V7} = {{7, "Key1"}, term_to_binary("Value7")}, + {K8, V8} = {{8, "Key1"}, term_to_binary("Value8")}, + ok = leveled_cdb:cdb_put(CDB1, K1, V1), + ok = leveled_cdb:cdb_put(CDB1, K2, V2), + ok = leveled_cdb:cdb_put(CDB1, K3, V3), + ok = leveled_cdb:cdb_put(CDB1, K4, V4), + ok = leveled_cdb:cdb_put(CDB1, K5, V5), + ok = leveled_cdb:cdb_put(CDB1, K6, V6), + ok = leveled_cdb:cdb_put(CDB1, K7, V7), + ok = leveled_cdb:cdb_put(CDB1, K8, V8), + {ok, FN2} = leveled_cdb:cdb_complete(CDB1), + {ok, CDB2} = leveled_cdb:cdb_open_reader(FN2), + LedgerSrv1 = [{8, "Key1"}, {2, "Key2"}, {3, "Key3"}], + LedgerFun1 = fun(Srv, Key, ObjSQN) -> + case lists:keyfind(ObjSQN, 1, Srv) of + {ObjSQN, Key} -> + true; + _ -> + false + end end, + Score1 = check_single_file(CDB2, LedgerFun1, LedgerSrv1, 8, 4), + ?assertMatch(37.5, Score1), + LedgerFun2 = fun(_Srv, _Key, _ObjSQN) -> true end, + Score2 = check_single_file(CDB2, LedgerFun2, LedgerSrv1, 8, 4), + ?assertMatch(100.0, Score2), + Score3 = check_single_file(CDB2, LedgerFun1, LedgerSrv1, 8, 3), + ?assertMatch(37.5, Score3), + ok = leveled_cdb:cdb_destroy(CDB2). + + +-endif. \ No newline at end of file diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index b7a54a6..8cc8722 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -105,16 +105,20 @@ ink_loadpcl/4, ink_registersnapshot/2, ink_compactjournal/3, - ink_close/1, + ink_getmanifest/1, + ink_updatemanifest/4, ink_print_manifest/1, + ink_close/1, build_dummy_journal/0, simple_manifest_reader/2, - clean_testdir/1]). + clean_testdir/1, + filepath/3]). -include_lib("eunit/include/eunit.hrl"). -define(MANIFEST_FP, "journal_manifest"). -define(FILES_FP, "journal_files"). +-define(COMPACT_FP, "post_compact"). -define(JOURNAL_FILEX, "cdb"). -define(MANIFEST_FILEX, "man"). -define(PENDING_FILEX, "pnd"). @@ -126,7 +130,7 @@ journal_sqn = 0 :: integer(), active_journaldb :: pid(), active_journaldb_sqn :: integer(), - removed_journaldbs = [] :: list(), + pending_removals = [] :: list(), registered_snapshots = [] :: list(), root_path :: string(), cdb_options :: #cdb_options{}, @@ -161,6 +165,17 @@ ink_loadpcl(Pid, MinSQN, FilterFun, Penciller) -> ink_compactjournal(Pid, Penciller, Timeout) -> gen_server:call(Pid, {compact_journal, Penciller, Timeout}, infinty). +ink_getmanifest(Pid) -> + gen_server:call(Pid, get_manifest, infinity). + +ink_updatemanifest(Pid, ManifestSnippet, PromptDeletion, DeletedFiles) -> + gen_server:call(Pid, + {update_manifest, + ManifestSnippet, + PromptDeletion, + DeletedFiles}, + infinity). + ink_print_manifest(Pid) -> gen_server:call(Pid, print_manifest, infinity). @@ -233,11 +248,45 @@ handle_call({register_snapshot, Requestor}, _From , State) -> {reply, {State#state.manifest, State#state.active_journaldb}, State#state{registered_snapshots=Rs}}; +handle_call(get_manifest, _From, State) -> + {reply, State#state.manifest, State}; +handle_call({update_manifest, + ManifestSnippet, + PromptDeletion, + DeletedFiles}, _From, State) -> + Man0 = lists:foldl(fun(ManEntry, AccMan) -> + Check = lists:member(ManEntry, DeletedFiles), + if + Check == false -> + lists:append(AccMan, ManEntry) + end + end, + [], + State#state.manifest), + Man1 = lists:foldl(fun(ManEntry, AccMan) -> + add_to_manifest(AccMan, ManEntry) end, + Man0, + ManifestSnippet), + NewManifestSQN = State#state.manifest_sqn + 1, + ok = simple_manifest_writer(Man1, NewManifestSQN, State#state.root_path), + PendingRemovals = case PromptDeletion of + true -> + State#state.pending_removals ++ + {NewManifestSQN, DeletedFiles}; + _ -> + State#state.pending_removals + end, + {reply, ok, State#state{manifest=Man1, + manifest_sqn=NewManifestSQN, + pending_removals=PendingRemovals}}; handle_call(print_manifest, _From, State) -> manifest_printer(State#state.manifest), {reply, ok, State}; handle_call({compact_journal, Penciller, Timeout}, _From, State) -> - leveled_iclerk:clerk_compact(Penciller, Timeout), + leveled_iclerk:clerk_compact(State#state.clerk, + self(), + Penciller, + Timeout), {reply, ok, State}; handle_call(close, _From, State) -> {stop, normal, ok, State}. @@ -265,7 +314,6 @@ code_change(_OldVsn, State, _Extra) -> %%%============================================================================ start_from_file(InkerOpts) -> - {ok, Clerk} = leveled_iclerk:clerk_new(self()), RootPath = InkerOpts#inker_options.root_path, CDBopts = InkerOpts#inker_options.cdb_options, JournalFP = filepath(RootPath, journal_dir), @@ -284,6 +332,14 @@ start_from_file(InkerOpts) -> filelib:ensure_dir(ManifestFP), {ok, []} end, + + CompactFP = filepath(RootPath, journal_compact_dir), + filelib:ensure_dir(CompactFP), + IClerkCDBOpts = CDBopts#cdb_options{file_path = CompactFP}, + IClerkOpts = #iclerk_options{inker = self(), + cdb_options=IClerkCDBOpts}, + {ok, Clerk} = leveled_iclerk:clerk_new(IClerkOpts), + {Manifest, {ActiveJournal, LowActiveSQN}, JournalSQN, @@ -597,8 +653,9 @@ find_in_manifest(SQN, [_Head|Tail]) -> filepath(RootPath, journal_dir) -> RootPath ++ "/" ++ ?FILES_FP ++ "/"; filepath(RootPath, manifest_dir) -> - RootPath ++ "/" ++ ?MANIFEST_FP ++ "/". - + RootPath ++ "/" ++ ?MANIFEST_FP ++ "/"; +filepath(RootPath, journal_compact_dir) -> + filepath(RootPath, journal_dir) ++ "/" ++ ?COMPACT_FP ++ "/". filepath(RootPath, NewSQN, new_journal) -> filename:join(filepath(RootPath, journal_dir), diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 793edc5..ac1bf68 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -1393,6 +1393,7 @@ findremainder(BitStr, Factor) -> %%%============================================================================ +-ifdef(TEST). generate_randomkeys({Count, StartSQN}) -> generate_randomkeys(Count, StartSQN, []); @@ -1760,4 +1761,5 @@ big_iterator_test() -> ?assertMatch(NumFoundKeys3, 4 * 128), ok = file:close(Handle), ok = file:delete(Filename). - \ No newline at end of file + +-endif. \ No newline at end of file From 50b50ba486a311fc0e9b6bd126745f3bd077e8ae Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 28 Sep 2016 11:41:56 +0100 Subject: [PATCH 050/167] Inker Clerk - Further Testing Expanded the unit tetsing of the Inker Clerk actor. Still WIP --- src/leveled_iclerk.erl | 112 +++++++++++++++++++++++++++++------------ src/leveled_inker.erl | 7 ++- 2 files changed, 87 insertions(+), 32 deletions(-) diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 2ead967..f6d85d7 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -87,6 +87,11 @@ handle_cast({compact, Penciller, Inker, _Timeout}, State) -> FilterFun = fun leveled_penciller:pcl_checksequencenumber/3, FilterServer = leveled_penciller:pcl_start(PclOpts), ok = leveled_penciller:pcl_loadsnapshot(FilterServer, []), + + CDBopts = State#state.cdb_options, + FP = CDBopts#cdb_options.file_path, + ok = filelib:ensure_dir(FP), + Candidates = scan_all_files(Manifest, FilterFun, FilterServer), @@ -94,7 +99,6 @@ handle_cast({compact, Penciller, Inker, _Timeout}, State) -> case score_run(BestRun, MaxRunLength) of Score when Score > 0 -> print_compaction_run(BestRun, MaxRunLength), - CDBopts = State#state.cdb_options, {ManifestSlice, PromptDelete} = compact_files(BestRun, CDBopts, @@ -274,10 +278,10 @@ compact_files(BestRun, CDBopts, FilterFun, FilterServer) -> [], true). -compact_files([], _CDBopts, _ActiveJournal0, _FilterFun, _FilterServer, +compact_files([], _CDBopts, ActiveJournal0, _FilterFun, _FilterServer, ManSlice0, PromptDelete0) -> - %% Need to close the active file - {ManSlice0, PromptDelete0}; + ManSlice1 = ManSlice0 ++ generate_manifest_entry(ActiveJournal0), + {ManSlice1, PromptDelete0}; compact_files([Batch|T], CDBopts, ActiveJournal0, FilterFun, FilterServer, ManSlice0, PromptDelete0) -> {SrcJournal, PositionList} = Batch, @@ -306,13 +310,20 @@ get_all_positions([], PositionBatches) -> get_all_positions([HeadRef|RestOfBest], PositionBatches) -> SrcJournal = HeadRef#candidate.journal, Positions = leveled_cdb:cdb_getpositions(SrcJournal, all), - Batches = split_positions_into_batches(Positions, SrcJournal, []), + Batches = split_positions_into_batches(lists:sort(Positions), + SrcJournal, + []), get_all_positions(RestOfBest, PositionBatches ++ Batches). split_positions_into_batches([], _Journal, Batches) -> Batches; split_positions_into_batches(Positions, Journal, Batches) -> - {ThisBatch, Tail} = lists:split(?BATCH_SIZE, Positions), + {ThisBatch, Tail} = if + length(Positions) > ?BATCH_SIZE -> + lists:split(?BATCH_SIZE, Positions); + true -> + {Positions, []} + end, split_positions_into_batches(Tail, Journal, Batches ++ [{Journal, ThisBatch}]). @@ -337,32 +348,36 @@ filter_output(KVCs, FilterFun, FilterServer) -> KVCs). -write_values([KVC|Rest], CDBopts, ActiveJournal0, ManSlice0) -> +write_values([], _CDBopts, Journal0, ManSlice0) -> + {Journal0, ManSlice0}; +write_values([KVC|Rest], CDBopts, Journal0, ManSlice0) -> {{SQN, PK}, V, _CrcCheck} = KVC, - {ok, ActiveJournal1} = case ActiveJournal0 of - null -> - FP = CDBopts#cdb_options.file_path, - FN = leveled_inker:filepath(FP, - SQN, - new_journal), - leveled_cdb:cdb_open_writer(FN, - CDBopts); - _ -> - {ok, ActiveJournal0} - end, - R = leveled_cdb:cdb_put(ActiveJournal1, {SQN, PK}, V), + {ok, Journal1} = case Journal0 of + null -> + FP = CDBopts#cdb_options.file_path, + FN = leveled_inker:filepath(FP, + SQN, + compact_journal), + leveled_cdb:cdb_open_writer(FN, + CDBopts); + _ -> + {ok, Journal0} + end, + R = leveled_cdb:cdb_put(Journal1, {SQN, PK}, V), case R of ok -> - write_values(Rest, CDBopts, ActiveJournal1, ManSlice0); + write_values(Rest, CDBopts, Journal1, ManSlice0); roll -> - {ok, NewFN} = leveled_cdb:cdb_complete(ActiveJournal1), - {ok, PidR} = leveled_cdb:cdb_open_reader(NewFN), - {StartSQN, _PK} = leveled_cdb:cdb_firstkey(PidR), - ManSlice1 = ManSlice0 ++ [{StartSQN, NewFN, PidR}], + ManSlice1 = ManSlice0 ++ generate_manifest_entry(Journal1), write_values(Rest, CDBopts, null, ManSlice1) end. +generate_manifest_entry(ActiveJournal) -> + {ok, NewFN} = leveled_cdb:cdb_complete(ActiveJournal), + {ok, PidR} = leveled_cdb:cdb_open_reader(NewFN), + {StartSQN, _PK} = leveled_cdb:cdb_firstkey(PidR), + [{StartSQN, NewFN, PidR}]. @@ -451,8 +466,7 @@ find_bestrun_test() -> #candidate{compaction_perc = 65.0}], assess_candidates(CList0, 6)). -check_single_file_test() -> - RP = "../test/journal", +fetch_testcdb(RP) -> FN1 = leveled_inker:filepath(RP, 1, new_journal), {ok, CDB1} = leveled_cdb:cdb_open_writer(FN1, #cdb_options{}), {K1, V1} = {{1, "Key1"}, term_to_binary("Value1")}, @@ -472,7 +486,11 @@ check_single_file_test() -> ok = leveled_cdb:cdb_put(CDB1, K7, V7), ok = leveled_cdb:cdb_put(CDB1, K8, V8), {ok, FN2} = leveled_cdb:cdb_complete(CDB1), - {ok, CDB2} = leveled_cdb:cdb_open_reader(FN2), + leveled_cdb:cdb_open_reader(FN2). + +check_single_file_test() -> + RP = "../test/journal", + {ok, CDB} = fetch_testcdb(RP), LedgerSrv1 = [{8, "Key1"}, {2, "Key2"}, {3, "Key3"}], LedgerFun1 = fun(Srv, Key, ObjSQN) -> case lists:keyfind(ObjSQN, 1, Srv) of @@ -481,14 +499,46 @@ check_single_file_test() -> _ -> false end end, - Score1 = check_single_file(CDB2, LedgerFun1, LedgerSrv1, 8, 4), + Score1 = check_single_file(CDB, LedgerFun1, LedgerSrv1, 8, 4), ?assertMatch(37.5, Score1), LedgerFun2 = fun(_Srv, _Key, _ObjSQN) -> true end, - Score2 = check_single_file(CDB2, LedgerFun2, LedgerSrv1, 8, 4), + Score2 = check_single_file(CDB, LedgerFun2, LedgerSrv1, 8, 4), ?assertMatch(100.0, Score2), - Score3 = check_single_file(CDB2, LedgerFun1, LedgerSrv1, 8, 3), + Score3 = check_single_file(CDB, LedgerFun1, LedgerSrv1, 8, 3), ?assertMatch(37.5, Score3), - ok = leveled_cdb:cdb_destroy(CDB2). + ok = leveled_cdb:cdb_destroy(CDB). +compact_single_file_test() -> + RP = "../test/journal", + {ok, CDB} = fetch_testcdb(RP), + Candidate = #candidate{journal = CDB, + low_sqn = 1, + filename = "test", + compaction_perc = 37.5}, + LedgerSrv1 = [{8, "Key1"}, {2, "Key2"}, {3, "Key3"}], + LedgerFun1 = fun(Srv, Key, ObjSQN) -> + case lists:keyfind(ObjSQN, 1, Srv) of + {ObjSQN, Key} -> + true; + _ -> + false + end end, + CompactFP = leveled_inker:filepath(RP, journal_compact_dir), + ok = filelib:ensure_dir(CompactFP), + R1 = compact_files([Candidate], + #cdb_options{file_path=CompactFP}, + LedgerFun1, + LedgerSrv1), + {ManSlice1, PromptDelete1} = R1, + ?assertMatch(true, PromptDelete1), + [{LowSQN, FN, PidR}] = ManSlice1, + io:format("FN of ~s~n", [FN]), + ?assertMatch(2, LowSQN), + ?assertMatch(probably, leveled_cdb:cdb_keycheck(PidR, {8, "Key1"})), + ?assertMatch(missing, leveled_cdb:cdb_get(PidR, {7, "Key1"})), + ?assertMatch(missing, leveled_cdb:cdb_get(PidR, {1, "Key1"})), + {_RK1, RV1} = leveled_cdb:cdb_get(PidR, {2, "Key2"}), + ?assertMatch("Value2", binary_to_term(RV1)). + -endif. \ No newline at end of file diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 8cc8722..8142799 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -112,6 +112,7 @@ build_dummy_journal/0, simple_manifest_reader/2, clean_testdir/1, + filepath/2, filepath/3]). -include_lib("eunit/include/eunit.hrl"). @@ -659,11 +660,15 @@ filepath(RootPath, journal_compact_dir) -> filepath(RootPath, NewSQN, new_journal) -> filename:join(filepath(RootPath, journal_dir), + "nursery_" + ++ integer_to_list(NewSQN) + ++ "." ++ ?PENDING_FILEX); +filepath(CompactFilePath, NewSQN, compact_journal) -> + filename:join(CompactFilePath, "nursery_" ++ integer_to_list(NewSQN) ++ "." ++ ?PENDING_FILEX). - simple_manifest_reader(SQN, RootPath) -> ManifestPath = filepath(RootPath, manifest_dir), io:format("Opening manifest file at ~s with SQN ~w~n", From 15f57a0b4aa1cd5228426e8c44e2b089228587a4 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 28 Sep 2016 18:26:52 +0100 Subject: [PATCH 051/167] Further Journal compaction tests Improved unit testing --- src/leveled_cdb.erl | 6 ++- src/leveled_iclerk.erl | 34 +++++++------ src/leveled_inker.erl | 112 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 125 insertions(+), 27 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index b1eb11a..40bf5ef 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -336,8 +336,10 @@ handle_call(cdb_complete, _From, State) -> %% Rename file NewName = filename:rootname(State#state.filename, ".pnd") ++ ".cdb", - io:format("Renaming file from ~s to ~s~n", - [State#state.filename, NewName]), + io:format("Renaming file from ~s to ~s " ++ + "for which existence is ~w~n", + [State#state.filename, NewName, + filelib:is_file(NewName)]), ok = file:rename(State#state.filename, NewName), {stop, normal, {ok, NewName}, State}; false -> diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index f6d85d7..49a4f43 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -12,7 +12,7 @@ handle_info/2, terminate/2, clerk_new/1, - clerk_compact/4, + clerk_compact/6, clerk_remove/2, clerk_stop/1, code_change/3]). @@ -51,8 +51,14 @@ clerk_remove(Pid, Removals) -> gen_server:cast(Pid, {remove, Removals}), ok. -clerk_compact(Pid, Penciller, Inker, Timeout) -> - clerk_compact(Pid, Penciller, Inker, Timeout). +clerk_compact(Pid, Checker, InitiateFun, FilterFun, Inker, Timeout) -> + gen_server:cast(Pid, + {compact, + Checker, + InitiateFun, + FilterFun, + Inker, + Timeout}). clerk_stop(Pid) -> gen_server:cast(Pid, stop). @@ -76,25 +82,18 @@ init([IClerkOpts]) -> handle_call(_Msg, _From, State) -> {reply, not_supprted, State}. -handle_cast({compact, Penciller, Inker, _Timeout}, State) -> +handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, + State) -> % Need to fetch manifest at start rather than have it be passed in % Don't want to process a queued call waiting on an old manifest Manifest = leveled_inker:ink_getmanifest(Inker), MaxRunLength = State#state.max_run_length, - PclOpts = #penciller_options{start_snapshot = true, - source_penciller = Penciller, - requestor = self()}, - FilterFun = fun leveled_penciller:pcl_checksequencenumber/3, - FilterServer = leveled_penciller:pcl_start(PclOpts), - ok = leveled_penciller:pcl_loadsnapshot(FilterServer, []), - + FilterServer = InitiateFun(Checker), CDBopts = State#state.cdb_options, FP = CDBopts#cdb_options.file_path, ok = filelib:ensure_dir(FP), - Candidates = scan_all_files(Manifest, - FilterFun, - FilterServer), + Candidates = scan_all_files(Manifest, FilterFun, FilterServer), BestRun = assess_candidates(Candidates, MaxRunLength), case score_run(BestRun, MaxRunLength) of Score when Score > 0 -> @@ -278,6 +277,10 @@ compact_files(BestRun, CDBopts, FilterFun, FilterServer) -> [], true). + +compact_files([], _CDBopts, null, _FilterFun, _FilterServer, + ManSlice0, PromptDelete0) -> + {ManSlice0, PromptDelete0}; compact_files([], _CDBopts, ActiveJournal0, _FilterFun, _FilterServer, ManSlice0, PromptDelete0) -> ManSlice1 = ManSlice0 ++ generate_manifest_entry(ActiveJournal0), @@ -538,7 +541,8 @@ compact_single_file_test() -> ?assertMatch(missing, leveled_cdb:cdb_get(PidR, {7, "Key1"})), ?assertMatch(missing, leveled_cdb:cdb_get(PidR, {1, "Key1"})), {_RK1, RV1} = leveled_cdb:cdb_get(PidR, {2, "Key2"}), - ?assertMatch("Value2", binary_to_term(RV1)). + ?assertMatch("Value2", binary_to_term(RV1)), + ok = leveled_cdb:cdb_destroy(CDB). -endif. \ No newline at end of file diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 8142799..56c1664 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -164,7 +164,26 @@ ink_loadpcl(Pid, MinSQN, FilterFun, Penciller) -> gen_server:call(Pid, {load_pcl, MinSQN, FilterFun, Penciller}, infinity). ink_compactjournal(Pid, Penciller, Timeout) -> - gen_server:call(Pid, {compact_journal, Penciller, Timeout}, infinty). + CheckerInitiateFun = fun initiate_penciller_snapshot/1, + CheckerFilterFun = fun leveled_penciller:pcl_checksequencenumber/3, + gen_server:call(Pid, + {compact, + Penciller, + CheckerInitiateFun, + CheckerFilterFun, + Timeout}, + infiniy). + +%% Allows the Checker to be overriden in test, use something other than a +%% penciller +ink_compactjournal(Pid, Checker, InitiateFun, FilterFun, Timeout) -> + gen_server:call(Pid, + {compact, + Checker, + InitiateFun, + FilterFun, + Timeout}, + infinity). ink_getmanifest(Pid) -> gen_server:call(Pid, get_manifest, infinity). @@ -259,21 +278,23 @@ handle_call({update_manifest, Check = lists:member(ManEntry, DeletedFiles), if Check == false -> - lists:append(AccMan, ManEntry) + AccMan ++ [ManEntry]; + true -> + AccMan end end, [], State#state.manifest), Man1 = lists:foldl(fun(ManEntry, AccMan) -> - add_to_manifest(AccMan, ManEntry) end, - Man0, - ManifestSnippet), + add_to_manifest(AccMan, ManEntry) end, + Man0, + ManifestSnippet), NewManifestSQN = State#state.manifest_sqn + 1, ok = simple_manifest_writer(Man1, NewManifestSQN, State#state.root_path), PendingRemovals = case PromptDeletion of true -> State#state.pending_removals ++ - {NewManifestSQN, DeletedFiles}; + [{NewManifestSQN, DeletedFiles}]; _ -> State#state.pending_removals end, @@ -283,14 +304,24 @@ handle_call({update_manifest, handle_call(print_manifest, _From, State) -> manifest_printer(State#state.manifest), {reply, ok, State}; -handle_call({compact_journal, Penciller, Timeout}, _From, State) -> +handle_call({compact, + Checker, + InitiateFun, + FilterFun, + Timeout}, + _From, State) -> leveled_iclerk:clerk_compact(State#state.clerk, + Checker, + InitiateFun, + FilterFun, self(), - Penciller, Timeout), {reply, ok, State}; handle_call(close, _From, State) -> - {stop, normal, ok, State}. + {stop, normal, ok, State}; +handle_call(Msg, _From, State) -> + io:format("Unexpected message ~w~n", [Msg]), + {reply, error, State}. handle_cast(_Msg, State) -> {noreply, State}. @@ -712,6 +743,15 @@ manifest_printer(Manifest) -> [SQN, FN]) end, Manifest). + +initiate_penciller_snapshot(Penciller) -> + PclOpts = #penciller_options{start_snapshot = true, + source_penciller = Penciller, + requestor = self()}, + FilterServer = leveled_penciller:pcl_start(PclOpts), + ok = leveled_penciller:pcl_loadsnapshot(FilterServer, []), + FilterServer. + %%%============================================================================ %%% Test %%%============================================================================ @@ -720,6 +760,7 @@ manifest_printer(Manifest) -> build_dummy_journal() -> RootPath = "../test/journal", + clean_testdir(RootPath), JournalFP = filepath(RootPath, journal_dir), ManifestFP = filepath(RootPath, manifest_dir), ok = filelib:ensure_dir(RootPath), @@ -753,7 +794,13 @@ clean_testdir(RootPath) -> clean_subdir(DirPath) -> {ok, Files} = file:list_dir(DirPath), - lists:foreach(fun(FN) -> file:delete(filename:join(DirPath, FN)) end, + lists:foreach(fun(FN) -> + File = filename:join(DirPath, FN), + case file:delete(File) of + ok -> io:format("Success deleting ~s~n", [File]); + _ -> io:format("Error deleting ~s~n", [File]) + end + end, Files). simple_buildmanifest_test() -> @@ -872,8 +919,53 @@ rollafile_simplejournal_test() -> ?assertMatch(R1, {{5, "KeyAA"}, {"TestValueAA", []}}), R2 = ink_get(Ink1, "KeyBB", 54), ?assertMatch(R2, {{54, "KeyBB"}, {"TestValueBB", []}}), + Man = ink_getmanifest(Ink1), + FakeMan = [{3, "test", dummy}, {1, "other_test", dummy}], + ok = ink_updatemanifest(Ink1, FakeMan, true, Man), + ?assertMatch(FakeMan, ink_getmanifest(Ink1)), + ok = ink_updatemanifest(Ink1, Man, true, FakeMan), + ?assertMatch({{5, "KeyAA"}, {"TestValueAA", []}}, + ink_get(Ink1, "KeyAA", 5)), + ?assertMatch({{54, "KeyBB"}, {"TestValueBB", []}}, + ink_get(Ink1, "KeyBB", 54)), ink_close(Ink1), clean_testdir(RootPath). +compact_journal_test() -> + RootPath = "../test/journal", + build_dummy_journal(), + CDBopts = #cdb_options{max_size=300000}, + {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, + cdb_options=CDBopts}), + FunnyLoop = lists:seq(1, 48), + {ok, NewSQN1, _ObjSize} = ink_put(Ink1, "KeyAA", "TestValueAA", []), + ?assertMatch(NewSQN1, 5), + ok = ink_print_manifest(Ink1), + R0 = ink_get(Ink1, "KeyAA", 5), + ?assertMatch(R0, {{5, "KeyAA"}, {"TestValueAA", []}}), + Checker = lists:map(fun(X) -> + PK = "KeyZ" ++ integer_to_list(X), + {ok, SQN, _} = ink_put(Ink1, + PK, + crypto:rand_bytes(10000), + []), + {SQN, PK} + end, + FunnyLoop), + {ok, NewSQN2, _ObjSize} = ink_put(Ink1, "KeyBB", "TestValueBB", []), + ?assertMatch(NewSQN2, 54), + ActualManifest = ink_getmanifest(Ink1), + ?assertMatch(2, length(ActualManifest)), + ok = ink_compactjournal(Ink1, + Checker, + fun(X) -> X end, + fun(L, K, SQN) -> lists:member({SQN, K}, L) end, + 5000), + timer:sleep(1000), + CompactedManifest = ink_getmanifest(Ink1), + ?assertMatch(1, length(CompactedManifest)), + ink_updatemanifest(Ink1, ActualManifest, true, CompactedManifest), + ink_close(Ink1), + clean_testdir(RootPath). -endif. \ No newline at end of file From 507428bd0b6762ab48bcf8673e481b43c56696ba Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 3 Oct 2016 23:34:28 +0100 Subject: [PATCH 052/167] Add initial system test Add some initial system tests. This highlighted issues: - That files deleted by compaction would be left orphaned and not close, and would not in fact delete (now deleted by closure only) - There was an issue on stratup that the first few keys in each journal would not be re-loaded into the ledger --- include/leveled.hrl | 1 + src/leveled_bookie.erl | 49 +++++++- src/leveled_cdb.erl | 22 +++- src/leveled_iclerk.erl | 18 ++- src/leveled_inker.erl | 112 ++++++++++++------ src/leveled_iterator.erl | 197 -------------------------------- src/leveled_penciller.erl | 37 +++--- src/leveled_sft.erl | 43 ++++--- test/end_to_end/basic_SUITE.erl | 135 ++++++++++++++++++++++ 9 files changed, 339 insertions(+), 275 deletions(-) delete mode 100644 src/leveled_iterator.erl create mode 100644 test/end_to_end/basic_SUITE.erl diff --git a/include/leveled.hrl b/include/leveled.hrl index 321ff7c..0debd70 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -42,6 +42,7 @@ -record(bookie_options, {root_path :: string(), cache_size :: integer(), + max_journalsize :: integer(), metadata_extractor :: function(), indexspec_converter :: function()}). diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 948ea66..a3679d6 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -11,7 +11,7 @@ %% and frequent use of iterators) %% - The Journal is an extended nursery log in leveldb terms. It is keyed %% on the sequence number of the write -%% - The ledger is a LSM tree, where the key is the actaul object key, and +%% - The ledger is a merge tree, where the key is the actaul object key, and %% the value is the metadata of the object including the sequence number %% %% @@ -140,6 +140,7 @@ book_riakhead/3, book_snapshotstore/3, book_snapshotledger/3, + book_compactjournal/2, book_close/1, strip_to_keyonly/1, strip_to_keyseqonly/1, @@ -152,6 +153,8 @@ -define(CACHE_SIZE, 1000). -define(JOURNAL_FP, "journal"). -define(LEDGER_FP, "ledger"). +-define(SHUTDOWN_WAITS, 60). +-define(SHUTDOWN_PAUSE, 10000). -record(state, {inker :: pid(), penciller :: pid(), @@ -188,6 +191,9 @@ book_snapshotstore(Pid, Requestor, Timeout) -> book_snapshotledger(Pid, Requestor, Timeout) -> gen_server:call(Pid, {snapshot, Requestor, ledger, Timeout}, infinity). +book_compactjournal(Pid, Timeout) -> + gen_server:call(Pid, {compact_journal, Timeout}, infinity). + book_close(Pid) -> gen_server:call(Pid, close, infinity). @@ -289,6 +295,11 @@ handle_call({snapshot, Requestor, SnapType, _Timeout}, _From, State) -> ledger -> {reply, {ok, LedgerSnapshot, null}, State} end; +handle_call({compact_journal, Timeout}, _From, State) -> + ok = leveled_inker:ink_compactjournal(State#state.inker, + State#state.penciller, + Timeout), + {reply, ok, State}; handle_call(close, _From, State) -> {stop, normal, ok, State}. @@ -300,7 +311,16 @@ handle_info(_Info, State) -> terminate(Reason, State) -> io:format("Bookie closing for reason ~w~n", [Reason]), - ok = leveled_inker:ink_close(State#state.inker), + WaitList = lists:duplicate(?SHUTDOWN_WAITS, ?SHUTDOWN_PAUSE), + ok = case shutdown_wait(WaitList, State#state.inker) of + false -> + io:format("Forcing close of inker following wait of " + ++ "~w milliseconds~n", + [lists:sum(WaitList)]), + leveled_inker:ink_forceclose(State#state.inker); + true -> + ok + end, ok = leveled_penciller:pcl_close(State#state.penciller). code_change(_OldVsn, State, _Extra) -> @@ -311,11 +331,30 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================ +shutdown_wait([], _Inker) -> + false; +shutdown_wait([TopPause|Rest], Inker) -> + case leveled_inker:ink_close(Inker) of + ok -> + true; + pause -> + io:format("Inker shutdown stil waiting process to complete~n"), + ok = timer:sleep(TopPause), + shutdown_wait(Rest, Inker) + end. + + set_options(Opts) -> %% TODO: Change the max size default, and allow setting through options + MaxJournalSize = case Opts#bookie_options.max_journalsize of + undefined -> + 30000; + MS -> + MS + end, {#inker_options{root_path = Opts#bookie_options.root_path ++ "/" ++ ?JOURNAL_FP, - cdb_options = #cdb_options{max_size=30000}}, + cdb_options = #cdb_options{max_size=MaxJournalSize}}, #penciller_options{root_path=Opts#bookie_options.root_path ++ "/" ++ ?LEDGER_FP}}. @@ -442,10 +481,10 @@ maybepush_ledgercache(MaxCacheSize, Cache, Penciller) -> load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> {MinSQN, MaxSQN, Output} = Acc0, {SQN, PK} = KeyInLedger, - io:format("Reloading changes with SQN=~w PK=~w~n", [SQN, PK]), {Obj, IndexSpecs} = binary_to_term(ExtractFun(ValueInLedger)), case SQN of SQN when SQN < MinSQN -> + io:format("Skipping due to low SQN ~w~n", [SQN]), {loop, Acc0}; SQN when SQN =< MaxSQN -> %% TODO - get correct size in a more efficient manner @@ -454,6 +493,8 @@ load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> Changes = preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs), {loop, {MinSQN, MaxSQN, Output ++ Changes}}; SQN when SQN > MaxSQN -> + io:format("Skipping as exceeded MaxSQN ~w with SQN ~w~n", + [MaxSQN, SQN]), {stop, Acc0} end. diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 40bf5ef..31b4c53 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -66,7 +66,8 @@ cdb_scan/4, cdb_close/1, cdb_complete/1, - cdb_destroy/1]). + cdb_destroy/1, + cdb_deletepending/1]). -include_lib("eunit/include/eunit.hrl"). @@ -84,7 +85,8 @@ filename :: string(), handle :: file:fd(), writer :: boolean(), - max_size :: integer()}). + max_size :: integer(), + pending_delete = false :: boolean()}). %%%============================================================================ @@ -138,6 +140,9 @@ cdb_complete(Pid) -> cdb_destroy(Pid) -> gen_server:cast(Pid, destroy). +cdb_deletepending(Pid) -> + gen_server:cast(Pid, delete_pending). + %% cdb_scan returns {LastPosition, Acc}. Use LastPosition as StartPosiiton to %% continue from that point (calling function has to protect against) double %% counting. @@ -355,6 +360,8 @@ handle_cast(destroy, State) -> ok = file:close(State#state.handle), ok = file:delete(State#state.filename), {noreply, State}; +handle_cast(delete_pending, State) -> + {noreply, State#state{pending_delete = true}}; handle_cast(_Msg, State) -> {noreply, State}. @@ -362,11 +369,14 @@ handle_info(_Info, State) -> {noreply, State}. terminate(_Reason, State) -> - case State#state.handle of - undefined -> + case {State#state.handle, State#state.pending_delete} of + {undefined, _} -> ok; - Handle -> - file:close(Handle) + {Handle, false} -> + file:close(Handle); + {Handle, true} -> + file:close(Handle), + file:delete(State#state.filename) end. code_change(_OldVsn, State, _Extra) -> diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 49a4f43..756d96b 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -80,7 +80,7 @@ init([IClerkOpts]) -> end. handle_call(_Msg, _From, State) -> - {reply, not_supprted, State}. + {reply, not_supported, State}. handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, State) -> @@ -111,11 +111,20 @@ handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, BestRun), ok = leveled_inker:ink_updatemanifest(Inker, ManifestSlice, - PromptDelete, FilesToDelete), - {noreply, State}; + ok = leveled_inker:ink_compactioncomplete(Inker), + case PromptDelete of + true -> + lists:foreach(fun({_SQN, _FN, J2D}) -> + leveled_cdb:cdb_deletepending(J2D) end, + FilesToDelete), + {noreply, State}; + false -> + {noreply, State} + end; Score -> io:format("No compaction run as highest score=~w~n", [Score]), + ok = leveled_inker:ink_compactioncomplete(Inker), {noreply, State} end; handle_cast({remove, _Removals}, State) -> @@ -361,6 +370,9 @@ write_values([KVC|Rest], CDBopts, Journal0, ManSlice0) -> FN = leveled_inker:filepath(FP, SQN, compact_journal), + io:format("Generate journal for compaction" + ++ " with filename ~s~n", + [FN]), leveled_cdb:cdb_open_writer(FN, CDBopts); _ -> diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 56c1664..3fab0fa 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -105,10 +105,12 @@ ink_loadpcl/4, ink_registersnapshot/2, ink_compactjournal/3, + ink_compactioncomplete/1, ink_getmanifest/1, - ink_updatemanifest/4, + ink_updatemanifest/3, ink_print_manifest/1, ink_close/1, + ink_forceclose/1, build_dummy_journal/0, simple_manifest_reader/2, clean_testdir/1, @@ -135,7 +137,9 @@ registered_snapshots = [] :: list(), root_path :: string(), cdb_options :: #cdb_options{}, - clerk :: pid()}). + clerk :: pid(), + compaction_pending = false :: boolean(), + is_snapshot = false :: boolean()}). %%%============================================================================ @@ -158,7 +162,10 @@ ink_registersnapshot(Pid, Requestor) -> gen_server:call(Pid, {snapshot, Requestor}, infinity). ink_close(Pid) -> - gen_server:call(Pid, close, infinity). + gen_server:call(Pid, {close, false}, infinity). + +ink_forceclose(Pid) -> + gen_server:call(Pid, {close, true}, infinity). ink_loadpcl(Pid, MinSQN, FilterFun, Penciller) -> gen_server:call(Pid, {load_pcl, MinSQN, FilterFun, Penciller}, infinity). @@ -172,7 +179,7 @@ ink_compactjournal(Pid, Penciller, Timeout) -> CheckerInitiateFun, CheckerFilterFun, Timeout}, - infiniy). + infinity). %% Allows the Checker to be overriden in test, use something other than a %% penciller @@ -185,14 +192,16 @@ ink_compactjournal(Pid, Checker, InitiateFun, FilterFun, Timeout) -> Timeout}, infinity). +ink_compactioncomplete(Pid) -> + gen_server:call(Pid, compaction_complete, infinity). + ink_getmanifest(Pid) -> gen_server:call(Pid, get_manifest, infinity). -ink_updatemanifest(Pid, ManifestSnippet, PromptDeletion, DeletedFiles) -> +ink_updatemanifest(Pid, ManifestSnippet, DeletedFiles) -> gen_server:call(Pid, {update_manifest, ManifestSnippet, - PromptDeletion, DeletedFiles}, infinity). @@ -213,7 +222,8 @@ init([InkerOpts]) -> {ActiveJournalDB, Manifest}} = ink_registersnapshot(SrcInker, Requestor), {ok, #state{manifest=Manifest, - active_journaldb=ActiveJournalDB}}; + active_journaldb=ActiveJournalDB, + is_snapshot=true}}; %% Need to do something about timeout {_RootPath, false} -> start_from_file(InkerOpts) @@ -272,7 +282,6 @@ handle_call(get_manifest, _From, State) -> {reply, State#state.manifest, State}; handle_call({update_manifest, ManifestSnippet, - PromptDeletion, DeletedFiles}, _From, State) -> Man0 = lists:foldl(fun(ManEntry, AccMan) -> Check = lists:member(ManEntry, DeletedFiles), @@ -291,13 +300,7 @@ handle_call({update_manifest, ManifestSnippet), NewManifestSQN = State#state.manifest_sqn + 1, ok = simple_manifest_writer(Man1, NewManifestSQN, State#state.root_path), - PendingRemovals = case PromptDeletion of - true -> - State#state.pending_removals ++ - [{NewManifestSQN, DeletedFiles}]; - _ -> - State#state.pending_removals - end, + PendingRemovals = [{NewManifestSQN, DeletedFiles}], {reply, ok, State#state{manifest=Man1, manifest_sqn=NewManifestSQN, pending_removals=PendingRemovals}}; @@ -316,9 +319,16 @@ handle_call({compact, FilterFun, self(), Timeout), - {reply, ok, State}; -handle_call(close, _From, State) -> - {stop, normal, ok, State}; + {reply, ok, State#state{compaction_pending=true}}; +handle_call(compaction_complete, _From, State) -> + {reply, ok, State#state{compaction_pending=false}}; +handle_call({close, Force}, _From, State) -> + case {State#state.compaction_pending, Force} of + {true, false} -> + {reply, pause, State}; + _ -> + {stop, normal, ok, State} + end; handle_call(Msg, _From, State) -> io:format("Unexpected message ~w~n", [Msg]), {reply, error, State}. @@ -330,12 +340,21 @@ handle_info(_Info, State) -> {noreply, State}. terminate(Reason, State) -> - io:format("Inker closing journal for reason ~w~n", [Reason]), - io:format("Close triggered with journal_sqn=~w and manifest_sqn=~w~n", - [State#state.journal_sqn, State#state.manifest_sqn]), - io:format("Manifest when closing is: ~n"), - manifest_printer(State#state.manifest), - close_allmanifest(State#state.manifest, State#state.active_journaldb). + case State#state.is_snapshot of + true -> + ok; + false -> + io:format("Inker closing journal for reason ~w~n", [Reason]), + io:format("Close triggered with journal_sqn=~w and manifest_sqn=~w~n", + [State#state.journal_sqn, State#state.manifest_sqn]), + io:format("Manifest when closing is: ~n"), + leveled_iclerk:clerk_stop(State#state.clerk), + lists:foreach(fun({Snap, _SQN}) -> ok = ink_close(Snap) end, + State#state.registered_snapshots), + manifest_printer(State#state.manifest), + close_allmanifest(State#state.manifest, State#state.active_journaldb), + close_allremovals(State#state.pending_removals) + end. code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -597,6 +616,25 @@ close_allmanifest([H|ManifestT], ActiveJournal) -> ok = leveled_cdb:cdb_close(Pid), close_allmanifest(ManifestT, ActiveJournal). +close_allremovals([]) -> + ok; +close_allremovals([{ManifestSQN, Removals}|Tail]) -> + io:format("Closing removals at ManifestSQN=~w~n", [ManifestSQN]), + lists:foreach(fun({LowSQN, FN, Handle}) -> + io:format("Closing removed file with LowSQN=~w" ++ + " and filename ~s~n", + [LowSQN, FN]), + if + is_pid(Handle) == true -> + ok = leveled_cdb:cdb_close(Handle); + true -> + io:format("Non pid in removal ~w - test~n", + [Handle]) + end + end, + Removals), + close_allremovals(Tail). + roll_pending_journals([TopJournalSQN], Manifest, _RootPath) when is_integer(TopJournalSQN) -> @@ -621,22 +659,22 @@ load_from_sequence(_MinSQN, _FilterFun, _Penciller, []) -> load_from_sequence(MinSQN, FilterFun, Penciller, [{LowSQN, FN, Pid}|ManTail]) when LowSQN >= MinSQN -> io:format("Loading from filename ~s from SQN ~w~n", [FN, MinSQN]), - ok = load_between_sequence(MinSQN, - MinSQN + ?LOADING_BATCH, - FilterFun, - Penciller, - Pid, - undefined), - load_from_sequence(MinSQN, FilterFun, Penciller, ManTail); + {ok, LastMinSQN} = load_between_sequence(MinSQN, + MinSQN + ?LOADING_BATCH, + FilterFun, + Penciller, + Pid, + undefined), + load_from_sequence(LastMinSQN, FilterFun, Penciller, ManTail); load_from_sequence(MinSQN, FilterFun, Penciller, [_H|ManTail]) -> load_from_sequence(MinSQN, FilterFun, Penciller, ManTail). load_between_sequence(MinSQN, MaxSQN, FilterFun, Penciller, CDBpid, StartPos) -> InitAcc = {MinSQN, MaxSQN, []}, case leveled_cdb:cdb_scan(CDBpid, FilterFun, InitAcc, StartPos) of - {eof, {_AccMinSQN, _AccMaxSQN, AccKL}} -> + {eof, {AccMinSQN, _AccMaxSQN, AccKL}} -> ok = push_to_penciller(Penciller, AccKL), - ok; + {ok, AccMinSQN}; {LastPosition, {_AccMinSQN, _AccMaxSQN, AccKL}} -> ok = push_to_penciller(Penciller, AccKL), load_between_sequence(MaxSQN + 1, @@ -748,7 +786,7 @@ initiate_penciller_snapshot(Penciller) -> PclOpts = #penciller_options{start_snapshot = true, source_penciller = Penciller, requestor = self()}, - FilterServer = leveled_penciller:pcl_start(PclOpts), + {ok, FilterServer} = leveled_penciller:pcl_start(PclOpts), ok = leveled_penciller:pcl_loadsnapshot(FilterServer, []), FilterServer. @@ -793,6 +831,7 @@ clean_testdir(RootPath) -> clean_subdir(filepath(RootPath, manifest_dir)). clean_subdir(DirPath) -> + ok = filelib:ensure_dir(DirPath), {ok, Files} = file:list_dir(DirPath), lists:foreach(fun(FN) -> File = filename:join(DirPath, FN), @@ -921,9 +960,9 @@ rollafile_simplejournal_test() -> ?assertMatch(R2, {{54, "KeyBB"}, {"TestValueBB", []}}), Man = ink_getmanifest(Ink1), FakeMan = [{3, "test", dummy}, {1, "other_test", dummy}], - ok = ink_updatemanifest(Ink1, FakeMan, true, Man), + ok = ink_updatemanifest(Ink1, FakeMan, Man), ?assertMatch(FakeMan, ink_getmanifest(Ink1)), - ok = ink_updatemanifest(Ink1, Man, true, FakeMan), + ok = ink_updatemanifest(Ink1, Man, FakeMan), ?assertMatch({{5, "KeyAA"}, {"TestValueAA", []}}, ink_get(Ink1, "KeyAA", 5)), ?assertMatch({{54, "KeyBB"}, {"TestValueBB", []}}, @@ -964,7 +1003,6 @@ compact_journal_test() -> timer:sleep(1000), CompactedManifest = ink_getmanifest(Ink1), ?assertMatch(1, length(CompactedManifest)), - ink_updatemanifest(Ink1, ActualManifest, true, CompactedManifest), ink_close(Ink1), clean_testdir(RootPath). diff --git a/src/leveled_iterator.erl b/src/leveled_iterator.erl deleted file mode 100644 index e065918..0000000 --- a/src/leveled_iterator.erl +++ /dev/null @@ -1,197 +0,0 @@ --module(leveled_iterator). - --export([termiterator/3]). - --include_lib("eunit/include/eunit.hrl"). - - -%% Takes a list of terms to iterate - the terms being sorted in Erlang term -%% order -%% -%% Helper Functions should have free functions - -%% {FolderFun, CompareFun, PointerCheck, PointerFetch} -%% FolderFun - function which takes the next item and the accumulator and -%% returns an updated accumulator. Note FolderFun can only increase the -%% accumulator by one entry each time -%% CompareFun - function which should be able to compare two keys (which are -%% not pointers), and return a winning item (or combination of items) -%% PointerCheck - function for differentiating between keys and pointer -%% PointerFetch - function that takes a pointer an EndKey (which may be -%% infinite) and returns a ne wslice of ordered results from that pointer -%% -%% Range can be for the form -%% {StartKey, EndKey, MaxKeys} where EndKey or MaxKeys can be infinite (but -%% not both) - - -termiterator(ListToIterate, HelperFuns, Range) -> - case Range of - {_, infinte, infinite} -> - bad_iterator; - _ -> - termiterator(null, ListToIterate, [], HelperFuns, Range) - end. - - -termiterator(HeadItem, [], Acc, HelperFuns, _) -> - case HeadItem of - null -> - Acc; - _ -> - {FolderFun, _, _, _} = HelperFuns, - FolderFun(Acc, HeadItem) - end; -termiterator(null, [NextItem|TailList], Acc, HelperFuns, Range) -> - %% Check that the NextItem is not a pointer before promoting to HeadItem - %% Cannot now promote a HeadItem which is a pointer - {_, _, PointerCheck, PointerFetch} = HelperFuns, - case PointerCheck(NextItem) of - {true, Pointer} -> - {_, EndKey, _} = Range, - NewSlice = PointerFetch(Pointer, EndKey), - ExtendedList = lists:merge(NewSlice, TailList), - termiterator(null, ExtendedList, Acc, HelperFuns, Range); - false -> - termiterator(NextItem, TailList, Acc, HelperFuns, Range) - end; -termiterator(HeadItem, [NextItem|TailList], Acc, HelperFuns, Range) -> - {FolderFun, CompareFun, PointerCheck, PointerFetch} = HelperFuns, - {_, EndKey, MaxItems} = Range, - %% HeadItem cannot be pointer, but NextItem might be, so check before - %% comparison - case PointerCheck(NextItem) of - {true, Pointer} -> - NewSlice = PointerFetch(Pointer, EndKey), - ExtendedList = lists:merge(NewSlice, [HeadItem|TailList]), - termiterator(null, ExtendedList, Acc, HelperFuns, Range); - false -> - %% Compare to see if Head and Next match, or if Head is a winner - %% to be added to accumulator - case CompareFun(HeadItem, NextItem) of - {match, StrongItem, _WeakItem} -> - %% Discard WeakItem, Strong Item might be an aggregation of - %% the items - termiterator(StrongItem, TailList, Acc, HelperFuns, Range); - {winner, HeadItem} -> - %% Add next item to accumulator, and proceed with next item - AccPlus = FolderFun(Acc, HeadItem), - case length(AccPlus) of - MaxItems -> - AccPlus; - _ -> - termiterator(NextItem, TailList, AccPlus, - HelperFuns, - {HeadItem, EndKey, MaxItems}) - end - end - end. - - -%% Initial forms of keys supported are Index Keys and Object Keys -%% -%% All keys are of the form {Key, Value, SequenceNumber, State} -%% -%% The Key will be of the form: -%% {o, Bucket, Key} - for an Object Key -%% {i, Bucket, IndexName, IndexTerm, Key} - for an Index Key -%% -%% The value will be of the form: -%% {o, ObjectHash, [vector-clocks]} - for an Object Key -%% null - for an Index Key -%% -%% Sequence number is the sequence number the key was added, and the highest -%% sequence number in the list of keys for an index key. -%% -%% State can be one of the following: -%% live - an active key -%% tomb - a tombstone key -%% {timestamp, TS} - an active key to a certain timestamp -%% {pointer, Pointer} - to be added by iterators to indicate further data -%% available in the range from a particular source - - -pointercheck_indexkey(IndexKey) -> - case IndexKey of - {_Key, _Values, _Sequence, {pointer, Pointer}} -> - {true, Pointer}; - _ -> - false - end. - -folder_indexkey(Acc, IndexKey) -> - case IndexKey of - {_Key, _Value, _Sequence, tomb} -> - Acc; - {Key, _Value, _Sequence, live} -> - {i, _, _, _, ObjectKey} = Key, - lists:append(Acc, [ObjectKey]) - end. - -compare_indexkey(IndexKey1, IndexKey2) -> - {{i, Bucket1, Index1, Term1, Key1}, _Val1, Sequence1, _St1} = IndexKey1, - {{i, Bucket2, Index2, Term2, Key2}, _Val2, Sequence2, _St2} = IndexKey2, - case {Bucket1, Index1, Term1, Key1} of - {Bucket2, Index2, Term2, Key2} when Sequence1 >= Sequence2 -> - {match, IndexKey1, IndexKey2}; - {Bucket2, Index2, Term2, Key2} -> - {match, IndexKey2, IndexKey1}; - _ when IndexKey2 >= IndexKey1 -> - {winner, IndexKey1}; - _ -> - {winner, IndexKey2} - end. - - - -%% Unit testsß - -getnextslice(Pointer, _EndKey) -> - case Pointer of - {test, NewList} -> - NewList; - _ -> - [] - end. - - -iterateoverindexkeyswithnopointer_test() -> - Key1 = {{i, "pdsRecord", "familyName_bin", "1972SMITH", "10001"}, - null, 1, live}, - Key2 = {{i, "pdsRecord", "familyName_bin", "1972SMITH", "10001"}, - null, 2, tomb}, - Key3 = {{i, "pdsRecord", "familyName_bin", "1971SMITH", "10002"}, - null, 2, live}, - Key4 = {{i, "pdsRecord", "familyName_bin", "1972JONES", "10003"}, - null, 2, live}, - KeyList = lists:sort([Key1, Key2, Key3, Key4]), - HelperFuns = {fun folder_indexkey/2, fun compare_indexkey/2, - fun pointercheck_indexkey/1, fun getnextslice/2}, - ?assertMatch(["10002", "10003"], - termiterator(KeyList, HelperFuns, {"1971", "1973", infinite})). - -iterateoverindexkeyswithpointer_test() -> - Key1 = {{i, "pdsRecord", "familyName_bin", "1972SMITH", "10001"}, - null, 1, live}, - Key2 = {{i, "pdsRecord", "familyName_bin", "1972SMITH", "10001"}, - null, 2, tomb}, - Key3 = {{i, "pdsRecord", "familyName_bin", "1971SMITH", "10002"}, - null, 2, live}, - Key4 = {{i, "pdsRecord", "familyName_bin", "1972JONES", "10003"}, - null, 2, live}, - Key5 = {{i, "pdsRecord", "familyName_bin", "1972ZAFRIDI", "10004"}, - null, 2, live}, - Key6 = {{i, "pdsRecord", "familyName_bin", "1972JONES", "10004"}, - null, 0, {pointer, {test, [Key5]}}}, - KeyList = lists:sort([Key1, Key2, Key3, Key4, Key6]), - HelperFuns = {fun folder_indexkey/2, fun compare_indexkey/2, - fun pointercheck_indexkey/1, fun getnextslice/2}, - ?assertMatch(["10002", "10003", "10004"], - termiterator(KeyList, HelperFuns, {"1971", "1973", infinite})), - ?assertMatch(["10002", "10003"], - termiterator(KeyList, HelperFuns, {"1971", "1973", 2})). - - - - - - diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 9747b9a..a1b1249 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -7,11 +7,16 @@ %% Ledger. %% - The Penciller provides re-write (compaction) work up to be managed by %% the Penciller's Clerk -%% - The Penciller mainatins a register of iterators who have requested +%% - The Penciller maintains a register of iterators who have requested %% snapshots of the Ledger %% - The accepts new dumps (in the form of lists of keys) from the Bookie, and %% calls the Bookie once the process of pencilling this data in the Ledger is %% complete - and the Bookie is free to forget about the data +%% - The Penciller's persistence of the ledger may not be reliable, in that it +%% may lose data but only in sequence from a particular sequence number. On +%% startup the Penciller will inform the Bookie of the highest sequence number +%% it has, and the Bookie should load any missing data from that point out of +%5 the journal. %% %% -------- LEDGER --------- %% @@ -78,18 +83,21 @@ %% %% ---------- SNAPSHOT ---------- %% -%% Iterators may request a snapshot of the database. To provide a snapshot -%% the Penciller must snapshot the ETS table, and then send this with a copy -%% of the manifest. +%% Iterators may request a snapshot of the database. A snapshot is a cloned +%% Penciller seeded not from disk, but by the in-memory ETS table and the +%% in-memory manifest. + +%% To provide a snapshot the Penciller must snapshot the ETS table. The +%% snapshot of the ETS table is managed by the Penciller storing a list of the +%% batches of Keys which have been pushed to the Penciller, and it is expected +%% that this will be converted by the clone into a gb_tree. The clone may +%% then update the master Penciller with the gb_tree to be cached and used by +%% other cloned processes. %% -%% Iterators requesting snapshots are registered by the Penciller, so that SFT -%% files valid at the point of the snapshot until either the iterator is +%% Clones formed to support snapshots are registered by the Penciller, so that +%% SFT files valid at the point of the snapshot until either the iterator is %% completed or has timed out. %% -%% Snapshot requests may request a filtered view of the ETS table (whihc may -%% be quicker than requesting the full table), or requets a snapshot of only -%% the persisted part of the Ledger -%% %% ---------- ON STARTUP ---------- %% %% On Startup the Bookie with ask the Penciller to initiate the Ledger first. @@ -105,15 +113,14 @@ %% ---------- ON SHUTDOWN ---------- %% %% On a controlled shutdown the Penciller should attempt to write any in-memory -%% ETS table to disk into the special ..on_shutdown folder +%% ETS table to a L0 SFT file, assuming one is nto already pending. If one is +%% already pending then the Penciller will not persist this part of the Ledger. %% %% ---------- FOLDER STRUCTURE ---------- %% %% The following folders are used by the Penciller -%% $ROOT/ledger_manifest/ - used for keeping manifest files -%% $ROOT/ledger_onshutdown/ - containing the persisted view of the ETS table -%% written on controlled shutdown -%% $ROOT/ledger_files/ - containing individual SFT files +%% $ROOT/ledger/ledger_manifest/ - used for keeping manifest files +%% $ROOT/ledger/ledger_files/ - containing individual SFT files %% %% In larger stores there could be a large number of files in the ledger_file %% folder - perhaps o(1000). It is assumed that modern file systems should diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index ac1bf68..743d93e 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -408,19 +408,7 @@ create_levelzero(Inp1, Filename) -> false -> ets:tab2list(Inp1) end, - Ext = filename:extension(Filename), - Components = filename:split(Filename), - {TmpFilename, PrmFilename} = case Ext of - [] -> - {filename:join(Components) ++ ".pnd", - filename:join(Components) ++ ".sft"}; - Ext -> - %% This seems unnecessarily hard - DN = filename:dirname(Filename), - FP = lists:last(Components), - FP_NOEXT = lists:sublist(FP, 1, 1 + length(FP) - length(Ext)), - {DN ++ "/" ++ FP_NOEXT ++ ".pnd", DN ++ "/" ++ FP_NOEXT ++ ".sft"} - end, + {TmpFilename, PrmFilename} = generate_filenames(Filename), case create_file(TmpFilename) of {error, Reason} -> {error, @@ -442,6 +430,23 @@ create_levelzero(Inp1, Filename) -> oversized_file=InputSize>?MAX_KEYS}} end. + +generate_filenames(RootFilename) -> + Ext = filename:extension(RootFilename), + Components = filename:split(RootFilename), + case Ext of + [] -> + {filename:join(Components) ++ ".pnd", + filename:join(Components) ++ ".sft"}; + Ext -> + %% This seems unnecessarily hard + DN = filename:dirname(RootFilename), + FP = lists:last(Components), + FP_NOEXT = lists:sublist(FP, 1, 1 + length(FP) - length(Ext)), + {DN ++ "/" ++ FP_NOEXT ++ "pnd", DN ++ "/" ++ FP_NOEXT ++ "sft"} + end. + + %% Start a bare file with an initial header and no further details %% Return the {Handle, metadata record} create_file(FileName) when is_list(FileName) -> @@ -1762,4 +1767,16 @@ big_iterator_test() -> ok = file:close(Handle), ok = file:delete(Filename). +filename_test() -> + FN1 = "../tmp/filename", + FN2 = "../tmp/filename.pnd", + FN3 = "../tmp/subdir/file_name.pend", + ?assertMatch({"../tmp/filename.pnd", "../tmp/filename.sft"}, + generate_filenames(FN1)), + ?assertMatch({"../tmp/filename.pnd", "../tmp/filename.sft"}, + generate_filenames(FN2)), + ?assertMatch({"../tmp/subdir/file_name.pnd", + "../tmp/subdir/file_name.sft"}, + generate_filenames(FN3)). + -endif. \ No newline at end of file diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl new file mode 100644 index 0000000..602514c --- /dev/null +++ b/test/end_to_end/basic_SUITE.erl @@ -0,0 +1,135 @@ +-module(basic_SUITE). +-include_lib("common_test/include/ct.hrl"). +-include("../include/leveled.hrl"). +-export([all/0]). +-export([simple_put_fetch/1, + journal_compaction/1]). + +all() -> [journal_compaction, simple_put_fetch]. + +simple_put_fetch(_Config) -> + RootPath = reset_filestructure(), + StartOpts1 = #bookie_options{root_path=RootPath}, + {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), + {TestObject, TestSpec} = generate_testobject(), + ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), + {ok, TestObject} = leveled_bookie:book_riakget(Bookie1, + TestObject#r_object.bucket, + TestObject#r_object.key), + ok = leveled_bookie:book_close(Bookie1), + StartOpts2 = #bookie_options{root_path=RootPath, + max_journalsize=3000000}, + {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), + {ok, TestObject} = leveled_bookie:book_riakget(Bookie2, + TestObject#r_object.bucket, + TestObject#r_object.key), + ObjList1 = generate_multiple_objects(5000, 2), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, + ObjList1), + ChkList1 = lists:sublist(lists:sort(ObjList1), 100), + lists:foreach(fun({_RN, Obj, _Spc}) -> + R = leveled_bookie:book_riakget(Bookie2, + Obj#r_object.bucket, + Obj#r_object.key), + R = {ok, Obj} end, + ChkList1), + {ok, TestObject} = leveled_bookie:book_riakget(Bookie2, + TestObject#r_object.bucket, + TestObject#r_object.key), + ok = leveled_bookie:book_close(Bookie2), + reset_filestructure(). + +journal_compaction(_Config) -> + RootPath = reset_filestructure(), + StartOpts1 = #bookie_options{root_path=RootPath, + max_journalsize=4000000}, + {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), + {TestObject, TestSpec} = generate_testobject(), + ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), + {ok, TestObject} = leveled_bookie:book_riakget(Bookie1, + TestObject#r_object.bucket, + TestObject#r_object.key), + ObjList1 = generate_multiple_objects(5000, 2), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, + ObjList1), + ChkList1 = lists:sublist(lists:sort(ObjList1), 100), + lists:foreach(fun({_RN, Obj, _Spc}) -> + R = leveled_bookie:book_riakget(Bookie1, + Obj#r_object.bucket, + Obj#r_object.key), + R = {ok, Obj} end, + ChkList1), + {ok, TestObject} = leveled_bookie:book_riakget(Bookie1, + TestObject#r_object.bucket, + TestObject#r_object.key), + %% Now replace all the objects + ObjList2 = generate_multiple_objects(5000, 2), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, + ObjList2), + ok = leveled_bookie:book_compactjournal(Bookie1, 30000), + ChkList3 = lists:sublist(lists:sort(ObjList2), 500), + lists:foreach(fun({_RN, Obj, _Spc}) -> + R = leveled_bookie:book_riakget(Bookie1, + Obj#r_object.bucket, + Obj#r_object.key), + R = {ok, Obj} end, + ChkList3), + ok = leveled_bookie:book_close(Bookie1), + % Restart + {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), + {ok, TestObject} = leveled_bookie:book_riakget(Bookie2, + TestObject#r_object.bucket, + TestObject#r_object.key), + lists:foreach(fun({_RN, Obj, _Spc}) -> + R = leveled_bookie:book_riakget(Bookie2, + Obj#r_object.bucket, + Obj#r_object.key), + R = {ok, Obj} end, + ChkList3), + ok = leveled_bookie:book_close(Bookie2), + reset_filestructure(). + + +reset_filestructure() -> + RootPath = "test", + filelib:ensure_dir(RootPath ++ "/journal/"), + filelib:ensure_dir(RootPath ++ "/ledger/"), + leveled_inker:clean_testdir(RootPath ++ "/journal"), + leveled_penciller:clean_testdir(RootPath ++ "/ledger"), + RootPath. + +generate_testobject() -> + {B1, K1, V1, Spec1, MD} = {"Bucket1", + "Key1", + "Value1", + [], + {"MDK1", "MDV1"}}, + Content = #r_content{metadata=MD, value=V1}, + {#r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]}, + Spec1}. + +generate_multiple_objects(Count, KeyNumber) -> + generate_multiple_objects(Count, KeyNumber, [], crypto:rand_bytes(4096)). + +generate_multiple_objects(0, _KeyNumber, ObjL, _Value) -> + ObjL; +generate_multiple_objects(Count, KeyNumber, ObjL, Value) -> + Obj = {"Bucket", + "Key" ++ integer_to_list(KeyNumber), + Value, + [], + [{"MDK", "MDV" ++ integer_to_list(KeyNumber)}, + {"MDK2", "MDV" ++ integer_to_list(KeyNumber)}]}, + {B1, K1, V1, Spec1, MD} = Obj, + Content = #r_content{metadata=MD, value=V1}, + Obj1 = #r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]}, + generate_multiple_objects(Count - 1, + KeyNumber + 1, + ObjL ++ [{random:uniform(), Obj1, Spec1}], + Value). + + + \ No newline at end of file From d903f184fd75c9541d0f196f30b0c9c2ff851310 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 5 Oct 2016 09:54:53 +0100 Subject: [PATCH 053/167] Add initial end-to-end common tests These tests highlighted some logical issues when scanning over databases on startup, so fixes are wrapped in here. --- include/leveled.hrl | 3 +- src/leveled_bookie.erl | 86 +++++++++++++------- src/leveled_inker.erl | 94 ++++++++++++++------- src/leveled_pclerk.erl | 40 +++++---- src/leveled_penciller.erl | 25 +++++- test/end_to_end/basic_SUITE.erl | 140 +++++++++++++++++++++++++++----- 6 files changed, 294 insertions(+), 94 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 0debd70..c04a44f 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -44,7 +44,8 @@ cache_size :: integer(), max_journalsize :: integer(), metadata_extractor :: function(), - indexspec_converter :: function()}). + indexspec_converter :: function(), + snapshot_bookie :: pid()}). -record(iclerk_options, {inker :: pid(), diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index a3679d6..c11c8e0 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -155,6 +155,7 @@ -define(LEDGER_FP, "ledger"). -define(SHUTDOWN_WAITS, 60). -define(SHUTDOWN_PAUSE, 10000). +-define(SNAPSHOT_TIMEOUT, 300000). -record(state, {inker :: pid(), penciller :: pid(), @@ -162,7 +163,8 @@ indexspec_converter :: function(), cache_size :: integer(), back_pressure :: boolean(), - ledger_cache :: gb_trees:tree()}). + ledger_cache :: gb_trees:tree(), + is_snapshot :: boolean()}). @@ -202,32 +204,46 @@ book_close(Pid) -> %%%============================================================================ init([Opts]) -> - {InkerOpts, PencillerOpts} = set_options(Opts), - {Inker, Penciller} = startup(InkerOpts, PencillerOpts), - Extractor = if - Opts#bookie_options.metadata_extractor == undefined -> - fun extract_metadata/2; - true -> - Opts#bookie_options.metadata_extractor - end, - Converter = if - Opts#bookie_options.indexspec_converter == undefined -> - fun convert_indexspecs/3; - true -> - Opts#bookie_options.indexspec_converter - end, - CacheSize = if - Opts#bookie_options.cache_size == undefined -> - ?CACHE_SIZE; - true -> - Opts#bookie_options.cache_size - end, - {ok, #state{inker=Inker, - penciller=Penciller, - metadata_extractor=Extractor, - indexspec_converter=Converter, - cache_size=CacheSize, - ledger_cache=gb_trees:empty()}}. + case Opts#bookie_options.snapshot_bookie of + undefined -> + % Start from file not snapshot + {InkerOpts, PencillerOpts} = set_options(Opts), + {Inker, Penciller} = startup(InkerOpts, PencillerOpts), + Extractor = if + Opts#bookie_options.metadata_extractor == undefined -> + fun extract_metadata/2; + true -> + Opts#bookie_options.metadata_extractor + end, + Converter = if + Opts#bookie_options.indexspec_converter == undefined -> + fun convert_indexspecs/3; + true -> + Opts#bookie_options.indexspec_converter + end, + CacheSize = if + Opts#bookie_options.cache_size == undefined -> + ?CACHE_SIZE; + true -> + Opts#bookie_options.cache_size + end, + {ok, #state{inker=Inker, + penciller=Penciller, + metadata_extractor=Extractor, + indexspec_converter=Converter, + cache_size=CacheSize, + ledger_cache=gb_trees:empty(), + is_snapshot=false}}; + Bookie -> + {ok, + {Penciller, LedgerCache}, + Inker} = book_snapshotstore(Bookie, self(), ?SNAPSHOT_TIMEOUT), + ok = leveled_penciller:pcl_loadsnapshot(Penciller, []), + {ok, #state{penciller=Penciller, + inker=Inker, + ledger_cache=LedgerCache, + is_snapshot=true}} + end. handle_call({put, PrimaryKey, Object, IndexSpecs}, From, State) -> @@ -289,11 +305,21 @@ handle_call({snapshot, Requestor, SnapType, _Timeout}, _From, State) -> {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), case SnapType of store -> - InkerOpts = #inker_options{}, + InkerOpts = #inker_options{start_snapshot=true, + source_inker=State#state.inker, + requestor=Requestor}, {ok, JournalSnapshot} = leveled_inker:ink_start(InkerOpts), - {reply, {ok, LedgerSnapshot, JournalSnapshot}, State}; + {reply, + {ok, + {LedgerSnapshot, State#state.ledger_cache}, + JournalSnapshot}, + State}; ledger -> - {reply, {ok, LedgerSnapshot, null}, State} + {reply, + {ok, + {LedgerSnapshot, State#state.ledger_cache}, + null}, + State} end; handle_call({compact_journal, Timeout}, _From, State) -> ok = leveled_inker:ink_compactjournal(State#state.inker, diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 3fab0fa..32cf340 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -159,7 +159,7 @@ ink_fetch(Pid, PrimaryKey, SQN) -> gen_server:call(Pid, {fetch, PrimaryKey, SQN}, infinity). ink_registersnapshot(Pid, Requestor) -> - gen_server:call(Pid, {snapshot, Requestor}, infinity). + gen_server:call(Pid, {register_snapshot, Requestor}, infinity). ink_close(Pid) -> gen_server:call(Pid, {close, false}, infinity). @@ -218,11 +218,12 @@ init([InkerOpts]) -> {undefined, true} -> SrcInker = InkerOpts#inker_options.source_inker, Requestor = InkerOpts#inker_options.requestor, - {ok, - {ActiveJournalDB, - Manifest}} = ink_registersnapshot(SrcInker, Requestor), + {Manifest, + ActiveJournalDB, + ActiveJournalSQN} = ink_registersnapshot(SrcInker, Requestor), {ok, #state{manifest=Manifest, active_journaldb=ActiveJournalDB, + active_journaldb_sqn=ActiveJournalSQN, is_snapshot=true}}; %% Need to do something about timeout {_RootPath, false} -> @@ -276,7 +277,8 @@ handle_call({register_snapshot, Requestor}, _From , State) -> Rs = [{Requestor, State#state.manifest_sqn}|State#state.registered_snapshots], {reply, {State#state.manifest, - State#state.active_journaldb}, + State#state.active_journaldb, + State#state.active_journaldb_sqn}, State#state{registered_snapshots=Rs}}; handle_call(get_manifest, _From, State) -> {reply, State#state.manifest, State}; @@ -656,33 +658,67 @@ roll_pending_journals([JournalSQN|T], Manifest, RootPath) -> load_from_sequence(_MinSQN, _FilterFun, _Penciller, []) -> ok; -load_from_sequence(MinSQN, FilterFun, Penciller, [{LowSQN, FN, Pid}|ManTail]) +load_from_sequence(MinSQN, FilterFun, Penciller, [{LowSQN, FN, Pid}|Rest]) when LowSQN >= MinSQN -> - io:format("Loading from filename ~s from SQN ~w~n", [FN, MinSQN]), - {ok, LastMinSQN} = load_between_sequence(MinSQN, - MinSQN + ?LOADING_BATCH, - FilterFun, - Penciller, - Pid, - undefined), - load_from_sequence(LastMinSQN, FilterFun, Penciller, ManTail); -load_from_sequence(MinSQN, FilterFun, Penciller, [_H|ManTail]) -> - load_from_sequence(MinSQN, FilterFun, Penciller, ManTail). - -load_between_sequence(MinSQN, MaxSQN, FilterFun, Penciller, CDBpid, StartPos) -> - InitAcc = {MinSQN, MaxSQN, []}, - case leveled_cdb:cdb_scan(CDBpid, FilterFun, InitAcc, StartPos) of - {eof, {AccMinSQN, _AccMaxSQN, AccKL}} -> - ok = push_to_penciller(Penciller, AccKL), - {ok, AccMinSQN}; - {LastPosition, {_AccMinSQN, _AccMaxSQN, AccKL}} -> - ok = push_to_penciller(Penciller, AccKL), - load_between_sequence(MaxSQN + 1, - MaxSQN + 1 + ?LOADING_BATCH, + load_between_sequence(MinSQN, + MinSQN + ?LOADING_BATCH, + FilterFun, + Penciller, + Pid, + undefined, + FN, + Rest); +load_from_sequence(MinSQN, FilterFun, Penciller, [{_LowSQN, FN, Pid}|Rest]) -> + case Rest of + [] -> + load_between_sequence(MinSQN, + MinSQN + ?LOADING_BATCH, FilterFun, Penciller, - CDBpid, - LastPosition) + Pid, + undefined, + FN, + Rest); + [{NextSQN, _FN, Pid}|_Rest] when NextSQN > MinSQN -> + load_between_sequence(MinSQN, + MinSQN + ?LOADING_BATCH, + FilterFun, + Penciller, + Pid, + undefined, + FN, + Rest); + _ -> + load_from_sequence(MinSQN, FilterFun, Penciller, Rest) + end. + + + +load_between_sequence(MinSQN, MaxSQN, FilterFun, Penciller, + CDBpid, StartPos, FN, Rest) -> + io:format("Loading from filename ~s from SQN ~w~n", [FN, MinSQN]), + InitAcc = {MinSQN, MaxSQN, []}, + Res = case leveled_cdb:cdb_scan(CDBpid, FilterFun, InitAcc, StartPos) of + {eof, {AccMinSQN, _AccMaxSQN, AccKL}} -> + ok = push_to_penciller(Penciller, AccKL), + {ok, AccMinSQN}; + {LastPosition, {_AccMinSQN, _AccMaxSQN, AccKL}} -> + ok = push_to_penciller(Penciller, AccKL), + NextSQN = MaxSQN + 1, + load_between_sequence(NextSQN, + NextSQN + ?LOADING_BATCH, + FilterFun, + Penciller, + CDBpid, + LastPosition, + FN, + Rest) + end, + case Res of + {ok, LMSQN} -> + load_from_sequence(LMSQN, FilterFun, Penciller, Rest); + ok -> + ok end. push_to_penciller(Penciller, KeyList) -> diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index c54ba01..47467a3 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -117,19 +117,7 @@ merge(WI) -> {SrcF, UpdMFest1} = select_filetomerge(SrcLevel, WI#penciller_work.manifest), SinkFiles = get_item(SrcLevel + 1, UpdMFest1, []), - Splits = lists:splitwith(fun(Ref) -> - case {Ref#manifest_entry.start_key, - Ref#manifest_entry.end_key} of - {_, EK} when SrcF#manifest_entry.start_key > EK -> - false; - {SK, _} when SrcF#manifest_entry.end_key < SK -> - false; - _ -> - true - end end, - SinkFiles), - {Candidates, Others} = Splits, - + {Candidates, Others} = check_for_merge_candidates(SrcF, SinkFiles), %% TODO: %% Need to work out if this is the top level %% And then tell merge process to create files at the top level @@ -185,7 +173,20 @@ mark_for_delete([Head|Tail], Penciller) -> leveled_sft:sft_setfordelete(Head#manifest_entry.owner, Penciller), mark_for_delete(Tail, Penciller). - + +check_for_merge_candidates(SrcF, SinkFiles) -> + lists:partition(fun(Ref) -> + case {Ref#manifest_entry.start_key, + Ref#manifest_entry.end_key} of + {_, EK} when SrcF#manifest_entry.start_key > EK -> + false; + {SK, _} when SrcF#manifest_entry.end_key < SK -> + false; + _ -> + true + end end, + SinkFiles). + %% An algorithm for discovering which files to merge .... %% We can find the most optimal file: @@ -375,6 +376,17 @@ merge_file_test() -> leveled_sft:sft_clear(ManEntry#manifest_entry.owner) end, Result). +select_merge_candidates_test() -> + Sink1 = #manifest_entry{start_key = {o, "Bucket", "Key1"}, + end_key = {o, "Bucket", "Key20000"}}, + Sink2 = #manifest_entry{start_key = {o, "Bucket", "Key20001"}, + end_key = {o, "Bucket1", "Key1"}}, + Src1 = #manifest_entry{start_key = {o, "Bucket", "Key40001"}, + end_key = {o, "Bucket", "Key60000"}}, + {Candidates, Others} = check_for_merge_candidates(Src1, [Sink1, Sink2]), + ?assertMatch([Sink2], Candidates), + ?assertMatch([Sink1], Others). + select_merge_file_test() -> L0 = [{{o, "B1", "K1"}, {o, "B3", "K3"}, dummy_pid}], diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index a1b1249..84745ad 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -862,10 +862,14 @@ fetch(Key, Manifest, Level, FetchFun) -> not_present, LevelManifest) of not_present -> + io:format("Key ~w out of range at level ~w with manifest ~w~n", + [Key, Level, LevelManifest]), fetch(Key, Manifest, Level + 1, FetchFun); FileToCheck -> case FetchFun(FileToCheck, Key) of not_present -> + io:format("Key ~w not found checking file at level ~w~n", + [Key, Level]), fetch(Key, Manifest, Level + 1, FetchFun); ObjectFound -> ObjectFound @@ -1022,6 +1026,25 @@ open_all_filesinmanifest({Manifest, TopSQN}, Level) -> UpdManifest = lists:keystore(Level, 1, Manifest, {Level, LvlFL}), open_all_filesinmanifest({UpdManifest, max(TopSQN, LvlSQN)}, Level + 1). +print_manifest(Manifest) -> + lists:foreach(fun(L) -> + io:format("Manifest at Level ~w~n", [L]), + Level = get_item(L, Manifest, []), + lists:foreach(fun(M) -> + {_, SB, SK} = M#manifest_entry.start_key, + {_, EB, EK} = M#manifest_entry.end_key, + io:format("Manifest entry of " ++ + "startkey ~s ~s " ++ + "endkey ~s ~s " ++ + "filename=~s~n", + [SB, SK, EB, EK, + M#manifest_entry.filename]) + end, + Level) + end, + lists:seq(1, ?MAX_LEVELS - 1)). + + assess_workqueue(WorkQ, ?MAX_LEVELS - 1, _Manifest) -> WorkQ; assess_workqueue(WorkQ, LevelToAssess, Manifest)-> @@ -1087,7 +1110,7 @@ commit_manifest_change(ReturnedWorkItem, State) -> io:format("Merge has been commmitted at sequence number ~w~n", [NewMSN]), NewManifest = ReturnedWorkItem#penciller_work.new_manifest, - %% io:format("Updated manifest is ~w~n", [NewManifest]), + print_manifest(NewManifest), {ok, State#state{ongoing_work=[], manifest_sqn=NewMSN, manifest=NewManifest, diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 602514c..5e73ac9 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -2,44 +2,112 @@ -include_lib("common_test/include/ct.hrl"). -include("../include/leveled.hrl"). -export([all/0]). --export([simple_put_fetch/1, - journal_compaction/1]). +-export([simple_put_fetch_head/1, + many_put_fetch_head/1, + journal_compaction/1, + simple_snapshot/1]). -all() -> [journal_compaction, simple_put_fetch]. +all() -> [simple_put_fetch_head, + many_put_fetch_head, + journal_compaction, + simple_snapshot]. -simple_put_fetch(_Config) -> +simple_put_fetch_head(_Config) -> RootPath = reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), - {ok, TestObject} = leveled_bookie:book_riakget(Bookie1, - TestObject#r_object.bucket, - TestObject#r_object.key), + check_bookie_forobject(Bookie1, TestObject), ok = leveled_bookie:book_close(Bookie1), StartOpts2 = #bookie_options{root_path=RootPath, max_journalsize=3000000}, {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), - {ok, TestObject} = leveled_bookie:book_riakget(Bookie2, - TestObject#r_object.bucket, - TestObject#r_object.key), + check_bookie_forobject(Bookie2, TestObject), ObjList1 = generate_multiple_objects(5000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, ObjList1), ChkList1 = lists:sublist(lists:sort(ObjList1), 100), - lists:foreach(fun({_RN, Obj, _Spc}) -> - R = leveled_bookie:book_riakget(Bookie2, - Obj#r_object.bucket, - Obj#r_object.key), - R = {ok, Obj} end, - ChkList1), - {ok, TestObject} = leveled_bookie:book_riakget(Bookie2, - TestObject#r_object.bucket, - TestObject#r_object.key), + check_bookie_forlist(Bookie2, ChkList1), + check_bookie_forobject(Bookie2, TestObject), ok = leveled_bookie:book_close(Bookie2), reset_filestructure(). +many_put_fetch_head(_Config) -> + RootPath = reset_filestructure(), + StartOpts1 = #bookie_options{root_path=RootPath}, + {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), + {TestObject, TestSpec} = generate_testobject(), + ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), + check_bookie_forobject(Bookie1, TestObject), + ok = leveled_bookie:book_close(Bookie1), + StartOpts2 = #bookie_options{root_path=RootPath, + max_journalsize=1000000000}, + {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), + check_bookie_forobject(Bookie2, TestObject), + GenList = [2, 20002, 40002, 60002, 80002, + 100002, 120002, 140002, 160002, 180002], + CLs = lists:map(fun(KN) -> + ObjListA = generate_multiple_smallobjects(20000, KN), + StartWatchA = os:timestamp(), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie2, Obj, Spc) + end, + ObjListA), + Time = timer:now_diff(os:timestamp(), StartWatchA), + io:format("20,000 objects loaded in ~w seconds~n", + [Time/1000000]), + check_bookie_forobject(Bookie2, TestObject), + lists:sublist(ObjListA, 1000) end, + GenList), + CL1A = lists:nth(1, CLs), + ChkListFixed = lists:nth(length(CLs), CLs), + check_bookie_forlist(Bookie2, CL1A), + ObjList2A = generate_multiple_objects(5000, 2), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, + ObjList2A), + ChkList2A = lists:sublist(lists:sort(ObjList2A), 1000), + check_bookie_forlist(Bookie2, ChkList2A), + check_bookie_forlist(Bookie2, ChkListFixed), + check_bookie_forobject(Bookie2, TestObject), + check_bookie_forlist(Bookie2, ChkList2A), + check_bookie_forlist(Bookie2, ChkListFixed), + check_bookie_forobject(Bookie2, TestObject), + ok = leveled_bookie:book_close(Bookie2), + {ok, Bookie3} = leveled_bookie:book_start(StartOpts2), + check_bookie_forlist(Bookie3, ChkList2A), + check_bookie_forobject(Bookie3, TestObject), + ok = leveled_bookie:book_close(Bookie3), + reset_filestructure(). + + +check_bookie_forlist(Bookie, ChkList) -> + lists:foreach(fun({_RN, Obj, _Spc}) -> + R = leveled_bookie:book_riakget(Bookie, + Obj#r_object.bucket, + Obj#r_object.key), + io:format("Checking key ~s~n", [Obj#r_object.key]), + R = {ok, Obj} end, + ChkList). + +check_bookie_forobject(Bookie, TestObject) -> + {ok, TestObject} = leveled_bookie:book_riakget(Bookie, + TestObject#r_object.bucket, + TestObject#r_object.key), + {ok, HeadObject} = leveled_bookie:book_riakhead(Bookie, + TestObject#r_object.bucket, + TestObject#r_object.key), + ok = case {HeadObject#r_object.bucket, + HeadObject#r_object.key, + HeadObject#r_object.vclock} of + {B1, K1, VC1} when B1 == TestObject#r_object.bucket, + K1 == TestObject#r_object.key, + VC1 == TestObject#r_object.vclock -> + ok + end. + journal_compaction(_Config) -> RootPath = reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath, @@ -93,6 +161,37 @@ journal_compaction(_Config) -> reset_filestructure(). +simple_snapshot(_Config) -> + RootPath = reset_filestructure(), + StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=3000000}, + {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), + {TestObject, TestSpec} = generate_testobject(), + ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), + ObjList1 = generate_multiple_objects(5000, 2), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, + ObjList1), + SnapOpts = #bookie_options{snapshot_bookie=Bookie1}, + {ok, SnapBookie} = leveled_bookie:book_start(SnapOpts), + ChkList1 = lists:sublist(lists:sort(ObjList1), 100), + lists:foreach(fun({_RN, Obj, _Spc}) -> + R = leveled_bookie:book_riakget(Bookie1, + Obj#r_object.bucket, + Obj#r_object.key), + R = {ok, Obj} end, + ChkList1), + lists:foreach(fun({_RN, Obj, _Spc}) -> + R = leveled_bookie:book_riakget(SnapBookie, + Obj#r_object.bucket, + Obj#r_object.key), + io:format("Finding key ~s~n", [Obj#r_object.key]), + R = {ok, Obj} end, + ChkList1), + ok = leveled_bookie:book_close(SnapBookie), + ok = leveled_bookie:book_close(Bookie1), + reset_filestructure(). + + reset_filestructure() -> RootPath = "test", filelib:ensure_dir(RootPath ++ "/journal/"), @@ -111,6 +210,9 @@ generate_testobject() -> {#r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]}, Spec1}. +generate_multiple_smallobjects(Count, KeyNumber) -> + generate_multiple_objects(Count, KeyNumber, [], crypto:rand_bytes(512)). + generate_multiple_objects(Count, KeyNumber) -> generate_multiple_objects(Count, KeyNumber, [], crypto:rand_bytes(4096)). From ad5aebe93eecc77e80ba569b8515b2d864330540 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 5 Oct 2016 18:28:31 +0100 Subject: [PATCH 054/167] Further work on system tests Another issue exposed with laziness in the using an incomplete ledger when checking for presence during compaction. --- src/leveled_bookie.erl | 37 +++++++------- src/leveled_iclerk.erl | 86 +++++++++++++++++++-------------- src/leveled_inker.erl | 22 +++++---- src/leveled_penciller.erl | 4 -- test/end_to_end/basic_SUITE.erl | 69 ++++++++++++++------------ 5 files changed, 115 insertions(+), 103 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index c11c8e0..ba1a333 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -159,8 +159,6 @@ -record(state, {inker :: pid(), penciller :: pid(), - metadata_extractor :: function(), - indexspec_converter :: function(), cache_size :: integer(), back_pressure :: boolean(), ledger_cache :: gb_trees:tree(), @@ -209,18 +207,6 @@ init([Opts]) -> % Start from file not snapshot {InkerOpts, PencillerOpts} = set_options(Opts), {Inker, Penciller} = startup(InkerOpts, PencillerOpts), - Extractor = if - Opts#bookie_options.metadata_extractor == undefined -> - fun extract_metadata/2; - true -> - Opts#bookie_options.metadata_extractor - end, - Converter = if - Opts#bookie_options.indexspec_converter == undefined -> - fun convert_indexspecs/3; - true -> - Opts#bookie_options.indexspec_converter - end, CacheSize = if Opts#bookie_options.cache_size == undefined -> ?CACHE_SIZE; @@ -229,8 +215,6 @@ init([Opts]) -> end, {ok, #state{inker=Inker, penciller=Penciller, - metadata_extractor=Extractor, - indexspec_converter=Converter, cache_size=CacheSize, ledger_cache=gb_trees:empty(), is_snapshot=false}}; @@ -311,19 +295,21 @@ handle_call({snapshot, Requestor, SnapType, _Timeout}, _From, State) -> {ok, JournalSnapshot} = leveled_inker:ink_start(InkerOpts), {reply, {ok, - {LedgerSnapshot, State#state.ledger_cache}, + {LedgerSnapshot, + State#state.ledger_cache}, JournalSnapshot}, State}; ledger -> {reply, {ok, - {LedgerSnapshot, State#state.ledger_cache}, + {LedgerSnapshot, + State#state.ledger_cache}, null}, State} end; handle_call({compact_journal, Timeout}, _From, State) -> ok = leveled_inker:ink_compactjournal(State#state.inker, - State#state.penciller, + self(), Timeout), {reply, ok, State}; handle_call(close, _From, State) -> @@ -510,7 +496,6 @@ load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> {Obj, IndexSpecs} = binary_to_term(ExtractFun(ValueInLedger)), case SQN of SQN when SQN < MinSQN -> - io:format("Skipping due to low SQN ~w~n", [SQN]), {loop, Acc0}; SQN when SQN =< MaxSQN -> %% TODO - get correct size in a more efficient manner @@ -631,4 +616,16 @@ multi_key_test() -> ok = book_close(Bookie2), reset_filestructure(). +indexspecs_test() -> + IndexSpecs = [{add, "t1_int", 456}, + {add, "t1_bin", "adbc123"}, + {remove, "t1_bin", "abdc456"}], + Changes = convert_indexspecs(IndexSpecs, 1, {o, "Bucket", "Key2"}), + ?assertMatch({{i, "Bucket", "t1_int", 456, "Key2"}, + {1, {active, infinity}, null}}, lists:nth(1, Changes)), + ?assertMatch({{i, "Bucket", "t1_bin", "adbc123", "Key2"}, + {1, {active, infinity}, null}}, lists:nth(2, Changes)), + ?assertMatch({{i, "Bucket", "t1_bin", "abdc456", "Key2"}, + {1, {tomb, infinity}, null}}, lists:nth(3, Changes)). + -endif. \ No newline at end of file diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 756d96b..6f187a3 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -13,7 +13,6 @@ terminate/2, clerk_new/1, clerk_compact/6, - clerk_remove/2, clerk_stop/1, code_change/3]). @@ -47,10 +46,6 @@ clerk_new(InkerClerkOpts) -> gen_server:start(?MODULE, [InkerClerkOpts], []). -clerk_remove(Pid, Removals) -> - gen_server:cast(Pid, {remove, Removals}), - ok. - clerk_compact(Pid, Checker, InitiateFun, FilterFun, Inker, Timeout) -> gen_server:cast(Pid, {compact, @@ -88,12 +83,12 @@ handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, % Don't want to process a queued call waiting on an old manifest Manifest = leveled_inker:ink_getmanifest(Inker), MaxRunLength = State#state.max_run_length, - FilterServer = InitiateFun(Checker), + {FilterServer, MaxSQN} = InitiateFun(Checker), CDBopts = State#state.cdb_options, FP = CDBopts#cdb_options.file_path, ok = filelib:ensure_dir(FP), - Candidates = scan_all_files(Manifest, FilterFun, FilterServer), + Candidates = scan_all_files(Manifest, FilterFun, FilterServer, MaxSQN), BestRun = assess_candidates(Candidates, MaxRunLength), case score_run(BestRun, MaxRunLength) of Score when Score > 0 -> @@ -102,7 +97,8 @@ handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, PromptDelete} = compact_files(BestRun, CDBopts, FilterFun, - FilterServer), + FilterServer, + MaxSQN), FilesToDelete = lists:map(fun(C) -> {C#candidate.low_sqn, C#candidate.filename, @@ -127,8 +123,6 @@ handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, ok = leveled_inker:ink_compactioncomplete(Inker), {noreply, State} end; -handle_cast({remove, _Removals}, State) -> - {noreply, State}; handle_cast(stop, State) -> {stop, normal, State}. @@ -147,38 +141,45 @@ code_change(_OldVsn, State, _Extra) -> %%%============================================================================ -check_single_file(CDB, FilterFun, FilterServer, SampleSize, BatchSize) -> +check_single_file(CDB, FilterFun, FilterServer, MaxSQN, SampleSize, BatchSize) -> + FN = leveled_cdb:cdb_filename(CDB), PositionList = leveled_cdb:cdb_getpositions(CDB, SampleSize), KeySizeList = fetch_inbatches(PositionList, BatchSize, CDB, []), R0 = lists:foldl(fun(KS, {ActSize, RplSize}) -> {{SQN, PK}, Size} = KS, Check = FilterFun(FilterServer, PK, SQN), - case Check of - true -> + case {Check, SQN > MaxSQN} of + {true, _} -> {ActSize + Size, RplSize}; - false -> + {false, true} -> + {ActSize + Size, RplSize}; + _ -> {ActSize, RplSize + Size} end end, {0, 0}, KeySizeList), {ActiveSize, ReplacedSize} = R0, - 100 * ActiveSize / (ActiveSize + ReplacedSize). + Score = 100 * ActiveSize / (ActiveSize + ReplacedSize), + io:format("Score for filename ~s is ~w~n", [FN, Score]), + Score. -scan_all_files(Manifest, FilterFun, FilterServer) -> - scan_all_files(Manifest, FilterFun, FilterServer, []). +scan_all_files(Manifest, FilterFun, FilterServer, MaxSQN) -> + scan_all_files(Manifest, FilterFun, FilterServer, MaxSQN, []). -scan_all_files([], _FilterFun, _FilterServer, CandidateList) -> +scan_all_files([], _FilterFun, _FilterServer, _MaxSQN, CandidateList) -> CandidateList; -scan_all_files([Entry|Tail], FilterFun, FilterServer, CandidateList) -> +scan_all_files([Entry|Tail], FilterFun, FilterServer, MaxSQN, CandidateList) -> {LowSQN, FN, JournalP} = Entry, CpctPerc = check_single_file(JournalP, FilterFun, FilterServer, + MaxSQN, ?SAMPLE_SIZE, ?BATCH_SIZE), scan_all_files(Tail, FilterFun, FilterServer, + MaxSQN, CandidateList ++ [#candidate{low_sqn = LowSQN, filename = FN, @@ -274,27 +275,29 @@ print_compaction_run(BestRun, MaxRunLength) -> end, BestRun). -compact_files([], _CDBopts, _FilterFun, _FilterServer) -> +compact_files([], _CDBopts, _FilterFun, _FilterServer, _MaxSQN) -> {[], 0}; -compact_files(BestRun, CDBopts, FilterFun, FilterServer) -> +compact_files(BestRun, CDBopts, FilterFun, FilterServer, MaxSQN) -> BatchesOfPositions = get_all_positions(BestRun, []), compact_files(BatchesOfPositions, CDBopts, null, FilterFun, FilterServer, + MaxSQN, [], true). -compact_files([], _CDBopts, null, _FilterFun, _FilterServer, +compact_files([], _CDBopts, null, _FilterFun, _FilterServer, _MaxSQN, ManSlice0, PromptDelete0) -> {ManSlice0, PromptDelete0}; -compact_files([], _CDBopts, ActiveJournal0, _FilterFun, _FilterServer, +compact_files([], _CDBopts, ActiveJournal0, _FilterFun, _FilterServer, _MaxSQN, ManSlice0, PromptDelete0) -> ManSlice1 = ManSlice0 ++ generate_manifest_entry(ActiveJournal0), {ManSlice1, PromptDelete0}; -compact_files([Batch|T], CDBopts, ActiveJournal0, FilterFun, FilterServer, +compact_files([Batch|T], CDBopts, ActiveJournal0, + FilterFun, FilterServer, MaxSQN, ManSlice0, PromptDelete0) -> {SrcJournal, PositionList} = Batch, KVCs0 = leveled_cdb:cdb_directfetch(SrcJournal, @@ -302,7 +305,8 @@ compact_files([Batch|T], CDBopts, ActiveJournal0, FilterFun, FilterServer, key_value_check), R0 = filter_output(KVCs0, FilterFun, - FilterServer), + FilterServer, + MaxSQN), {KVCs1, PromptDelete1} = R0, PromptDelete2 = case {PromptDelete0, PromptDelete1} of {true, true} -> @@ -314,7 +318,7 @@ compact_files([Batch|T], CDBopts, ActiveJournal0, FilterFun, FilterServer, CDBopts, ActiveJournal0, ManSlice0), - compact_files(T, CDBopts, ActiveJournal1, FilterFun, FilterServer, + compact_files(T, CDBopts, ActiveJournal1, FilterFun, FilterServer, MaxSQN, ManSlice1, PromptDelete2). get_all_positions([], PositionBatches) -> @@ -341,16 +345,18 @@ split_positions_into_batches(Positions, Journal, Batches) -> Batches ++ [{Journal, ThisBatch}]). -filter_output(KVCs, FilterFun, FilterServer) -> +filter_output(KVCs, FilterFun, FilterServer, MaxSQN) -> lists:foldl(fun(KVC, {Acc, PromptDelete}) -> {{SQN, PK}, _V, CrcCheck} = KVC, KeyValid = FilterFun(FilterServer, PK, SQN), - case {KeyValid, CrcCheck} of - {true, true} -> + case {KeyValid, CrcCheck, SQN > MaxSQN} of + {true, true, _} -> {Acc ++ [KVC], PromptDelete}; - {false, _} -> + {false, true, true} -> + {Acc ++ [KVC], PromptDelete}; + {false, true, false} -> {Acc, PromptDelete}; - {_, false} -> + {_, false, _} -> io:format("Corrupted value found for " ++ " Key ~w at SQN ~w~n", [PK, SQN]), {Acc, false} @@ -415,7 +421,9 @@ simple_score_test() -> ?assertMatch(6.0, score_run(Run1, 4)), Run2 = [#candidate{compaction_perc = 75.0}], ?assertMatch(-15.0, score_run(Run2, 4)), - ?assertMatch(0.0, score_run([], 4)). + ?assertMatch(0.0, score_run([], 4)), + Run3 = [#candidate{compaction_perc = 100.0}], + ?assertMatch(-40.0, score_run(Run3, 4)). score_compare_test() -> Run1 = [#candidate{compaction_perc = 75.0}, @@ -514,15 +522,18 @@ check_single_file_test() -> _ -> false end end, - Score1 = check_single_file(CDB, LedgerFun1, LedgerSrv1, 8, 4), + Score1 = check_single_file(CDB, LedgerFun1, LedgerSrv1, 9, 8, 4), ?assertMatch(37.5, Score1), LedgerFun2 = fun(_Srv, _Key, _ObjSQN) -> true end, - Score2 = check_single_file(CDB, LedgerFun2, LedgerSrv1, 8, 4), + Score2 = check_single_file(CDB, LedgerFun2, LedgerSrv1, 9, 8, 4), ?assertMatch(100.0, Score2), - Score3 = check_single_file(CDB, LedgerFun1, LedgerSrv1, 8, 3), + Score3 = check_single_file(CDB, LedgerFun1, LedgerSrv1, 9, 8, 3), ?assertMatch(37.5, Score3), + Score4 = check_single_file(CDB, LedgerFun1, LedgerSrv1, 4, 8, 4), + ?assertMatch(75.0, Score4), ok = leveled_cdb:cdb_destroy(CDB). - + + compact_single_file_test() -> RP = "../test/journal", {ok, CDB} = fetch_testcdb(RP), @@ -543,7 +554,8 @@ compact_single_file_test() -> R1 = compact_files([Candidate], #cdb_options{file_path=CompactFP}, LedgerFun1, - LedgerSrv1), + LedgerSrv1, + 9), {ManSlice1, PromptDelete1} = R1, ?assertMatch(true, PromptDelete1), [{LowSQN, FN, PidR}] = ManSlice1, diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 32cf340..2161730 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -170,12 +170,12 @@ ink_forceclose(Pid) -> ink_loadpcl(Pid, MinSQN, FilterFun, Penciller) -> gen_server:call(Pid, {load_pcl, MinSQN, FilterFun, Penciller}, infinity). -ink_compactjournal(Pid, Penciller, Timeout) -> +ink_compactjournal(Pid, Bookie, Timeout) -> CheckerInitiateFun = fun initiate_penciller_snapshot/1, CheckerFilterFun = fun leveled_penciller:pcl_checksequencenumber/3, gen_server:call(Pid, {compact, - Penciller, + Bookie, CheckerInitiateFun, CheckerFilterFun, Timeout}, @@ -818,13 +818,14 @@ manifest_printer(Manifest) -> Manifest). -initiate_penciller_snapshot(Penciller) -> - PclOpts = #penciller_options{start_snapshot = true, - source_penciller = Penciller, - requestor = self()}, - {ok, FilterServer} = leveled_penciller:pcl_start(PclOpts), - ok = leveled_penciller:pcl_loadsnapshot(FilterServer, []), - FilterServer. +initiate_penciller_snapshot(Bookie) -> + {ok, + {LedgerSnap, LedgerCache}, + _} = leveled_bookie:book_snapshotledger(Bookie, self(), undefined), + ok = leveled_penciller:pcl_loadsnapshot(LedgerSnap, + gb_trees:to_list(LedgerCache)), + MaxSQN = leveled_penciller:pcl_getstartupsequencenumber(LedgerSnap), + {LedgerSnap, MaxSQN}. %%%============================================================================ %%% Test @@ -864,6 +865,7 @@ build_dummy_journal() -> clean_testdir(RootPath) -> clean_subdir(filepath(RootPath, journal_dir)), + clean_subdir(filepath(RootPath, journal_compact_dir)), clean_subdir(filepath(RootPath, manifest_dir)). clean_subdir(DirPath) -> @@ -1033,7 +1035,7 @@ compact_journal_test() -> ?assertMatch(2, length(ActualManifest)), ok = ink_compactjournal(Ink1, Checker, - fun(X) -> X end, + fun(X) -> {X, 55} end, fun(L, K, SQN) -> lists:member({SQN, K}, L) end, 5000), timer:sleep(1000), diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 84745ad..fcb0657 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -862,14 +862,10 @@ fetch(Key, Manifest, Level, FetchFun) -> not_present, LevelManifest) of not_present -> - io:format("Key ~w out of range at level ~w with manifest ~w~n", - [Key, Level, LevelManifest]), fetch(Key, Manifest, Level + 1, FetchFun); FileToCheck -> case FetchFun(FileToCheck, Key) of not_present -> - io:format("Key ~w not found checking file at level ~w~n", - [Key, Level]), fetch(Key, Manifest, Level + 1, FetchFun); ObjectFound -> ObjectFound diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 5e73ac9..3173d05 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -12,6 +12,7 @@ all() -> [simple_put_fetch_head, journal_compaction, simple_snapshot]. + simple_put_fetch_head(_Config) -> RootPath = reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath}, @@ -19,6 +20,7 @@ simple_put_fetch_head(_Config) -> {TestObject, TestSpec} = generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), check_bookie_forobject(Bookie1, TestObject), + check_bookie_formissingobject(Bookie1, "Bucket1", "Key2"), ok = leveled_bookie:book_close(Bookie1), StartOpts2 = #bookie_options{root_path=RootPath, max_journalsize=3000000}, @@ -31,6 +33,7 @@ simple_put_fetch_head(_Config) -> ChkList1 = lists:sublist(lists:sort(ObjList1), 100), check_bookie_forlist(Bookie2, ChkList1), check_bookie_forobject(Bookie2, TestObject), + check_bookie_formissingobject(Bookie2, "Bucket1", "Key2"), ok = leveled_bookie:book_close(Bookie2), reset_filestructure(). @@ -88,7 +91,6 @@ check_bookie_forlist(Bookie, ChkList) -> R = leveled_bookie:book_riakget(Bookie, Obj#r_object.bucket, Obj#r_object.key), - io:format("Checking key ~s~n", [Obj#r_object.key]), R = {ok, Obj} end, ChkList). @@ -108,6 +110,10 @@ check_bookie_forobject(Bookie, TestObject) -> ok end. +check_bookie_formissingobject(Bookie, Bucket, Key) -> + not_found = leveled_bookie:book_riakget(Bookie, Bucket, Key), + not_found = leveled_bookie:book_riakhead(Bookie, Bucket, Key). + journal_compaction(_Config) -> RootPath = reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath, @@ -115,23 +121,30 @@ journal_compaction(_Config) -> {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), - {ok, TestObject} = leveled_bookie:book_riakget(Bookie1, - TestObject#r_object.bucket, - TestObject#r_object.key), + check_bookie_forobject(Bookie1, TestObject), ObjList1 = generate_multiple_objects(5000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjList1), - ChkList1 = lists:sublist(lists:sort(ObjList1), 100), - lists:foreach(fun({_RN, Obj, _Spc}) -> - R = leveled_bookie:book_riakget(Bookie1, - Obj#r_object.bucket, - Obj#r_object.key), - R = {ok, Obj} end, - ChkList1), - {ok, TestObject} = leveled_bookie:book_riakget(Bookie1, - TestObject#r_object.bucket, - TestObject#r_object.key), + ChkList1 = lists:sublist(lists:sort(ObjList1), 1000), + check_bookie_forlist(Bookie1, ChkList1), + check_bookie_forobject(Bookie1, TestObject), + {B2, K2, V2, Spec2, MD} = {"Bucket1", + "Key1", + "Value1", + [], + {"MDK1", "MDV1"}}, + {TestObject2, TestSpec2} = generate_testobject(B2, K2, V2, Spec2, MD), + ok = leveled_bookie:book_riakput(Bookie1, TestObject2, TestSpec2), + ok = leveled_bookie:book_compactjournal(Bookie1, 30000), + check_bookie_forlist(Bookie1, ChkList1), + check_bookie_forobject(Bookie1, TestObject), + check_bookie_forobject(Bookie1, TestObject2), + timer:sleep(5000), % Allow for compaction to complete + io:format("Has journal completed?~n"), + check_bookie_forlist(Bookie1, ChkList1), + check_bookie_forobject(Bookie1, TestObject), + check_bookie_forobject(Bookie1, TestObject2), %% Now replace all the objects ObjList2 = generate_multiple_objects(5000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> @@ -139,24 +152,12 @@ journal_compaction(_Config) -> ObjList2), ok = leveled_bookie:book_compactjournal(Bookie1, 30000), ChkList3 = lists:sublist(lists:sort(ObjList2), 500), - lists:foreach(fun({_RN, Obj, _Spc}) -> - R = leveled_bookie:book_riakget(Bookie1, - Obj#r_object.bucket, - Obj#r_object.key), - R = {ok, Obj} end, - ChkList3), + check_bookie_forlist(Bookie1, ChkList3), ok = leveled_bookie:book_close(Bookie1), % Restart {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), - {ok, TestObject} = leveled_bookie:book_riakget(Bookie2, - TestObject#r_object.bucket, - TestObject#r_object.key), - lists:foreach(fun({_RN, Obj, _Spc}) -> - R = leveled_bookie:book_riakget(Bookie2, - Obj#r_object.bucket, - Obj#r_object.key), - R = {ok, Obj} end, - ChkList3), + check_bookie_forobject(Bookie2, TestObject), + check_bookie_forlist(Bookie2, ChkList3), ok = leveled_bookie:book_close(Bookie2), reset_filestructure(). @@ -200,15 +201,19 @@ reset_filestructure() -> leveled_penciller:clean_testdir(RootPath ++ "/ledger"), RootPath. + generate_testobject() -> {B1, K1, V1, Spec1, MD} = {"Bucket1", "Key1", "Value1", [], {"MDK1", "MDV1"}}, - Content = #r_content{metadata=MD, value=V1}, - {#r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]}, - Spec1}. + generate_testobject(B1, K1, V1, Spec1, MD). + +generate_testobject(B, K, V, Spec, MD) -> + Content = #r_content{metadata=MD, value=V}, + {#r_object{bucket=B, key=K, contents=[Content], vclock=[{'a',1}]}, + Spec}. generate_multiple_smallobjects(Count, KeyNumber) -> generate_multiple_objects(Count, KeyNumber, [], crypto:rand_bytes(512)). From f58f4d0ea57a503bc66d59854ba4596bd7723670 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 6 Oct 2016 13:23:20 +0100 Subject: [PATCH 055/167] Mini Refactor Thought about the mess, thought about swithcing to a FSM, throught about just sorting a bit of the mess instead. --- src/leveled_penciller.erl | 286 +++++++++++++++++--------------------- 1 file changed, 127 insertions(+), 159 deletions(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index fcb0657..3402aa8 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -232,7 +232,6 @@ terminate/2, code_change/3, pcl_start/1, - pcl_quickstart/1, pcl_pushmem/2, pcl_fetch/2, pcl_checksequencenumber/3, @@ -292,8 +291,6 @@ %%% API %%%============================================================================ -pcl_quickstart(RootPath) -> - pcl_start(#penciller_options{root_path=RootPath}). pcl_start(PCLopts) -> gen_server:start(?MODULE, [PCLopts], []). @@ -361,34 +358,139 @@ init([PCLopts]) -> end. -handle_call({push_mem, DumpList}, From, State) -> - if - State#state.is_snapshot == true -> - {reply, bad_request, State}; - true -> - writer_call({push_mem, DumpList}, From, State) +handle_call({push_mem, DumpList}, From, State=#state{is_snapshot=Snap}) + when Snap == false -> + % The process for pushing to memory is as follows + % - Check that the inbound list does not contain any Keys with a lower + % sequence number than any existing keys (assess_sqn/1) + % - Check that any file that had been sent to be written to L0 previously + % is now completed. If it is wipe out the in-memory view as this is now + % safely persisted. This will block waiting for this to complete if it + % hasn't (checkready_pushmem/1). + % - Quick check to see if there is a need to write a L0 file + % (quickcheck_pushmem/3). If there clearly isn't, then we can reply, and + % then add to memory in the background before updating the loop state + % - Push the update into memory (do_pushtomem/3) + % - If we haven't got through quickcheck now need to check if there is a + % definite need to write a new L0 file (roll_memory/2). If all clear this + % will write the file in the background and allow a response to the user. + % If not the change has still been made but the the L0 file will not have + % been prompted - so the reply does not indicate failure but returns the + % atom 'pause' to signal a loose desire for back-pressure to be applied. + % The only reason in this case why there should be a pause is if the + % manifest is locked pending completion of a manifest change - so reacting + % to the pause signal may not be sensible + StartWatch = os:timestamp(), + case assess_sqn(DumpList) of + {MinSQN, MaxSQN} when MaxSQN >= MinSQN, + MinSQN >= State#state.ledger_sqn -> + MaxTableSize = State#state.memtable_maxsize, + {TableSize0, State1} = checkready_pushtomem(State), + case quickcheck_pushtomem(DumpList, + TableSize0, + MaxTableSize) of + {twist, TableSize1} -> + gen_server:reply(From, ok), + io:format("Reply made on push in ~w microseconds~n", + [timer:now_diff(os:timestamp(), StartWatch)]), + L0Snap = do_pushtomem(DumpList, + State1#state.memtable, + State1#state.memtable_copy, + MaxSQN), + io:format("Push completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(), StartWatch)]), + {noreply, + State1#state{memtable_copy=L0Snap, + table_size=TableSize1, + ledger_sqn=MaxSQN}}; + {maybe_roll, TableSize1} -> + L0Snap = do_pushtomem(DumpList, + State1#state.memtable, + State1#state.memtable_copy, + MaxSQN), + + case roll_memory(State1, MaxTableSize) of + {ok, L0Pend, ManSN, TableSize2} -> + io:format("Push completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(), StartWatch)]), + {reply, + ok, + State1#state{levelzero_pending=L0Pend, + table_size=TableSize2, + manifest_sqn=ManSN, + memtable_copy=L0Snap, + ledger_sqn=MaxSQN, + backlog=false}}; + {pause, Reason, Details} -> + io:format("Excess work due to - " ++ Reason, + Details), + {reply, + pause, + State1#state{backlog=true, + memtable_copy=L0Snap, + table_size=TableSize1, + ledger_sqn=MaxSQN}} + end + end; + {MinSQN, MaxSQN} -> + io:format("Mismatch of sequence number expectations with push " + ++ "having sequence numbers between ~w and ~w " + ++ "but current sequence number is ~w~n", + [MinSQN, MaxSQN, State#state.ledger_sqn]), + {reply, refused, State}; + empty -> + io:format("Empty request pushed to Penciller~n"), + {reply, ok, State} end; -handle_call({confirm_delete, FileName}, _From, State) -> - if - State#state.is_snapshot == true -> - {reply, bad_request, State}; +handle_call({confirm_delete, FileName}, _From, State=#state{is_snapshot=Snap}) + when Snap == false -> + Reply = confirm_delete(FileName, + State#state.unreferenced_files, + State#state.registered_snapshots), + case Reply of true -> - writer_call({confirm_delete, FileName}, _From, State) + UF1 = lists:keydelete(FileName, 1, State#state.unreferenced_files), + {reply, true, State#state{unreferenced_files=UF1}}; + _ -> + {reply, Reply, State} end; -handle_call(prompt_compaction, _From, State) -> +handle_call(prompt_compaction, _From, State=#state{is_snapshot=Snap}) + when Snap == false -> + %% If there is a prompt immediately after a L0 async write event then + %% there exists the potential for the prompt to stall the database. + %% Should only accept prompts if there has been a safe wait from the + %% last L0 write event. + Proceed = case State#state.levelzero_pending of + {true, _Pid, TS} -> + TD = timer:now_diff(os:timestamp(),TS), + if + TD < ?PROMPT_WAIT_ONL0 * 1000000 -> false; + true -> true + end; + ?L0PEND_RESET -> + true + end, if - State#state.is_snapshot == true -> - {reply, bad_request, State}; + Proceed -> + {_TableSize, State1} = checkready_pushtomem(State), + case roll_memory(State1, State1#state.memtable_maxsize) of + {ok, L0Pend, MSN, TableSize} -> + io:format("Prompted push completed~n"), + {reply, ok, State1#state{levelzero_pending=L0Pend, + table_size=TableSize, + manifest_sqn=MSN, + backlog=false}}; + {pause, Reason, Details} -> + io:format("Excess work due to - " ++ Reason, Details), + {reply, pause, State1#state{backlog=true}} + end; true -> - writer_call(prompt_compaction, _From, State) - end; -handle_call({manifest_change, WI}, _From, State) -> - if - State#state.is_snapshot == true -> - {reply, bad_request, State}; - true -> - writer_call({manifest_change, WI}, _From, State) + {reply, ok, State#state{backlog=false}} end; +handle_call({manifest_change, WI}, _From, State=#state{is_snapshot=Snap}) + when Snap == false -> + {ok, UpdState} = commit_manifest_change(WI, State), + {reply, ok, UpdState}; handle_call({check_sqn, Key, SQN}, _From, State) -> Obj = if State#state.is_snapshot == true -> @@ -529,138 +631,6 @@ code_change(_OldVsn, State, _Extra) -> {ok, State}. -writer_call({push_mem, DumpList}, From, State0) -> - % The process for pushing to memory is as follows - % - Check that the inbound list does not contain any Keys with a lower - % sequence number than any existing keys (assess_sqn/1) - % - Check that any file that had been sent to be written to L0 previously - % is now completed. If it is wipe out the in-memory view as this is now - % safely persisted. This will block waiting for this to complete if it - % hasn't (checkready_pushmem/1). - % - Quick check to see if there is a need to write a L0 file - % (quickcheck_pushmem/3). If there clearly isn't, then we can reply, and - % then add to memory in the background before updating the loop state - % - Push the update into memory (do_pushtomem/3) - % - If we haven't got through quickcheck now need to check if there is a - % definite need to write a new L0 file (roll_memory/2). If all clear this - % will write the file in the background and allow a response to the user. - % If not the change has still been made but the the L0 file will not have - % been prompted - so the reply does not indicate failure but returns the - % atom 'pause' to signal a loose desire for back-pressure to be applied. - % The only reason in this case why there should be a pause is if the - % manifest is locked pending completion of a manifest change - so reacting - % to the pause signal may not be sensible - StartWatch = os:timestamp(), - case assess_sqn(DumpList) of - {MinSQN, MaxSQN} when MaxSQN >= MinSQN, - MinSQN >= State0#state.ledger_sqn -> - MaxTableSize = State0#state.memtable_maxsize, - {TableSize0, State1} = checkready_pushtomem(State0), - case quickcheck_pushtomem(DumpList, - TableSize0, - MaxTableSize) of - {twist, TableSize1} -> - gen_server:reply(From, ok), - io:format("Reply made on push in ~w microseconds~n", - [timer:now_diff(os:timestamp(), StartWatch)]), - L0Snap = do_pushtomem(DumpList, - State1#state.memtable, - State1#state.memtable_copy, - MaxSQN), - io:format("Push completed in ~w microseconds~n", - [timer:now_diff(os:timestamp(), StartWatch)]), - {noreply, - State1#state{memtable_copy=L0Snap, - table_size=TableSize1, - ledger_sqn=MaxSQN}}; - {maybe_roll, TableSize1} -> - L0Snap = do_pushtomem(DumpList, - State1#state.memtable, - State1#state.memtable_copy, - MaxSQN), - - case roll_memory(State1, MaxTableSize) of - {ok, L0Pend, ManSN, TableSize2} -> - io:format("Push completed in ~w microseconds~n", - [timer:now_diff(os:timestamp(), StartWatch)]), - {reply, - ok, - State1#state{levelzero_pending=L0Pend, - table_size=TableSize2, - manifest_sqn=ManSN, - memtable_copy=L0Snap, - ledger_sqn=MaxSQN, - backlog=false}}; - {pause, Reason, Details} -> - io:format("Excess work due to - " ++ Reason, - Details), - {reply, - pause, - State1#state{backlog=true, - memtable_copy=L0Snap, - table_size=TableSize1, - ledger_sqn=MaxSQN}} - end - end; - {MinSQN, MaxSQN} -> - io:format("Mismatch of sequence number expectations with push " - ++ "having sequence numbers between ~w and ~w " - ++ "but current sequence number is ~w~n", - [MinSQN, MaxSQN, State0#state.ledger_sqn]), - {reply, refused, State0}; - empty -> - io:format("Empty request pushed to Penciller~n"), - {reply, ok, State0} - end; -writer_call({confirm_delete, FileName}, _From, State) -> - Reply = confirm_delete(FileName, - State#state.unreferenced_files, - State#state.registered_snapshots), - case Reply of - true -> - UF1 = lists:keydelete(FileName, 1, State#state.unreferenced_files), - {reply, true, State#state{unreferenced_files=UF1}}; - _ -> - {reply, Reply, State} - end; -writer_call(prompt_compaction, _From, State) -> - %% If there is a prompt immediately after a L0 async write event then - %% there exists the potential for the prompt to stall the database. - %% Should only accept prompts if there has been a safe wait from the - %% last L0 write event. - Proceed = case State#state.levelzero_pending of - {true, _Pid, TS} -> - TD = timer:now_diff(os:timestamp(),TS), - if - TD < ?PROMPT_WAIT_ONL0 * 1000000 -> false; - true -> true - end; - ?L0PEND_RESET -> - true - end, - if - Proceed -> - {_TableSize, State1} = checkready_pushtomem(State), - case roll_memory(State1, State1#state.memtable_maxsize) of - {ok, L0Pend, MSN, TableSize} -> - io:format("Prompted push completed~n"), - {reply, ok, State1#state{levelzero_pending=L0Pend, - table_size=TableSize, - manifest_sqn=MSN, - backlog=false}}; - {pause, Reason, Details} -> - io:format("Excess work due to - " ++ Reason, Details), - {reply, pause, State1#state{backlog=true}} - end; - true -> - {reply, ok, State#state{backlog=false}} - end; -writer_call({manifest_change, WI}, _From, State) -> - {ok, UpdState} = commit_manifest_change(WI, State), - {reply, ok, UpdState}. - - - %%%============================================================================ %%% Internal functions %%%============================================================================ @@ -1286,8 +1256,6 @@ simple_server_test() -> ok; _ -> io:format("Unexpected sequence number on restart ~w~n", [TopSQN]), - ok = pcl_close(PCLr), - clean_testdir(RootPath), error end, ?assertMatch(Check, ok), From 2055f8ed3f6a17844f459ca9430be40fad340b2b Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 7 Oct 2016 10:04:48 +0100 Subject: [PATCH 056/167] Add more complex snapshot test This exposed another off-by-one error on startup. This commit also includes an unsafe change to reply early from a rolling CDB file (with lots of objects writing the hash table can take too long). This is bad, but will be resolved through a refactor of the manifest writing: essentially we deferred writing of the manifest update which was an unnecessary performance optimisation. If instead we wait on this, the process is made substantially simpler, and it is safer to perform the roll of the complete CDB journal asynchronously. If the manifest update takes too long, an append-only log may be used instead. --- include/leveled.hrl | 6 +- src/leveled_bookie.erl | 21 +++- src/leveled_cdb.erl | 73 ++++++++---- src/leveled_inker.erl | 29 +++-- src/leveled_penciller.erl | 163 +++++++++++++------------- test/end_to_end/basic_SUITE.erl | 198 ++++++++++++++++++++++---------- 6 files changed, 307 insertions(+), 183 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index c04a44f..37694ed 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -29,15 +29,13 @@ root_path :: string(), cdb_options :: #cdb_options{}, start_snapshot = false :: boolean(), - source_inker :: pid(), - requestor :: pid()}). + source_inker :: pid()}). -record(penciller_options, {root_path :: string(), max_inmemory_tablesize :: integer(), start_snapshot = false :: boolean(), - source_penciller :: pid(), - requestor :: pid()}). + source_penciller :: pid()}). -record(bookie_options, {root_path :: string(), diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index ba1a333..bafba89 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -213,6 +213,8 @@ init([Opts]) -> true -> Opts#bookie_options.cache_size end, + io:format("Bookie starting with Pcl ~w Ink ~w~n", + [Penciller, Inker]), {ok, #state{inker=Inker, penciller=Penciller, cache_size=CacheSize, @@ -223,6 +225,8 @@ init([Opts]) -> {Penciller, LedgerCache}, Inker} = book_snapshotstore(Bookie, self(), ?SNAPSHOT_TIMEOUT), ok = leveled_penciller:pcl_loadsnapshot(Penciller, []), + io:format("Snapshot starting with Pcl ~w Ink ~w~n", + [Penciller, Inker]), {ok, #state{penciller=Penciller, inker=Inker, ledger_cache=LedgerCache, @@ -282,16 +286,14 @@ handle_call({head, Key}, _From, State) -> {reply, {ok, OMD}, State} end end; -handle_call({snapshot, Requestor, SnapType, _Timeout}, _From, State) -> +handle_call({snapshot, _Requestor, SnapType, _Timeout}, _From, State) -> PCLopts = #penciller_options{start_snapshot=true, - source_penciller=State#state.penciller, - requestor=Requestor}, + source_penciller=State#state.penciller}, {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), case SnapType of store -> InkerOpts = #inker_options{start_snapshot=true, - source_inker=State#state.inker, - requestor=Requestor}, + source_inker=State#state.inker}, {ok, JournalSnapshot} = leveled_inker:ink_start(InkerOpts), {reply, {ok, @@ -497,12 +499,19 @@ load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> case SQN of SQN when SQN < MinSQN -> {loop, Acc0}; - SQN when SQN =< MaxSQN -> + SQN when SQN < MaxSQN -> %% TODO - get correct size in a more efficient manner %% Need to have compressed size Size = byte_size(term_to_binary(ValueInLedger, [compressed])), Changes = preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs), {loop, {MinSQN, MaxSQN, Output ++ Changes}}; + MaxSQN -> + %% TODO - get correct size in a more efficient manner + %% Need to have compressed size + io:format("Reached end of load batch with SQN ~w~n", [SQN]), + Size = byte_size(term_to_binary(ValueInLedger, [compressed])), + Changes = preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs), + {stop, {MinSQN, MaxSQN, Output ++ Changes}}; SQN when SQN > MaxSQN -> io:format("Skipping as exceeded MaxSQN ~w with SQN ~w~n", [MaxSQN, SQN]), diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 31b4c53..668b1c8 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -66,6 +66,7 @@ cdb_scan/4, cdb_close/1, cdb_complete/1, + cdb_roll/1, cdb_destroy/1, cdb_deletepending/1]). @@ -137,6 +138,9 @@ cdb_close(Pid) -> cdb_complete(Pid) -> gen_server:call(Pid, cdb_complete, infinity). +cdb_roll(Pid) -> + gen_server:call(Pid, cdb_roll, infinity). + cdb_destroy(Pid) -> gen_server:cast(Pid, destroy). @@ -197,9 +201,7 @@ handle_call({open_writer, Filename}, _From, State) -> writer=true}}; handle_call({open_reader, Filename}, _From, State) -> io:format("Opening file for reading with filename ~s~n", [Filename]), - {ok, Handle} = file:open(Filename, [binary, raw, read]), - Index = load_index(Handle), - LastKey = find_lastkey(Handle, Index), + {Handle, Index, LastKey} = open_for_readonly(Filename), {reply, ok, State#state{handle=Handle, last_key=LastKey, filename=Filename, @@ -332,29 +334,33 @@ handle_call({cdb_scan, FilterFun, Acc, StartPos}, _From, State) -> handle_call(cdb_close, _From, State) -> ok = file:close(State#state.handle), {stop, normal, ok, State#state{handle=undefined}}; +handle_call(cdb_complete, _From, State=#state{writer=Writer}) + when Writer == true -> + NewName = determine_new_filename(State#state.filename), + ok = close_file(State#state.handle, + State#state.hashtree, + State#state.last_position), + ok = rename_for_read(State#state.filename, NewName), + {stop, normal, {ok, NewName}, State}; handle_call(cdb_complete, _From, State) -> - case State#state.writer of - true -> - ok = close_file(State#state.handle, - State#state.hashtree, - State#state.last_position), - %% Rename file - NewName = filename:rootname(State#state.filename, ".pnd") - ++ ".cdb", - io:format("Renaming file from ~s to ~s " ++ - "for which existence is ~w~n", - [State#state.filename, NewName, - filelib:is_file(NewName)]), - ok = file:rename(State#state.filename, NewName), - {stop, normal, {ok, NewName}, State}; - false -> - ok = file:close(State#state.handle), - {stop, normal, {ok, State#state.filename}, State}; - undefined -> - ok = file:close(State#state.handle), - {stop, normal, {ok, State#state.filename}, State} - end. - + ok = file:close(State#state.handle), + {stop, normal, {ok, State#state.filename}, State}; +handle_call(cdb_roll, From, State=#state{writer=Writer}) + when Writer == true -> + NewName = determine_new_filename(State#state.filename), + gen_server:reply(From, {ok, NewName}), + ok = close_file(State#state.handle, + State#state.hashtree, + State#state.last_position), + ok = rename_for_read(State#state.filename, NewName), + io:format("Opening file for reading with filename ~s~n", [NewName]), + {Handle, Index, LastKey} = open_for_readonly(NewName), + {noreply, State#state{handle=Handle, + last_key=LastKey, + filename=NewName, + writer=false, + hash_index=Index}}. + handle_cast(destroy, State) -> ok = file:close(State#state.handle), @@ -659,6 +665,23 @@ fold_keys(Handle, FoldFun, Acc0) -> %% Internal functions %%%%%%%%%%%%%%%%%%%% +determine_new_filename(Filename) -> + filename:rootname(Filename, ".pnd") ++ ".cdb". + +rename_for_read(Filename, NewName) -> + %% Rename file + io:format("Renaming file from ~s to ~s " ++ + "for which existence is ~w~n", + [Filename, NewName, + filelib:is_file(NewName)]), + file:rename(Filename, NewName). + +open_for_readonly(Filename) -> + {ok, Handle} = file:open(Filename, [binary, raw, read]), + Index = load_index(Handle), + LastKey = find_lastkey(Handle, Index), + {Handle, Index, LastKey}. + load_index(Handle) -> Index = lists:seq(0, 255), lists:map(fun(X) -> diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 2161730..561ff64 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -139,7 +139,8 @@ cdb_options :: #cdb_options{}, clerk :: pid(), compaction_pending = false :: boolean(), - is_snapshot = false :: boolean()}). + is_snapshot = false :: boolean(), + source_inker :: pid()}). %%%============================================================================ @@ -161,6 +162,9 @@ ink_fetch(Pid, PrimaryKey, SQN) -> ink_registersnapshot(Pid, Requestor) -> gen_server:call(Pid, {register_snapshot, Requestor}, infinity). +ink_releasesnapshot(Pid, Snapshot) -> + gen_server:call(Pid, {release_snapshot, Snapshot}, infinity). + ink_close(Pid) -> gen_server:call(Pid, {close, false}, infinity). @@ -217,13 +221,13 @@ init([InkerOpts]) -> InkerOpts#inker_options.start_snapshot} of {undefined, true} -> SrcInker = InkerOpts#inker_options.source_inker, - Requestor = InkerOpts#inker_options.requestor, {Manifest, ActiveJournalDB, - ActiveJournalSQN} = ink_registersnapshot(SrcInker, Requestor), + ActiveJournalSQN} = ink_registersnapshot(SrcInker, self()), {ok, #state{manifest=Manifest, active_journaldb=ActiveJournalDB, active_journaldb_sqn=ActiveJournalSQN, + source_inker=SrcInker, is_snapshot=true}}; %% Need to do something about timeout {_RootPath, false} -> @@ -276,10 +280,17 @@ handle_call({load_pcl, StartSQN, FilterFun, Penciller}, _From, State) -> handle_call({register_snapshot, Requestor}, _From , State) -> Rs = [{Requestor, State#state.manifest_sqn}|State#state.registered_snapshots], + io:format("Inker snapshot ~w registered at SQN ~w~n", + [Requestor, State#state.manifest_sqn]), {reply, {State#state.manifest, State#state.active_journaldb, State#state.active_journaldb_sqn}, State#state{registered_snapshots=Rs}}; +handle_call({release_snapshot, Snapshot}, _From , State) -> + Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots), + io:format("Snapshot ~w released~n", [Snapshot]), + io:format("Remaining snapshots are ~w~n", [Rs]), + {reply, ok, State#state{registered_snapshots=Rs}}; handle_call(get_manifest, _From, State) -> {reply, State#state.manifest, State}; handle_call({update_manifest, @@ -344,7 +355,7 @@ handle_info(_Info, State) -> terminate(Reason, State) -> case State#state.is_snapshot of true -> - ok; + ok = ink_releasesnapshot(State#state.source_inker, self()); false -> io:format("Inker closing journal for reason ~w~n", [Reason]), io:format("Close triggered with journal_sqn=~w and manifest_sqn=~w~n", @@ -444,16 +455,16 @@ put_object(PrimaryKey, Object, KeyChanges, State) -> end end. -roll_active_file(OldActiveJournal, Manifest, ManifestSQN, RootPath) -> +roll_active_file(ActiveJournal, Manifest, ManifestSQN, RootPath) -> SW = os:timestamp(), - io:format("Rolling old journal ~w~n", [OldActiveJournal]), - {ok, NewFilename} = leveled_cdb:cdb_complete(OldActiveJournal), - {ok, PidR} = leveled_cdb:cdb_open_reader(NewFilename), + io:format("Rolling old journal ~w~n", [ActiveJournal]), + {ok, NewFilename} = leveled_cdb:cdb_roll(ActiveJournal), JournalRegex2 = "nursery_(?[0-9]+)\\." ++ ?JOURNAL_FILEX, [JournalSQN] = sequencenumbers_fromfilenames([NewFilename], JournalRegex2, 'SQN'), - NewManifest = add_to_manifest(Manifest, {JournalSQN, NewFilename, PidR}), + NewManifest = add_to_manifest(Manifest, + {JournalSQN, NewFilename, ActiveJournal}), NewManifestSQN = ManifestSQN + 1, ok = simple_manifest_writer(NewManifest, NewManifestSQN, RootPath), io:format("Rolling old journal completed in ~w microseconds~n", diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 3402aa8..26b70e8 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -491,42 +491,40 @@ handle_call({manifest_change, WI}, _From, State=#state{is_snapshot=Snap}) when Snap == false -> {ok, UpdState} = commit_manifest_change(WI, State), {reply, ok, UpdState}; -handle_call({check_sqn, Key, SQN}, _From, State) -> - Obj = if - State#state.is_snapshot == true -> - fetch_snap(Key, +handle_call({fetch, Key}, _From, State=#state{is_snapshot=Snap}) + when Snap == false -> + {reply, + fetch(Key, + State#state.manifest, + State#state.memtable), + State}; +handle_call({check_sqn, Key, SQN}, _From, State=#state{is_snapshot=Snap}) + when Snap == false -> + {reply, + compare_to_sqn(fetch(Key, State#state.manifest, - State#state.levelzero_snapshot); - true -> - fetch(Key, - State#state.manifest, - State#state.memtable) - end, - Reply = case Obj of - not_present -> - false; - Obj -> - SQNToCompare = leveled_bookie:strip_to_seqonly(Obj), - if - SQNToCompare > SQN -> - false; - true -> - true - end - end, - {reply, Reply, State}; -handle_call({fetch, Key}, _From, State) -> - Reply = if - State#state.is_snapshot == true -> - fetch_snap(Key, - State#state.manifest, - State#state.levelzero_snapshot); - true -> - fetch(Key, - State#state.manifest, - State#state.memtable) - end, - {reply, Reply, State}; + State#state.memtable), + SQN), + State}; +handle_call({fetch, Key}, + _From, + State=#state{snapshot_fully_loaded=Ready}) + when Ready == true -> + {reply, + fetch_snap(Key, + State#state.manifest, + State#state.levelzero_snapshot), + State}; +handle_call({check_sqn, Key, SQN}, + _From, + State=#state{snapshot_fully_loaded=Ready}) + when Ready == true -> + {reply, + compare_to_sqn(fetch_snap(Key, + State#state.manifest, + State#state.levelzero_snapshot), + SQN), + State}; handle_call(work_for_clerk, From, State) -> {UpdState, Work} = return_work(State, From), {reply, {Work, UpdState#state.backlog}, UpdState}; @@ -568,6 +566,8 @@ handle_cast(_Msg, State) -> handle_info(_Info, State) -> {noreply, State}. +terminate(_Reason, _State=#state{is_snapshot=Snap}) when Snap == true -> + ok; terminate(_Reason, State) -> %% When a Penciller shuts down it isn't safe to try an manage the safe %% finishing of any outstanding work. The last commmitted manifest will @@ -585,46 +585,41 @@ terminate(_Reason, State) -> %% The cast may not succeed as the clerk could be synchronously calling %% the penciller looking for a manifest commit %% - if - State#state.is_snapshot == true -> - ok; - true -> - leveled_pclerk:clerk_stop(State#state.clerk), - Dump = ets:tab2list(State#state.memtable), - case {State#state.levelzero_pending, - get_item(0, State#state.manifest, []), length(Dump)} of - {?L0PEND_RESET, [], L} when L > 0 -> - MSN = State#state.manifest_sqn + 1, - FileName = State#state.root_path - ++ "/" ++ ?FILES_FP ++ "/" - ++ integer_to_list(MSN) ++ "_0_0", - NewSFT = leveled_sft:sft_new(FileName ++ ".pnd", - Dump, - [], - 0), - {ok, L0Pid, {{[], []}, _SK, _HK}} = NewSFT, - io:format("Dump of memory on close to filename ~s~n", - [FileName]), - leveled_sft:sft_close(L0Pid), - file:rename(FileName ++ ".pnd", FileName ++ ".sft"); - {?L0PEND_RESET, [], L} when L == 0 -> - io:format("No keys to dump from memory when closing~n"); - {{true, L0Pid, _TS}, _, _} -> - leveled_sft:sft_close(L0Pid), - io:format("No opportunity to persist memory before closing" - ++ " with ~w keys discarded~n", - [length(Dump)]); - _ -> - io:format("No opportunity to persist memory before closing" - ++ " with ~w keys discarded~n", - [length(Dump)]) - end, - ok = close_files(0, State#state.manifest), - lists:foreach(fun({_FN, Pid, _SN}) -> - leveled_sft:sft_close(Pid) end, - State#state.unreferenced_files), - ok - end. + leveled_pclerk:clerk_stop(State#state.clerk), + Dump = ets:tab2list(State#state.memtable), + case {State#state.levelzero_pending, + get_item(0, State#state.manifest, []), length(Dump)} of + {?L0PEND_RESET, [], L} when L > 0 -> + MSN = State#state.manifest_sqn + 1, + FileName = State#state.root_path + ++ "/" ++ ?FILES_FP ++ "/" + ++ integer_to_list(MSN) ++ "_0_0", + NewSFT = leveled_sft:sft_new(FileName ++ ".pnd", + Dump, + [], + 0), + {ok, L0Pid, {{[], []}, _SK, _HK}} = NewSFT, + io:format("Dump of memory on close to filename ~s~n", + [FileName]), + leveled_sft:sft_close(L0Pid), + file:rename(FileName ++ ".pnd", FileName ++ ".sft"); + {?L0PEND_RESET, [], L} when L == 0 -> + io:format("No keys to dump from memory when closing~n"); + {{true, L0Pid, _TS}, _, _} -> + leveled_sft:sft_close(L0Pid), + io:format("No opportunity to persist memory before closing" + ++ " with ~w keys discarded~n", + [length(Dump)]); + _ -> + io:format("No opportunity to persist memory before closing" + ++ " with ~w keys discarded~n", + [length(Dump)]) + end, + ok = close_files(0, State#state.manifest), + lists:foreach(fun({_FN, Pid, _SN}) -> + leveled_sft:sft_close(Pid) end, + State#state.unreferenced_files), + ok. code_change(_OldVsn, State, _Extra) -> @@ -843,6 +838,21 @@ fetch(Key, Manifest, Level, FetchFun) -> end. +compare_to_sqn(Obj, SQN) -> + case Obj of + not_present -> + false; + Obj -> + SQNToCompare = leveled_bookie:strip_to_seqonly(Obj), + if + SQNToCompare > SQN -> + false; + true -> + true + end + end. + + %% Manifest lock - don't have two changes to the manifest happening %% concurrently @@ -1280,8 +1290,7 @@ simple_server_test() -> ?assertMatch(R13, Key3), ?assertMatch(R14, Key4), SnapOpts = #penciller_options{start_snapshot = true, - source_penciller = PCLr, - requestor = self()}, + source_penciller = PCLr}, {ok, PclSnap} = pcl_start(SnapOpts), ok = pcl_loadsnapshot(PclSnap, []), ?assertMatch(Key1, pcl_fetch(PclSnap, {o,"Bucket0001", "Key0001"})), diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 3173d05..9496e2f 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -5,12 +5,12 @@ -export([simple_put_fetch_head/1, many_put_fetch_head/1, journal_compaction/1, - simple_snapshot/1]). + fetchput_snapshot/1]). all() -> [simple_put_fetch_head, many_put_fetch_head, journal_compaction, - simple_snapshot]. + fetchput_snapshot]. simple_put_fetch_head(_Config) -> @@ -51,19 +51,8 @@ many_put_fetch_head(_Config) -> check_bookie_forobject(Bookie2, TestObject), GenList = [2, 20002, 40002, 60002, 80002, 100002, 120002, 140002, 160002, 180002], - CLs = lists:map(fun(KN) -> - ObjListA = generate_multiple_smallobjects(20000, KN), - StartWatchA = os:timestamp(), - lists:foreach(fun({_RN, Obj, Spc}) -> - leveled_bookie:book_riakput(Bookie2, Obj, Spc) - end, - ObjListA), - Time = timer:now_diff(os:timestamp(), StartWatchA), - io:format("20,000 objects loaded in ~w seconds~n", - [Time/1000000]), - check_bookie_forobject(Bookie2, TestObject), - lists:sublist(ObjListA, 1000) end, - GenList), + CLs = load_objects(20000, GenList, Bookie2, TestObject, + fun generate_multiple_smallobjects/2), CL1A = lists:nth(1, CLs), ChkListFixed = lists:nth(length(CLs), CLs), check_bookie_forlist(Bookie2, CL1A), @@ -85,35 +74,6 @@ many_put_fetch_head(_Config) -> ok = leveled_bookie:book_close(Bookie3), reset_filestructure(). - -check_bookie_forlist(Bookie, ChkList) -> - lists:foreach(fun({_RN, Obj, _Spc}) -> - R = leveled_bookie:book_riakget(Bookie, - Obj#r_object.bucket, - Obj#r_object.key), - R = {ok, Obj} end, - ChkList). - -check_bookie_forobject(Bookie, TestObject) -> - {ok, TestObject} = leveled_bookie:book_riakget(Bookie, - TestObject#r_object.bucket, - TestObject#r_object.key), - {ok, HeadObject} = leveled_bookie:book_riakhead(Bookie, - TestObject#r_object.bucket, - TestObject#r_object.key), - ok = case {HeadObject#r_object.bucket, - HeadObject#r_object.key, - HeadObject#r_object.vclock} of - {B1, K1, VC1} when B1 == TestObject#r_object.bucket, - K1 == TestObject#r_object.key, - VC1 == TestObject#r_object.vclock -> - ok - end. - -check_bookie_formissingobject(Bookie, Bucket, Key) -> - not_found = leveled_bookie:book_riakget(Bookie, Bucket, Key), - not_found = leveled_bookie:book_riakhead(Bookie, Bucket, Key). - journal_compaction(_Config) -> RootPath = reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath, @@ -162,7 +122,7 @@ journal_compaction(_Config) -> reset_filestructure(). -simple_snapshot(_Config) -> +fetchput_snapshot(_Config) -> RootPath = reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=3000000}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), @@ -172,27 +132,82 @@ simple_snapshot(_Config) -> lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjList1), - SnapOpts = #bookie_options{snapshot_bookie=Bookie1}, - {ok, SnapBookie} = leveled_bookie:book_start(SnapOpts), + SnapOpts1 = #bookie_options{snapshot_bookie=Bookie1}, + {ok, SnapBookie1} = leveled_bookie:book_start(SnapOpts1), ChkList1 = lists:sublist(lists:sort(ObjList1), 100), - lists:foreach(fun({_RN, Obj, _Spc}) -> - R = leveled_bookie:book_riakget(Bookie1, - Obj#r_object.bucket, - Obj#r_object.key), - R = {ok, Obj} end, - ChkList1), - lists:foreach(fun({_RN, Obj, _Spc}) -> - R = leveled_bookie:book_riakget(SnapBookie, - Obj#r_object.bucket, - Obj#r_object.key), - io:format("Finding key ~s~n", [Obj#r_object.key]), - R = {ok, Obj} end, - ChkList1), - ok = leveled_bookie:book_close(SnapBookie), + check_bookie_forlist(Bookie1, ChkList1), + check_bookie_forlist(SnapBookie1, ChkList1), + ok = leveled_bookie:book_close(SnapBookie1), + check_bookie_forlist(Bookie1, ChkList1), ok = leveled_bookie:book_close(Bookie1), + io:format("Closed initial bookies~n"), + + {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), + SnapOpts2 = #bookie_options{snapshot_bookie=Bookie2}, + {ok, SnapBookie2} = leveled_bookie:book_start(SnapOpts2), + io:format("Bookies restarted~n"), + + check_bookie_forlist(Bookie2, ChkList1), + io:format("Check active bookie still contains original data~n"), + check_bookie_forlist(SnapBookie2, ChkList1), + io:format("Check snapshot still contains original data~n"), + + + ObjList2 = generate_multiple_objects(5000, 2), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, + ObjList2), + io:format("Replacement objects put~n"), + + ChkList2 = lists:sublist(lists:sort(ObjList2), 100), + check_bookie_forlist(Bookie2, ChkList2), + check_bookie_forlist(SnapBookie2, ChkList1), + io:format("Checked for replacement objects in active bookie" ++ + ", old objects in snapshot~n"), + + {ok, FNsA} = file:list_dir(RootPath ++ "/ledger/ledger_files"), + ObjList3 = generate_multiple_objects(15000, 5002), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, + ObjList3), + ChkList3 = lists:sublist(lists:sort(ObjList3), 100), + check_bookie_forlist(Bookie2, ChkList3), + check_bookie_formissinglist(SnapBookie2, ChkList3), + GenList = [20002, 40002, 60002, 80002, 100002, 120002], + CLs2 = load_objects(20000, GenList, Bookie2, TestObject, + fun generate_multiple_smallobjects/2), + io:format("Loaded significant numbers of new objects~n"), + + check_bookie_forlist(Bookie2, lists:nth(length(CLs2), CLs2)), + io:format("Checked active bookie has new objects~n"), + + {ok, SnapBookie3} = leveled_bookie:book_start(SnapOpts2), + check_bookie_forlist(SnapBookie3, lists:nth(length(CLs2), CLs2)), + check_bookie_formissinglist(SnapBookie2, ChkList3), + check_bookie_formissinglist(SnapBookie2, lists:nth(length(CLs2), CLs2)), + check_bookie_forlist(SnapBookie3, ChkList2), + check_bookie_forlist(SnapBookie2, ChkList1), + io:format("Started new snapshot and check for new objects~n"), + + CLs3 = load_objects(20000, GenList, Bookie2, TestObject, + fun generate_multiple_smallobjects/2), + check_bookie_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), + check_bookie_forlist(Bookie2, lists:nth(1, CLs3)), + {ok, FNsB} = file:list_dir(RootPath ++ "/ledger/ledger_files"), + ok = leveled_bookie:book_close(SnapBookie2), + check_bookie_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), + ok = leveled_bookie:book_close(SnapBookie3), + check_bookie_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), + check_bookie_forlist(Bookie2, lists:nth(1, CLs3)), + timer:sleep(90000), + {ok, FNsC} = file:list_dir(RootPath ++ "/ledger/ledger_files"), + true = length(FNsB) > length(FNsA), + true = length(FNsB) > length(FNsC), + ok = leveled_bookie:book_close(Bookie2), reset_filestructure(). + reset_filestructure() -> RootPath = "test", filelib:ensure_dir(RootPath ++ "/journal/"), @@ -202,6 +217,52 @@ reset_filestructure() -> RootPath. +check_bookie_forlist(Bookie, ChkList) -> + check_bookie_forlist(Bookie, ChkList, false). + +check_bookie_forlist(Bookie, ChkList, Log) -> + lists:foreach(fun({_RN, Obj, _Spc}) -> + if + Log == true -> + io:format("Fetching Key ~w~n", [Obj#r_object.key]); + true -> + ok + end, + R = leveled_bookie:book_riakget(Bookie, + Obj#r_object.bucket, + Obj#r_object.key), + R = {ok, Obj} end, + ChkList). + +check_bookie_formissinglist(Bookie, ChkList) -> + lists:foreach(fun({_RN, Obj, _Spc}) -> + R = leveled_bookie:book_riakget(Bookie, + Obj#r_object.bucket, + Obj#r_object.key), + R = not_found end, + ChkList). + +check_bookie_forobject(Bookie, TestObject) -> + {ok, TestObject} = leveled_bookie:book_riakget(Bookie, + TestObject#r_object.bucket, + TestObject#r_object.key), + {ok, HeadObject} = leveled_bookie:book_riakhead(Bookie, + TestObject#r_object.bucket, + TestObject#r_object.key), + ok = case {HeadObject#r_object.bucket, + HeadObject#r_object.key, + HeadObject#r_object.vclock} of + {B1, K1, VC1} when B1 == TestObject#r_object.bucket, + K1 == TestObject#r_object.key, + VC1 == TestObject#r_object.vclock -> + ok + end. + +check_bookie_formissingobject(Bookie, Bucket, Key) -> + not_found = leveled_bookie:book_riakget(Bookie, Bucket, Key), + not_found = leveled_bookie:book_riakhead(Bookie, Bucket, Key). + + generate_testobject() -> {B1, K1, V1, Spec1, MD} = {"Bucket1", "Key1", @@ -239,4 +300,17 @@ generate_multiple_objects(Count, KeyNumber, ObjL, Value) -> Value). - \ No newline at end of file +load_objects(ChunkSize, GenList, Bookie, TestObject, Generator) -> + lists:map(fun(KN) -> + ObjListA = Generator(ChunkSize, KN), + StartWatchA = os:timestamp(), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie, Obj, Spc) + end, + ObjListA), + Time = timer:now_diff(os:timestamp(), StartWatchA), + io:format("~w objects loaded in ~w seconds~n", + [ChunkSize, Time/1000000]), + check_bookie_forobject(Bookie, TestObject), + lists:sublist(ObjListA, 1000) end, + GenList). From 8dfeb520ef985b2583fb17f9a2899bbd65dc6267 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 7 Oct 2016 18:07:03 +0100 Subject: [PATCH 057/167] Inker Refactor Inker refactored to block on manifest write. If this is inefficient the manifets write can be converted ot an append only operation. Waiting on the manifest write makes the logic at startup much easier to manage. --- src/leveled_cdb.erl | 26 +- src/leveled_iclerk.erl | 15 +- src/leveled_inker.erl | 677 ++++++++++++++++------------------------- 3 files changed, 282 insertions(+), 436 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 668b1c8..04adfda 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -139,7 +139,7 @@ cdb_complete(Pid) -> gen_server:call(Pid, cdb_complete, infinity). cdb_roll(Pid) -> - gen_server:call(Pid, cdb_roll, infinity). + gen_server:cast(Pid, cdb_roll). cdb_destroy(Pid) -> gen_server:cast(Pid, destroy). @@ -344,11 +344,17 @@ handle_call(cdb_complete, _From, State=#state{writer=Writer}) {stop, normal, {ok, NewName}, State}; handle_call(cdb_complete, _From, State) -> ok = file:close(State#state.handle), - {stop, normal, {ok, State#state.filename}, State}; -handle_call(cdb_roll, From, State=#state{writer=Writer}) - when Writer == true -> + {stop, normal, {ok, State#state.filename}, State}. + + +handle_cast(destroy, State) -> + ok = file:close(State#state.handle), + ok = file:delete(State#state.filename), + {noreply, State}; +handle_cast(delete_pending, State) -> + {noreply, State#state{pending_delete = true}}; +handle_cast(cdb_roll, State=#state{writer=Writer}) when Writer == true -> NewName = determine_new_filename(State#state.filename), - gen_server:reply(From, {ok, NewName}), ok = close_file(State#state.handle, State#state.hashtree, State#state.last_position), @@ -359,15 +365,7 @@ handle_call(cdb_roll, From, State=#state{writer=Writer}) last_key=LastKey, filename=NewName, writer=false, - hash_index=Index}}. - - -handle_cast(destroy, State) -> - ok = file:close(State#state.handle), - ok = file:delete(State#state.filename), - {noreply, State}; -handle_cast(delete_pending, State) -> - {noreply, State#state{pending_delete = true}}; + hash_index=Index}}; handle_cast(_Msg, State) -> {noreply, State}. diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 6f187a3..9403ef4 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -81,7 +81,12 @@ handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, State) -> % Need to fetch manifest at start rather than have it be passed in % Don't want to process a queued call waiting on an old manifest - Manifest = leveled_inker:ink_getmanifest(Inker), + Manifest = case leveled_inker:ink_getmanifest(Inker) of + [] -> + []; + [_Active|Tail] -> + Tail + end, MaxRunLength = State#state.max_run_length, {FilterServer, MaxSQN} = InitiateFun(Checker), CDBopts = State#state.cdb_options, @@ -159,7 +164,12 @@ check_single_file(CDB, FilterFun, FilterServer, MaxSQN, SampleSize, BatchSize) - {0, 0}, KeySizeList), {ActiveSize, ReplacedSize} = R0, - Score = 100 * ActiveSize / (ActiveSize + ReplacedSize), + Score = case ActiveSize + ReplacedSize of + 0 -> + 100.0; + _ -> + 100 * ActiveSize / (ActiveSize + ReplacedSize) + end, io:format("Score for filename ~s is ~w~n", [FN, Score]), Score. @@ -219,6 +229,7 @@ assess_candidates(AllCandidates, MaxRunLength) -> end. assess_candidates([], _MaxRunLength, _CurrentRun0, BestAssessment) -> + io:format("Best run of ~w~n", [BestAssessment]), BestAssessment; assess_candidates([HeadC|Tail], MaxRunLength, CurrentRun0, BestAssessment) -> CurrentRun1 = choose_best_assessment(CurrentRun0 ++ [HeadC], diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 561ff64..471ea0b 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -9,33 +9,43 @@ %% responsible for scheduling compaction work to be carried out by the Inker's %% clerk. %% -%% -------- Journal --------- +%% -------- Journal Files --------- %% -%% The Journal is a series of files originally named as _nursery.cdb +%% The Journal is a series of files originally named as _ %% where the sequence number is the first object sequence number (key) within %% the given database file. The files will be named *.cdb at the point they %% have been made immutable (through a rename operation). Prior to this, they %% will originally start out as a *.pnd file. %% %% At some stage in the future compacted versions of old journal cdb files may -%% be produced. These files will be named -.cdb, and once -%% the manifest is updated the original _nursery.cdb (or -%% _.cdb) files they replace will be erased. +%% be produced. These files will be named -.cdb, and once +%% the manifest is updated the original _.cdb (or +%% _.cdb) files they replace will be erased. %% -%% The current Journal is made up of a set of files referenced in the manifest, -%% combined with a set of files of the form _nursery.[cdb|pnd] with -%% a higher Sequence Number compared to the files in the manifest. +%% The current Journal is made up of a set of files referenced in the manifest. +%% No PUTs are made to files which are not in the manifest. %% %% The Journal is ordered by sequence number from front to back both within %% and across files. %% %% On startup the Inker should open the manifest with the highest sequence %% number, and this will contain the list of filenames that make up the -%% non-recent part of the Journal. The Manifest is completed by opening these -%% files plus any other files with a higher sequence number. The file with -%% the highest sequence number is assumed to to be the active writer. Any file -%% with a lower sequence number and a *.pnd extension should be re-rolled into -%% a *.cdb file. +%% non-recent part of the Journal. All the filenames should then be opened. +%% How they are opened depends on the file extension: +%% +%% - If the file extension is *.cdb the file is opened read only +%% - If the file extension is *.pnd and the file is not the most recent in the +%% manifest, then the file should be completed bfore being opened read-only +%% - If the file extension is *.pnd the file is opened for writing +%% +%% -------- Manifest Files --------- +%% +%% The manifest is just saved as a straight term_to_binary blob, with a +%% filename ordered by the Manifest SQN. The Manifest is first saved with a +%% *.pnd extension, and then renamed to one with a *.man extension. +%% +%% On startup the *.man manifest file with the highest manifest sequence +%% number should be used. %% %% -------- Objects --------- %% @@ -45,47 +55,23 @@ %% - An object (an Erlang term) %% - A set of Key Deltas associated with the change %% -%% -------- Manifest --------- -%% -%% The Journal has a manifest which is the current record of which cdb files -%% are currently active in the Journal (i.e. following compaction). The -%% manifest holds this information through two lists - a list of files which -%% are definitely in the current manifest, and a list of files which have been -%% removed, but may still be present on disk. The use of two lists is to -%% avoid any circumsatnces where a compaction event has led to the deletion of -%% a Journal file with a higher sequence number than any in the remaining -%% manifest. -%% -%% A new manifest file is saved for every compaction event. The manifest files -%% are saved using the filename .man once saved. The ManifestSQN -%% is incremented once for every compaction event. -%% %% -------- Compaction --------- %% %% Compaction is a process whereby an Inker's clerk will: -%% - Request a snapshot of the Ledger, as well as the lowest sequence number -%% that is currently registerd by another snapshot owner -%% - Picks a Journal database file at random (not including the current -%% nursery log) -%% - Performs a random walk on keys and sequence numbers in the chosen CDB -%% file to extract a subset of 100 key and sequence number combinations from -%% the database -%% - Looks up the current sequence number for those keys in the Ledger -%% - If more than % (default n=20) of the keys are now at a higher sequence -%% number, then the database file is a candidate for compaction. In this case -%% each of the next 8 files in sequence should be checked until all those 8 -%% files have been checked or one of the files has been found to be below the -%% threshold. -%% - If a set of below-the-threshold files is found, the files are re-written -%% without any superceded values -%%- The clerk should then request that the Inker commit the manifest change -%% -%% -------- Inker's Clerk --------- -%% -%% +%% - Request a view of the current Inker manifest and a snaphot of the Ledger +%% - Test all files within the Journal to find th eapproximate comapction +%% potential percentage (the volume of the Journal that has been replaced) +%% - Attempts to find the optimal "run" of files to compact +%% - Compacts those files in the run, by rolling over the files re-writing +%% to a new Journal if and only if the Key is still present in the Ledger (or +%% the sequence number of the Key is higher than the SQN of the snapshot) +%% - Requests the Inker update the manifest with the new changes +%% - Instructs the files to destroy themselves when they are next closed %% +%% TODO: how to instruct the files to close is tbd %% + -module(leveled_inker). -behaviour(gen_server). @@ -132,7 +118,6 @@ manifest_sqn = 0 :: integer(), journal_sqn = 0 :: integer(), active_journaldb :: pid(), - active_journaldb_sqn :: integer(), pending_removals = [] :: list(), registered_snapshots = [] :: list(), root_path :: string(), @@ -222,11 +207,9 @@ init([InkerOpts]) -> {undefined, true} -> SrcInker = InkerOpts#inker_options.source_inker, {Manifest, - ActiveJournalDB, - ActiveJournalSQN} = ink_registersnapshot(SrcInker, self()), + ActiveJournalDB} = ink_registersnapshot(SrcInker, self()), {ok, #state{manifest=Manifest, active_journaldb=ActiveJournalDB, - active_journaldb_sqn=ActiveJournalSQN, source_inker=SrcInker, is_snapshot=true}}; %% Need to do something about timeout @@ -235,28 +218,16 @@ init([InkerOpts]) -> end. -handle_call({put, Key, Object, KeyChanges}, From, State) -> +handle_call({put, Key, Object, KeyChanges}, _From, State) -> case put_object(Key, Object, KeyChanges, State) of {ok, UpdState, ObjSize} -> {reply, {ok, UpdState#state.journal_sqn, ObjSize}, UpdState}; {rolling, UpdState, ObjSize} -> - gen_server:reply(From, {ok, UpdState#state.journal_sqn, ObjSize}), - {NewManifest, - NewManifestSQN} = roll_active_file(State#state.active_journaldb, - State#state.manifest, - State#state.manifest_sqn, - State#state.root_path), - {noreply, UpdState#state{manifest=NewManifest, - manifest_sqn=NewManifestSQN}}; - {blocked, UpdState} -> - {reply, blocked, UpdState} + ok = leveled_cdb:cdb_roll(State#state.active_journaldb), + {reply, {ok, UpdState#state.journal_sqn, ObjSize}, UpdState} end; handle_call({fetch, Key, SQN}, _From, State) -> - case get_object(Key, - SQN, - State#state.manifest, - State#state.active_journaldb, - State#state.active_journaldb_sqn) of + case get_object(Key, SQN, State#state.manifest) of {{SQN, Key}, {Value, _IndexSpecs}} -> {reply, {ok, Value}, State}; Other -> @@ -265,16 +236,9 @@ handle_call({fetch, Key, SQN}, _From, State) -> {reply, not_present, State} end; handle_call({get, Key, SQN}, _From, State) -> - {reply, get_object(Key, - SQN, - State#state.manifest, - State#state.active_journaldb, - State#state.active_journaldb_sqn), State}; + {reply, get_object(Key, SQN, State#state.manifest), State}; handle_call({load_pcl, StartSQN, FilterFun, Penciller}, _From, State) -> - Manifest = lists:reverse(State#state.manifest) - ++ [{State#state.active_journaldb_sqn, - dummy, - State#state.active_journaldb}], + Manifest = lists:reverse(State#state.manifest), Reply = load_from_sequence(StartSQN, FilterFun, Penciller, Manifest), {reply, Reply, State}; handle_call({register_snapshot, Requestor}, _From , State) -> @@ -283,8 +247,7 @@ handle_call({register_snapshot, Requestor}, _From , State) -> io:format("Inker snapshot ~w registered at SQN ~w~n", [Requestor, State#state.manifest_sqn]), {reply, {State#state.manifest, - State#state.active_journaldb, - State#state.active_journaldb_sqn}, + State#state.active_journaldb}, State#state{registered_snapshots=Rs}}; handle_call({release_snapshot, Snapshot}, _From , State) -> Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots), @@ -297,21 +260,16 @@ handle_call({update_manifest, ManifestSnippet, DeletedFiles}, _From, State) -> Man0 = lists:foldl(fun(ManEntry, AccMan) -> - Check = lists:member(ManEntry, DeletedFiles), - if - Check == false -> - AccMan ++ [ManEntry]; - true -> - AccMan - end + remove_from_manifest(AccMan, ManEntry) end, - [], - State#state.manifest), + State#state.manifest, + DeletedFiles), Man1 = lists:foldl(fun(ManEntry, AccMan) -> add_to_manifest(AccMan, ManEntry) end, Man0, ManifestSnippet), NewManifestSQN = State#state.manifest_sqn + 1, + manifest_printer(Man1), ok = simple_manifest_writer(Man1, NewManifestSQN, State#state.root_path), PendingRemovals = [{NewManifestSQN, DeletedFiles}], {reply, ok, State#state{manifest=Man1, @@ -365,8 +323,8 @@ terminate(Reason, State) -> lists:foreach(fun({Snap, _SQN}) -> ok = ink_close(Snap) end, State#state.registered_snapshots), manifest_printer(State#state.manifest), - close_allmanifest(State#state.manifest, State#state.active_journaldb), - close_allremovals(State#state.pending_removals) + ok = close_allmanifest(State#state.manifest), + ok = close_allremovals(State#state.pending_removals) end. code_change(_OldVsn, State, _Extra) -> @@ -381,13 +339,10 @@ start_from_file(InkerOpts) -> RootPath = InkerOpts#inker_options.root_path, CDBopts = InkerOpts#inker_options.cdb_options, JournalFP = filepath(RootPath, journal_dir), - {ok, JournalFilenames} = case filelib:is_dir(JournalFP) of - true -> - file:list_dir(JournalFP); - false -> - filelib:ensure_dir(JournalFP), - {ok, []} - end, + filelib:ensure_dir(JournalFP), + CompactFP = filepath(RootPath, journal_compact_dir), + filelib:ensure_dir(CompactFP), + ManifestFP = filepath(RootPath, manifest_dir), {ok, ManifestFilenames} = case filelib:is_dir(ManifestFP) of true -> @@ -397,26 +352,21 @@ start_from_file(InkerOpts) -> {ok, []} end, - CompactFP = filepath(RootPath, journal_compact_dir), - filelib:ensure_dir(CompactFP), IClerkCDBOpts = CDBopts#cdb_options{file_path = CompactFP}, IClerkOpts = #iclerk_options{inker = self(), cdb_options=IClerkCDBOpts}, {ok, Clerk} = leveled_iclerk:clerk_new(IClerkOpts), {Manifest, - {ActiveJournal, LowActiveSQN}, + ManifestSQN, JournalSQN, - ManifestSQN} = build_manifest(ManifestFilenames, - JournalFilenames, - fun simple_manifest_reader/2, + ActiveJournal} = build_manifest(ManifestFilenames, RootPath, CDBopts), - {ok, #state{manifest = lists:reverse(lists:keysort(1, Manifest)), + {ok, #state{manifest = Manifest, manifest_sqn = ManifestSQN, journal_sqn = JournalSQN, active_journaldb = ActiveJournal, - active_journaldb_sqn = LowActiveSQN, root_path = RootPath, cdb_options = CDBopts, clerk = Clerk}}. @@ -436,56 +386,32 @@ put_object(PrimaryKey, Object, KeyChanges, State) -> ok -> {ok, State#state{journal_sqn=NewSQN}, ObjSize}; roll -> - FileName = filepath(State#state.root_path, NewSQN, new_journal), + SW = os:timestamp(), CDBopts = State#state.cdb_options, - {ok, NewJournalP} = leveled_cdb:cdb_open_writer(FileName, CDBopts), - case leveled_cdb:cdb_put(NewJournalP, - {NewSQN, PrimaryKey}, - Bin1) of - ok -> - {rolling, - State#state{journal_sqn=NewSQN, - active_journaldb=NewJournalP, - active_journaldb_sqn=NewSQN}, - ObjSize}; - roll -> - {blocked, State#state{journal_sqn=NewSQN, - active_journaldb=NewJournalP, - active_journaldb_sqn=NewSQN}} - end + ManEntry = start_new_activejournal(NewSQN, + State#state.root_path, + CDBopts), + {_, _, NewJournalP} = ManEntry, + NewManifest = add_to_manifest(State#state.manifest, ManEntry), + ok = simple_manifest_writer(NewManifest, + State#state.manifest_sqn + 1, + State#state.root_path), + ok = leveled_cdb:cdb_put(NewJournalP, {NewSQN, PrimaryKey}, Bin1), + io:format("Put to new active journal " ++ + "with manifest write took ~w microseconds~n", + [timer:now_diff(os:timestamp(),SW)]), + {rolling, + State#state{journal_sqn=NewSQN, + manifest=NewManifest, + manifest_sqn = State#state.manifest_sqn + 1, + active_journaldb=NewJournalP}, + ObjSize} end. -roll_active_file(ActiveJournal, Manifest, ManifestSQN, RootPath) -> - SW = os:timestamp(), - io:format("Rolling old journal ~w~n", [ActiveJournal]), - {ok, NewFilename} = leveled_cdb:cdb_roll(ActiveJournal), - JournalRegex2 = "nursery_(?[0-9]+)\\." ++ ?JOURNAL_FILEX, - [JournalSQN] = sequencenumbers_fromfilenames([NewFilename], - JournalRegex2, - 'SQN'), - NewManifest = add_to_manifest(Manifest, - {JournalSQN, NewFilename, ActiveJournal}), - NewManifestSQN = ManifestSQN + 1, - ok = simple_manifest_writer(NewManifest, NewManifestSQN, RootPath), - io:format("Rolling old journal completed in ~w microseconds~n", - [timer:now_diff(os:timestamp(),SW)]), - {NewManifest, NewManifestSQN}. -get_object(PrimaryKey, SQN, Manifest, ActiveJournal, ActiveJournalSQN) -> - Obj = if - SQN < ActiveJournalSQN -> - JournalP = find_in_manifest(SQN, Manifest), - if - JournalP == error -> - io:format("Unable to find SQN~w in Manifest~w~n", - [SQN, Manifest]), - error; - true -> - leveled_cdb:cdb_get(JournalP, {SQN, PrimaryKey}) - end; - true -> - leveled_cdb:cdb_get(ActiveJournal, {SQN, PrimaryKey}) - end, +get_object(PrimaryKey, SQN, Manifest) -> + JournalP = find_in_manifest(SQN, Manifest), + Obj = leveled_cdb:cdb_get(JournalP, {SQN, PrimaryKey}), case Obj of {{SQN, PK}, Bin} -> {{SQN, PK}, binary_to_term(Bin)}; @@ -495,139 +421,129 @@ get_object(PrimaryKey, SQN, Manifest, ActiveJournal, ActiveJournalSQN) -> build_manifest(ManifestFilenames, - JournalFilenames, - ManifestRdrFun, - RootPath) -> - build_manifest(ManifestFilenames, - JournalFilenames, - ManifestRdrFun, - RootPath, - #cdb_options{}). - -build_manifest(ManifestFilenames, - JournalFilenames, - ManifestRdrFun, RootPath, CDBopts) -> - %% Find the manifest with a highest Manifest sequence number - %% Open it and read it to get the current Confirmed Manifest + % Find the manifest with a highest Manifest sequence number + % Open it and read it to get the current Confirmed Manifest ManifestRegex = "(?[0-9]+)\\." ++ ?MANIFEST_FILEX, ValidManSQNs = sequencenumbers_fromfilenames(ManifestFilenames, ManifestRegex, 'MSQN'), - {JournalSQN1, - ConfirmedManifest, - Removed, + {Manifest, ManifestSQN} = case length(ValidManSQNs) of 0 -> - {0, [], [], 0}; + {[], 1}; _ -> PersistedManSQN = lists:max(ValidManSQNs), - M1 = ManifestRdrFun(PersistedManSQN, RootPath), - J1 = lists:foldl(fun({JSQN, _FN}, Acc) -> - max(JSQN, Acc) end, - 0, - M1), - {J1, M1, [], PersistedManSQN} + M1 = simple_manifest_reader(PersistedManSQN, + RootPath), + {M1, PersistedManSQN} end, - %% Find any more recent immutable files that have a higher sequence number - %% - the immutable files have already been rolled, and so have a completed - %% hashtree lookup - JournalRegex1 = "nursery_(?[0-9]+)\\." ++ ?JOURNAL_FILEX, - UnremovedJournalFiles = lists:foldl(fun(FN, Acc) -> - case lists:member(FN, Removed) of - true -> - Acc; - false -> - Acc ++ [FN] - end end, - [], - JournalFilenames), - OtherSQNs_imm = sequencenumbers_fromfilenames(UnremovedJournalFiles, - JournalRegex1, - 'SQN'), - ExtendManifestFun = fun(X, Acc) -> - if - X > JournalSQN1 - -> - FN = filepath(RootPath, journal_dir) - ++ "nursery_" ++ - integer_to_list(X) - ++ "." ++ - ?JOURNAL_FILEX, - add_to_manifest(Acc, {X, FN}); - true - -> Acc - end end, - Manifest1 = lists:foldl(ExtendManifestFun, - ConfirmedManifest, - lists:sort(OtherSQNs_imm)), + % Open the manifest files, completing if necessary and ensure there is + % a valid active journal at the head of the manifest + OpenManifest = open_all_manifest(Manifest, RootPath, CDBopts), + {ActiveLowSQN, _FN, ActiveJournal} = lists:nth(1, OpenManifest), + JournalSQN = case leveled_cdb:cdb_lastkey(ActiveJournal) of + empty -> + ActiveLowSQN; + {JSQN, _LastKey} -> + JSQN + end, - %% Enrich the manifest so it contains the Pid of any of the immutable - %% entries - io:format("Manifest on startup is: ~n"), - manifest_printer(Manifest1), - Manifest2 = lists:map(fun({LowSQN, FN}) -> - {ok, Pid} = leveled_cdb:cdb_open_reader(FN), - {LowSQN, FN, Pid} end, - Manifest1), - - %% Find any more recent mutable files that have a higher sequence number - %% Roll any mutable files which do not have the highest sequence number - %% to create the hashtree and complete the header entries - JournalRegex2 = "nursery_(?[0-9]+)\\." ++ ?PENDING_FILEX, - OtherSQNs_pnd = sequencenumbers_fromfilenames(JournalFilenames, - JournalRegex2, - 'SQN'), - - case length(OtherSQNs_pnd) of - 0 -> - %% Need to create a new active writer, but also find the highest - %% SQN from within the confirmed manifest - TopSQNInManifest = - case length(Manifest2) of - 0 -> - %% Manifest is empty and no active writers - %% can be found so database is empty - 0; - _ -> - TM = lists:last(lists:keysort(1,Manifest2)), - {_SQN, _FN, TMPid} = TM, - {HighSQN, _HighKey} = leveled_cdb:cdb_lastkey(TMPid), - HighSQN - end, - LowActiveSQN = TopSQNInManifest + 1, - ActiveFN = filepath(RootPath, LowActiveSQN, new_journal), - {ok, ActiveJournal} = leveled_cdb:cdb_open_writer(ActiveFN, - CDBopts), - {Manifest2, - {ActiveJournal, LowActiveSQN}, - TopSQNInManifest, - ManifestSQN}; - _ -> - {ActiveJournalSQN, - Manifest3} = roll_pending_journals(lists:sort(OtherSQNs_pnd), - Manifest2, - RootPath), - %% Need to work out highest sequence number in tail file to feed - %% into opening of pending journal - ActiveFN = filepath(RootPath, ActiveJournalSQN, new_journal), - {ok, ActiveJournal} = leveled_cdb:cdb_open_writer(ActiveFN, - CDBopts), - {HighestSQN, _HighestKey} = leveled_cdb:cdb_lastkey(ActiveJournal), - {Manifest3, - {ActiveJournal, ActiveJournalSQN}, - HighestSQN, - ManifestSQN} - end. + % Update the manifest if it has been changed by the process of laoding + % the manifest (must also increment the manifest SQN). + UpdManifestSQN = if + length(OpenManifest) > length(Manifest) -> + io:format("Updated manifest on startup: ~n"), + manifest_printer(OpenManifest), + simple_manifest_writer(OpenManifest, + ManifestSQN + 1, + RootPath), + ManifestSQN + 1; + true -> + io:format("Unchanged manifest on startup: ~n"), + manifest_printer(OpenManifest), + ManifestSQN + end, + {OpenManifest, UpdManifestSQN, JournalSQN, ActiveJournal}. -close_allmanifest([], ActiveJournal) -> - leveled_cdb:cdb_close(ActiveJournal); -close_allmanifest([H|ManifestT], ActiveJournal) -> + +close_allmanifest([]) -> + ok; +close_allmanifest([H|ManifestT]) -> {_, _, Pid} = H, ok = leveled_cdb:cdb_close(Pid), - close_allmanifest(ManifestT, ActiveJournal). + close_allmanifest(ManifestT). + + +open_all_manifest([], RootPath, CDBOpts) -> + io:format("Manifest is empty, starting from manifest SQN 1~n"), + add_to_manifest([], start_new_activejournal(1, RootPath, CDBOpts)); +open_all_manifest(Man0, RootPath, CDBOpts) -> + Man1 = lists:reverse(lists:sort(Man0)), + [{HeadSQN, HeadFN}|ManifestTail] = Man1, + CompleteHeadFN = HeadFN ++ "." ++ ?JOURNAL_FILEX, + PendingHeadFN = HeadFN ++ "." ++ ?PENDING_FILEX, + Man2 = case filelib:is_file(CompleteHeadFN) of + true -> + io:format("Head manifest entry ~s is complete~n", + [HeadFN]), + {ok, HeadR} = leveled_cdb:cdb_open_reader(CompleteHeadFN), + {LastSQN, _LastPK} = leveled_cdb:cdb_lastkey(HeadR), + add_to_manifest(add_to_manifest(ManifestTail, + {HeadSQN, HeadFN, HeadR}), + start_new_activejournal(LastSQN + 1, + RootPath, + CDBOpts)); + false -> + {ok, HeadW} = leveled_cdb:cdb_open_writer(PendingHeadFN, + CDBOpts), + add_to_manifest(ManifestTail, {HeadSQN, HeadFN, HeadW}) + end, + lists:map(fun(ManEntry) -> + case ManEntry of + {LowSQN, FN} -> + CFN = FN ++ "." ++ ?JOURNAL_FILEX, + PFN = FN ++ "." ++ ?PENDING_FILEX, + case filelib:is_file(CFN) of + true -> + {ok, + Pid} = leveled_cdb:cdb_open_reader(CFN), + {LowSQN, FN, Pid}; + false -> + {ok, + Pid} = leveled_cdb:cdb_open_reader(PFN), + {LowSQN, FN, Pid} + end; + _ -> + ManEntry + end end, + Man2). + + +start_new_activejournal(SQN, RootPath, CDBOpts) -> + Filename = filepath(RootPath, SQN, new_journal), + {ok, PidW} = leveled_cdb:cdb_open_writer(Filename, CDBOpts), + {SQN, Filename, PidW}. + +add_to_manifest(Manifest, Entry) -> + {SQN, FN, PidR} = Entry, + StrippedName = filename:rootname(FN), + lists:reverse(lists:sort([{SQN, StrippedName, PidR}|Manifest])). + +remove_from_manifest(Manifest, Entry) -> + {SQN, FN, _PidR} = Entry, + io:format("File ~s to be removed from manifest~n", [FN]), + lists:keydelete(SQN, 1, Manifest). + +find_in_manifest(_SQN, []) -> + error; +find_in_manifest(SQN, [{LowSQN, _FN, Pid}|_Tail]) when SQN >= LowSQN -> + Pid; +find_in_manifest(SQN, [_Head|Tail]) -> + find_in_manifest(SQN, Tail). + close_allremovals([]) -> ok; @@ -649,20 +565,6 @@ close_allremovals([{ManifestSQN, Removals}|Tail]) -> close_allremovals(Tail). -roll_pending_journals([TopJournalSQN], Manifest, _RootPath) - when is_integer(TopJournalSQN) -> - {TopJournalSQN, Manifest}; -roll_pending_journals([JournalSQN|T], Manifest, RootPath) -> - Filename = filepath(RootPath, JournalSQN, new_journal), - {ok, PidW} = leveled_cdb:cdb_open_writer(Filename), - {ok, NewFilename} = leveled_cdb:cdb_complete(PidW), - {ok, PidR} = leveled_cdb:cdb_open_reader(NewFilename), - roll_pending_journals(T, - add_to_manifest(Manifest, - {JournalSQN, NewFilename, PidR}), - RootPath). - - %% Scan between sequence numbers applying FilterFun to each entry where %% FilterFun{K, V, Acc} -> Penciller Key List %% Load the output for the CDB file into the Penciller. @@ -757,15 +659,6 @@ sequencenumbers_fromfilenames(Filenames, Regex, IntName) -> [], Filenames). -add_to_manifest(Manifest, Entry) -> - lists:reverse(lists:sort([Entry|Manifest])). - -find_in_manifest(_SQN, []) -> - error; -find_in_manifest(SQN, [{LowSQN, _FN, Pid}|_Tail]) when SQN >= LowSQN -> - Pid; -find_in_manifest(SQN, [_Head|Tail]) -> - find_in_manifest(SQN, Tail). filepath(RootPath, journal_dir) -> RootPath ++ "/" ++ ?FILES_FP ++ "/"; @@ -776,15 +669,22 @@ filepath(RootPath, journal_compact_dir) -> filepath(RootPath, NewSQN, new_journal) -> filename:join(filepath(RootPath, journal_dir), - "nursery_" - ++ integer_to_list(NewSQN) + integer_to_list(NewSQN) ++ "_" + ++ generate_uuid() ++ "." ++ ?PENDING_FILEX); filepath(CompactFilePath, NewSQN, compact_journal) -> filename:join(CompactFilePath, - "nursery_" - ++ integer_to_list(NewSQN) + integer_to_list(NewSQN) ++ "_" + ++ generate_uuid() ++ "." ++ ?PENDING_FILEX). +%% Credit to +%% https://github.com/afiskon/erlang-uuid-v4/blob/master/src/uuid.erl +generate_uuid() -> + <> = crypto:rand_bytes(16), + io_lib:format("~8.16.0b-~4.16.0b-4~3.16.0b-~4.16.0b-~12.16.0b", + [A, B, C band 16#0fff, D band 16#3fff bor 16#8000, E]). + simple_manifest_reader(SQN, RootPath) -> ManifestPath = filepath(RootPath, manifest_dir), io:format("Opening manifest file at ~s with SQN ~w~n", @@ -858,7 +758,9 @@ build_dummy_journal() -> {K2, V2} = {"Key2", "TestValue2"}, ok = leveled_cdb:cdb_put(J1, {1, K1}, term_to_binary({V1, []})), ok = leveled_cdb:cdb_put(J1, {2, K2}, term_to_binary({V2, []})), - {ok, _} = leveled_cdb:cdb_complete(J1), + ok = leveled_cdb:cdb_roll(J1), + _LK = leveled_cdb:cdb_lastkey(J1), + ok = leveled_cdb:cdb_close(J1), F2 = filename:join(JournalFP, "nursery_3.pnd"), {ok, J2} = leveled_cdb:cdb_open_writer(F2), {K1, V3} = {"Key1", "TestValue3"}, @@ -866,7 +768,8 @@ build_dummy_journal() -> ok = leveled_cdb:cdb_put(J2, {3, K1}, term_to_binary({V3, []})), ok = leveled_cdb:cdb_put(J2, {4, K4}, term_to_binary({V4, []})), ok = leveled_cdb:cdb_close(J2), - Manifest = [{1, "../test/journal/journal_files/nursery_1.cdb"}], + Manifest = [{1, "../test/journal/journal_files/nursery_1"}, + {3, "../test/journal/journal_files/nursery_3"}], ManifestBin = term_to_binary(Manifest), {ok, MF1} = file:open(filename:join(ManifestFP, "1.man"), [binary, raw, read, write]), @@ -891,146 +794,33 @@ clean_subdir(DirPath) -> end, Files). -simple_buildmanifest_test() -> - RootPath = "../test/journal", - build_dummy_journal(), - Res = build_manifest(["1.man"], - ["../test/journal/journal_files/nursery_1.cdb", - "../test/journal/journal_files/nursery_3.pnd"], - fun simple_manifest_reader/2, - RootPath), - io:format("Build manifest output is ~w~n", [Res]), - {Man, {ActJournal, ActJournalSQN}, HighSQN, ManSQN} = Res, - ?assertMatch(HighSQN, 4), - ?assertMatch(ManSQN, 1), - ?assertMatch([{1, "../test/journal/journal_files/nursery_1.cdb", _}], Man), - {ActSQN, _ActK} = leveled_cdb:cdb_lastkey(ActJournal), - ?assertMatch(ActSQN, 4), - ?assertMatch(ActJournalSQN, 3), - close_allmanifest(Man, ActJournal), - clean_testdir(RootPath). -another_buildmanifest_test() -> - %% There is a rolled jounral file which is not yet in the manifest - RootPath = "../test/journal", - build_dummy_journal(), - FN = filepath(RootPath, 3, new_journal), - {ok, FileToRoll} = leveled_cdb:cdb_open_writer(FN), - {ok, _} = leveled_cdb:cdb_complete(FileToRoll), - FN2 = filepath(RootPath, 5, new_journal), - {ok, NewActiveJN} = leveled_cdb:cdb_open_writer(FN2), - {K5, V5} = {"Key5", "TestValue5"}, - {K6, V6} = {"Key6", "TestValue6"}, - ok = leveled_cdb:cdb_put(NewActiveJN, {5, K5}, term_to_binary({V5, []})), - ok = leveled_cdb:cdb_put(NewActiveJN, {6, K6}, term_to_binary({V6, []})), - ok = leveled_cdb:cdb_close(NewActiveJN), - %% Test setup - now build manifest - Res = build_manifest(["1.man"], - ["../test/journal/journal_files/nursery_1.cdb", - "../test/journal/journal_files/nursery_3.cdb", - "../test/journal/journal_files/nursery_5.pnd"], - fun simple_manifest_reader/2, - RootPath), - io:format("Build manifest output is ~w~n", [Res]), - {Man, {ActJournal, ActJournalSQN}, HighSQN, ManSQN} = Res, - ?assertMatch(HighSQN, 6), - ?assertMatch(ManSQN, 1), - ?assertMatch([{3, "../test/journal/journal_files/nursery_3.cdb", _}, - {1, "../test/journal/journal_files/nursery_1.cdb", _}], Man), - {ActSQN, _ActK} = leveled_cdb:cdb_lastkey(ActJournal), - ?assertMatch(ActSQN, 6), - ?assertMatch(ActJournalSQN, 5), - close_allmanifest(Man, ActJournal), - clean_testdir(RootPath). - - -empty_buildmanifest_test() -> - RootPath = "../test/journal", - Res = build_manifest([], - [], - fun simple_manifest_reader/2, - RootPath), - io:format("Build manifest output is ~w~n", [Res]), - {Man, {ActJournal, ActJournalSQN}, HighSQN, ManSQN} = Res, - ?assertMatch(Man, []), - ?assertMatch(ManSQN, 0), - ?assertMatch(HighSQN, 0), - ?assertMatch(ActJournalSQN, 1), - empty = leveled_cdb:cdb_lastkey(ActJournal), - FN = leveled_cdb:cdb_filename(ActJournal), - %% The filename should be based on the next journal SQN (1) not 0 - ?assertMatch(FN, filepath(RootPath, 1, new_journal)), - close_allmanifest(Man, ActJournal), - clean_testdir(RootPath). - -simplejournal_test() -> - %% build up a database, and then open it through the gen_server wrap - %% Get and Put some keys - RootPath = "../test/journal", - build_dummy_journal(), - {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, - cdb_options=#cdb_options{}}), - R1 = ink_get(Ink1, "Key1", 1), - ?assertMatch(R1, {{1, "Key1"}, {"TestValue1", []}}), - R2 = ink_get(Ink1, "Key1", 3), - ?assertMatch(R2, {{3, "Key1"}, {"TestValue3", []}}), - {ok, NewSQN1, _ObjSize} = ink_put(Ink1, "Key99", "TestValue99", []), - ?assertMatch(NewSQN1, 5), - R3 = ink_get(Ink1, "Key99", 5), - io:format("Result 3 is ~w~n", [R3]), - ?assertMatch(R3, {{5, "Key99"}, {"TestValue99", []}}), - ink_close(Ink1), - clean_testdir(RootPath). - -rollafile_simplejournal_test() -> +simple_inker_test() -> RootPath = "../test/journal", build_dummy_journal(), CDBopts = #cdb_options{max_size=300000}, {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, cdb_options=CDBopts}), - FunnyLoop = lists:seq(1, 48), - {ok, NewSQN1, _ObjSize} = ink_put(Ink1, "KeyAA", "TestValueAA", []), - ?assertMatch(NewSQN1, 5), - ok = ink_print_manifest(Ink1), - R0 = ink_get(Ink1, "KeyAA", 5), - ?assertMatch(R0, {{5, "KeyAA"}, {"TestValueAA", []}}), - lists:foreach(fun(X) -> - {ok, _, _} = ink_put(Ink1, - "KeyZ" ++ integer_to_list(X), - crypto:rand_bytes(10000), - []) end, - FunnyLoop), - {ok, NewSQN2, _ObjSize} = ink_put(Ink1, "KeyBB", "TestValueBB", []), - ?assertMatch(NewSQN2, 54), - ok = ink_print_manifest(Ink1), - R1 = ink_get(Ink1, "KeyAA", 5), - ?assertMatch(R1, {{5, "KeyAA"}, {"TestValueAA", []}}), - R2 = ink_get(Ink1, "KeyBB", 54), - ?assertMatch(R2, {{54, "KeyBB"}, {"TestValueBB", []}}), - Man = ink_getmanifest(Ink1), - FakeMan = [{3, "test", dummy}, {1, "other_test", dummy}], - ok = ink_updatemanifest(Ink1, FakeMan, Man), - ?assertMatch(FakeMan, ink_getmanifest(Ink1)), - ok = ink_updatemanifest(Ink1, Man, FakeMan), - ?assertMatch({{5, "KeyAA"}, {"TestValueAA", []}}, - ink_get(Ink1, "KeyAA", 5)), - ?assertMatch({{54, "KeyBB"}, {"TestValueBB", []}}, - ink_get(Ink1, "KeyBB", 54)), + Obj1 = ink_get(Ink1, "Key1", 1), + ?assertMatch({{1, "Key1"}, {"TestValue1", []}}, Obj1), + Obj2 = ink_get(Ink1, "Key4", 4), + ?assertMatch({{4, "Key4"}, {"TestValue4", []}}, Obj2), ink_close(Ink1), clean_testdir(RootPath). + compact_journal_test() -> RootPath = "../test/journal", build_dummy_journal(), CDBopts = #cdb_options{max_size=300000}, {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, cdb_options=CDBopts}), - FunnyLoop = lists:seq(1, 48), {ok, NewSQN1, _ObjSize} = ink_put(Ink1, "KeyAA", "TestValueAA", []), ?assertMatch(NewSQN1, 5), ok = ink_print_manifest(Ink1), R0 = ink_get(Ink1, "KeyAA", 5), ?assertMatch(R0, {{5, "KeyAA"}, {"TestValueAA", []}}), + FunnyLoop = lists:seq(1, 48), Checker = lists:map(fun(X) -> PK = "KeyZ" ++ integer_to_list(X), {ok, SQN, _} = ink_put(Ink1, @@ -1043,16 +833,63 @@ compact_journal_test() -> {ok, NewSQN2, _ObjSize} = ink_put(Ink1, "KeyBB", "TestValueBB", []), ?assertMatch(NewSQN2, 54), ActualManifest = ink_getmanifest(Ink1), - ?assertMatch(2, length(ActualManifest)), + ok = ink_print_manifest(Ink1), + ?assertMatch(3, length(ActualManifest)), ok = ink_compactjournal(Ink1, Checker, fun(X) -> {X, 55} end, fun(L, K, SQN) -> lists:member({SQN, K}, L) end, 5000), timer:sleep(1000), - CompactedManifest = ink_getmanifest(Ink1), - ?assertMatch(1, length(CompactedManifest)), + CompactedManifest1 = ink_getmanifest(Ink1), + ?assertMatch(2, length(CompactedManifest1)), + Checker2 = lists:sublist(Checker, 16), + ok = ink_compactjournal(Ink1, + Checker2, + fun(X) -> {X, 55} end, + fun(L, K, SQN) -> lists:member({SQN, K}, L) end, + 5000), + timer:sleep(1000), + CompactedManifest2 = ink_getmanifest(Ink1), + R = lists:foldl(fun({_SQN, FN, _P}, Acc) -> + case string:str(FN, "post_compact") of + N when N > 0 -> + true; + 0 -> + Acc + end end, + false, + CompactedManifest2), + ?assertMatch(true, R), + ?assertMatch(2, length(CompactedManifest2)), ink_close(Ink1), clean_testdir(RootPath). +empty_manifest_test() -> + RootPath = "../test/journal", + clean_testdir(RootPath), + CDBopts = #cdb_options{max_size=300000}, + {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, + cdb_options=CDBopts}), + ?assertMatch(not_present, ink_fetch(Ink1, "Key1", 1)), + ok = ink_compactjournal(Ink1, + [], + fun(X) -> {X, 55} end, + fun(L, K, SQN) -> lists:member({SQN, K}, L) end, + 5000), + timer:sleep(1000), + ?assertMatch(1, length(ink_getmanifest(Ink1))), + ok = ink_close(Ink1), + {ok, Ink2} = ink_start(#inker_options{root_path=RootPath, + cdb_options=CDBopts}), + ?assertMatch(not_present, ink_fetch(Ink2, "Key1", 1)), + {ok, SQN, Size} = ink_put(Ink2, "Key1", "Value1", []), + ?assertMatch(2, SQN), + ?assertMatch(true, Size > 0), + {ok, V} = ink_fetch(Ink2, "Key1", 2), + ?assertMatch("Value1", V), + ink_close(Ink2), + clean_testdir(RootPath). + + -endif. \ No newline at end of file From 4a8a2c1555ad69826cc38a07ed2d19fae1e90801 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 8 Oct 2016 22:15:48 +0100 Subject: [PATCH 058/167] Code reduction refactor An attempt to refactor out more complex code. The Penciller clerk and Penciller have been re-shaped so that there relationship is much simpler, and also to make sure that they shut down much more neatly when the clerk is busy to avoid crashdumps in ct tests. The CDB now has a binary_mode - so that we don't do binary_to_term twice ... although this may have made things slower ??!!? Perhaps the is_binary check now required on read is an overhead. Perhaps it is some other mystery. There is now a more effiicient fetching of the size on pcl_load now as well. --- include/leveled.hrl | 3 +- src/leveled_bookie.erl | 24 ++--- src/leveled_cdb.erl | 61 ++++++++----- src/leveled_iclerk.erl | 27 +++++- src/leveled_inker.erl | 21 ++++- src/leveled_pclerk.erl | 111 +++++++++++------------ src/leveled_penciller.erl | 150 +++++++++++--------------------- test/end_to_end/basic_SUITE.erl | 10 ++- 8 files changed, 216 insertions(+), 191 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 37694ed..cd82d2a 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -22,7 +22,8 @@ -record(cdb_options, {max_size :: integer(), - file_path :: string()}). + file_path :: string(), + binary_mode = false :: boolean()}). -record(inker_options, {cdb_max_size :: integer(), diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index bafba89..a055cdf 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -368,7 +368,8 @@ set_options(Opts) -> end, {#inker_options{root_path = Opts#bookie_options.root_path ++ "/" ++ ?JOURNAL_FP, - cdb_options = #cdb_options{max_size=MaxJournalSize}}, + cdb_options = #cdb_options{max_size=MaxJournalSize, + binary_mode=true}}, #penciller_options{root_path=Opts#bookie_options.root_path ++ "/" ++ ?LEDGER_FP}}. @@ -495,22 +496,25 @@ maybepush_ledgercache(MaxCacheSize, Cache, Penciller) -> load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> {MinSQN, MaxSQN, Output} = Acc0, {SQN, PK} = KeyInLedger, - {Obj, IndexSpecs} = binary_to_term(ExtractFun(ValueInLedger)), + % VBin may already be a term + % TODO: Should VSize include CRC? + % Using ExtractFun means we ignore simple way of getting size (from length) + {VBin, VSize} = ExtractFun(ValueInLedger), + {Obj, IndexSpecs} = case is_binary(VBin) of + true -> + binary_to_term(VBin); + false -> + VBin + end, case SQN of SQN when SQN < MinSQN -> {loop, Acc0}; SQN when SQN < MaxSQN -> - %% TODO - get correct size in a more efficient manner - %% Need to have compressed size - Size = byte_size(term_to_binary(ValueInLedger, [compressed])), - Changes = preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs), + Changes = preparefor_ledgercache(PK, SQN, Obj, VSize, IndexSpecs), {loop, {MinSQN, MaxSQN, Output ++ Changes}}; MaxSQN -> - %% TODO - get correct size in a more efficient manner - %% Need to have compressed size io:format("Reached end of load batch with SQN ~w~n", [SQN]), - Size = byte_size(term_to_binary(ValueInLedger, [compressed])), - Changes = preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs), + Changes = preparefor_ledgercache(PK, SQN, Obj, VSize, IndexSpecs), {stop, {MinSQN, MaxSQN, Output ++ Changes}}; SQN when SQN > MaxSQN -> io:format("Skipping as exceeded MaxSQN ~w with SQN ~w~n", diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 04adfda..45f2a6a 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -76,6 +76,7 @@ -define(WORD_SIZE, 4). -define(CRC_CHECK, true). -define(MAX_FILE_SIZE, 3221225472). +-define(BINARY_MODE, false). -define(BASE_POSITION, 2048). -define(WRITE_OPS, [binary, raw, read, write]). @@ -87,7 +88,8 @@ handle :: file:fd(), writer :: boolean(), max_size :: integer(), - pending_delete = false :: boolean()}). + pending_delete = false :: boolean(), + binary_mode = false :: boolean()}). %%%============================================================================ @@ -187,7 +189,7 @@ init([Opts]) -> M -> M end, - {ok, #state{max_size=MaxSize}}. + {ok, #state{max_size=MaxSize, binary_mode=Opts#cdb_options.binary_mode}}. handle_call({open_writer, Filename}, _From, State) -> io:format("Opening file for writing with filename ~s~n", [Filename]), @@ -251,6 +253,7 @@ handle_call({put_kv, Key, Value}, _From, State) -> Result = put(State#state.handle, Key, Value, {State#state.last_position, State#state.hashtree}, + State#state.binary_mode, State#state.max_size), case Result of roll -> @@ -478,23 +481,32 @@ open_active_file(FileName) when is_list(FileName) -> %% Append to an active file a new key/value pair returning an updated %% dictionary of Keys and positions. Returns an updated Position %% -put(FileName, Key, Value, {LastPosition, HashTree}, MaxSize) when is_list(FileName) -> - {ok, Handle} = file:open(FileName, ?WRITE_OPS), - put(Handle, Key, Value, {LastPosition, HashTree}, MaxSize); -put(Handle, Key, Value, {LastPosition, HashTree}, MaxSize) -> - Bin = key_value_to_record({Key, Value}), - PotentialNewSize = LastPosition + byte_size(Bin), - if PotentialNewSize > MaxSize -> - roll; - true -> - ok = file:pwrite(Handle, LastPosition, Bin), - {Handle, PotentialNewSize, put_hashtree(Key, LastPosition, HashTree)} - end. +put(FileName, + Key, + Value, + {LastPosition, HashTree}, + BinaryMode, + MaxSize) when is_list(FileName) -> + {ok, Handle} = file:open(FileName, ?WRITE_OPS), + put(Handle, Key, Value, {LastPosition, HashTree}, BinaryMode, MaxSize); +put(Handle, Key, Value, {LastPosition, HashTree}, BinaryMode, MaxSize) -> + Bin = key_value_to_record({Key, Value}, BinaryMode), + PotentialNewSize = LastPosition + byte_size(Bin), + if + PotentialNewSize > MaxSize -> + roll; + true -> + ok = file:pwrite(Handle, LastPosition, Bin), + {Handle, + PotentialNewSize, + put_hashtree(Key, LastPosition, HashTree)} + end. %% Should not be used for non-test PUTs by the inker - as the Max File Size %% should be taken from the startup options not the default put(FileName, Key, Value, {LastPosition, HashTree}) -> - put(FileName, Key, Value, {LastPosition, HashTree}, ?MAX_FILE_SIZE). + put(FileName, Key, Value, {LastPosition, HashTree}, + ?BINARY_MODE, ?MAX_FILE_SIZE). %% @@ -864,7 +876,7 @@ scan_over_file(Handle, Position, FilterFun, Output, LastKey) -> ValueAsBin, Position, Output, - fun extract_value/1) of + fun extract_valueandsize/1) of {stop, UpdOutput} -> {NewPosition, UpdOutput}; {loop, UpdOutput} -> @@ -993,10 +1005,10 @@ read_next_term(Handle, Length, crc, Check) -> {unchecked, binary_to_term(Bin)} end. -%% Extract value from binary containing CRC -extract_value(ValueAsBin) -> +%% Extract value and size from binary containing CRC +extract_valueandsize(ValueAsBin) -> <<_CRC:32/integer, Bin/binary>> = ValueAsBin, - binary_to_term(Bin). + {binary_to_term(Bin), byte_size(Bin)}. %% Used for reading lengths @@ -1234,9 +1246,14 @@ hash_to_slot(Hash, L) -> %% Create a binary of the LengthKeyLengthValue, adding a CRC check %% at the front of the value -key_value_to_record({Key, Value}) -> - BK = term_to_binary(Key), - BV = term_to_binary(Value), +key_value_to_record({Key, Value}, BinaryMode) -> + BK = term_to_binary(Key), + BV = case BinaryMode of + true -> + Value; + false -> + term_to_binary(Value) + end, LK = byte_size(BK), LV = byte_size(BV), LK_FL = endian_flip(LK), diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 9403ef4..96926c7 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -28,6 +28,7 @@ %% Sliding scale to allow preference of longer runs up to maximum -define(SINGLEFILE_COMPACTION_TARGET, 60.0). -define(MAXRUN_COMPACTION_TARGET, 80.0). +-define(CRC_SIZE, 4). -record(state, {inker :: pid(), max_run_length :: integer(), @@ -150,16 +151,17 @@ check_single_file(CDB, FilterFun, FilterServer, MaxSQN, SampleSize, BatchSize) - FN = leveled_cdb:cdb_filename(CDB), PositionList = leveled_cdb:cdb_getpositions(CDB, SampleSize), KeySizeList = fetch_inbatches(PositionList, BatchSize, CDB, []), + io:format("KeySizeList ~w~n", [KeySizeList]), R0 = lists:foldl(fun(KS, {ActSize, RplSize}) -> {{SQN, PK}, Size} = KS, Check = FilterFun(FilterServer, PK, SQN), case {Check, SQN > MaxSQN} of {true, _} -> - {ActSize + Size, RplSize}; + {ActSize + Size - ?CRC_SIZE, RplSize}; {false, true} -> - {ActSize + Size, RplSize}; + {ActSize + Size - ?CRC_SIZE, RplSize}; _ -> - {ActSize, RplSize + Size} + {ActSize, RplSize + Size - ?CRC_SIZE} end end, {0, 0}, KeySizeList), @@ -580,4 +582,23 @@ compact_single_file_test() -> ok = leveled_cdb:cdb_destroy(CDB). +compact_empty_file_test() -> + RP = "../test/journal", + FN1 = leveled_inker:filepath(RP, 1, new_journal), + CDBopts = #cdb_options{binary_mode=true}, + {ok, CDB1} = leveled_cdb:cdb_open_writer(FN1, CDBopts), + ok = leveled_cdb:cdb_put(CDB1, {1, "Key1"}, <<>>), + {ok, FN2} = leveled_cdb:cdb_complete(CDB1), + {ok, CDB2} = leveled_cdb:cdb_open_reader(FN2), + LedgerSrv1 = [{8, "Key1"}, {2, "Key2"}, {3, "Key3"}], + LedgerFun1 = fun(Srv, Key, ObjSQN) -> + case lists:keyfind(ObjSQN, 1, Srv) of + {ObjSQN, Key} -> + true; + _ -> + false + end end, + Score1 = check_single_file(CDB2, LedgerFun1, LedgerSrv1, 9, 8, 4), + ?assertMatch(100.0, Score1). + -endif. \ No newline at end of file diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 471ea0b..7957d94 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -413,8 +413,10 @@ get_object(PrimaryKey, SQN, Manifest) -> JournalP = find_in_manifest(SQN, Manifest), Obj = leveled_cdb:cdb_get(JournalP, {SQN, PrimaryKey}), case Obj of - {{SQN, PK}, Bin} -> + {{SQN, PK}, Bin} when is_binary(Bin) -> {{SQN, PK}, binary_to_term(Bin)}; + {{SQN, PK}, Term} -> + {{SQN, PK}, Term}; _ -> Obj end. @@ -807,6 +809,23 @@ simple_inker_test() -> ?assertMatch({{4, "Key4"}, {"TestValue4", []}}, Obj2), ink_close(Ink1), clean_testdir(RootPath). + +simple_inker_completeactivejournal_test() -> + RootPath = "../test/journal", + build_dummy_journal(), + CDBopts = #cdb_options{max_size=300000}, + {ok, PidW} = leveled_cdb:cdb_open_writer(filepath(RootPath, + 3, + new_journal)), + {ok, _FN} = leveled_cdb:cdb_complete(PidW), + {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, + cdb_options=CDBopts}), + Obj1 = ink_get(Ink1, "Key1", 1), + ?assertMatch({{1, "Key1"}, {"TestValue1", []}}, Obj1), + Obj2 = ink_get(Ink1, "Key4", 4), + ?assertMatch({{4, "Key4"}, {"TestValue4", []}}, Obj2), + ink_close(Ink1), + clean_testdir(RootPath). compact_journal_test() -> diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 47467a3..42001c1 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -14,17 +14,20 @@ handle_info/2, terminate/2, clerk_new/1, - clerk_prompt/2, - clerk_stop/1, + clerk_prompt/1, + clerk_returnmanifestchange/2, code_change/3, perform_merge/4]). -include_lib("eunit/include/eunit.hrl"). -define(INACTIVITY_TIMEOUT, 2000). +-define(QUICK_TIMEOUT, 500). -define(HAPPYTIME_MULTIPLIER, 5). --record(state, {owner :: pid()}). +-record(state, {owner :: pid(), + change_pending=false :: boolean(), + work_item :: #penciller_work{}}). %%%============================================================================ %%% API @@ -34,13 +37,12 @@ clerk_new(Owner) -> {ok, Pid} = gen_server:start(?MODULE, [], []), ok = gen_server:call(Pid, {register, Owner}, infinity), {ok, Pid}. - -clerk_prompt(Pid, penciller) -> - gen_server:cast(Pid, penciller_prompt), - ok. -clerk_stop(Pid) -> - gen_server:cast(Pid, stop). +clerk_returnmanifestchange(Pid, Closing) -> + gen_server:call(Pid, {return_manifest_change, Closing}). + +clerk_prompt(Pid) -> + gen_server:cast(Pid, prompt). %%%============================================================================ %%% gen_server callbacks @@ -50,21 +52,41 @@ init([]) -> {ok, #state{}}. handle_call({register, Owner}, _From, State) -> - {reply, ok, State#state{owner=Owner}, ?INACTIVITY_TIMEOUT}. + {reply, ok, State#state{owner=Owner}, ?INACTIVITY_TIMEOUT}; +handle_call({return_manifest_change, Closing}, From, State) -> + case {State#state.change_pending, Closing} of + {true, true} -> + WI = State#state.work_item, + ok = mark_for_delete(WI#penciller_work.unreferenced_files, + State#state.owner), + {stop, normal, {ok, WI}, State}; + {true, false} -> + WI = State#state.work_item, + gen_server:reply(From, {ok, WI}), + mark_for_delete(WI#penciller_work.unreferenced_files, + State#state.owner), + {noreply, + State#state{work_item=null, change_pending=false}, + ?INACTIVITY_TIMEOUT}; + {false, true} -> + {stop, normal, no_change_required, State} + end. -handle_cast(penciller_prompt, State) -> - Timeout = requestandhandle_work(State), - {noreply, State, Timeout}; -handle_cast(stop, State) -> - {stop, normal, State}. +handle_cast(prompt, State) -> + io:format("Clerk reducing timeout due to prompt~n"), + {noreply, State, ?QUICK_TIMEOUT}; +handle_cast(_Msg, State) -> + {noreply, State}. -handle_info(timeout, State) -> - case leveled_penciller:pcl_prompt(State#state.owner) of - ok -> - Timeout = requestandhandle_work(State), +handle_info(timeout, State=#state{change_pending=Pnd}) when Pnd == false -> + case requestandhandle_work(State) of + {false, Timeout} -> {noreply, State, Timeout}; - pause -> - {noreply, State, ?INACTIVITY_TIMEOUT} + {true, WI} -> + % No timeout now as will wait for call to return manifest + % change + {noreply, + State#state{change_pending=true, work_item=WI}} end; handle_info(_Info, State) -> {noreply, State}. @@ -86,29 +108,16 @@ requestandhandle_work(State) -> io:format("Work prompted but none needed~n"), case Backlog of false -> - ?INACTIVITY_TIMEOUT * ?HAPPYTIME_MULTIPLIER; + {false, ?INACTIVITY_TIMEOUT * ?HAPPYTIME_MULTIPLIER}; _ -> - ?INACTIVITY_TIMEOUT + {false, ?INACTIVITY_TIMEOUT} end; {WI, _} -> {NewManifest, FilesToDelete} = merge(WI), UpdWI = WI#penciller_work{new_manifest=NewManifest, unreferenced_files=FilesToDelete}, - R = leveled_penciller:pcl_requestmanifestchange(State#state.owner, - UpdWI), - case R of - ok -> - %% Request for manifest change must be a synchronous call - %% Otherwise cannot mark files for deletion (may erase - %% without manifest change on close) - mark_for_delete(FilesToDelete, State#state.owner), - ?INACTIVITY_TIMEOUT; - _ -> - %% New files will forever remain in an undetermined state - %% The disconnected files should be logged at start-up for - %% Manual clear-up - ?INACTIVITY_TIMEOUT - end + ok = leveled_penciller:pcl_promptmanifestchange(State#state.owner), + {true, UpdWI} end. @@ -252,22 +261,16 @@ do_merge(KL1, KL2, Level, {Filepath, MSN}, FileCounter, OutList) -> io:format("File to be created as part of MSN=~w Filename=~s~n", [MSN, FileName]), TS1 = os:timestamp(), - case leveled_sft:sft_new(FileName, KL1, KL2, Level + 1) of - {ok, _Pid, {error, Reason}} -> - io:format("Exiting due to error~w~n", [Reason]), - error; - {ok, Pid, Reply} -> - {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, - ExtMan = lists:append(OutList, - [#manifest_entry{start_key=SmallestKey, - end_key=HighestKey, - owner=Pid, - filename=FileName}]), - MTime = timer:now_diff(os:timestamp(), TS1), - io:format("File creation took ~w microseconds ~n", [MTime]), - do_merge(KL1Rem, KL2Rem, Level, {Filepath, MSN}, - FileCounter + 1, ExtMan) - end. + {ok, Pid, Reply} = leveled_sft:sft_new(FileName, KL1, KL2, Level + 1), + {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, + ExtMan = lists:append(OutList, + [#manifest_entry{start_key=SmallestKey, + end_key=HighestKey, + owner=Pid, + filename=FileName}]), + MTime = timer:now_diff(os:timestamp(), TS1), + io:format("File creation took ~w microseconds ~n", [MTime]), + do_merge(KL1Rem, KL2Rem, Level, {Filepath, MSN}, FileCounter + 1, ExtMan). get_item(Index, List, Default) -> diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 26b70e8..fb20819 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -236,9 +236,8 @@ pcl_fetch/2, pcl_checksequencenumber/3, pcl_workforclerk/1, - pcl_requestmanifestchange/2, + pcl_promptmanifestchange/1, pcl_confirmdelete/2, - pcl_prompt/1, pcl_close/1, pcl_registersnapshot/2, pcl_updatesnapshotcache/3, @@ -308,15 +307,12 @@ pcl_checksequencenumber(Pid, Key, SQN) -> pcl_workforclerk(Pid) -> gen_server:call(Pid, work_for_clerk, infinity). -pcl_requestmanifestchange(Pid, WorkItem) -> - gen_server:call(Pid, {manifest_change, WorkItem}, infinity). +pcl_promptmanifestchange(Pid) -> + gen_server:cast(Pid, manifest_change). pcl_confirmdelete(Pid, FileName) -> gen_server:call(Pid, {confirm_delete, FileName}, infinity). -pcl_prompt(Pid) -> - gen_server:call(Pid, prompt_compaction, infinity). - pcl_getstartupsequencenumber(Pid) -> gen_server:call(Pid, get_startup_sqn, infinity). @@ -454,43 +450,6 @@ handle_call({confirm_delete, FileName}, _From, State=#state{is_snapshot=Snap}) _ -> {reply, Reply, State} end; -handle_call(prompt_compaction, _From, State=#state{is_snapshot=Snap}) - when Snap == false -> - %% If there is a prompt immediately after a L0 async write event then - %% there exists the potential for the prompt to stall the database. - %% Should only accept prompts if there has been a safe wait from the - %% last L0 write event. - Proceed = case State#state.levelzero_pending of - {true, _Pid, TS} -> - TD = timer:now_diff(os:timestamp(),TS), - if - TD < ?PROMPT_WAIT_ONL0 * 1000000 -> false; - true -> true - end; - ?L0PEND_RESET -> - true - end, - if - Proceed -> - {_TableSize, State1} = checkready_pushtomem(State), - case roll_memory(State1, State1#state.memtable_maxsize) of - {ok, L0Pend, MSN, TableSize} -> - io:format("Prompted push completed~n"), - {reply, ok, State1#state{levelzero_pending=L0Pend, - table_size=TableSize, - manifest_sqn=MSN, - backlog=false}}; - {pause, Reason, Details} -> - io:format("Excess work due to - " ++ Reason, Details), - {reply, pause, State1#state{backlog=true}} - end; - true -> - {reply, ok, State#state{backlog=false}} - end; -handle_call({manifest_change, WI}, _From, State=#state{is_snapshot=Snap}) - when Snap == false -> - {ok, UpdState} = commit_manifest_change(WI, State), - {reply, ok, UpdState}; handle_call({fetch, Key}, _From, State=#state{is_snapshot=Snap}) when Snap == false -> {reply, @@ -498,14 +457,6 @@ handle_call({fetch, Key}, _From, State=#state{is_snapshot=Snap}) State#state.manifest, State#state.memtable), State}; -handle_call({check_sqn, Key, SQN}, _From, State=#state{is_snapshot=Snap}) - when Snap == false -> - {reply, - compare_to_sqn(fetch(Key, - State#state.manifest, - State#state.memtable), - SQN), - State}; handle_call({fetch, Key}, _From, State=#state{snapshot_fully_loaded=Ready}) @@ -560,6 +511,11 @@ handle_call(close, _From, State) -> handle_cast({update_snapshotcache, Tree, SQN}, State) -> MemTableC = cache_tree_in_memcopy(State#state.memtable_copy, Tree, SQN), {noreply, State#state{memtable_copy=MemTableC}}; +handle_cast(manifest_change, State) -> + {ok, WI} = leveled_pclerk:clerk_returnmanifestchange(State#state.clerk, + false), + {ok, UpdState} = commit_manifest_change(WI, State), + {noreply, UpdState}; handle_cast(_Msg, State) -> {noreply, State}. @@ -585,13 +541,20 @@ terminate(_Reason, State) -> %% The cast may not succeed as the clerk could be synchronously calling %% the penciller looking for a manifest commit %% - leveled_pclerk:clerk_stop(State#state.clerk), - Dump = ets:tab2list(State#state.memtable), - case {State#state.levelzero_pending, - get_item(0, State#state.manifest, []), length(Dump)} of + MC = leveled_pclerk:clerk_returnmanifestchange(State#state.clerk, true), + UpdState = case MC of + {ok, WI} -> + {ok, NewState} = commit_manifest_change(WI, State), + NewState; + no_change_required -> + State + end, + Dump = ets:tab2list(UpdState#state.memtable), + case {UpdState#state.levelzero_pending, + get_item(0, UpdState#state.manifest, []), length(Dump)} of {?L0PEND_RESET, [], L} when L > 0 -> - MSN = State#state.manifest_sqn + 1, - FileName = State#state.root_path + MSN = UpdState#state.manifest_sqn + 1, + FileName = UpdState#state.root_path ++ "/" ++ ?FILES_FP ++ "/" ++ integer_to_list(MSN) ++ "_0_0", NewSFT = leveled_sft:sft_new(FileName ++ ".pnd", @@ -615,10 +578,10 @@ terminate(_Reason, State) -> ++ " with ~w keys discarded~n", [length(Dump)]) end, - ok = close_files(0, State#state.manifest), + ok = close_files(0, UpdState#state.manifest), lists:foreach(fun({_FN, Pid, _SN}) -> leveled_sft:sft_close(Pid) end, - State#state.unreferenced_files), + UpdState#state.unreferenced_files), ok. @@ -732,6 +695,8 @@ checkready_pushtomem(State) -> end_key=EndKey, owner=Pid, filename=SrcFN}, + % Prompt clerk to ask about work - do this for every L0 roll + ok = leveled_pclerk:clerk_prompt(State#state.clerk), {0, State#state{manifest=lists:keystore(0, 1, @@ -742,9 +707,6 @@ checkready_pushtomem(State) -> ?L0PEND_RESET -> {State#state.table_size, State} end, - - %% Prompt clerk to ask about work - do this for every push_mem - ok = leveled_pclerk:clerk_prompt(UpdState#state.clerk, penciller), {TableSize, UpdState}. quickcheck_pushtomem(DumpList, TableSize, MaxSize) -> @@ -1216,6 +1178,15 @@ confirm_delete_test() -> ?assertMatch(R3, false). +maybe_pause_push(R) -> + if + R == pause -> + io:format("Pausing push~n"), + timer:sleep(1000); + true -> + ok + end. + simple_server_test() -> RootPath = "../test/ledger", clean_testdir(RootPath), @@ -1230,27 +1201,17 @@ simple_server_test() -> Key4 = {{o,"Bucket0004", "Key0004"}, {3002, {active, infinity}, null}}, KL4 = lists:sort(leveled_sft:generate_randomkeys({1000, 3002})), ok = pcl_pushmem(PCL, [Key1]), - R1 = pcl_fetch(PCL, {o,"Bucket0001", "Key0001"}), - ?assertMatch(R1, Key1), + ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001"})), ok = pcl_pushmem(PCL, KL1), - R2 = pcl_fetch(PCL, {o,"Bucket0001", "Key0001"}), - ?assertMatch(R2, Key1), - S1 = pcl_pushmem(PCL, [Key2]), - if S1 == pause -> timer:sleep(2); true -> ok end, - R3 = pcl_fetch(PCL, {o,"Bucket0001", "Key0001"}), - R4 = pcl_fetch(PCL, {o,"Bucket0002", "Key0002"}), - ?assertMatch(R3, Key1), - ?assertMatch(R4, Key2), - S2 = pcl_pushmem(PCL, KL2), - if S2 == pause -> timer:sleep(1000); true -> ok end, - S3 = pcl_pushmem(PCL, [Key3]), - if S3 == pause -> timer:sleep(1000); true -> ok end, - R5 = pcl_fetch(PCL, {o,"Bucket0001", "Key0001"}), - R6 = pcl_fetch(PCL, {o,"Bucket0002", "Key0002"}), - R7 = pcl_fetch(PCL, {o,"Bucket0003", "Key0003"}), - ?assertMatch(R5, Key1), - ?assertMatch(R6, Key2), - ?assertMatch(R7, Key3), + ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001"})), + maybe_pause_push(pcl_pushmem(PCL, [Key2])), + ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001"})), + ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002"})), + maybe_pause_push(pcl_pushmem(PCL, KL2)), + maybe_pause_push(pcl_pushmem(PCL, [Key3])), + ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001"})), + ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002"})), + ?assertMatch(Key3, pcl_fetch(PCL, {o,"Bucket0003", "Key0003"})), ok = pcl_close(PCL), {ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath, max_inmemory_tablesize=1000}), @@ -1268,27 +1229,20 @@ simple_server_test() -> io:format("Unexpected sequence number on restart ~w~n", [TopSQN]), error end, - ?assertMatch(Check, ok), - R8 = pcl_fetch(PCLr, {o,"Bucket0001", "Key0001"}), - R9 = pcl_fetch(PCLr, {o,"Bucket0002", "Key0002"}), - R10 = pcl_fetch(PCLr, {o,"Bucket0003", "Key0003"}), - ?assertMatch(R8, Key1), - ?assertMatch(R9, Key2), - ?assertMatch(R10, Key3), + ?assertMatch(ok, Check), + ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001"})), + ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002"})), + ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003"})), S4 = pcl_pushmem(PCLr, KL3), if S4 == pause -> timer:sleep(1000); true -> ok end, S5 = pcl_pushmem(PCLr, [Key4]), if S5 == pause -> timer:sleep(1000); true -> ok end, S6 = pcl_pushmem(PCLr, KL4), if S6 == pause -> timer:sleep(1000); true -> ok end, - R11 = pcl_fetch(PCLr, {o,"Bucket0001", "Key0001"}), - R12 = pcl_fetch(PCLr, {o,"Bucket0002", "Key0002"}), - R13 = pcl_fetch(PCLr, {o,"Bucket0003", "Key0003"}), - R14 = pcl_fetch(PCLr, {o,"Bucket0004", "Key0004"}), - ?assertMatch(R11, Key1), - ?assertMatch(R12, Key2), - ?assertMatch(R13, Key3), - ?assertMatch(R14, Key4), + ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001"})), + ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002"})), + ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003"})), + ?assertMatch(Key4, pcl_fetch(PCLr, {o,"Bucket0004", "Key0004"})), SnapOpts = #penciller_options{start_snapshot = true, source_penciller = PCLr}, {ok, PclSnap} = pcl_start(SnapOpts), diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 9496e2f..3f1560a 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -221,6 +221,7 @@ check_bookie_forlist(Bookie, ChkList) -> check_bookie_forlist(Bookie, ChkList, false). check_bookie_forlist(Bookie, ChkList, Log) -> + SW = os:timestamp(), lists:foreach(fun({_RN, Obj, _Spc}) -> if Log == true -> @@ -232,15 +233,20 @@ check_bookie_forlist(Bookie, ChkList, Log) -> Obj#r_object.bucket, Obj#r_object.key), R = {ok, Obj} end, - ChkList). + ChkList), + io:format("Fetch check took ~w microseconds checking list of length ~w~n", + [timer:now_diff(os:timestamp(), SW), length(ChkList)]). check_bookie_formissinglist(Bookie, ChkList) -> + SW = os:timestamp(), lists:foreach(fun({_RN, Obj, _Spc}) -> R = leveled_bookie:book_riakget(Bookie, Obj#r_object.bucket, Obj#r_object.key), R = not_found end, - ChkList). + ChkList), + io:format("Miss check took ~w microseconds checking list of length ~w~n", + [timer:now_diff(os:timestamp(), SW), length(ChkList)]). check_bookie_forobject(Bookie, TestObject) -> {ok, TestObject} = leveled_bookie:book_riakget(Bookie, From d2cc07a9ebe8b1c15cb0da12d2fc8f1d1fba4ad6 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sun, 9 Oct 2016 22:33:45 +0100 Subject: [PATCH 059/167] Doc update and clerk<->penciller changes Reviewing code to update comments revealed a weakness in the sequence of events between penciller and clerk committing a manifest change wherby an ill-timed crash could lead to files being deleted without the manifest changing. A different, and safer pattern now used between theses two actors. --- src/leveled_bookie.erl | 43 ++++++++--------- src/leveled_inker.erl | 2 +- src/leveled_pclerk.erl | 98 ++++++++++++++++++++++++++++++--------- src/leveled_penciller.erl | 42 +++++++++-------- 4 files changed, 119 insertions(+), 66 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index a055cdf..4116ad0 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -18,9 +18,11 @@ %% -------- The actors --------- %% %% The store is fronted by a Bookie, who takes support from different actors: -%% - An Inker who persists new data into the jornal, and returns items from +%% - An Inker who persists new data into the journal, and returns items from %% the journal based on sequence number -%% - A Penciller who periodically redraws the ledger +%% - A Penciller who periodically redraws the ledger, that associates keys with +%% sequence numbers and other metadata, as well as secondary keys (for index +%% queries) %% - One or more Clerks, who may be used by either the inker or the penciller %% to fulfill background tasks %% @@ -61,11 +63,10 @@ %% well as the object size on disk within the Journal. %% %% Once the object has been persisted to the Journal, the Ledger can be updated. -%% The Ledger is updated by the Bookie applying a function (passed in at -%% startup) to the Value to return the Object Metadata, a function to generate -%% a hash of the Value and also taking the Primary Key, the IndexSpecs, the -%% Sequence Number in the Journal and the Object Size (returned from the -%% Inker). +%% The Ledger is updated by the Bookie applying a function (extract_metadata/4) +%% to the Value to return the Object Metadata, a function to generate a hash +%% of the Value and also taking the Primary Key, the IndexSpecs, the Sequence +%% Number in the Journal and the Object Size (returned from the Inker). %% %% The Bookie should generate a series of ledger key changes from this %% information, using a function passed in at startup. For Riak this will be @@ -79,20 +80,20 @@ %% null, %% {active, TS}|{tomb, TS}} %% -%% Recent Ledger changes are retained initially in the Bookies' memory (in an -%% in-memory ets table). Periodically, the current table is pushed to the -%% Penciller for eventual persistence, and a new table is started. +%% Recent Ledger changes are retained initially in the Bookies' memory (in a +%% small generally balanced tree). Periodically, the current table is pushed to +%% the Penciller for eventual persistence, and a new table is started. %% %% This completes the non-deferrable work associated with a PUT %% %% -------- Snapshots (Key & Metadata Only) -------- %% %% If there is a snapshot request (e.g. to iterate over the keys) the Bookie -%% must first produce a tree representing the results of the request which are -%% present in its in-memory view of the ledger. The Bookie then requests -%% a copy of the current Ledger manifest from the Penciller, and the Penciller -%5 should interest of the iterator at the manifest sequence number at the time -%% of the request. +%% may request a clone of the Penciller, or the Penciller and the Inker. +%% +%% The clone is seeded with the manifest. Teh clone should be registered with +%% the real Inker/Penciller, so that the real Inker/Penciller may prevent the +%% deletion of files still in use by a snapshot clone. %% %% Iterators should de-register themselves from the Penciller on completion. %% Iterators should be automatically release after a timeout period. A file @@ -100,10 +101,6 @@ %% there are no registered iterators from before the point the file was %% removed from the manifest. %% -%% Snapshots may be non-recent, if recency is unimportant. Non-recent -%% snapshots do no require the Bookie to return the results of the in-memory -%% table, the Penciller alone cna be asked. -%% %% -------- Special Ops -------- %% %% e.g. Get all for SegmentID/Partition @@ -115,10 +112,11 @@ %% On startup the Bookie must restart both the Inker to load the Journal, and %% the Penciller to load the Ledger. Once the Penciller has started, the %% Bookie should request the highest sequence number in the Ledger, and then -%% and try and rebuild any missing information from the Journal +%% and try and rebuild any missing information from the Journal. %% %% To rebuild the Ledger it requests the Inker to scan over the files from -%% the sequence number and re-generate the Ledger changes. +%% the sequence number and re-generate the Ledger changes - pushing the changes +%% directly back into the Ledger. @@ -359,7 +357,6 @@ shutdown_wait([TopPause|Rest], Inker) -> set_options(Opts) -> - %% TODO: Change the max size default, and allow setting through options MaxJournalSize = case Opts#bookie_options.max_journalsize of undefined -> 30000; @@ -497,8 +494,6 @@ load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> {MinSQN, MaxSQN, Output} = Acc0, {SQN, PK} = KeyInLedger, % VBin may already be a term - % TODO: Should VSize include CRC? - % Using ExtractFun means we ignore simple way of getting size (from length) {VBin, VSize} = ExtractFun(ValueInLedger), {Obj, IndexSpecs} = case is_binary(VBin) of true -> diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 7957d94..b54c2d2 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -59,7 +59,7 @@ %% %% Compaction is a process whereby an Inker's clerk will: %% - Request a view of the current Inker manifest and a snaphot of the Ledger -%% - Test all files within the Journal to find th eapproximate comapction +%% - Test all files within the Journal to find the approximate comapction %% potential percentage (the volume of the Journal that has been replaced) %% - Attempts to find the optimal "run" of files to compact %% - Compacts those files in the run, by rolling over the files re-writing diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 42001c1..307eab6 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -1,6 +1,52 @@ -%% Controlling asynchronous work in leveleddb to manage compaction within a -%% level and cleaning out of old files across a level - +%% -------- PENCILLER's CLERK --------- +%% +%% The Penciller's clerk is responsible for compaction work within the Ledger. +%% +%% The Clerk will periodically poll the Penciller to see if there is work for +%% it to complete, except if the Clerk has informed the Penciller that it has +%% readied a manifest change to be committed - in which case it will wait to +%% be called by the Penciller. +%% +%% -------- COMMITTING MANIFEST CHANGES --------- +%% +%% Once the Penciller has taken a manifest change, the SFT file owners which no +%% longer form part of the manifest will be marked for delete. By marking for +%% deletion, the owners will poll to confirm when it is safe for them to be +%% deleted. +%% +%% It is imperative that the file is not marked for deletion until it is +%% certain that the manifest change has been committed. Some uncollected +%% garbage is considered acceptable. +%% +%% The process of committing a manifest change is as follows: +%% +%% A - The Clerk completes a merge, and casts a prompt to the Penciller with +%% a work item describing the change +%% +%% B - The Penciller commits the change to disk, and then calls the Clerk to +%% confirm the manifest change +%% +%% C - The Clerk replies immediately to acknowledge this call, then marks the +%% removed files for deletion +%% +%% Shutdown < A/B - If the Penciller starts the shutdown process before the +%% merge is complete, in the shutdown the Penciller will call a request for the +%% manifest change which will pick up the pending change. It will then confirm +%% the change, and now the Clerk will mark the files for delete before it +%% replies to the Penciller so it can complete the shutdown process (which will +%% prompt erasing of the removed files). +%% +%% The clerk will not request work on timeout if the committing of a manifest +%5 change is pending confirmation. +%% +%% -------- TIMEOUTS --------- +%% +%% The Penciller may prompt the Clerk to callback soon (i.e. reduce the +%% Timeout) if it has urgent work ready (i.e. it has written a L0 file). +%% +%% There will also be a natural quick timeout once the committing of a manifest +%% change has occurred. +%% -module(leveled_pclerk). @@ -15,15 +61,15 @@ terminate/2, clerk_new/1, clerk_prompt/1, - clerk_returnmanifestchange/2, + clerk_manifestchange/3, code_change/3, perform_merge/4]). -include_lib("eunit/include/eunit.hrl"). --define(INACTIVITY_TIMEOUT, 2000). +-define(INACTIVITY_TIMEOUT, 5000). -define(QUICK_TIMEOUT, 500). --define(HAPPYTIME_MULTIPLIER, 5). +-define(HAPPYTIME_MULTIPLIER, 2). -record(state, {owner :: pid(), change_pending=false :: boolean(), @@ -38,8 +84,8 @@ clerk_new(Owner) -> ok = gen_server:call(Pid, {register, Owner}, infinity), {ok, Pid}. -clerk_returnmanifestchange(Pid, Closing) -> - gen_server:call(Pid, {return_manifest_change, Closing}). +clerk_manifestchange(Pid, Action, Closing) -> + gen_server:call(Pid, {manifest_change, Action, Closing}, infinity). clerk_prompt(Pid) -> gen_server:cast(Pid, prompt). @@ -53,23 +99,29 @@ init([]) -> handle_call({register, Owner}, _From, State) -> {reply, ok, State#state{owner=Owner}, ?INACTIVITY_TIMEOUT}; -handle_call({return_manifest_change, Closing}, From, State) -> - case {State#state.change_pending, Closing} of - {true, true} -> +handle_call({manifest_change, return, true}, _From, State) -> + case State#state.change_pending of + true -> + WI = State#state.work_item, + {reply, {ok, WI}, State}; + false -> + {reply, no_change, State} + end; +handle_call({manifest_change, confirm, Closing}, From, State) -> + case Closing of + true -> WI = State#state.work_item, ok = mark_for_delete(WI#penciller_work.unreferenced_files, State#state.owner), - {stop, normal, {ok, WI}, State}; - {true, false} -> + {stop, normal, ok, State}; + false -> + gen_server:reply(From, ok), WI = State#state.work_item, - gen_server:reply(From, {ok, WI}), mark_for_delete(WI#penciller_work.unreferenced_files, State#state.owner), {noreply, State#state{work_item=null, change_pending=false}, - ?INACTIVITY_TIMEOUT}; - {false, true} -> - {stop, normal, no_change_required, State} + ?QUICK_TIMEOUT} end. handle_cast(prompt, State) -> @@ -116,7 +168,8 @@ requestandhandle_work(State) -> {NewManifest, FilesToDelete} = merge(WI), UpdWI = WI#penciller_work{new_manifest=NewManifest, unreferenced_files=FilesToDelete}, - ok = leveled_penciller:pcl_promptmanifestchange(State#state.owner), + ok = leveled_penciller:pcl_promptmanifestchange(State#state.owner, + UpdWI), {true, UpdWI} end. @@ -203,8 +256,8 @@ check_for_merge_candidates(SrcF, SinkFiles) -> %% - The one that overlaps with the fewest files below? %% - The smallest file? %% We could try and be fair in some way (merge oldest first) -%% Ultimately, there is alack of certainty that being fair or optimal is -%% genuinely better - ultimately every file has to be compacted. +%% Ultimately, there is a lack of certainty that being fair or optimal is +%% genuinely better - eventually every file has to be compacted. %% %% Hence, the initial implementation is to select files to merge at random @@ -286,6 +339,7 @@ get_item(Index, List, Default) -> %%% Test %%%============================================================================ +-ifdef(TEST). generate_randomkeys(Count, BucketRangeLow, BucketRangeHigh) -> generate_randomkeys(Count, [], BucketRangeLow, BucketRangeHigh). @@ -398,4 +452,6 @@ select_merge_file_test() -> Manifest = [{0, L0}, {1, L1}], {FileRef, NewManifest} = select_filetomerge(0, Manifest), ?assertMatch(FileRef, {{o, "B1", "K1"}, {o, "B3", "K3"}, dummy_pid}), - ?assertMatch(NewManifest, [{0, []}, {1, L1}]). \ No newline at end of file + ?assertMatch(NewManifest, [{0, []}, {1, L1}]). + +-endif. \ No newline at end of file diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index fb20819..dda82cc 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -7,8 +7,8 @@ %% Ledger. %% - The Penciller provides re-write (compaction) work up to be managed by %% the Penciller's Clerk -%% - The Penciller maintains a register of iterators who have requested -%% snapshots of the Ledger +%% - The Penciller can be cloned and maintains a register of clones who have +%% requested snapshots of the Ledger %% - The accepts new dumps (in the form of lists of keys) from the Bookie, and %% calls the Bookie once the process of pencilling this data in the Ledger is %% complete - and the Bookie is free to forget about the data @@ -236,7 +236,7 @@ pcl_fetch/2, pcl_checksequencenumber/3, pcl_workforclerk/1, - pcl_promptmanifestchange/1, + pcl_promptmanifestchange/2, pcl_confirmdelete/2, pcl_close/1, pcl_registersnapshot/2, @@ -307,8 +307,8 @@ pcl_checksequencenumber(Pid, Key, SQN) -> pcl_workforclerk(Pid) -> gen_server:call(Pid, work_for_clerk, infinity). -pcl_promptmanifestchange(Pid) -> - gen_server:cast(Pid, manifest_change). +pcl_promptmanifestchange(Pid, WI) -> + gen_server:cast(Pid, {manifest_change, WI}). pcl_confirmdelete(Pid, FileName) -> gen_server:call(Pid, {confirm_delete, FileName}, infinity). @@ -511,10 +511,11 @@ handle_call(close, _From, State) -> handle_cast({update_snapshotcache, Tree, SQN}, State) -> MemTableC = cache_tree_in_memcopy(State#state.memtable_copy, Tree, SQN), {noreply, State#state{memtable_copy=MemTableC}}; -handle_cast(manifest_change, State) -> - {ok, WI} = leveled_pclerk:clerk_returnmanifestchange(State#state.clerk, - false), +handle_cast({manifest_change, WI}, State) -> {ok, UpdState} = commit_manifest_change(WI, State), + ok = leveled_pclerk:clerk_manifestchange(State#state.clerk, + confirm, + false), {noreply, UpdState}; handle_cast(_Msg, State) -> {noreply, State}. @@ -541,12 +542,18 @@ terminate(_Reason, State) -> %% The cast may not succeed as the clerk could be synchronously calling %% the penciller looking for a manifest commit %% - MC = leveled_pclerk:clerk_returnmanifestchange(State#state.clerk, true), + MC = leveled_pclerk:clerk_manifestchange(State#state.clerk, + return, + true), UpdState = case MC of {ok, WI} -> {ok, NewState} = commit_manifest_change(WI, State), + Clerk = State#state.clerk, + ok = leveled_pclerk:clerk_manifestchange(Clerk, + confirm, + true), NewState; - no_change_required -> + no_change -> State end, Dump = ets:tab2list(UpdState#state.memtable), @@ -1233,12 +1240,9 @@ simple_server_test() -> ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001"})), ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002"})), ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003"})), - S4 = pcl_pushmem(PCLr, KL3), - if S4 == pause -> timer:sleep(1000); true -> ok end, - S5 = pcl_pushmem(PCLr, [Key4]), - if S5 == pause -> timer:sleep(1000); true -> ok end, - S6 = pcl_pushmem(PCLr, KL4), - if S6 == pause -> timer:sleep(1000); true -> ok end, + maybe_pause_push(pcl_pushmem(PCLr, KL3)), + maybe_pause_push(pcl_pushmem(PCLr, [Key4])), + maybe_pause_push(pcl_pushmem(PCLr, KL4)), ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001"})), ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002"})), ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003"})), @@ -1268,10 +1272,8 @@ simple_server_test() -> % in a new snapshot Key1A = {{o,"Bucket0001", "Key0001"}, {4002, {active, infinity}, null}}, KL1A = lists:sort(leveled_sft:generate_randomkeys({4002, 2})), - S7 = pcl_pushmem(PCLr, [Key1A]), - if S7 == pause -> timer:sleep(1000); true -> ok end, - S8 = pcl_pushmem(PCLr, KL1A), - if S8 == pause -> timer:sleep(1000); true -> ok end, + maybe_pause_push(pcl_pushmem(PCLr, [Key1A])), + maybe_pause_push(pcl_pushmem(PCLr, KL1A)), ?assertMatch(true, pcl_checksequencenumber(PclSnap, {o,"Bucket0001", "Key0001"}, 1)), From 0a088672805dcae4a9f549b89cb4e6848c95b8f6 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 12 Oct 2016 17:12:49 +0100 Subject: [PATCH 060/167] Iterator support Add iterator support, used initially only for retrieving bucket statistics. The iterator is supported by exporting a function, and when the function is claled it will take a snapshot of the ledger, run the iterator and hten close the snapshot. This required a numbe rof underlying changes, in particular to get key comparison to work as "expected". The code had previously misunderstood how comparison worked between Erlang terms, and in particular did not account for tuple length being compared first by size of the tuple (and not just by each element in order). --- src/leveled_bookie.erl | 108 ++++++- src/leveled_inker.erl | 4 +- src/leveled_pclerk.erl | 4 + src/leveled_penciller.erl | 551 ++++++++++++++++++++++++++++---- src/leveled_sft.erl | 267 ++++++++-------- test/end_to_end/basic_SUITE.erl | 30 +- 6 files changed, 762 insertions(+), 202 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 4116ad0..46ce7e2 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -71,7 +71,7 @@ %% The Bookie should generate a series of ledger key changes from this %% information, using a function passed in at startup. For Riak this will be %% of the form: -%% {{o, Bucket, Key}, +%% {{o, Bucket, Key, SubKey|null}, %% SQN, %% {Hash, Size, {Riak_Metadata}}, %% {active, TS}|{tomb, TS}} or @@ -136,6 +136,7 @@ book_riakput/3, book_riakget/3, book_riakhead/3, + book_returnfolder/2, book_snapshotstore/3, book_snapshotledger/3, book_compactjournal/2, @@ -144,7 +145,11 @@ strip_to_keyseqonly/1, strip_to_seqonly/1, strip_to_statusonly/1, - striphead_to_details/1]). + strip_to_keyseqstatusonly/1, + striphead_to_details/1, + key_compare/3, + key_dominates/2, + print_key/1]). -include_lib("eunit/include/eunit.hrl"). @@ -172,17 +177,20 @@ book_start(Opts) -> gen_server:start(?MODULE, [Opts], []). book_riakput(Pid, Object, IndexSpecs) -> - PrimaryKey = {o, Object#r_object.bucket, Object#r_object.key}, + PrimaryKey = {o, Object#r_object.bucket, Object#r_object.key, null}, gen_server:call(Pid, {put, PrimaryKey, Object, IndexSpecs}, infinity). book_riakget(Pid, Bucket, Key) -> - PrimaryKey = {o, Bucket, Key}, + PrimaryKey = {o, Bucket, Key, null}, gen_server:call(Pid, {get, PrimaryKey}, infinity). book_riakhead(Pid, Bucket, Key) -> - PrimaryKey = {o, Bucket, Key}, + PrimaryKey = {o, Bucket, Key, null}, gen_server:call(Pid, {head, PrimaryKey}, infinity). +book_returnfolder(Pid, FolderType) -> + gen_server:call(Pid, {return_folder, FolderType}, infinity). + book_snapshotstore(Pid, Requestor, Timeout) -> gen_server:call(Pid, {snapshot, Requestor, store, Timeout}, infinity). @@ -307,6 +315,15 @@ handle_call({snapshot, _Requestor, SnapType, _Timeout}, _From, State) -> null}, State} end; +handle_call({return_folder, FolderType}, _From, State) -> + case FolderType of + {bucket_stats, Bucket} -> + {reply, + bucket_stats(State#state.penciller, + State#state.ledger_cache, + Bucket), + State} + end; handle_call({compact_journal, Timeout}, _From, State) -> ok = leveled_inker:ink_compactjournal(State#state.inker, self(), @@ -343,6 +360,26 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================ +bucket_stats(Penciller, LedgerCache, Bucket) -> + Folder = fun() -> + PCLopts = #penciller_options{start_snapshot=true, + source_penciller=Penciller}, + {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), + Increment = gb_trees:to_list(LedgerCache), + ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, + Increment), + StartKey = {o, Bucket, null, null}, + EndKey = {o, Bucket, null, null}, + Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, + StartKey, + EndKey, + fun accumulate_size/3, + {0, 0}), + ok = leveled_penciller:pcl_close(LedgerSnapshot), + Acc + end, + {async, Folder}. + shutdown_wait([], _Inker) -> false; shutdown_wait([TopPause|Rest], Inker) -> @@ -411,12 +448,30 @@ strip_to_keyonly({K, _V}) -> K. strip_to_keyseqonly({K, {SeqN, _, _}}) -> {K, SeqN}. +strip_to_keyseqstatusonly({K, {SeqN, St, _MD}}) -> {K, SeqN, St}. + strip_to_statusonly({_, {_, St, _}}) -> St. strip_to_seqonly({_, {SeqN, _, _}}) -> SeqN. striphead_to_details({SeqN, St, MD}) -> {SeqN, St, MD}. +key_dominates(LeftKey, RightKey) -> + case {LeftKey, RightKey} of + {{LK, _LVAL}, {RK, _RVAL}} when LK < RK -> + left_hand_first; + {{LK, _LVAL}, {RK, _RVAL}} when RK < LK -> + right_hand_first; + {{LK, {LSN, _LST, _LMD}}, {RK, {RSN, _RST, _RMD}}} + when LK == RK, LSN >= RSN -> + left_hand_dominant; + {{LK, {LSN, _LST, _LMD}}, {RK, {RSN, _RST, _RMD}}} + when LK == RK, LSN < RSN -> + right_hand_dominant + end. + + + get_metadatas(#r_object{contents=Contents}) -> [Content#r_content.metadata || Content <- Contents]. @@ -435,8 +490,13 @@ hash(Obj=#r_object{}) -> extract_metadata(Obj, Size) -> {get_metadatas(Obj), vclock(Obj), hash(Obj), Size}. +accumulate_size(_Key, Value, {Size, Count}) -> + {_, _, MD} = Value, + {_, _, _, ObjSize} = MD, + {Size + ObjSize, Count + 1}. + build_metadata_object(PrimaryKey, Head) -> - {o, Bucket, Key} = PrimaryKey, + {o, Bucket, Key, null} = PrimaryKey, {MD, VC, _, _} = Head, Contents = lists:foldl(fun(X, Acc) -> Acc ++ [#r_content{metadata=X}] end, [], @@ -453,12 +513,36 @@ convert_indexspecs(IndexSpecs, SQN, PrimaryKey) -> %% TODO: timestamps for delayed reaping {tomb, infinity} end, - {o, B, K} = PrimaryKey, - {{i, B, IndexField, IndexValue, K}, + {o, B, K, _SK} = PrimaryKey, + {{i, B, {IndexField, IndexValue}, K}, {SQN, Status, null}} end, IndexSpecs). +% Return a tuple of string to ease the printing of keys to logs +print_key(Key) -> + case Key of + {o, B, K, _SK} -> + {"Object", B, K}; + {i, B, {F, _V}, _K} -> + {"Index", B, F} + end. + +% Compare a key against a query key, only comparing elements that are non-null +% in the Query key +key_compare(QueryKey, CheckingKey, gt) -> + key_compare(QueryKey, CheckingKey, fun(X,Y) -> X > Y end); +key_compare(QueryKey, CheckingKey, lt) -> + key_compare(QueryKey, CheckingKey, fun(X,Y) -> X < Y end); +key_compare({QK1, null, null, null}, {CK1, _, _, _}, CompareFun) -> + CompareFun(QK1, CK1); +key_compare({QK1, QK2, null, null}, {CK1, CK2, _, _}, CompareFun) -> + CompareFun({QK1, QK2}, {CK1, CK2}); +key_compare({QK1, QK2, QK3, null}, {CK1, CK2, CK3, _}, CompareFun) -> + CompareFun({QK1, QK2, QK3}, {CK1, CK2, CK3}); +key_compare(QueryKey, CheckingKey, CompareFun) -> + CompareFun(QueryKey, CheckingKey). + preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs) -> PrimaryChange = {PK, @@ -628,12 +712,12 @@ indexspecs_test() -> IndexSpecs = [{add, "t1_int", 456}, {add, "t1_bin", "adbc123"}, {remove, "t1_bin", "abdc456"}], - Changes = convert_indexspecs(IndexSpecs, 1, {o, "Bucket", "Key2"}), - ?assertMatch({{i, "Bucket", "t1_int", 456, "Key2"}, + Changes = convert_indexspecs(IndexSpecs, 1, {o, "Bucket", "Key2", null}), + ?assertMatch({{i, "Bucket", {"t1_int", 456}, "Key2"}, {1, {active, infinity}, null}}, lists:nth(1, Changes)), - ?assertMatch({{i, "Bucket", "t1_bin", "adbc123", "Key2"}, + ?assertMatch({{i, "Bucket", {"t1_bin", "adbc123"}, "Key2"}, {1, {active, infinity}, null}}, lists:nth(2, Changes)), - ?assertMatch({{i, "Bucket", "t1_bin", "abdc456", "Key2"}, + ?assertMatch({{i, "Bucket", {"t1_bin", "abdc456"}, "Key2"}, {1, {tomb, infinity}, null}}, lists:nth(3, Changes)). -endif. \ No newline at end of file diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index b54c2d2..bcf1023 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -251,8 +251,8 @@ handle_call({register_snapshot, Requestor}, _From , State) -> State#state{registered_snapshots=Rs}}; handle_call({release_snapshot, Snapshot}, _From , State) -> Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots), - io:format("Snapshot ~w released~n", [Snapshot]), - io:format("Remaining snapshots are ~w~n", [Rs]), + io:format("Ledger snapshot ~w released~n", [Snapshot]), + io:format("Remaining ledger snapshots are ~w~n", [Rs]), {reply, ok, State#state{registered_snapshots=Rs}}; handle_call(get_manifest, _From, State) -> {reply, State#state.manifest, State}; diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 307eab6..cff4a45 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -100,6 +100,7 @@ init([]) -> handle_call({register, Owner}, _From, State) -> {reply, ok, State#state{owner=Owner}, ?INACTIVITY_TIMEOUT}; handle_call({manifest_change, return, true}, _From, State) -> + io:format("Request for manifest change from clerk on closing~n"), case State#state.change_pending of true -> WI = State#state.work_item, @@ -110,11 +111,13 @@ handle_call({manifest_change, return, true}, _From, State) -> handle_call({manifest_change, confirm, Closing}, From, State) -> case Closing of true -> + io:format("Confirmation of manifest change on closing~n"), WI = State#state.work_item, ok = mark_for_delete(WI#penciller_work.unreferenced_files, State#state.owner), {stop, normal, ok, State}; false -> + io:format("Prompted confirmation of manifest change~n"), gen_server:reply(From, ok), WI = State#state.work_item, mark_for_delete(WI#penciller_work.unreferenced_files, @@ -168,6 +171,7 @@ requestandhandle_work(State) -> {NewManifest, FilesToDelete} = merge(WI), UpdWI = WI#penciller_work{new_manifest=NewManifest, unreferenced_files=FilesToDelete}, + io:format("Clerk prompting Penciller regarding manifest change~n"), ok = leveled_penciller:pcl_promptmanifestchange(State#state.owner, UpdWI), {true, UpdWI} diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index dda82cc..05ea41f 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -234,12 +234,14 @@ pcl_start/1, pcl_pushmem/2, pcl_fetch/2, + pcl_fetchkeys/5, pcl_checksequencenumber/3, pcl_workforclerk/1, pcl_promptmanifestchange/2, pcl_confirmdelete/2, pcl_close/1, pcl_registersnapshot/2, + pcl_releasesnapshot/2, pcl_updatesnapshotcache/3, pcl_loadsnapshot/2, pcl_getstartupsequencenumber/1, @@ -284,7 +286,6 @@ snapshot_fully_loaded = false :: boolean(), source_penciller :: pid()}). - %%%============================================================================ %%% API @@ -301,6 +302,11 @@ pcl_pushmem(Pid, DumpList) -> pcl_fetch(Pid, Key) -> gen_server:call(Pid, {fetch, Key}, infinity). +pcl_fetchkeys(Pid, StartKey, EndKey, AccFun, InitAcc) -> + gen_server:call(Pid, + {fetch_keys, StartKey, EndKey, AccFun, InitAcc}, + infinity). + pcl_checksequencenumber(Pid, Key, SQN) -> gen_server:call(Pid, {check_sqn, Key, SQN}, infinity). @@ -319,6 +325,9 @@ pcl_getstartupsequencenumber(Pid) -> pcl_registersnapshot(Pid, Snapshot) -> gen_server:call(Pid, {register_snapshot, Snapshot}, infinity). +pcl_releasesnapshot(Pid, Snapshot) -> + gen_server:cast(Pid, {release_snapshot, Snapshot}). + pcl_updatesnapshotcache(Pid, Tree, SQN) -> gen_server:cast(Pid, {update_snapshotcache, Tree, SQN}). @@ -476,6 +485,16 @@ handle_call({check_sqn, Key, SQN}, State#state.levelzero_snapshot), SQN), State}; +handle_call({fetch_keys, StartKey, EndKey, AccFun, InitAcc}, + _From, + State=#state{snapshot_fully_loaded=Ready}) + when Ready == true -> + L0iter = gb_trees:iterator_from(StartKey, State#state.levelzero_snapshot), + SFTiter = initiate_rangequery_frommanifest(StartKey, + EndKey, + State#state.manifest), + Acc = keyfolder(L0iter, SFTiter, StartKey, EndKey, {AccFun, InitAcc}), + {reply, Acc, State}; handle_call(work_for_clerk, From, State) -> {UpdState, Work} = return_work(State, From), {reply, {Work, UpdState#state.backlog}, UpdState}; @@ -498,7 +517,10 @@ handle_call({load_snapshot, Increment}, _From, State) -> TreeSQN0 > MemTableCopy#l0snapshot.ledger_sqn -> pcl_updatesnapshotcache(State#state.source_penciller, Tree0, - TreeSQN0) + TreeSQN0); + true -> + io:format("No update required to snapshot cache~n"), + ok end, {Tree1, TreeSQN1} = roll_new_tree(Tree0, [Increment], TreeSQN0), io:format("Snapshot loaded to start at SQN~w~n", [TreeSQN1]), @@ -517,15 +539,20 @@ handle_cast({manifest_change, WI}, State) -> confirm, false), {noreply, UpdState}; -handle_cast(_Msg, State) -> - {noreply, State}. +handle_cast({release_snapshot, Snapshot}, State) -> + Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots), + io:format("Penciller snapshot ~w released~n", [Snapshot]), + {noreply, State#state{registered_snapshots=Rs}}. handle_info(_Info, State) -> {noreply, State}. -terminate(_Reason, _State=#state{is_snapshot=Snap}) when Snap == true -> +terminate(Reason, State=#state{is_snapshot=Snap}) when Snap == true -> + ok = pcl_releasesnapshot(State#state.source_penciller, self()), + io:format("Sent release message for snapshot following close for " + ++ "reason ~w~n", [Reason]), ok; -terminate(_Reason, State) -> +terminate(Reason, State) -> %% When a Penciller shuts down it isn't safe to try an manage the safe %% finishing of any outstanding work. The last commmitted manifest will %% be used. @@ -542,6 +569,7 @@ terminate(_Reason, State) -> %% The cast may not succeed as the clerk could be synchronously calling %% the penciller looking for a manifest commit %% + io:format("Penciller closing for reason - ~w~n", [Reason]), MC = leveled_pclerk:clerk_manifestchange(State#state.clerk, return, true), @@ -976,19 +1004,216 @@ print_manifest(Manifest) -> io:format("Manifest at Level ~w~n", [L]), Level = get_item(L, Manifest, []), lists:foreach(fun(M) -> - {_, SB, SK} = M#manifest_entry.start_key, - {_, EB, EK} = M#manifest_entry.end_key, - io:format("Manifest entry of " ++ - "startkey ~s ~s " ++ - "endkey ~s ~s " ++ - "filename=~s~n", - [SB, SK, EB, EK, - M#manifest_entry.filename]) - end, + print_manifest_entry(M) end, Level) end, lists:seq(1, ?MAX_LEVELS - 1)). +print_manifest_entry(Entry) -> + {S1, S2, S3} = leveled_bookie:print_key(Entry#manifest_entry.start_key), + {E1, E2, E3} = leveled_bookie:print_key(Entry#manifest_entry.end_key), + io:format("Manifest entry of " ++ + "startkey ~s ~s ~s " ++ + "endkey ~s ~s ~s " ++ + "filename=~s~n", + [S1, S2, S3, E1, E2, E3, + Entry#manifest_entry.filename]). + +initiate_rangequery_frommanifest(StartKey, EndKey, Manifest) -> + CompareFun = fun(M) -> + C1 = leveled_bookie:key_compare(StartKey, + M#manifest_entry.end_key, + gt), + C2 = leveled_bookie:key_compare(EndKey, + M#manifest_entry.start_key, + lt), + not (C1 or C2) end, + lists:foldl(fun(L, AccL) -> + Level = get_item(L, Manifest, []), + FL = lists:foldl(fun(M, Acc) -> + case CompareFun(M) of + true -> + Acc ++ [{next_file, M}]; + false -> + Acc + end end, + [], + Level), + case FL of + [] -> AccL; + FL -> AccL ++ [{L, FL}] + end + end, + [], + lists:seq(1, ?MAX_LEVELS - 1)). + +%% Looks to find the best choice for the next key across the levels (other +%% than in-memory table) +%% In finding the best choice, the next key in a given level may be a next +%% block or next file pointer which will need to be expanded + +find_nextkey(QueryArray, StartKey, EndKey) -> + find_nextkey(QueryArray, + 1, + {null, null}, + {fun leveled_sft:sft_getkvrange/4, StartKey, EndKey, 1}). + +find_nextkey(_QueryArray, LCnt, {null, null}, _QueryFunT) + when LCnt > ?MAX_LEVELS -> + % The array has been scanned wihtout finding a best key - must be + % exhausted - respond to indicate no more keys to be found by the + % iterator + no_more_keys; +find_nextkey(QueryArray, LCnt, {BKL, BestKV}, _QueryFunT) + when LCnt > ?MAX_LEVELS -> + % All levels have been scanned, so need to remove the best result from + % the array, and return that array along with the best key/sqn/status + % combination + {BKL, [BestKV|Tail]} = lists:keyfind(BKL, 1, QueryArray), + {lists:keyreplace(BKL, 1, QueryArray, {BKL, Tail}), BestKV}; +find_nextkey(QueryArray, LCnt, {BestKeyLevel, BestKV}, QueryFunT) -> + % Get the next key at this level + {NextKey, RestOfKeys} = case lists:keyfind(LCnt, 1, QueryArray) of + false -> + {null, null}; + {LCnt, []} -> + {null, null}; + {LCnt, [NK|ROfKs]} -> + {NK, ROfKs} + end, + % Compare the next key at this level with the best key + case {NextKey, BestKeyLevel, BestKV} of + {null, BKL, BKV} -> + % There is no key at this level - go to the next level + find_nextkey(QueryArray, LCnt + 1, {BKL, BKV}, QueryFunT); + {{next_file, ManifestEntry}, BKL, BKV} -> + % The first key at this level is pointer to a file - need to query + % the file to expand this level out before proceeding + Owner = ManifestEntry#manifest_entry.owner, + {QueryFun, StartKey, EndKey, ScanSize} = QueryFunT, + QueryResult = QueryFun(Owner, StartKey, EndKey, ScanSize), + NewEntry = {LCnt, QueryResult ++ RestOfKeys}, + % Need to loop around at this level (LCnt) as we have not yet + % examined a real key at this level + find_nextkey(lists:keyreplace(LCnt, 1, QueryArray, NewEntry), + LCnt, + {BKL, BKV}, + QueryFunT); + {{next, SFTpid, NewStartKey}, BKL, BKV} -> + % The first key at this level is pointer within a file - need to + % query the file to expand this level out before proceeding + {QueryFun, _StartKey, EndKey, ScanSize} = QueryFunT, + QueryResult = QueryFun(SFTpid, NewStartKey, EndKey, ScanSize), + NewEntry = {LCnt, QueryResult ++ RestOfKeys}, + % Need to loop around at this level (LCnt) as we have not yet + % examined a real key at this level + find_nextkey(lists:keyreplace(LCnt, 1, QueryArray, NewEntry), + LCnt, + {BKL, BKV}, + QueryFunT); + {{Key, Val}, null, null} -> + % No best key set - so can assume that this key is the best key, + % and check the higher levels + find_nextkey(QueryArray, + LCnt + 1, + {LCnt, {Key, Val}}, + QueryFunT); + {{Key, Val}, _BKL, {BestKey, _BestVal}} when Key < BestKey -> + % There is a real key and a best key to compare, and the real key + % at this level is before the best key, and so is now the new best + % key + % The QueryArray is not modified until we have checked all levels + find_nextkey(QueryArray, + LCnt + 1, + {LCnt, {Key, Val}}, + QueryFunT); + {{Key, Val}, BKL, {BestKey, BestVal}} when Key == BestKey -> + SQN = leveled_bookie:strip_to_seqonly({Key, Val}), + BestSQN = leveled_bookie:strip_to_seqonly({BestKey, BestVal}), + if + SQN =< BestSQN -> + % This is a dominated key, so we need to skip over it + NewEntry = {LCnt, RestOfKeys}, + find_nextkey(lists:keyreplace(LCnt, 1, QueryArray, NewEntry), + LCnt + 1, + {BKL, {BestKey, BestVal}}, + QueryFunT); + SQN > BestSQN -> + % There is a real key at the front of this level and it has + % a higher SQN than the best key, so we should use this as + % the best key + % But we also need to remove the dominated key from the + % lower level in the query array + io:format("Key at level ~w with SQN ~w is better than " ++ + "key at lower level ~w with SQN ~w~n", + [LCnt, SQN, BKL, BestSQN]), + OldBestEntry = lists:keyfind(BKL, 1, QueryArray), + {BKL, [{BestKey, BestVal}|BestTail]} = OldBestEntry, + find_nextkey(lists:keyreplace(BKL, + 1, + QueryArray, + {BKL, BestTail}), + LCnt + 1, + {LCnt, {Key, Val}}, + QueryFunT) + end; + {_, BKL, BKV} -> + % This is not the best key + find_nextkey(QueryArray, LCnt + 1, {BKL, BKV}, QueryFunT) + end. + + +keyfolder(null, SFTiterator, StartKey, EndKey, {AccFun, Acc}) -> + case find_nextkey(SFTiterator, StartKey, EndKey) of + no_more_keys -> + Acc; + {NxtSFTiterator, {SFTKey, SFTVal}} -> + Acc1 = AccFun(SFTKey, SFTVal, Acc), + keyfolder(null, NxtSFTiterator, StartKey, EndKey, {AccFun, Acc1}) + end; +keyfolder(IMMiterator, SFTiterator, StartKey, EndKey, {AccFun, Acc}) -> + case gb_trees:next(IMMiterator) of + none -> + % There are no more keys in the in-memory iterator, so now + % iterate only over the remaining keys in the SFT iterator + keyfolder(null, SFTiterator, StartKey, EndKey, {AccFun, Acc}); + {IMMKey, IMMVal, NxtIMMiterator} -> + case {leveled_bookie:key_compare(EndKey, IMMKey, lt), + find_nextkey(SFTiterator, StartKey, EndKey)} of + {true, _} -> + % There are no more keys in-range in the in-memory + % iterator, so take action as if this iterator is empty + % (see above) + keyfolder(null, SFTiterator, + StartKey, EndKey, {AccFun, Acc}); + {false, no_more_keys} -> + % No more keys in range in the persisted store, so use the + % in-memory KV as the next + Acc1 = AccFun(IMMKey, IMMVal, Acc), + keyfolder(NxtIMMiterator, SFTiterator, + StartKey, EndKey, {AccFun, Acc1}); + {false, {NxtSFTiterator, {SFTKey, SFTVal}}} -> + % There is a next key, so need to know which is the next + % key between the two (and handle two keys with different + % sequence numbers). + case leveled_bookie:key_dominates({IMMKey, IMMVal}, + {SFTKey, SFTVal}) of + left_hand_first -> + Acc1 = AccFun(IMMKey, IMMVal, Acc), + keyfolder(NxtIMMiterator, SFTiterator, + StartKey, EndKey, {AccFun, Acc1}); + right_hand_first -> + Acc1 = AccFun(SFTKey, SFTVal, Acc), + keyfolder(IMMiterator, NxtSFTiterator, + StartKey, EndKey, {AccFun, Acc1}); + left_hand_dominant -> + Acc1 = AccFun(IMMKey, IMMVal, Acc), + keyfolder(NxtIMMiterator, NxtSFTiterator, + StartKey, EndKey, {AccFun, Acc1}) + end + end + end. + assess_workqueue(WorkQ, ?MAX_LEVELS - 1, _Manifest) -> WorkQ; @@ -1069,8 +1294,14 @@ commit_manifest_change(ReturnedWorkItem, State) -> rename_manifest_files(RootPath, NewMSN) -> - file:rename(filepath(RootPath, NewMSN, pending_manifest), - filepath(RootPath, NewMSN, current_manifest)). + OldFN = filepath(RootPath, NewMSN, pending_manifest), + NewFN = filepath(RootPath, NewMSN, current_manifest), + io:format("Rename of manifest from ~s ~w to ~s ~w~n", + [OldFN, + filelib:is_file(OldFN), + NewFN, + filelib:is_file(NewFN)]), + file:rename(OldFN,NewFN). filepath(RootPath, manifest) -> RootPath ++ "/" ++ ?MANIFEST_FP; @@ -1152,20 +1383,27 @@ clean_subdir(DirPath) -> end. compaction_work_assessment_test() -> - L0 = [{{o, "B1", "K1"}, {o, "B3", "K3"}, dummy_pid}], - L1 = [{{o, "B1", "K1"}, {o, "B2", "K2"}, dummy_pid}, - {{o, "B2", "K3"}, {o, "B4", "K4"}, dummy_pid}], + L0 = [{{o, "B1", "K1", null}, {o, "B3", "K3", null}, dummy_pid}], + L1 = [{{o, "B1", "K1", null}, {o, "B2", "K2", null}, dummy_pid}, + {{o, "B2", "K3", null}, {o, "B4", "K4", null}, dummy_pid}], Manifest = [{0, L0}, {1, L1}], WorkQ1 = assess_workqueue([], 0, Manifest), ?assertMatch(WorkQ1, [{0, Manifest}]), L1Alt = lists:append(L1, - [{{o, "B5", "K0001"}, {o, "B5", "K9999"}, dummy_pid}, - {{o, "B6", "K0001"}, {o, "B6", "K9999"}, dummy_pid}, - {{o, "B7", "K0001"}, {o, "B7", "K9999"}, dummy_pid}, - {{o, "B8", "K0001"}, {o, "B8", "K9999"}, dummy_pid}, - {{o, "B9", "K0001"}, {o, "B9", "K9999"}, dummy_pid}, - {{o, "BA", "K0001"}, {o, "BA", "K9999"}, dummy_pid}, - {{o, "BB", "K0001"}, {o, "BB", "K9999"}, dummy_pid}]), + [{{o, "B5", "K0001", null}, {o, "B5", "K9999", null}, + dummy_pid}, + {{o, "B6", "K0001", null}, {o, "B6", "K9999", null}, + dummy_pid}, + {{o, "B7", "K0001", null}, {o, "B7", "K9999", null}, + dummy_pid}, + {{o, "B8", "K0001", null}, {o, "B8", "K9999", null}, + dummy_pid}, + {{o, "B9", "K0001", null}, {o, "B9", "K9999", null}, + dummy_pid}, + {{o, "BA", "K0001", null}, {o, "BA", "K9999", null}, + dummy_pid}, + {{o, "BB", "K0001", null}, {o, "BB", "K9999", null}, + dummy_pid}]), Manifest3 = [{0, []}, {1, L1Alt}], WorkQ3 = assess_workqueue([], 0, Manifest3), ?assertMatch(WorkQ3, [{1, Manifest3}]). @@ -1199,26 +1437,26 @@ simple_server_test() -> clean_testdir(RootPath), {ok, PCL} = pcl_start(#penciller_options{root_path=RootPath, max_inmemory_tablesize=1000}), - Key1 = {{o,"Bucket0001", "Key0001"}, {1, {active, infinity}, null}}, + Key1 = {{o,"Bucket0001", "Key0001", null}, {1, {active, infinity}, null}}, KL1 = lists:sort(leveled_sft:generate_randomkeys({1000, 2})), - Key2 = {{o,"Bucket0002", "Key0002"}, {1002, {active, infinity}, null}}, + Key2 = {{o,"Bucket0002", "Key0002", null}, {1002, {active, infinity}, null}}, KL2 = lists:sort(leveled_sft:generate_randomkeys({1000, 1002})), - Key3 = {{o,"Bucket0003", "Key0003"}, {2002, {active, infinity}, null}}, + Key3 = {{o,"Bucket0003", "Key0003", null}, {2002, {active, infinity}, null}}, KL3 = lists:sort(leveled_sft:generate_randomkeys({1000, 2002})), - Key4 = {{o,"Bucket0004", "Key0004"}, {3002, {active, infinity}, null}}, + Key4 = {{o,"Bucket0004", "Key0004", null}, {3002, {active, infinity}, null}}, KL4 = lists:sort(leveled_sft:generate_randomkeys({1000, 3002})), ok = pcl_pushmem(PCL, [Key1]), - ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001"})), + ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), ok = pcl_pushmem(PCL, KL1), - ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001"})), + ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), maybe_pause_push(pcl_pushmem(PCL, [Key2])), - ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001"})), - ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002"})), + ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), + ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), maybe_pause_push(pcl_pushmem(PCL, KL2)), maybe_pause_push(pcl_pushmem(PCL, [Key3])), - ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001"})), - ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002"})), - ?assertMatch(Key3, pcl_fetch(PCL, {o,"Bucket0003", "Key0003"})), + ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), + ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), + ?assertMatch(Key3, pcl_fetch(PCL, {o,"Bucket0003", "Key0003", null})), ok = pcl_close(PCL), {ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath, max_inmemory_tablesize=1000}), @@ -1233,61 +1471,86 @@ simple_server_test() -> %% everything got persisted ok; _ -> - io:format("Unexpected sequence number on restart ~w~n", [TopSQN]), + io:format("Unexpected sequence number on restart ~w~n", + [TopSQN]), error end, ?assertMatch(ok, Check), - ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001"})), - ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002"})), - ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003"})), + ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})), + ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002", null})), + ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003", null})), maybe_pause_push(pcl_pushmem(PCLr, KL3)), maybe_pause_push(pcl_pushmem(PCLr, [Key4])), maybe_pause_push(pcl_pushmem(PCLr, KL4)), - ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001"})), - ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002"})), - ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003"})), - ?assertMatch(Key4, pcl_fetch(PCLr, {o,"Bucket0004", "Key0004"})), + ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})), + ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002", null})), + ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003", null})), + ?assertMatch(Key4, pcl_fetch(PCLr, {o,"Bucket0004", "Key0004", null})), SnapOpts = #penciller_options{start_snapshot = true, source_penciller = PCLr}, {ok, PclSnap} = pcl_start(SnapOpts), ok = pcl_loadsnapshot(PclSnap, []), - ?assertMatch(Key1, pcl_fetch(PclSnap, {o,"Bucket0001", "Key0001"})), - ?assertMatch(Key2, pcl_fetch(PclSnap, {o,"Bucket0002", "Key0002"})), - ?assertMatch(Key3, pcl_fetch(PclSnap, {o,"Bucket0003", "Key0003"})), - ?assertMatch(Key4, pcl_fetch(PclSnap, {o,"Bucket0004", "Key0004"})), + ?assertMatch(Key1, pcl_fetch(PclSnap, {o,"Bucket0001", "Key0001", null})), + ?assertMatch(Key2, pcl_fetch(PclSnap, {o,"Bucket0002", "Key0002", null})), + ?assertMatch(Key3, pcl_fetch(PclSnap, {o,"Bucket0003", "Key0003", null})), + ?assertMatch(Key4, pcl_fetch(PclSnap, {o,"Bucket0004", "Key0004", null})), ?assertMatch(true, pcl_checksequencenumber(PclSnap, - {o,"Bucket0001", "Key0001"}, + {o, + "Bucket0001", + "Key0001", + null}, 1)), ?assertMatch(true, pcl_checksequencenumber(PclSnap, - {o,"Bucket0002", "Key0002"}, + {o, + "Bucket0002", + "Key0002", + null}, 1002)), ?assertMatch(true, pcl_checksequencenumber(PclSnap, - {o,"Bucket0003", "Key0003"}, + {o, + "Bucket0003", + "Key0003", + null}, 2002)), ?assertMatch(true, pcl_checksequencenumber(PclSnap, - {o,"Bucket0004", "Key0004"}, + {o, + "Bucket0004", + "Key0004", + null}, 3002)), % Add some more keys and confirm that chekc sequence number still % sees the old version in the previous snapshot, but will see the new version % in a new snapshot - Key1A = {{o,"Bucket0001", "Key0001"}, {4002, {active, infinity}, null}}, + Key1A = {{o,"Bucket0001", "Key0001", null}, {4002, {active, infinity}, null}}, KL1A = lists:sort(leveled_sft:generate_randomkeys({4002, 2})), maybe_pause_push(pcl_pushmem(PCLr, [Key1A])), maybe_pause_push(pcl_pushmem(PCLr, KL1A)), ?assertMatch(true, pcl_checksequencenumber(PclSnap, - {o,"Bucket0001", "Key0001"}, + {o, + "Bucket0001", + "Key0001", + null}, 1)), ok = pcl_close(PclSnap), {ok, PclSnap2} = pcl_start(SnapOpts), ok = pcl_loadsnapshot(PclSnap2, []), ?assertMatch(false, pcl_checksequencenumber(PclSnap2, - {o,"Bucket0001", "Key0001"}, + {o, + "Bucket0001", + "Key0001", + null}, 1)), ?assertMatch(true, pcl_checksequencenumber(PclSnap2, - {o,"Bucket0001", "Key0001"}, + {o, + "Bucket0001", + "Key0001", + null}, 4002)), ?assertMatch(true, pcl_checksequencenumber(PclSnap2, - {o,"Bucket0002", "Key0002"}, + {o, + "Bucket0002", + "Key0002", + null}, 1002)), ok = pcl_close(PclSnap2), ok = pcl_close(PCLr), @@ -1334,4 +1597,174 @@ memcopy_updatecache_test() -> ?assertMatch(2000, Size2), ?assertMatch(3000, MemCopy4#l0snapshot.ledger_sqn). +rangequery_manifest_test() -> + {E1, + E2, + E3} = {#manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, + end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, + filename="Z1"}, + #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, + end_key={o, "Bucket1", "K71", null}, + filename="Z2"}, + #manifest_entry{start_key={o, "Bucket1", "K75", null}, + end_key={o, "Bucket1", "K993", null}, + filename="Z3"}}, + {E4, + E5, + E6} = {#manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, + end_key={i, "Bucket1", {"Idx1", "Fld7"}, "K93"}, + filename="Z4"}, + #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld7"}, "K97"}, + end_key={o, "Bucket1", "K78", null}, + filename="Z5"}, + #manifest_entry{start_key={o, "Bucket1", "K81", null}, + end_key={o, "Bucket1", "K996", null}, + filename="Z6"}}, + Man = [{1, [E1, E2, E3]}, {2, [E4, E5, E6]}], + R1 = initiate_rangequery_frommanifest({o, "Bucket1", "K711", null}, + {o, "Bucket1", "K999", null}, + Man), + ?assertMatch([{1, [{next_file, E3}]}, + {2, [{next_file, E5}, {next_file, E6}]}], + R1), + R2 = initiate_rangequery_frommanifest({i, "Bucket1", {"Idx1", "Fld8"}, null}, + {i, "Bucket1", {"Idx1", "Fld8"}, null}, + Man), + ?assertMatch([{1, [{next_file, E1}]}, {2, [{next_file, E5}]}], + R2), + R3 = initiate_rangequery_frommanifest({i, "Bucket1", {"Idx0", "Fld8"}, null}, + {i, "Bucket1", {"Idx0", "Fld9"}, null}, + Man), + ?assertMatch([], R3). + +simple_findnextkey_test() -> + QueryArray = [ + {2, [{{o, "Bucket1", "Key1"}, {5, {active, infinity}, null}}, + {{o, "Bucket1", "Key5"}, {4, {active, infinity}, null}}]}, + {3, [{{o, "Bucket1", "Key3"}, {3, {active, infinity}, null}}]}, + {5, [{{o, "Bucket1", "Key2"}, {2, {active, infinity}, null}}]} + ], + {Array2, KV1} = find_nextkey(QueryArray, + {o, "Bucket1", "Key0"}, + {o, "Bucket1", "Key5"}), + ?assertMatch({{o, "Bucket1", "Key1"}, {5, {active, infinity}, null}}, KV1), + {Array3, KV2} = find_nextkey(Array2, + {o, "Bucket1", "Key0"}, + {o, "Bucket1", "Key5"}), + ?assertMatch({{o, "Bucket1", "Key2"}, {2, {active, infinity}, null}}, KV2), + {Array4, KV3} = find_nextkey(Array3, + {o, "Bucket1", "Key0"}, + {o, "Bucket1", "Key5"}), + ?assertMatch({{o, "Bucket1", "Key3"}, {3, {active, infinity}, null}}, KV3), + {Array5, KV4} = find_nextkey(Array4, + {o, "Bucket1", "Key0"}, + {o, "Bucket1", "Key5"}), + ?assertMatch({{o, "Bucket1", "Key5"}, {4, {active, infinity}, null}}, KV4), + ER = find_nextkey(Array5, + {o, "Bucket1", "Key0"}, + {o, "Bucket1", "Key5"}), + ?assertMatch(no_more_keys, ER). + +sqnoverlap_findnextkey_test() -> + QueryArray = [ + {2, [{{o, "Bucket1", "Key1"}, {5, {active, infinity}, null}}, + {{o, "Bucket1", "Key5"}, {4, {active, infinity}, null}}]}, + {3, [{{o, "Bucket1", "Key3"}, {3, {active, infinity}, null}}]}, + {5, [{{o, "Bucket1", "Key5"}, {2, {active, infinity}, null}}]} + ], + {Array2, KV1} = find_nextkey(QueryArray, + {o, "Bucket1", "Key0"}, + {o, "Bucket1", "Key5"}), + ?assertMatch({{o, "Bucket1", "Key1"}, {5, {active, infinity}, null}}, KV1), + {Array3, KV2} = find_nextkey(Array2, + {o, "Bucket1", "Key0"}, + {o, "Bucket1", "Key5"}), + ?assertMatch({{o, "Bucket1", "Key3"}, {3, {active, infinity}, null}}, KV2), + {Array4, KV3} = find_nextkey(Array3, + {o, "Bucket1", "Key0"}, + {o, "Bucket1", "Key5"}), + ?assertMatch({{o, "Bucket1", "Key5"}, {4, {active, infinity}, null}}, KV3), + ER = find_nextkey(Array4, + {o, "Bucket1", "Key0"}, + {o, "Bucket1", "Key5"}), + ?assertMatch(no_more_keys, ER). + +sqnoverlap_otherway_findnextkey_test() -> + QueryArray = [ + {2, [{{o, "Bucket1", "Key1"}, {5, {active, infinity}, null}}, + {{o, "Bucket1", "Key5"}, {1, {active, infinity}, null}}]}, + {3, [{{o, "Bucket1", "Key3"}, {3, {active, infinity}, null}}]}, + {5, [{{o, "Bucket1", "Key5"}, {2, {active, infinity}, null}}]} + ], + {Array2, KV1} = find_nextkey(QueryArray, + {o, "Bucket1", "Key0"}, + {o, "Bucket1", "Key5"}), + ?assertMatch({{o, "Bucket1", "Key1"}, {5, {active, infinity}, null}}, KV1), + {Array3, KV2} = find_nextkey(Array2, + {o, "Bucket1", "Key0"}, + {o, "Bucket1", "Key5"}), + ?assertMatch({{o, "Bucket1", "Key3"}, {3, {active, infinity}, null}}, KV2), + {Array4, KV3} = find_nextkey(Array3, + {o, "Bucket1", "Key0"}, + {o, "Bucket1", "Key5"}), + ?assertMatch({{o, "Bucket1", "Key5"}, {2, {active, infinity}, null}}, KV3), + ER = find_nextkey(Array4, + {o, "Bucket1", "Key0"}, + {o, "Bucket1", "Key5"}), + ?assertMatch(no_more_keys, ER). + +foldwithimm_simple_test() -> + QueryArray = [ + {2, [{{o, "Bucket1", "Key1"}, {5, {active, infinity}, null}}, + {{o, "Bucket1", "Key5"}, {1, {active, infinity}, null}}]}, + {3, [{{o, "Bucket1", "Key3"}, {3, {active, infinity}, null}}]}, + {5, [{{o, "Bucket1", "Key5"}, {2, {active, infinity}, null}}]} + ], + IMM0 = gb_trees:enter({o, "Bucket1", "Key6"}, + {7, {active, infinity}, null}, + gb_trees:empty()), + IMM1 = gb_trees:enter({o, "Bucket1", "Key1"}, + {8, {active, infinity}, null}, + IMM0), + IMM2 = gb_trees:enter({o, "Bucket1", "Key8"}, + {9, {active, infinity}, null}, + IMM1), + IMMiter = gb_trees:iterator_from({o, "Bucket1", "Key1"}, IMM2), + AccFun = fun(K, V, Acc) -> SQN= leveled_bookie:strip_to_seqonly({K, V}), + Acc ++ [{K, SQN}] end, + Acc = keyfolder(IMMiter, + QueryArray, + {o, "Bucket1", "Key1"}, {o, "Bucket1", "Key6"}, + {AccFun, []}), + ?assertMatch([{{o, "Bucket1", "Key1"}, 8}, + {{o, "Bucket1", "Key3"}, 3}, + {{o, "Bucket1", "Key5"}, 2}, + {{o, "Bucket1", "Key6"}, 7}], Acc), + + IMM1A = gb_trees:enter({o, "Bucket1", "Key1"}, + {8, {active, infinity}, null}, + gb_trees:empty()), + IMMiterA = gb_trees:iterator_from({o, "Bucket1", "Key1"}, IMM1A), + AccA = keyfolder(IMMiterA, + QueryArray, + {o, "Bucket1", "Key1"}, {o, "Bucket1", "Key6"}, + {AccFun, []}), + ?assertMatch([{{o, "Bucket1", "Key1"}, 8}, + {{o, "Bucket1", "Key3"}, 3}, + {{o, "Bucket1", "Key5"}, 2}], AccA), + + IMM3 = gb_trees:enter({o, "Bucket1", "Key4"}, + {10, {active, infinity}, null}, + IMM2), + IMMiterB = gb_trees:iterator_from({o, "Bucket1", "Key1"}, IMM3), + AccB = keyfolder(IMMiterB, + QueryArray, + {o, "Bucket1", "Key1"}, {o, "Bucket1", "Key6"}, + {AccFun, []}), + ?assertMatch([{{o, "Bucket1", "Key1"}, 8}, + {{o, "Bucket1", "Key3"}, 3}, + {{o, "Bucket1", "Key4"}, 10}, + {{o, "Bucket1", "Key5"}, 2}, + {{o, "Bucket1", "Key6"}, 7}], AccB). + -endif. \ No newline at end of file diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 743d93e..9cc3a68 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -14,8 +14,8 @@ %% %% All keys are not equal in sft files, keys are only expected in a specific %% series of formats -%% - {o, Bucket, Key} - Object Keys -%% - {i, Bucket, IndexName, IndexTerm, Key} - Postings +%% - {o, Bucket, Key, SubKey|null} - Object Keys +%% - {i, Bucket, {IndexName, IndexTerm}, Key} - Postings %% The {Bucket, Key} part of all types of keys are hashed for segment filters. %% For Postings the {Bucket, IndexName, IndexTerm} is also hashed. This %% causes a false positive on lookup of a segment, but allows for the presence @@ -155,7 +155,7 @@ sft_new/5, sft_open/1, sft_get/2, - sft_getkeyrange/4, + sft_getkvrange/4, sft_close/1, sft_clear/1, sft_checkready/1, @@ -243,15 +243,8 @@ sft_setfordelete(Pid, Penciller) -> sft_get(Pid, Key) -> gen_server:call(Pid, {get_kv, Key}, infinity). -sft_getkeyrange(Pid, StartKey, EndKey, ScanWidth) -> - gen_server:call(Pid, - {get_keyrange, StartKey, EndKey, ScanWidth}, - infinity). - sft_getkvrange(Pid, StartKey, EndKey, ScanWidth) -> - gen_server:call(Pid, - {get_kvrange, StartKey, EndKey, ScanWidth}, - infinity). + gen_server:call(Pid, {get_kvrange, StartKey, EndKey, ScanWidth}, infinity). sft_clear(Pid) -> gen_server:call(Pid, clear, infinity). @@ -313,15 +306,13 @@ handle_call({sft_open, Filename}, _From, _State) -> handle_call({get_kv, Key}, _From, State) -> Reply = fetch_keyvalue(State#state.handle, State, Key), {reply, Reply, State}; -handle_call({get_keyrange, StartKey, EndKey, ScanWidth}, _From, State) -> - Reply = fetch_range_keysonly(State#state.handle, State, - StartKey, EndKey, - ScanWidth), - {reply, Reply, State}; handle_call({get_kvrange, StartKey, EndKey, ScanWidth}, _From, State) -> - Reply = fetch_range_kv(State#state.handle, State, - StartKey, EndKey, - ScanWidth), + Reply = pointer_append_queryresults(fetch_range_kv(State#state.handle, + State, + StartKey, + EndKey, + ScanWidth), + self()), {reply, Reply, State}; handle_call(close, _From, State) -> {stop, normal, ok, State}; @@ -582,7 +573,7 @@ acc_list_keysonly(null, empty) -> acc_list_keysonly(null, RList) -> RList; acc_list_keysonly(R, RList) -> - lists:append(RList, [leveled_bookie:strip_to_keyseqonly(R)]). + lists:append(RList, [leveled_bookie:strip_to_keyseqstatusonly(R)]). acc_list_kv(null, empty) -> []; @@ -672,10 +663,12 @@ scan_block([], StartKey, _EndKey, _FunList, _AccFun, Acc) -> {partial, Acc, StartKey}; scan_block([HeadKV|T], StartKey, EndKey, FunList, AccFun, Acc) -> K = leveled_bookie:strip_to_keyonly(HeadKV), - case K of - K when K < StartKey, StartKey /= all -> + Pre = leveled_bookie:key_compare(StartKey, K, gt), + Post = leveled_bookie:key_compare(EndKey, K, lt), + case {Pre, Post} of + {true, _} when StartKey /= all -> scan_block(T, StartKey, EndKey, FunList, AccFun, Acc); - K when K > EndKey, EndKey /= all -> + {_, true} when EndKey /= all -> {complete, Acc}; _ -> case applyfuns(FunList, HeadKV) of @@ -1121,8 +1114,7 @@ maybe_expand_pointer([H|Tail]) -> case H of {next, SFTPid, StartKey} -> %% io:format("Scanning further on PID ~w ~w~n", [SFTPid, StartKey]), - QResult = sft_getkvrange(SFTPid, StartKey, all, ?MERGE_SCANWIDTH), - Acc = pointer_append_queryresults(QResult, SFTPid), + Acc = sft_getkvrange(SFTPid, StartKey, all, ?MERGE_SCANWIDTH), lists:append(Acc, Tail); _ -> [H|Tail] @@ -1409,8 +1401,9 @@ generate_randomkeys(0, _SQN, Acc) -> Acc; generate_randomkeys(Count, SQN, Acc) -> RandKey = {{o, - lists:concat(["Bucket", random:uniform(1024)]), - lists:concat(["Key", random:uniform(1024)])}, + lists:concat(["Bucket", random:uniform(1024)]), + lists:concat(["Key", random:uniform(1024)]), + null}, {SQN, {active, infinity}, null}}, generate_randomkeys(Count - 1, SQN + 1, [RandKey|Acc]). @@ -1423,73 +1416,74 @@ generate_sequentialkeys(Target, Incr, Acc) when Incr =:= Target -> generate_sequentialkeys(Target, Incr, Acc) -> KeyStr = string:right(integer_to_list(Incr), 8, $0), NextKey = {{o, - "BucketSeq", - lists:concat(["Key", KeyStr])}, + "BucketSeq", + lists:concat(["Key", KeyStr]), + null}, {5, {active, infinity}, null}}, generate_sequentialkeys(Target, Incr + 1, [NextKey|Acc]). simple_create_block_test() -> - KeyList1 = [{{o, "Bucket1", "Key1"}, {1, {active, infinity}, null}}, - {{o, "Bucket1", "Key3"}, {2, {active, infinity}, null}}], - KeyList2 = [{{o, "Bucket1", "Key2"}, {3, {active, infinity}, null}}], + KeyList1 = [{{o, "Bucket1", "Key1", null}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key3", null}, {2, {active, infinity}, null}}], + KeyList2 = [{{o, "Bucket1", "Key2", null}, {3, {active, infinity}, null}}], {MergedKeyList, ListStatus, SN, _, _, _} = create_block(KeyList1, KeyList2, 1), ?assertMatch(partial, ListStatus), [H1|T1] = MergedKeyList, - ?assertMatch(H1, {{o, "Bucket1", "Key1"}, {1, {active, infinity}, null}}), + ?assertMatch(H1, {{o, "Bucket1", "Key1", null}, {1, {active, infinity}, null}}), [H2|T2] = T1, - ?assertMatch(H2, {{o, "Bucket1", "Key2"}, {3, {active, infinity}, null}}), - ?assertMatch(T2, [{{o, "Bucket1", "Key3"}, {2, {active, infinity}, null}}]), + ?assertMatch(H2, {{o, "Bucket1", "Key2", null}, {3, {active, infinity}, null}}), + ?assertMatch(T2, [{{o, "Bucket1", "Key3", null}, {2, {active, infinity}, null}}]), ?assertMatch(SN, {1,3}). dominate_create_block_test() -> - KeyList1 = [{{o, "Bucket1", "Key1"}, {1, {active, infinity}, null}}, - {{o, "Bucket1", "Key2"}, {2, {active, infinity}, null}}], - KeyList2 = [{{o, "Bucket1", "Key2"}, {3, {tomb, infinity}, null}}], + KeyList1 = [{{o, "Bucket1", "Key1", null}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key2", null}, {2, {active, infinity}, null}}], + KeyList2 = [{{o, "Bucket1", "Key2", null}, {3, {tomb, infinity}, null}}], {MergedKeyList, ListStatus, SN, _, _, _} = create_block(KeyList1, KeyList2, 1), ?assertMatch(partial, ListStatus), [K1, K2] = MergedKeyList, - ?assertMatch(K1, {{o, "Bucket1", "Key1"}, {1, {active, infinity}, null}}), - ?assertMatch(K2, {{o, "Bucket1", "Key2"}, {3, {tomb, infinity}, null}}), + ?assertMatch(K1, {{o, "Bucket1", "Key1", null}, {1, {active, infinity}, null}}), + ?assertMatch(K2, {{o, "Bucket1", "Key2", null}, {3, {tomb, infinity}, null}}), ?assertMatch(SN, {1,3}). sample_keylist() -> - KeyList1 = [{{o, "Bucket1", "Key1"}, {1, {active, infinity}, null}}, - {{o, "Bucket1", "Key3"}, {1, {active, infinity}, null}}, - {{o, "Bucket1", "Key5"}, {1, {active, infinity}, null}}, - {{o, "Bucket1", "Key7"}, {1, {active, infinity}, null}}, - {{o, "Bucket1", "Key9"}, {1, {active, infinity}, null}}, - {{o, "Bucket2", "Key1"}, {1, {active, infinity}, null}}, - {{o, "Bucket2", "Key3"}, {1, {active, infinity}, null}}, - {{o, "Bucket2", "Key5"}, {1, {active, infinity}, null}}, - {{o, "Bucket2", "Key7"}, {1, {active, infinity}, null}}, - {{o, "Bucket2", "Key9"}, {1, {active, infinity}, null}}, - {{o, "Bucket3", "Key1"}, {1, {active, infinity}, null}}, - {{o, "Bucket3", "Key3"}, {1, {active, infinity}, null}}, - {{o, "Bucket3", "Key5"}, {1, {active, infinity}, null}}, - {{o, "Bucket3", "Key7"}, {1, {active, infinity}, null}}, - {{o, "Bucket3", "Key9"}, {1, {active, infinity}, null}}, - {{o, "Bucket4", "Key1"}, {1, {active, infinity}, null}}], - KeyList2 = [{{o, "Bucket1", "Key2"}, {1, {active, infinity}, null}}, - {{o, "Bucket1", "Key4"}, {1, {active, infinity}, null}}, - {{o, "Bucket1", "Key6"}, {1, {active, infinity}, null}}, - {{o, "Bucket1", "Key8"}, {1, {active, infinity}, null}}, - {{o, "Bucket1", "Key9a"}, {1, {active, infinity}, null}}, - {{o, "Bucket1", "Key9b"}, {1, {active, infinity}, null}}, - {{o, "Bucket1", "Key9c"}, {1, {active, infinity}, null}}, - {{o, "Bucket1", "Key9d"}, {1, {active, infinity}, null}}, - {{o, "Bucket2", "Key2"}, {1, {active, infinity}, null}}, - {{o, "Bucket2", "Key4"}, {1, {active, infinity}, null}}, - {{o, "Bucket2", "Key6"}, {1, {active, infinity}, null}}, - {{o, "Bucket2", "Key8"}, {1, {active, infinity}, null}}, - {{o, "Bucket3", "Key2"}, {1, {active, infinity}, null}}, - {{o, "Bucket3", "Key4"}, {3, {active, infinity}, null}}, - {{o, "Bucket3", "Key6"}, {2, {active, infinity}, null}}, - {{o, "Bucket3", "Key8"}, {1, {active, infinity}, null}}], + KeyList1 = [{{o, "Bucket1", "Key1", null}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key3", null}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key5", null}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key7", null}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key9", null}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key1", null}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key3", null}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key5", null}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key7", null}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key9", null}, {1, {active, infinity}, null}}, + {{o, "Bucket3", "Key1", null}, {1, {active, infinity}, null}}, + {{o, "Bucket3", "Key3", null}, {1, {active, infinity}, null}}, + {{o, "Bucket3", "Key5", null}, {1, {active, infinity}, null}}, + {{o, "Bucket3", "Key7", null}, {1, {active, infinity}, null}}, + {{o, "Bucket3", "Key9", null}, {1, {active, infinity}, null}}, + {{o, "Bucket4", "Key1", null}, {1, {active, infinity}, null}}], + KeyList2 = [{{o, "Bucket1", "Key2", null}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key4", null}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key6", null}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key8", null}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key9a", null}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key9b", null}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key9c", null}, {1, {active, infinity}, null}}, + {{o, "Bucket1", "Key9d", null}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key2", null}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key4", null}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key6", null}, {1, {active, infinity}, null}}, + {{o, "Bucket2", "Key8", null}, {1, {active, infinity}, null}}, + {{o, "Bucket3", "Key2", null}, {1, {active, infinity}, null}}, + {{o, "Bucket3", "Key4", null}, {3, {active, infinity}, null}}, + {{o, "Bucket3", "Key6", null}, {2, {active, infinity}, null}}, + {{o, "Bucket3", "Key8", null}, {1, {active, infinity}, null}}], {KeyList1, KeyList2}. alternating_create_block_test() -> @@ -1501,12 +1495,12 @@ alternating_create_block_test() -> ?assertMatch(BlockSize, 32), ?assertMatch(ListStatus, complete), K1 = lists:nth(1, MergedKeyList), - ?assertMatch(K1, {{o, "Bucket1", "Key1"}, {1, {active, infinity}, null}}), + ?assertMatch(K1, {{o, "Bucket1", "Key1", null}, {1, {active, infinity}, null}}), K11 = lists:nth(11, MergedKeyList), - ?assertMatch(K11, {{o, "Bucket1", "Key9b"}, {1, {active, infinity}, null}}), + ?assertMatch(K11, {{o, "Bucket1", "Key9b", null}, {1, {active, infinity}, null}}), K32 = lists:nth(32, MergedKeyList), - ?assertMatch(K32, {{o, "Bucket4", "Key1"}, {1, {active, infinity}, null}}), - HKey = {{o, "Bucket1", "Key0"}, {1, {active, infinity}, null}}, + ?assertMatch(K32, {{o, "Bucket4", "Key1", null}, {1, {active, infinity}, null}}), + HKey = {{o, "Bucket1", "Key0", null}, {1, {active, infinity}, null}}, {_, ListStatus2, _, _, _, _} = create_block([HKey|KeyList1], KeyList2, 1), ?assertMatch(ListStatus2, full). @@ -1565,17 +1559,17 @@ createslot_stage1_test() -> {{LowKey, SegFilter, _SerialisedSlot, _LengthList}, {{LSN, HSN}, LastKey, Status}, KL1, KL2} = Out, - ?assertMatch(LowKey, {o, "Bucket1", "Key1"}), - ?assertMatch(LastKey, {o, "Bucket4", "Key1"}), + ?assertMatch(LowKey, {o, "Bucket1", "Key1", null}), + ?assertMatch(LastKey, {o, "Bucket4", "Key1", null}), ?assertMatch(Status, partial), ?assertMatch(KL1, []), ?assertMatch(KL2, []), R0 = check_for_segments(serialise_segment_filter(SegFilter), - [hash_for_segmentid({keyonly, {o, "Bucket1", "Key1"}})], + [hash_for_segmentid({keyonly, {o, "Bucket1", "Key1", null}})], true), ?assertMatch(R0, {maybe_present, [0]}), R1 = check_for_segments(serialise_segment_filter(SegFilter), - [hash_for_segmentid({keyonly, {o, "Bucket1", "Key99"}})], + [hash_for_segmentid({keyonly, {o, "Bucket1", "Key99", null}})], true), ?assertMatch(R1, not_present), ?assertMatch(LSN, 1), @@ -1605,25 +1599,29 @@ createslot_stage3_test() -> Sum1 = lists:foldl(fun(X, Sum) -> Sum + X end, 0, LengthList), Sum2 = byte_size(SerialisedSlot), ?assertMatch(Sum1, Sum2), - ?assertMatch(LowKey, {o, "BucketSeq", "Key00000001"}), - ?assertMatch(LastKey, {o, "BucketSeq", "Key00000128"}), + ?assertMatch(LowKey, {o, "BucketSeq", "Key00000001", null}), + ?assertMatch(LastKey, {o, "BucketSeq", "Key00000128", null}), ?assertMatch(KL1, []), Rem = length(KL2), ?assertMatch(Rem, 72), R0 = check_for_segments(serialise_segment_filter(SegFilter), - [hash_for_segmentid({keyonly, {o, "BucketSeq", "Key00000100"}})], + [hash_for_segmentid({keyonly, + {o, "BucketSeq", "Key00000100", null}})], true), ?assertMatch(R0, {maybe_present, [3]}), R1 = check_for_segments(serialise_segment_filter(SegFilter), - [hash_for_segmentid({keyonly, {o, "Bucket1", "Key99"}})], + [hash_for_segmentid({keyonly, + {o, "Bucket1", "Key99", null}})], true), ?assertMatch(R1, not_present), R2 = check_for_segments(serialise_segment_filter(SegFilter), - [hash_for_segmentid({keyonly, {o, "BucketSeq", "Key00000040"}})], + [hash_for_segmentid({keyonly, + {o, "BucketSeq", "Key00000040", null}})], true), ?assertMatch(R2, {maybe_present, [1]}), R3 = check_for_segments(serialise_segment_filter(SegFilter), - [hash_for_segmentid({keyonly, {o, "BucketSeq", "Key00000004"}})], + [hash_for_segmentid({keyonly, + {o, "BucketSeq", "Key00000004", null}})], true), ?assertMatch(R3, {maybe_present, [0]}). @@ -1640,11 +1638,11 @@ writekeys_stage1_test() -> fun testwrite_function/2), {Handle, {_, PointerIndex}, SNExtremes, KeyExtremes} = FunOut, ?assertMatch(SNExtremes, {1,3}), - ?assertMatch(KeyExtremes, {{o, "Bucket1", "Key1"}, - {o, "Bucket4", "Key1"}}), + ?assertMatch(KeyExtremes, {{o, "Bucket1", "Key1", null}, + {o, "Bucket4", "Key1", null}}), [TopIndex|[]] = PointerIndex, {TopKey, _SegFilter, {LengthList, _Total}} = TopIndex, - ?assertMatch(TopKey, {o, "Bucket1", "Key1"}), + ?assertMatch(TopKey, {o, "Bucket1", "Key1", null}), TotalLength = lists:foldl(fun(X, Acc) -> Acc + X end, 0, LengthList), ActualLength = lists:foldl(fun(X, Acc) -> Acc + byte_size(X) end, @@ -1660,11 +1658,11 @@ initial_create_file_test() -> {KL1, KL2} = sample_keylist(), {Handle, FileMD} = create_file(Filename), {UpdHandle, UpdFileMD, {[], []}} = complete_file(Handle, FileMD, KL1, KL2, 1), - Result1 = fetch_keyvalue(UpdHandle, UpdFileMD, {o, "Bucket1", "Key8"}), + Result1 = fetch_keyvalue(UpdHandle, UpdFileMD, {o, "Bucket1", "Key8", null}), io:format("Result is ~w~n", [Result1]), - ?assertMatch(Result1, {{o, "Bucket1", "Key8"}, + ?assertMatch(Result1, {{o, "Bucket1", "Key8", null}, {1, {active, infinity}, null}}), - Result2 = fetch_keyvalue(UpdHandle, UpdFileMD, {o, "Bucket1", "Key88"}), + Result2 = fetch_keyvalue(UpdHandle, UpdFileMD, {o, "Bucket1", "Key88", null}), io:format("Result is ~w~n", [Result2]), ?assertMatch(Result2, not_present), ok = file:close(UpdHandle), @@ -1699,7 +1697,9 @@ big_create_file_test() -> SubList), io:format("FailedFinds of ~w~n", [FailedFinds]), ?assertMatch(FailedFinds, 0), - Result3 = fetch_keyvalue(Handle, FileMD, {o, "Bucket1024", "Key1024Alt"}), + Result3 = fetch_keyvalue(Handle, + FileMD, + {o, "Bucket1024", "Key1024Alt", null}), ?assertMatch(Result3, not_present), ok = file:close(Handle), ok = file:delete(Filename). @@ -1708,35 +1708,46 @@ initial_iterator_test() -> Filename = "../test/test2.sft", {KL1, KL2} = sample_keylist(), {Handle, FileMD} = create_file(Filename), - {UpdHandle, UpdFileMD, {[], []}} = complete_file(Handle, FileMD, KL1, KL2, 1), + {UpdHandle, UpdFileMD, {[], []}} = complete_file(Handle, + FileMD, + KL1, + KL2, + 1), Result1 = fetch_range_keysonly(UpdHandle, UpdFileMD, - {o, "Bucket1", "Key8"}, - {o, "Bucket1", "Key9d"}), + {o, "Bucket1", "Key8", null}, + {o, "Bucket1", "Key9d", null}), io:format("Result returned of ~w~n", [Result1]), - ?assertMatch(Result1, {complete, [{{o, "Bucket1", "Key8"}, 1}, - {{o, "Bucket1", "Key9"}, 1}, - {{o, "Bucket1", "Key9a"}, 1}, - {{o, "Bucket1", "Key9b"}, 1}, - {{o, "Bucket1", "Key9c"}, 1}, - {{o, "Bucket1", "Key9d"}, 1}]}), + ?assertMatch({complete, + [{{o, "Bucket1", "Key8", null}, 1, {active, infinity}}, + {{o, "Bucket1", "Key9", null}, 1, {active, infinity}}, + {{o, "Bucket1", "Key9a", null}, 1, {active, infinity}}, + {{o, "Bucket1", "Key9b", null}, 1, {active, infinity}}, + {{o, "Bucket1", "Key9c", null}, 1, {active, infinity}}, + {{o, "Bucket1", "Key9d", null}, 1, {active, infinity}} + ]}, + Result1), Result2 = fetch_range_keysonly(UpdHandle, UpdFileMD, - {o, "Bucket1", "Key8"}, - {o, "Bucket1", "Key9b"}), - ?assertMatch(Result2, {complete, [{{o, "Bucket1", "Key8"}, 1}, - {{o, "Bucket1", "Key9"}, 1}, - {{o, "Bucket1", "Key9a"}, 1}, - {{o, "Bucket1", "Key9b"}, 1}]}), + {o, "Bucket1", "Key8", null}, + {o, "Bucket1", "Key9b", null}), + ?assertMatch({complete, + [{{o, "Bucket1", "Key8", null}, 1, {active, infinity}}, + {{o, "Bucket1", "Key9", null}, 1, {active, infinity}}, + {{o, "Bucket1", "Key9a", null}, 1, {active, infinity}}, + {{o, "Bucket1", "Key9b", null}, 1, {active, infinity}} + ]}, + Result2), Result3 = fetch_range_keysonly(UpdHandle, UpdFileMD, - {o, "Bucket3", "Key4"}, + {o, "Bucket3", "Key4", null}, all), {partial, RL3, _} = Result3, - ?assertMatch(RL3, [{{o, "Bucket3", "Key4"}, 3}, - {{o, "Bucket3", "Key5"}, 1}, - {{o, "Bucket3", "Key6"}, 2}, - {{o, "Bucket3", "Key7"}, 1}, - {{o, "Bucket3", "Key8"}, 1}, - {{o, "Bucket3", "Key9"}, 1}, - {{o, "Bucket4", "Key1"}, 1}]), + ?assertMatch([{{o, "Bucket3", "Key4", null}, 3, {active, infinity}}, + {{o, "Bucket3", "Key5", null}, 1, {active, infinity}}, + {{o, "Bucket3", "Key6", null}, 2, {active, infinity}}, + {{o, "Bucket3", "Key7", null}, 1, {active, infinity}}, + {{o, "Bucket3", "Key8", null}, 1, {active, infinity}}, + {{o, "Bucket3", "Key9", null}, 1, {active, infinity}}, + {{o, "Bucket4", "Key1", null}, 1, {active, infinity}}], + RL3), ok = file:close(UpdHandle), ok = file:delete(Filename). @@ -1748,22 +1759,26 @@ big_iterator_test() -> InitFileMD, KL1, KL2, 1), io:format("Remainder lengths are ~w and ~w ~n", [length(KL1Rem), length(KL2Rem)]), - {complete, Result1} = fetch_range_keysonly(Handle, FileMD, {o, "Bucket0000", "Key0000"}, - {o, "Bucket9999", "Key9999"}, + {complete, Result1} = fetch_range_keysonly(Handle, + FileMD, + {o, "Bucket0000", "Key0000", null}, + {o, "Bucket9999", "Key9999", null}, 256), NumFoundKeys1 = length(Result1), NumAddedKeys = 10000 - length(KL1Rem), ?assertMatch(NumFoundKeys1, NumAddedKeys), - {partial, Result2, _} = fetch_range_keysonly(Handle, FileMD, {o, "Bucket0000", "Key0000"}, - {o, "Bucket9999", "Key9999"}, + {partial, Result2, _} = fetch_range_keysonly(Handle, + FileMD, + {o, "Bucket0000", "Key0000", null}, + {o, "Bucket9999", "Key9999", null}, 32), - NumFoundKeys2 = length(Result2), - ?assertMatch(NumFoundKeys2, 32 * 128), - {partial, Result3, _} = fetch_range_keysonly(Handle, FileMD, {o, "Bucket0000", "Key0000"}, - {o, "Bucket9999", "Key9999"}, + ?assertMatch(32 * 128, length(Result2)), + {partial, Result3, _} = fetch_range_keysonly(Handle, + FileMD, + {o, "Bucket0000", "Key0000", null}, + {o, "Bucket9999", "Key9999", null}, 4), - NumFoundKeys3 = length(Result3), - ?assertMatch(NumFoundKeys3, 4 * 128), + ?assertMatch(4 * 128, length(Result3)), ok = file:close(Handle), ok = file:delete(Filename). diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 3f1560a..df41333 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -7,9 +7,9 @@ journal_compaction/1, fetchput_snapshot/1]). -all() -> [simple_put_fetch_head, - many_put_fetch_head, - journal_compaction, +all() -> [% simple_put_fetch_head, + % many_put_fetch_head, + % journal_compaction, fetchput_snapshot]. @@ -203,6 +203,15 @@ fetchput_snapshot(_Config) -> {ok, FNsC} = file:list_dir(RootPath ++ "/ledger/ledger_files"), true = length(FNsB) > length(FNsA), true = length(FNsB) > length(FNsC), + + {B1Size, B1Count} = check_bucket_stats(Bookie2, "Bucket1"), + true = B1Size > 0, + true = B1Count == 1, + {B1Size, B1Count} = check_bucket_stats(Bookie2, "Bucket1"), + {BSize, BCount} = check_bucket_stats(Bookie2, "Bucket"), + true = BSize > 0, + true = BCount == 140000, + ok = leveled_bookie:book_close(Bookie2), reset_filestructure(). @@ -217,6 +226,21 @@ reset_filestructure() -> RootPath. + +check_bucket_stats(Bookie, Bucket) -> + FoldSW1 = os:timestamp(), + io:format("Checking bucket size~n"), + {async, Folder1} = leveled_bookie:book_returnfolder(Bookie, + {bucket_stats, + Bucket}), + {B1Size, B1Count} = Folder1(), + io:format("Bucket fold completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(), FoldSW1)]), + io:format("Bucket ~w has size ~w and count ~w~n", + [Bucket, B1Size, B1Count]), + {B1Size, B1Count}. + + check_bookie_forlist(Bookie, ChkList) -> check_bookie_forlist(Bookie, ChkList, false). From 938cc0fc166ba5dab05d29ce95cd3135a8a10369 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 12 Oct 2016 17:35:32 +0100 Subject: [PATCH 061/167] Re-add tests Oops - committed with tests commented out --- test/end_to_end/basic_SUITE.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index df41333..7029e0d 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -7,9 +7,9 @@ journal_compaction/1, fetchput_snapshot/1]). -all() -> [% simple_put_fetch_head, - % many_put_fetch_head, - % journal_compaction, +all() -> [simple_put_fetch_head, + many_put_fetch_head, + journal_compaction, fetchput_snapshot]. From 2d981cb2e7b3108b5f08783735d994ff87927c5a Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 12 Oct 2016 18:10:47 +0100 Subject: [PATCH 062/167] FAdvise Add fadvise magic to SFT files. Also delete unnecessary rice modeule --- src/leveled_rice.erl | 283 ------------------------------------------- src/leveled_sft.erl | 28 ++--- 2 files changed, 7 insertions(+), 304 deletions(-) delete mode 100644 src/leveled_rice.erl diff --git a/src/leveled_rice.erl b/src/leveled_rice.erl deleted file mode 100644 index f432944..0000000 --- a/src/leveled_rice.erl +++ /dev/null @@ -1,283 +0,0 @@ -%% Used for creating fixed-size self-regulating encoded bloom filters -%% -%% Normally a bloom filter in order to achieve optimium size increases the -%% number of hashes as the desired false positive rate increases. There is -%% a processing overhead for checking this bloom, both because of the number -%% of hash calculations required, and also because of the need to CRC check -%% the bloom to ensure a false negative result is not returned due to -%% corruption. -%% -%% A more space efficient bloom can be achieved through the compression of -%% bloom filters with less hashes (and in an optimal case a single hash). -%% This can be achieved using rice encoding. -%% -%% Rice-encoding and single hash blooms are used here in order to provide an -%% optimally space efficient solution, but also as the processing required to -%% support uncompression can be concurrently performing a checksum role. -%% -%% For this to work, the bloom is divided into 64 parts and a 32-bit hash is -%% required. Each hash is placed into one of 64 blooms based on the six least -%% significant bits of the hash, and the fmost significant 26-bits are used -%% to indicate the bit to be added to the bloom. -%% -%% The bloom is then created by calculating the differences between the ordered -%% elements of the hash list and representing the difference using an exponent -%% and a 13-bit remainder i.e. -%% 8000 -> 0 11111 01000000 -%% 10000 -> 10 00000 00010000 -%% 20000 -> 110 01110 00100000 -%% -%% Each bloom should have approximately 64 differences. -%% -%% Fronting the bloom is a bloom index, formed first by 16 pairs of 3-byte -%% max hash, 2-byte length (bits) - with then each of the encoded bitstrings -%% appended. The max hash is the total of all the differences (which should -%% be the highest hash in the bloom). -%% -%% To check a key against the bloom, hash it, take the four least signifcant -%% bits and read the start pointer, max hash end pointer from the expected -%% positions in the bloom index. Then roll through from the start pointer to -%% the end pointer, accumulating each difference. There is a possible match if -%% either the accumulator hits the expected hash or the max hash doesn't match -%% the final accumulator (to cover if the bloom has been corrupted by a bit -%% flip somwhere). A miss is more than twice as expensive (on average) than a -%% potential match - but still only requires around 64 integer additions -%% and the processing of <100 bytes of data. -%% -%% For 2048 keys, this takes up <4KB. The false positive rate is 0.000122 -%% This compares favourably for the equivalent size optimal bloom which -%% would require 11 hashes and have a false positive rate of 0.000459. -%% Checking with a positive match should take on average about 6 microseconds, -%% and a negative match should take around 11 microseconds. -%% -%% See ../test/rice_test.erl for proving timings and fpr. - - - --module(leveled_rice). - --export([create_bloom/1, - check_key/2, - check_keys/2]). - --include_lib("eunit/include/eunit.hrl"). - --define(SLOT_COUNT, 64). --define(MAX_HASH, 16777216). --define(DIVISOR_BITS, 13). --define(DIVISOR, 8092). - -%% Create a bitstring representing the bloom filter from a key list - -create_bloom(KeyList) -> - create_bloom(KeyList, ?SLOT_COUNT, ?MAX_HASH). - -create_bloom(KeyList, SlotCount, MaxHash) -> - HashLists = array:new(SlotCount, [{default, []}]), - OrdHashLists = create_hashlist(KeyList, HashLists, SlotCount, MaxHash), - serialise_bloom(OrdHashLists). - - -%% Checking for a key - -check_keys([], _) -> - true; -check_keys([Key|Rest], BitStr) -> - case check_key(Key, BitStr) of - false -> - false; - true -> - check_keys(Rest, BitStr) - end. - -check_key(Key, BitStr) -> - check_key(Key, BitStr, ?SLOT_COUNT, ?MAX_HASH, ?DIVISOR_BITS, ?DIVISOR). - -check_key(Key, BitStr, SlotCount, MaxHash, Factor, Divisor) -> - {Slot, Hash} = get_slothash(Key, MaxHash, SlotCount), - {StartPos, Length, TopHash} = find_position(Slot, BitStr, 0, 40 * SlotCount), - case BitStr of - <<_:StartPos/bitstring, Bloom:Length/bitstring, _/bitstring>> -> - check_hash(Hash, Bloom, Factor, Divisor, 0, TopHash); - _ -> - io:format("Possible corruption of bloom index ~n"), - true - end. - -find_position(Slot, BloomIndex, Counter, StartPosition) -> - <> = BloomIndex, - case Slot of - Counter -> - {StartPosition, Length, TopHash}; - _ -> - find_position(Slot, Rest, Counter + 1, StartPosition + Length) - end. - - -% Checking for a hash within a bloom - -check_hash(_, <<>>, _, _, Acc, MaxHash) -> - case Acc of - MaxHash -> - false; - _ -> - io:format("Failure of CRC check on bloom filter~n"), - true - end; -check_hash(HashToCheck, BitStr, Factor, Divisor, Acc, TopHash) -> - case findexponent(BitStr) of - {ok, Exponent, BitStrTail} -> - case findremainder(BitStrTail, Factor) of - {ok, Remainder, BitStrTail2} -> - NextHash = Acc + Divisor * Exponent + Remainder, - case NextHash of - HashToCheck -> - true; - _ -> - check_hash(HashToCheck, BitStrTail2, Factor, - Divisor, NextHash, TopHash) - end; - error -> - io:format("Failure of CRC check on bloom filter~n"), - true - end; - error -> - io:format("Failure of CRC check on bloom filter~n"), - true - end. - -%% Convert the key list into an array of sorted hash lists - -create_hashlist([], HashLists, _, _) -> - HashLists; -create_hashlist([HeadKey|Rest], HashLists, SlotCount, MaxHash) -> - {Slot, Hash} = get_slothash(HeadKey, MaxHash, SlotCount), - HashList = array:get(Slot, HashLists), - create_hashlist(Rest, - array:set(Slot, lists:usort([Hash|HashList]), HashLists), - SlotCount, MaxHash). - -%% Convert an array of hash lists into an serialsed bloom - -serialise_bloom(HashLists) -> - SlotCount = array:size(HashLists), - serialise_bloom(HashLists, SlotCount, 0, []). - -serialise_bloom(HashLists, SlotCount, Counter, Blooms) -> - case Counter of - SlotCount -> - finalise_bloom(Blooms); - _ -> - Bloom = serialise_singlebloom(array:get(Counter, HashLists)), - serialise_bloom(HashLists, SlotCount, Counter + 1, [Bloom|Blooms]) - end. - -serialise_singlebloom(HashList) -> - serialise_singlebloom(HashList, <<>>, 0, ?DIVISOR, ?DIVISOR_BITS). - -serialise_singlebloom([], BloomStr, TopHash, _, _) -> - % io:format("Single bloom created with bloom of ~w and top hash of ~w~n", [BloomStr, TopHash]), - {BloomStr, TopHash}; -serialise_singlebloom([Hash|Rest], BloomStr, TopHash, Divisor, Factor) -> - HashGap = Hash - TopHash, - Exponent = buildexponent(HashGap div Divisor), - Remainder = HashGap rem Divisor, - NewBloomStr = <>, - serialise_singlebloom(Rest, NewBloomStr, Hash, Divisor, Factor). - - -finalise_bloom(Blooms) -> - finalise_bloom(Blooms, {<<>>, <<>>}). - -finalise_bloom([], BloomAcc) -> - {BloomIndex, BloomStr} = BloomAcc, - <>; -finalise_bloom([Bloom|Rest], BloomAcc) -> - {BloomStr, TopHash} = Bloom, - {BloomIndexAcc, BloomStrAcc} = BloomAcc, - Length = bit_size(BloomStr), - UpdIdx = <>, - % io:format("Adding bloom string of ~w to bloom~n", [BloomStr]), - UpdBloomStr = <>, - finalise_bloom(Rest, {UpdIdx, UpdBloomStr}). - - - - -buildexponent(Exponent) -> - buildexponent(Exponent, <<0:1>>). - -buildexponent(0, OutputBits) -> - OutputBits; -buildexponent(Exponent, OutputBits) -> - buildexponent(Exponent - 1, <<1:1, OutputBits/bitstring>>). - - -findexponent(BitStr) -> - findexponent(BitStr, 0). - -findexponent(<<>>, _) -> - error; -findexponent(<>, Acc) -> - case H of - 1 -> findexponent(T, Acc + 1); - 0 -> {ok, Acc, T} - end. - - -findremainder(BitStr, Factor) -> - case BitStr of - <> -> - {ok, Remainder, BitStrTail}; - _ -> - error - end. - - -get_slothash(Key, MaxHash, SlotCount) -> - Hash = erlang:phash2(Key, MaxHash), - {Hash rem SlotCount, Hash div SlotCount}. - - -%%%%%%%%%%%%%%%% -% T E S T -%%%%%%%%%%%%%%% - -corrupt_bloom(Bloom) -> - Length = bit_size(Bloom), - Random = random:uniform(Length), - <> = Bloom, - case Bit of - 1 -> - <>; - 0 -> - <> - end. - -bloom_test() -> - KeyList = ["key1", "key2", "key3", "key4"], - Bloom = create_bloom(KeyList), - io:format("Bloom of ~w of length ~w ~n", [Bloom, bit_size(Bloom)]), - ?assertMatch(true, check_key("key1", Bloom)), - ?assertMatch(true, check_key("key2", Bloom)), - ?assertMatch(true, check_key("key3", Bloom)), - ?assertMatch(true, check_key("key4", Bloom)), - ?assertMatch(false, check_key("key5", Bloom)). - -bloom_corruption_test() -> - KeyList = ["key1", "key2", "key3", "key4"], - Bloom = create_bloom(KeyList), - Bloom1 = corrupt_bloom(Bloom), - ?assertMatch(true, check_keys(KeyList, Bloom1)), - Bloom2 = corrupt_bloom(Bloom), - ?assertMatch(true, check_keys(KeyList, Bloom2)), - Bloom3 = corrupt_bloom(Bloom), - ?assertMatch(true, check_keys(KeyList, Bloom3)), - Bloom4 = corrupt_bloom(Bloom), - ?assertMatch(true, check_keys(KeyList, Bloom4)), - Bloom5 = corrupt_bloom(Bloom), - ?assertMatch(true, check_keys(KeyList, Bloom5)), - Bloom6 = corrupt_bloom(Bloom), - ?assertMatch(true, check_keys(KeyList, Bloom6)). - - diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 9cc3a68..4a54eb6 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -159,7 +159,6 @@ sft_close/1, sft_clear/1, sft_checkready/1, - sft_getfilename/1, sft_setfordelete/2, sft_getmaxsequencenumber/1, generate_randomkeys/1]). @@ -255,9 +254,6 @@ sft_close(Pid) -> sft_checkready(Pid) -> gen_server:call(Pid, background_complete, infinity). -sft_getfilename(Pid) -> - gen_server:call(Pid, get_filename, infinty). - sft_getmaxsequencenumber(Pid) -> gen_server:call(Pid, get_maxsqn, infinity). @@ -330,8 +326,6 @@ handle_call(background_complete, _From, State) -> false -> {reply, {error, State#state.background_failure}, State} end; -handle_call(get_filename, _From, State) -> - {reply, State#state.filename, State}; handle_call({set_for_delete, Penciller}, _From, State) -> {reply, ok, @@ -362,9 +356,7 @@ handle_info(timeout, State) -> end; false -> {noreply, State} - end; -handle_info(_Info, State) -> - {noreply, State}. + end. terminate(Reason, State) -> io:format("Exit called for reason ~w on filename ~s~n", @@ -878,18 +870,12 @@ sftwrite_function(finalise, IndexLength:32/integer, FilterLength:32/integer, SummaryLength:32/integer>>), - file:close(Handle); -sftwrite_function(finalise, - {Handle, - SlotIndex, - SNExtremes, - KeyExtremes}) -> - {SlotFilters, PointerIndex} = convert_slotindex(SlotIndex), - sftwrite_function(finalise, - {Handle, - {SlotFilters, PointerIndex}, - SNExtremes, - KeyExtremes}). + {ok, _Position} = file:position(Handle, bof), + ok = file:advise(Handle, + BlocksLength + IndexLength, + FilterLength, + will_need), + file:close(Handle). %% Level 0 files are of variable (infinite) size to avoid issues with having %% any remainders when flushing from memory From de54a28328247a1ad42fb56f6945b09cdf9aca4b Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 13 Oct 2016 17:51:47 +0100 Subject: [PATCH 063/167] Load and Count test This test exposed two bugs: - Yet another set of off-by-one errors (really stupidly scanning the Manifest from Level 1 not Level 0) - The return of an old issue related to scanning the journal on load whereby we fail to go back to the previous file before the current SQN --- src/leveled_bookie.erl | 10 +- src/leveled_iclerk.erl | 1 - src/leveled_inker.erl | 2 +- src/leveled_pclerk.erl | 43 ++++---- src/leveled_penciller.erl | 167 +++++++++++++++++++------------- src/leveled_sft.erl | 2 - test/end_to_end/basic_SUITE.erl | 95 +++++++++++++++++- 7 files changed, 219 insertions(+), 101 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 46ce7e2..ebd6dc2 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -361,13 +361,15 @@ code_change(_OldVsn, State, _Extra) -> %%%============================================================================ bucket_stats(Penciller, LedgerCache, Bucket) -> + PCLopts = #penciller_options{start_snapshot=true, + source_penciller=Penciller}, + {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), Folder = fun() -> - PCLopts = #penciller_options{start_snapshot=true, - source_penciller=Penciller}, - {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), Increment = gb_trees:to_list(LedgerCache), + io:format("Length of increment in snapshot is ~w~n", + [length(Increment)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, - Increment), + {infinity, Increment}), StartKey = {o, Bucket, null, null}, EndKey = {o, Bucket, null, null}, Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 96926c7..33e5e9b 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -151,7 +151,6 @@ check_single_file(CDB, FilterFun, FilterServer, MaxSQN, SampleSize, BatchSize) - FN = leveled_cdb:cdb_filename(CDB), PositionList = leveled_cdb:cdb_getpositions(CDB, SampleSize), KeySizeList = fetch_inbatches(PositionList, BatchSize, CDB, []), - io:format("KeySizeList ~w~n", [KeySizeList]), R0 = lists:foldl(fun(KS, {ActSize, RplSize}) -> {{SQN, PK}, Size} = KS, Check = FilterFun(FilterServer, PK, SQN), diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index bcf1023..946a118 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -594,7 +594,7 @@ load_from_sequence(MinSQN, FilterFun, Penciller, [{_LowSQN, FN, Pid}|Rest]) -> undefined, FN, Rest); - [{NextSQN, _FN, Pid}|_Rest] when NextSQN > MinSQN -> + [{NextSQN, _NxtFN, _NxtPid}|_Rest] when NextSQN > MinSQN -> load_between_sequence(MinSQN, MinSQN + ?LOADING_BATCH, FilterFun, diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index cff4a45..413205b 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -73,7 +73,7 @@ -record(state, {owner :: pid(), change_pending=false :: boolean(), - work_item :: #penciller_work{}}). + work_item :: #penciller_work{}|null}). %%%============================================================================ %%% API @@ -208,28 +208,23 @@ merge(WI) -> {WI#penciller_work.ledger_filepath, WI#penciller_work.next_sqn}) end, - case MergedFiles of - error -> - merge_failure; - _ -> - NewLevel = lists:sort(lists:append(MergedFiles, Others)), - UpdMFest2 = lists:keystore(SrcLevel + 1, - 1, - UpdMFest1, - {SrcLevel + 1, NewLevel}), - - ok = filelib:ensure_dir(WI#penciller_work.manifest_file), - {ok, Handle} = file:open(WI#penciller_work.manifest_file, - [binary, raw, write]), - ok = file:write(Handle, term_to_binary(UpdMFest2)), - ok = file:close(Handle), - case lists:member(SrcF, MergedFiles) of - true -> - {UpdMFest2, Candidates}; - false -> - %% Can rub out src file as it is not part of output - {UpdMFest2, Candidates ++ [SrcF]} - end + NewLevel = lists:sort(lists:append(MergedFiles, Others)), + UpdMFest2 = lists:keystore(SrcLevel + 1, + 1, + UpdMFest1, + {SrcLevel + 1, NewLevel}), + + ok = filelib:ensure_dir(WI#penciller_work.manifest_file), + {ok, Handle} = file:open(WI#penciller_work.manifest_file, + [binary, raw, write]), + ok = file:write(Handle, term_to_binary(UpdMFest2)), + ok = file:close(Handle), + case lists:member(SrcF, MergedFiles) of + true -> + {UpdMFest2, Candidates}; + false -> + %% Can rub out src file as it is not part of output + {UpdMFest2, Candidates ++ [SrcF]} end. @@ -317,6 +312,8 @@ do_merge(KL1, KL2, Level, {Filepath, MSN}, FileCounter, OutList) -> [Level + 1, FileCounter])), io:format("File to be created as part of MSN=~w Filename=~s~n", [MSN, FileName]), + % Attempt to trace intermittent eaccess failures + false = filelib:is_file(FileName), TS1 = os:timestamp(), {ok, Pid, Reply} = leveled_sft:sft_new(FileName, KL1, KL2, Level + 1), {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 05ea41f..b785802 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -335,7 +335,7 @@ pcl_loadsnapshot(Pid, Increment) -> gen_server:call(Pid, {load_snapshot, Increment}, infinity). pcl_close(Pid) -> - gen_server:call(Pid, close). + gen_server:call(Pid, close, 60000). %%%============================================================================ @@ -493,6 +493,8 @@ handle_call({fetch_keys, StartKey, EndKey, AccFun, InitAcc}, SFTiter = initiate_rangequery_frommanifest(StartKey, EndKey, State#state.manifest), + io:format("Manifest for iterator of:~n"), + print_manifest(SFTiter), Acc = keyfolder(L0iter, SFTiter, StartKey, EndKey, {AccFun, InitAcc}), {reply, Acc, State}; handle_call(work_for_clerk, From, State) -> @@ -523,7 +525,8 @@ handle_call({load_snapshot, Increment}, _From, State) -> ok end, {Tree1, TreeSQN1} = roll_new_tree(Tree0, [Increment], TreeSQN0), - io:format("Snapshot loaded to start at SQN~w~n", [TreeSQN1]), + io:format("Snapshot loaded with increments to start at SQN=~w~n", + [TreeSQN1]), {reply, ok, State#state{levelzero_snapshot=Tree1, ledger_sqn=TreeSQN1, snapshot_fully_loaded=true}}; @@ -852,6 +855,7 @@ compare_to_sqn(Obj, SQN) -> %% Manifest lock - don't have two changes to the manifest happening %% concurrently +% TODO: Is this necessary now? manifest_locked(State) -> if @@ -930,11 +934,15 @@ return_work(State, From) -> roll_new_tree(Tree, [], HighSQN) -> {Tree, HighSQN}; roll_new_tree(Tree, [{SQN, KVList}|TailIncs], HighSQN) when SQN >= HighSQN -> - UpdTree = lists:foldl(fun({K, V}, TreeAcc) -> - gb_trees:enter(K, V, TreeAcc) end, - Tree, - KVList), - roll_new_tree(UpdTree, TailIncs, SQN); + R = lists:foldl(fun({Kx, Vx}, {TreeAcc, MaxSQN}) -> + UpdTree = gb_trees:enter(Kx, Vx, TreeAcc), + SQNx = leveled_bookie:strip_to_seqonly({Kx, Vx}), + {UpdTree, max(SQNx, MaxSQN)} + end, + {Tree, HighSQN}, + KVList), + {UpdTree, UpdSQN} = R, + roll_new_tree(UpdTree, TailIncs, UpdSQN); roll_new_tree(Tree, [_H|TailIncs], HighSQN) -> roll_new_tree(Tree, TailIncs, HighSQN). @@ -949,14 +957,14 @@ cache_tree_in_memcopy(MemCopy, Tree, SQN) -> SQN >= PushSQN -> Acc; true -> - Acc ++ {PushSQN, PushL} + Acc ++ [{PushSQN, PushL}] end end, [], MemCopy#l0snapshot.increments), #l0snapshot{ledger_sqn = SQN, increments = Incs, tree = Tree}; - _ -> + _CurrentSQN -> MemCopy end. @@ -1004,10 +1012,17 @@ print_manifest(Manifest) -> io:format("Manifest at Level ~w~n", [L]), Level = get_item(L, Manifest, []), lists:foreach(fun(M) -> - print_manifest_entry(M) end, + R = is_record(M, manifest_entry), + case R of + true -> + print_manifest_entry(M); + false -> + {_, M1} = M, + print_manifest_entry(M1) + end end, Level) end, - lists:seq(1, ?MAX_LEVELS - 1)). + lists:seq(0, ?MAX_LEVELS - 1)). print_manifest_entry(Entry) -> {S1, S2, S3} = leveled_bookie:print_key(Entry#manifest_entry.start_key), @@ -1045,7 +1060,7 @@ initiate_rangequery_frommanifest(StartKey, EndKey, Manifest) -> end end, [], - lists:seq(1, ?MAX_LEVELS - 1)). + lists:seq(0, ?MAX_LEVELS - 1)). %% Looks to find the best choice for the next key across the levels (other %% than in-memory table) @@ -1054,7 +1069,7 @@ initiate_rangequery_frommanifest(StartKey, EndKey, Manifest) -> find_nextkey(QueryArray, StartKey, EndKey) -> find_nextkey(QueryArray, - 1, + 0, {null, null}, {fun leveled_sft:sft_getkvrange/4, StartKey, EndKey, 1}). @@ -1178,40 +1193,47 @@ keyfolder(IMMiterator, SFTiterator, StartKey, EndKey, {AccFun, Acc}) -> % iterate only over the remaining keys in the SFT iterator keyfolder(null, SFTiterator, StartKey, EndKey, {AccFun, Acc}); {IMMKey, IMMVal, NxtIMMiterator} -> - case {leveled_bookie:key_compare(EndKey, IMMKey, lt), - find_nextkey(SFTiterator, StartKey, EndKey)} of - {true, _} -> + case leveled_bookie:key_compare(EndKey, IMMKey, lt) of + true -> % There are no more keys in-range in the in-memory % iterator, so take action as if this iterator is empty % (see above) keyfolder(null, SFTiterator, StartKey, EndKey, {AccFun, Acc}); - {false, no_more_keys} -> - % No more keys in range in the persisted store, so use the - % in-memory KV as the next - Acc1 = AccFun(IMMKey, IMMVal, Acc), - keyfolder(NxtIMMiterator, SFTiterator, - StartKey, EndKey, {AccFun, Acc1}); - {false, {NxtSFTiterator, {SFTKey, SFTVal}}} -> - % There is a next key, so need to know which is the next - % key between the two (and handle two keys with different - % sequence numbers). - case leveled_bookie:key_dominates({IMMKey, IMMVal}, - {SFTKey, SFTVal}) of - left_hand_first -> + false -> + case find_nextkey(SFTiterator, StartKey, EndKey) of + no_more_keys -> + % No more keys in range in the persisted store, so use the + % in-memory KV as the next Acc1 = AccFun(IMMKey, IMMVal, Acc), keyfolder(NxtIMMiterator, SFTiterator, StartKey, EndKey, {AccFun, Acc1}); - right_hand_first -> - Acc1 = AccFun(SFTKey, SFTVal, Acc), - keyfolder(IMMiterator, NxtSFTiterator, - StartKey, EndKey, {AccFun, Acc1}); - left_hand_dominant -> - Acc1 = AccFun(IMMKey, IMMVal, Acc), - keyfolder(NxtIMMiterator, NxtSFTiterator, - StartKey, EndKey, {AccFun, Acc1}) + {NxtSFTiterator, {SFTKey, SFTVal}} -> + % There is a next key, so need to know which is the + % next key between the two (and handle two keys + % with different sequence numbers). + case leveled_bookie:key_dominates({IMMKey, + IMMVal}, + {SFTKey, + SFTVal}) of + left_hand_first -> + Acc1 = AccFun(IMMKey, IMMVal, Acc), + keyfolder(NxtIMMiterator, SFTiterator, + StartKey, EndKey, + {AccFun, Acc1}); + right_hand_first -> + Acc1 = AccFun(SFTKey, SFTVal, Acc), + keyfolder(IMMiterator, NxtSFTiterator, + StartKey, EndKey, + {AccFun, Acc1}); + left_hand_dominant -> + Acc1 = AccFun(IMMKey, IMMVal, Acc), + keyfolder(NxtIMMiterator, NxtSFTiterator, + StartKey, EndKey, + {AccFun, Acc1}) + end end - end + end end. @@ -1373,10 +1395,12 @@ clean_subdir(DirPath) -> case filelib:is_dir(DirPath) of true -> {ok, Files} = file:list_dir(DirPath), - lists:foreach(fun(FN) -> file:delete(filename:join(DirPath, FN)), - io:format("Delete file ~s/~s~n", - [DirPath, FN]) - end, + lists:foreach(fun(FN) -> + File = filename:join(DirPath, FN), + case file:delete(File) of + ok -> io:format("Success deleting ~s~n", [File]); + _ -> io:format("Error deleting ~s~n", [File]) + end end, Files); false -> ok @@ -1556,35 +1580,19 @@ simple_server_test() -> ok = pcl_close(PCLr), clean_testdir(RootPath). -memcopy_test() -> +memcopy_updatecache1_test() -> KVL1 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), - "Value" ++ integer_to_list(X) ++ "A"} end, + {X, null, "Val" ++ integer_to_list(X) ++ "A"}} + end, lists:seq(1, 1000)), KVL2 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), - "Value" ++ integer_to_list(X) ++ "B"} end, + {X, null, "Val" ++ integer_to_list(X) ++ "B"}} + end, lists:seq(1001, 2000)), KVL3 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), - "Value" ++ integer_to_list(X) ++ "C"} end, - lists:seq(1, 1000)), - MemCopy0 = #l0snapshot{}, - MemCopy1 = add_increment_to_memcopy(MemCopy0, 1000, KVL1), - MemCopy2 = add_increment_to_memcopy(MemCopy1, 2000, KVL2), - MemCopy3 = add_increment_to_memcopy(MemCopy2, 3000, KVL3), - {Tree1, HighSQN1} = roll_new_tree(gb_trees:empty(), MemCopy3#l0snapshot.increments, 0), - Size1 = gb_trees:size(Tree1), - ?assertMatch(2000, Size1), - ?assertMatch(3000, HighSQN1). - -memcopy_updatecache_test() -> - KVL1 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), - "Value" ++ integer_to_list(X) ++ "A"} end, - lists:seq(1, 1000)), - KVL2 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), - "Value" ++ integer_to_list(X) ++ "B"} end, - lists:seq(1001, 2000)), - KVL3 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), - "Value" ++ integer_to_list(X) ++ "C"} end, - lists:seq(1, 1000)), + {X, null, "Val" ++ integer_to_list(X) ++ "C"}} + end, + lists:seq(2001, 3000)), MemCopy0 = #l0snapshot{}, MemCopy1 = add_increment_to_memcopy(MemCopy0, 1000, KVL1), MemCopy2 = add_increment_to_memcopy(MemCopy1, 2000, KVL2), @@ -1594,9 +1602,34 @@ memcopy_updatecache_test() -> MemCopy4 = cache_tree_in_memcopy(MemCopy3, Tree1, HighSQN1), ?assertMatch(0, length(MemCopy4#l0snapshot.increments)), Size2 = gb_trees:size(MemCopy4#l0snapshot.tree), - ?assertMatch(2000, Size2), + ?assertMatch(3000, Size2), ?assertMatch(3000, MemCopy4#l0snapshot.ledger_sqn). +memcopy_updatecache2_test() -> + KVL1 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), + {X, null, "Val" ++ integer_to_list(X) ++ "A"}} + end, + lists:seq(1, 1000)), + KVL2 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), + {X, null, "Val" ++ integer_to_list(X) ++ "B"}} + end, + lists:seq(1001, 2000)), + KVL3 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), + {X, null, "Val" ++ integer_to_list(X) ++ "C"}} + end, + lists:seq(1, 1000)), + MemCopy0 = #l0snapshot{}, + MemCopy1 = add_increment_to_memcopy(MemCopy0, 1000, KVL1), + MemCopy2 = add_increment_to_memcopy(MemCopy1, 2000, KVL2), + MemCopy3 = add_increment_to_memcopy(MemCopy2, 3000, KVL3), + ?assertMatch(0, MemCopy3#l0snapshot.ledger_sqn), + {Tree1, HighSQN1} = roll_new_tree(gb_trees:empty(), MemCopy3#l0snapshot.increments, 0), + MemCopy4 = cache_tree_in_memcopy(MemCopy3, Tree1, HighSQN1), + ?assertMatch(1, length(MemCopy4#l0snapshot.increments)), + Size2 = gb_trees:size(MemCopy4#l0snapshot.tree), + ?assertMatch(2000, Size2), + ?assertMatch(2000, MemCopy4#l0snapshot.ledger_sqn). + rangequery_manifest_test() -> {E1, E2, diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 4a54eb6..dda8607 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -278,8 +278,6 @@ handle_call({sft_new, Filename, KL1, KL2, Level}, _From, State) -> {error, Reason} -> {reply, {error, Reason}, State}; {Handle, FileMD} -> - io:format("Creating file with inputs of size ~w ~w~n", - [length(KL1), length(KL2)]), {ReadHandle, UpdFileMD, KeyRemainders} = complete_file(Handle, FileMD, KL1, KL2, diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 7029e0d..2f30975 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -5,12 +5,14 @@ -export([simple_put_fetch_head/1, many_put_fetch_head/1, journal_compaction/1, - fetchput_snapshot/1]). + fetchput_snapshot/1, + load_and_count/1]). all() -> [simple_put_fetch_head, many_put_fetch_head, journal_compaction, - fetchput_snapshot]. + fetchput_snapshot, + load_and_count]. simple_put_fetch_head(_Config) -> @@ -216,6 +218,73 @@ fetchput_snapshot(_Config) -> reset_filestructure(). +load_and_count(_Config) -> + % Use artificially small files, and the load keys, counting they're all + % present + RootPath = reset_filestructure(), + StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=50000000}, + {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), + {TestObject, TestSpec} = generate_testobject(), + ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), + check_bookie_forobject(Bookie1, TestObject), + io:format("Loading initial small objects~n"), + lists:foldl(fun(_X, Acc) -> + load_objects(5000, [Acc + 2], Bookie1, TestObject, + fun generate_multiple_smallobjects/2), + {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + if + Acc + 5000 == Count -> + ok + end, + Acc + 5000 end, + 0, + lists:seq(1, 20)), + check_bookie_forobject(Bookie1, TestObject), + io:format("Loading larger compressible objects~n"), + lists:foldl(fun(_X, Acc) -> + load_objects(5000, [Acc + 2], Bookie1, TestObject, + fun generate_multiple_compressibleobjects/2), + {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + if + Acc + 5000 == Count -> + ok + end, + Acc + 5000 end, + 100000, + lists:seq(1, 20)), + check_bookie_forobject(Bookie1, TestObject), + io:format("Replacing small objects~n"), + lists:foldl(fun(_X, Acc) -> + load_objects(5000, [Acc + 2], Bookie1, TestObject, + fun generate_multiple_smallobjects/2), + {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + if + Count == 200000 -> + ok + end, + Acc + 5000 end, + 0, + lists:seq(1, 20)), + check_bookie_forobject(Bookie1, TestObject), + io:format("Loading more small objects~n"), + lists:foldl(fun(_X, Acc) -> + load_objects(5000, [Acc + 2], Bookie1, TestObject, + fun generate_multiple_compressibleobjects/2), + {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + if + Acc + 5000 == Count -> + ok + end, + Acc + 5000 end, + 200000, + lists:seq(1, 20)), + check_bookie_forobject(Bookie1, TestObject), + ok = leveled_bookie:book_close(Bookie1), + {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), + {_BSize, 300000} = check_bucket_stats(Bookie2, "Bucket"), + ok = leveled_bookie:book_close(Bookie2), + reset_filestructure(). + reset_filestructure() -> RootPath = "test", @@ -236,7 +305,7 @@ check_bucket_stats(Bookie, Bucket) -> {B1Size, B1Count} = Folder1(), io:format("Bucket fold completed in ~w microseconds~n", [timer:now_diff(os:timestamp(), FoldSW1)]), - io:format("Bucket ~w has size ~w and count ~w~n", + io:format("Bucket ~s has size ~w and count ~w~n", [Bucket, B1Size, B1Count]), {B1Size, B1Count}. @@ -306,6 +375,26 @@ generate_testobject(B, K, V, Spec, MD) -> {#r_object{bucket=B, key=K, contents=[Content], vclock=[{'a',1}]}, Spec}. + +generate_multiple_compressibleobjects(Count, KeyNumber) -> + S1 = "111111111111111", + S2 = "222222222222222", + S3 = "333333333333333", + S4 = "aaaaaaaaaaaaaaa", + S5 = "AAAAAAAAAAAAAAA", + S6 = "GGGGGGGGGGGGGGG", + S7 = "===============", + S8 = "...............", + Selector = [{1, S1}, {2, S2}, {3, S3}, {4, S4}, + {5, S5}, {6, S6}, {7, S7}, {8, S8}], + L = lists:seq(1, 1024), + V = lists:foldl(fun(_X, Acc) -> + {_, Str} = lists:keyfind(random:uniform(8), 1, Selector), + Acc ++ Str end, + "", + L), + generate_multiple_objects(Count, KeyNumber, [], V). + generate_multiple_smallobjects(Count, KeyNumber) -> generate_multiple_objects(Count, KeyNumber, [], crypto:rand_bytes(512)). From bbdac65f8d832bc12a57230e13b436a56f3cc44b Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 13 Oct 2016 21:02:15 +0100 Subject: [PATCH 064/167] Split out key codec Aplit out key codec, and also saner approach to key comparison (although still awkward). --- src/leveled_bookie.erl | 129 ++-------------------------------- src/leveled_cdb.erl | 6 ++ src/leveled_codec.erl | 142 ++++++++++++++++++++++++++++++++++++++ src/leveled_pclerk.erl | 4 +- src/leveled_penciller.erl | 29 ++++---- src/leveled_sft.erl | 26 ++++--- 6 files changed, 181 insertions(+), 155 deletions(-) create mode 100644 src/leveled_codec.erl diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index ebd6dc2..ecf5f81 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -140,16 +140,7 @@ book_snapshotstore/3, book_snapshotledger/3, book_compactjournal/2, - book_close/1, - strip_to_keyonly/1, - strip_to_keyseqonly/1, - strip_to_seqonly/1, - strip_to_statusonly/1, - strip_to_keyseqstatusonly/1, - striphead_to_details/1, - key_compare/3, - key_dominates/2, - print_key/1]). + book_close/1]). -include_lib("eunit/include/eunit.hrl"). @@ -265,7 +256,7 @@ handle_call({get, Key}, _From, State) -> not_present -> {reply, not_found, State}; Head -> - {Seqn, Status, _MD} = striphead_to_details(Head), + {Seqn, Status, _MD} = leveled_codec:striphead_to_details(Head), case Status of {tomb, _} -> {reply, not_found, State}; @@ -283,12 +274,12 @@ handle_call({head, Key}, _From, State) -> not_present -> {reply, not_found, State}; Head -> - {_Seqn, Status, MD} = striphead_to_details(Head), + {_Seqn, Status, MD} = leveled_codec:striphead_to_details(Head), case Status of {tomb, _} -> {reply, not_found, State}; {active, _} -> - OMD = build_metadata_object(Key, MD), + OMD = leveled_codec:build_metadata_object(Key, MD), {reply, {ok, OMD}, State} end end; @@ -442,116 +433,20 @@ fetch_value(Key, SQN, Inker) -> not_present end. -%% Format of a Key within the ledger is -%% {PrimaryKey, SQN, Metadata, Status} - -strip_to_keyonly({keyonly, K}) -> K; -strip_to_keyonly({K, _V}) -> K. - -strip_to_keyseqonly({K, {SeqN, _, _}}) -> {K, SeqN}. - -strip_to_keyseqstatusonly({K, {SeqN, St, _MD}}) -> {K, SeqN, St}. - -strip_to_statusonly({_, {_, St, _}}) -> St. - -strip_to_seqonly({_, {SeqN, _, _}}) -> SeqN. - -striphead_to_details({SeqN, St, MD}) -> {SeqN, St, MD}. - -key_dominates(LeftKey, RightKey) -> - case {LeftKey, RightKey} of - {{LK, _LVAL}, {RK, _RVAL}} when LK < RK -> - left_hand_first; - {{LK, _LVAL}, {RK, _RVAL}} when RK < LK -> - right_hand_first; - {{LK, {LSN, _LST, _LMD}}, {RK, {RSN, _RST, _RMD}}} - when LK == RK, LSN >= RSN -> - left_hand_dominant; - {{LK, {LSN, _LST, _LMD}}, {RK, {RSN, _RST, _RMD}}} - when LK == RK, LSN < RSN -> - right_hand_dominant - end. - - - -get_metadatas(#r_object{contents=Contents}) -> - [Content#r_content.metadata || Content <- Contents]. - -set_vclock(Object=#r_object{}, VClock) -> Object#r_object{vclock=VClock}. - -vclock(#r_object{vclock=VClock}) -> VClock. - -to_binary(v0, Obj) -> - term_to_binary(Obj). - -hash(Obj=#r_object{}) -> - Vclock = vclock(Obj), - UpdObj = set_vclock(Obj, lists:sort(Vclock)), - erlang:phash2(to_binary(v0, UpdObj)). - -extract_metadata(Obj, Size) -> - {get_metadatas(Obj), vclock(Obj), hash(Obj), Size}. accumulate_size(_Key, Value, {Size, Count}) -> {_, _, MD} = Value, {_, _, _, ObjSize} = MD, {Size + ObjSize, Count + 1}. -build_metadata_object(PrimaryKey, Head) -> - {o, Bucket, Key, null} = PrimaryKey, - {MD, VC, _, _} = Head, - Contents = lists:foldl(fun(X, Acc) -> Acc ++ [#r_content{metadata=X}] end, - [], - MD), - #r_object{contents=Contents, bucket=Bucket, key=Key, vclock=VC}. - -convert_indexspecs(IndexSpecs, SQN, PrimaryKey) -> - lists:map(fun({IndexOp, IndexField, IndexValue}) -> - Status = case IndexOp of - add -> - %% TODO: timestamp support - {active, infinity}; - remove -> - %% TODO: timestamps for delayed reaping - {tomb, infinity} - end, - {o, B, K, _SK} = PrimaryKey, - {{i, B, {IndexField, IndexValue}, K}, - {SQN, Status, null}} - end, - IndexSpecs). - -% Return a tuple of string to ease the printing of keys to logs -print_key(Key) -> - case Key of - {o, B, K, _SK} -> - {"Object", B, K}; - {i, B, {F, _V}, _K} -> - {"Index", B, F} - end. - -% Compare a key against a query key, only comparing elements that are non-null -% in the Query key -key_compare(QueryKey, CheckingKey, gt) -> - key_compare(QueryKey, CheckingKey, fun(X,Y) -> X > Y end); -key_compare(QueryKey, CheckingKey, lt) -> - key_compare(QueryKey, CheckingKey, fun(X,Y) -> X < Y end); -key_compare({QK1, null, null, null}, {CK1, _, _, _}, CompareFun) -> - CompareFun(QK1, CK1); -key_compare({QK1, QK2, null, null}, {CK1, CK2, _, _}, CompareFun) -> - CompareFun({QK1, QK2}, {CK1, CK2}); -key_compare({QK1, QK2, QK3, null}, {CK1, CK2, CK3, _}, CompareFun) -> - CompareFun({QK1, QK2, QK3}, {CK1, CK2, CK3}); -key_compare(QueryKey, CheckingKey, CompareFun) -> - CompareFun(QueryKey, CheckingKey). preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs) -> PrimaryChange = {PK, {SQN, {active, infinity}, - extract_metadata(Obj, Size)}}, - SecChanges = convert_indexspecs(IndexSpecs, SQN, PK), + leveled_codec:extract_metadata(Obj, Size)}}, + SecChanges = leveled_codec:convert_indexspecs(IndexSpecs, SQN, PK), [PrimaryChange] ++ SecChanges. addto_ledgercache(Changes, Cache) -> @@ -709,17 +604,5 @@ multi_key_test() -> ?assertMatch(F2D, Obj2), ok = book_close(Bookie2), reset_filestructure(). - -indexspecs_test() -> - IndexSpecs = [{add, "t1_int", 456}, - {add, "t1_bin", "adbc123"}, - {remove, "t1_bin", "abdc456"}], - Changes = convert_indexspecs(IndexSpecs, 1, {o, "Bucket", "Key2", null}), - ?assertMatch({{i, "Bucket", {"t1_int", 456}, "Key2"}, - {1, {active, infinity}, null}}, lists:nth(1, Changes)), - ?assertMatch({{i, "Bucket", {"t1_bin", "adbc123"}, "Key2"}, - {1, {active, infinity}, null}}, lists:nth(2, Changes)), - ?assertMatch({{i, "Bucket", {"t1_bin", "abdc456"}, "Key2"}, - {1, {tomb, infinity}, null}}, lists:nth(3, Changes)). -endif. \ No newline at end of file diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 45f2a6a..9b9af5a 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -1098,12 +1098,18 @@ write_key_value_pairs(Handle, [HeadPair|TailList], Acc) -> write_hash_tables(Handle, HashTree) -> Seq = lists:seq(0, 255), {ok, StartPos} = file:position(Handle, cur), + SWC = os:timestamp(), {IndexList, HashTreeBin} = write_hash_tables(Seq, HashTree, StartPos, [], <<>>), + io:format("HashTree computed in ~w microseconds~n", + [timer:now_diff(os:timestamp(), SWC)]), + SWW = os:timestamp(), ok = file:write(Handle, HashTreeBin), + io:format("HashTree written in ~w microseconds~n", + [timer:now_diff(os:timestamp(), SWW)]), {ok, EndPos} = file:position(Handle, cur), ok = file:advise(Handle, StartPos, EndPos - StartPos, will_need), IndexList. diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl new file mode 100644 index 0000000..6ae0f6a --- /dev/null +++ b/src/leveled_codec.erl @@ -0,0 +1,142 @@ +%% -------- Key Codec --------- +%% +%% Functions for manipulating keys and values within leveled. These are +%% currently static functions, they cannot be overridden in the store other +%% than by changing them here. The formats are focused on the problem of +%% supporting Riak KV + +-module(leveled_codec). + +-include("../include/leveled.hrl"). + + +-include_lib("eunit/include/eunit.hrl"). + +-export([strip_to_keyonly/1, + strip_to_keyseqonly/1, + strip_to_seqonly/1, + strip_to_statusonly/1, + strip_to_keyseqstatusonly/1, + striphead_to_details/1, + endkey_passed/2, + key_dominates/2, + print_key/1, + extract_metadata/2, + build_metadata_object/2, + convert_indexspecs/3]). + + +strip_to_keyonly({keyonly, K}) -> K; +strip_to_keyonly({K, _V}) -> K. + +strip_to_keyseqonly({K, {SeqN, _, _}}) -> {K, SeqN}. + +strip_to_keyseqstatusonly({K, {SeqN, St, _MD}}) -> {K, SeqN, St}. + +strip_to_statusonly({_, {_, St, _}}) -> St. + +strip_to_seqonly({_, {SeqN, _, _}}) -> SeqN. + +striphead_to_details({SeqN, St, MD}) -> {SeqN, St, MD}. + +key_dominates(LeftKey, RightKey) -> + case {LeftKey, RightKey} of + {{LK, _LVAL}, {RK, _RVAL}} when LK < RK -> + left_hand_first; + {{LK, _LVAL}, {RK, _RVAL}} when RK < LK -> + right_hand_first; + {{LK, {LSN, _LST, _LMD}}, {RK, {RSN, _RST, _RMD}}} + when LK == RK, LSN >= RSN -> + left_hand_dominant; + {{LK, {LSN, _LST, _LMD}}, {RK, {RSN, _RST, _RMD}}} + when LK == RK, LSN < RSN -> + right_hand_dominant + end. + + + +get_metadatas(#r_object{contents=Contents}) -> + [Content#r_content.metadata || Content <- Contents]. + +set_vclock(Object=#r_object{}, VClock) -> Object#r_object{vclock=VClock}. + +vclock(#r_object{vclock=VClock}) -> VClock. + +to_binary(v0, Obj) -> + term_to_binary(Obj). + +hash(Obj=#r_object{}) -> + Vclock = vclock(Obj), + UpdObj = set_vclock(Obj, lists:sort(Vclock)), + erlang:phash2(to_binary(v0, UpdObj)). + +extract_metadata(Obj, Size) -> + {get_metadatas(Obj), vclock(Obj), hash(Obj), Size}. + + +build_metadata_object(PrimaryKey, Head) -> + {o, Bucket, Key, null} = PrimaryKey, + {MD, VC, _, _} = Head, + Contents = lists:foldl(fun(X, Acc) -> Acc ++ [#r_content{metadata=X}] end, + [], + MD), + #r_object{contents=Contents, bucket=Bucket, key=Key, vclock=VC}. + +convert_indexspecs(IndexSpecs, SQN, PrimaryKey) -> + lists:map(fun({IndexOp, IndexField, IndexValue}) -> + Status = case IndexOp of + add -> + %% TODO: timestamp support + {active, infinity}; + remove -> + %% TODO: timestamps for delayed reaping + {tomb, infinity} + end, + {o, B, K, _SK} = PrimaryKey, + {{i, B, {IndexField, IndexValue}, K}, + {SQN, Status, null}} + end, + IndexSpecs). + +% Return a tuple of string to ease the printing of keys to logs +print_key(Key) -> + case Key of + {o, B, K, _SK} -> + {"Object", B, K}; + {i, B, {F, _V}, _K} -> + {"Index", B, F} + end. + +% Compare a key against a query key, only comparing elements that are non-null +% in the Query key. This is used for comparing against end keys in queries. +endkey_passed({EK1, null, null, null}, {CK1, _, _, _}) -> + EK1 < CK1; +endkey_passed({EK1, EK2, null, null}, {CK1, CK2, _, _}) -> + {EK1, EK2} < {CK1, CK2}; +endkey_passed({EK1, EK2, EK3, null}, {CK1, CK2, CK3, _}) -> + {EK1, EK2, EK3} < {CK1, CK2, CK3}; +endkey_passed(EndKey, CheckingKey) -> + EndKey < CheckingKey. + + + +%%%============================================================================ +%%% Test +%%%============================================================================ + +-ifdef(TEST). + + +indexspecs_test() -> + IndexSpecs = [{add, "t1_int", 456}, + {add, "t1_bin", "adbc123"}, + {remove, "t1_bin", "abdc456"}], + Changes = convert_indexspecs(IndexSpecs, 1, {o, "Bucket", "Key2", null}), + ?assertMatch({{i, "Bucket", {"t1_int", 456}, "Key2"}, + {1, {active, infinity}, null}}, lists:nth(1, Changes)), + ?assertMatch({{i, "Bucket", {"t1_bin", "adbc123"}, "Key2"}, + {1, {active, infinity}, null}}, lists:nth(2, Changes)), + ?assertMatch({{i, "Bucket", {"t1_bin", "abdc456"}, "Key2"}, + {1, {tomb, infinity}, null}}, lists:nth(3, Changes)). + +-endif. \ No newline at end of file diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 413205b..32d97ee 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -370,7 +370,7 @@ find_randomkeys(_FList, 0, _Source) -> ok; find_randomkeys(FList, Count, Source) -> KV1 = lists:nth(random:uniform(length(Source)), Source), - K1 = leveled_bookie:strip_to_keyonly(KV1), + K1 = leveled_codec:strip_to_keyonly(KV1), P1 = choose_pid_toquery(FList, K1), FoundKV = leveled_sft:sft_get(P1, K1), Check = case FoundKV of @@ -378,7 +378,7 @@ find_randomkeys(FList, Count, Source) -> io:format("Failed to find ~w in ~w~n", [K1, P1]), error; _ -> - Found = leveled_bookie:strip_to_keyonly(FoundKV), + Found = leveled_codec:strip_to_keyonly(FoundKV), io:format("success finding ~w in ~w~n", [K1, P1]), ?assertMatch(K1, Found), ok diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index b785802..b1399ce 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -843,7 +843,7 @@ compare_to_sqn(Obj, SQN) -> not_present -> false; Obj -> - SQNToCompare = leveled_bookie:strip_to_seqonly(Obj), + SQNToCompare = leveled_codec:strip_to_seqonly(Obj), if SQNToCompare > SQN -> false; @@ -936,7 +936,7 @@ roll_new_tree(Tree, [], HighSQN) -> roll_new_tree(Tree, [{SQN, KVList}|TailIncs], HighSQN) when SQN >= HighSQN -> R = lists:foldl(fun({Kx, Vx}, {TreeAcc, MaxSQN}) -> UpdTree = gb_trees:enter(Kx, Vx, TreeAcc), - SQNx = leveled_bookie:strip_to_seqonly({Kx, Vx}), + SQNx = leveled_codec:strip_to_seqonly({Kx, Vx}), {UpdTree, max(SQNx, MaxSQN)} end, {Tree, HighSQN}, @@ -1025,8 +1025,8 @@ print_manifest(Manifest) -> lists:seq(0, ?MAX_LEVELS - 1)). print_manifest_entry(Entry) -> - {S1, S2, S3} = leveled_bookie:print_key(Entry#manifest_entry.start_key), - {E1, E2, E3} = leveled_bookie:print_key(Entry#manifest_entry.end_key), + {S1, S2, S3} = leveled_codec:print_key(Entry#manifest_entry.start_key), + {E1, E2, E3} = leveled_codec:print_key(Entry#manifest_entry.end_key), io:format("Manifest entry of " ++ "startkey ~s ~s ~s " ++ "endkey ~s ~s ~s " ++ @@ -1036,12 +1036,9 @@ print_manifest_entry(Entry) -> initiate_rangequery_frommanifest(StartKey, EndKey, Manifest) -> CompareFun = fun(M) -> - C1 = leveled_bookie:key_compare(StartKey, - M#manifest_entry.end_key, - gt), - C2 = leveled_bookie:key_compare(EndKey, - M#manifest_entry.start_key, - lt), + C1 = StartKey > M#manifest_entry.end_key, + C2 = leveled_codec:endkey_passed(EndKey, + M#manifest_entry.start_key), not (C1 or C2) end, lists:foldl(fun(L, AccL) -> Level = get_item(L, Manifest, []), @@ -1143,8 +1140,8 @@ find_nextkey(QueryArray, LCnt, {BestKeyLevel, BestKV}, QueryFunT) -> {LCnt, {Key, Val}}, QueryFunT); {{Key, Val}, BKL, {BestKey, BestVal}} when Key == BestKey -> - SQN = leveled_bookie:strip_to_seqonly({Key, Val}), - BestSQN = leveled_bookie:strip_to_seqonly({BestKey, BestVal}), + SQN = leveled_codec:strip_to_seqonly({Key, Val}), + BestSQN = leveled_codec:strip_to_seqonly({BestKey, BestVal}), if SQN =< BestSQN -> % This is a dominated key, so we need to skip over it @@ -1193,7 +1190,7 @@ keyfolder(IMMiterator, SFTiterator, StartKey, EndKey, {AccFun, Acc}) -> % iterate only over the remaining keys in the SFT iterator keyfolder(null, SFTiterator, StartKey, EndKey, {AccFun, Acc}); {IMMKey, IMMVal, NxtIMMiterator} -> - case leveled_bookie:key_compare(EndKey, IMMKey, lt) of + case leveled_codec:endkey_passed(EndKey, IMMKey) of true -> % There are no more keys in-range in the in-memory % iterator, so take action as if this iterator is empty @@ -1212,7 +1209,7 @@ keyfolder(IMMiterator, SFTiterator, StartKey, EndKey, {AccFun, Acc}) -> % There is a next key, so need to know which is the % next key between the two (and handle two keys % with different sequence numbers). - case leveled_bookie:key_dominates({IMMKey, + case leveled_codec:key_dominates({IMMKey, IMMVal}, {SFTKey, SFTVal}) of @@ -1377,7 +1374,7 @@ assess_sqn(DumpList) -> assess_sqn([], MinSQN, MaxSQN) -> {MinSQN, MaxSQN}; assess_sqn([HeadKey|Tail], MinSQN, MaxSQN) -> - {_K, SQN} = leveled_bookie:strip_to_keyseqonly(HeadKey), + {_K, SQN} = leveled_codec:strip_to_keyseqonly(HeadKey), assess_sqn(Tail, min(MinSQN, SQN), max(MaxSQN, SQN)). @@ -1763,7 +1760,7 @@ foldwithimm_simple_test() -> {9, {active, infinity}, null}, IMM1), IMMiter = gb_trees:iterator_from({o, "Bucket1", "Key1"}, IMM2), - AccFun = fun(K, V, Acc) -> SQN= leveled_bookie:strip_to_seqonly({K, V}), + AccFun = fun(K, V, Acc) -> SQN = leveled_codec:strip_to_seqonly({K, V}), Acc ++ [{K, SQN}] end, Acc = keyfolder(IMMiter, QueryArray, diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index dda8607..6cead92 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -563,7 +563,7 @@ acc_list_keysonly(null, empty) -> acc_list_keysonly(null, RList) -> RList; acc_list_keysonly(R, RList) -> - lists:append(RList, [leveled_bookie:strip_to_keyseqstatusonly(R)]). + lists:append(RList, [leveled_codec:strip_to_keyseqstatusonly(R)]). acc_list_kv(null, empty) -> []; @@ -652,10 +652,8 @@ fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, FunList, scan_block([], StartKey, _EndKey, _FunList, _AccFun, Acc) -> {partial, Acc, StartKey}; scan_block([HeadKV|T], StartKey, EndKey, FunList, AccFun, Acc) -> - K = leveled_bookie:strip_to_keyonly(HeadKV), - Pre = leveled_bookie:key_compare(StartKey, K, gt), - Post = leveled_bookie:key_compare(EndKey, K, lt), - case {Pre, Post} of + K = leveled_codec:strip_to_keyonly(HeadKV), + case {StartKey > K, leveled_codec:endkey_passed(EndKey, K)} of {true, _} when StartKey /= all -> scan_block(T, StartKey, EndKey, FunList, AccFun, Acc); {_, true} when EndKey /= all -> @@ -988,15 +986,15 @@ create_slot(KL1, KL2, Level, BlockCount, SegLists, SerialisedSlot, LengthList, TrackingMetadata = case LowKey of null -> [NewLowKeyV|_] = BlockKeyList, - {leveled_bookie:strip_to_keyonly(NewLowKeyV), + {leveled_codec:strip_to_keyonly(NewLowKeyV), min(LSN, LSNb), max(HSN, HSNb), - leveled_bookie:strip_to_keyonly(last(BlockKeyList, + leveled_codec:strip_to_keyonly(last(BlockKeyList, {last, LastKey})), Status}; _ -> {LowKey, min(LSN, LSNb), max(HSN, HSNb), - leveled_bookie:strip_to_keyonly(last(BlockKeyList, + leveled_codec:strip_to_keyonly(last(BlockKeyList, {last, LastKey})), Status} end, @@ -1035,7 +1033,7 @@ key_dominates(KL1, KL2, Level) -> Level). key_dominates_expanded([H1|T1], [], Level) -> - St1 = leveled_bookie:strip_to_statusonly(H1), + St1 = leveled_codec:strip_to_statusonly(H1), case maybe_reap_expiredkey(St1, Level) of true -> {skipped_key, maybe_expand_pointer(T1), []}; @@ -1043,7 +1041,7 @@ key_dominates_expanded([H1|T1], [], Level) -> {{next_key, H1}, maybe_expand_pointer(T1), []} end; key_dominates_expanded([], [H2|T2], Level) -> - St2 = leveled_bookie:strip_to_statusonly(H2), + St2 = leveled_codec:strip_to_statusonly(H2), case maybe_reap_expiredkey(St2, Level) of true -> {skipped_key, [], maybe_expand_pointer(T2)}; @@ -1052,8 +1050,8 @@ key_dominates_expanded([], [H2|T2], Level) -> end; key_dominates_expanded([H1|T1], [H2|T2], Level) -> {{K1, V1}, {K2, V2}} = {H1, H2}, - {Sq1, St1, _MD1} = leveled_bookie:striphead_to_details(V1), - {Sq2, St2, _MD2} = leveled_bookie:striphead_to_details(V2), + {Sq1, St1, _MD1} = leveled_codec:striphead_to_details(V1), + {Sq2, St2, _MD2} = leveled_codec:striphead_to_details(V2), case K1 of K2 -> case Sq1 > Sq2 of @@ -1116,7 +1114,7 @@ pointer_append_queryresults(Results, QueryPid) -> %% Update the sequence numbers update_sequencenumbers(Item, LSN, HSN) when is_tuple(Item) -> - update_sequencenumbers(leveled_bookie:strip_to_seqonly(Item), LSN, HSN); + update_sequencenumbers(leveled_codec:strip_to_seqonly(Item), LSN, HSN); update_sequencenumbers(SN, 0, 0) -> {SN, SN}; update_sequencenumbers(SN, LSN, HSN) when SN < LSN -> @@ -1227,7 +1225,7 @@ merge_seglists({SegList1, SegList2, SegList3, SegList4}) -> lists:sort(Stage4). hash_for_segmentid(KV) -> - erlang:phash2(leveled_bookie:strip_to_keyonly(KV), ?MAX_SEG_HASH). + erlang:phash2(leveled_codec:strip_to_keyonly(KV), ?MAX_SEG_HASH). %% Check for a given list of segments in the filter, returning in normal From 9be0f964061fb588dccc400caf06ab0d7102822a Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 14 Oct 2016 13:36:12 +0100 Subject: [PATCH 065/167] Or process calculation of the Hash Table When the journal CDB file is called to roll it now starts a new clerk to perform the hashtable calculation (which may take many seconds). This stops the store from getting blocked if there is an attempt to GET from the journal that has just been rolled. The journal file process now has anumber fo distinct states (reading, writing, pending_roll, closing). A future refactor may look to make leveled_cdb a gen_fsm rather than a gen_server. --- src/leveled_cdb.erl | 151 ++++++++++++++++++++++++----------------- src/leveled_iclerk.erl | 9 +++ 2 files changed, 98 insertions(+), 62 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 9b9af5a..e49138a 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -67,8 +67,10 @@ cdb_close/1, cdb_complete/1, cdb_roll/1, + cdb_returnhashtable/3, cdb_destroy/1, - cdb_deletepending/1]). + cdb_deletepending/1, + hashtable_calc/2]). -include_lib("eunit/include/eunit.hrl"). @@ -79,6 +81,7 @@ -define(BINARY_MODE, false). -define(BASE_POSITION, 2048). -define(WRITE_OPS, [binary, raw, read, write]). +-define(PENDING_ROLL_WAIT, 30). -record(state, {hashtree, last_position :: integer(), @@ -88,6 +91,7 @@ handle :: file:fd(), writer :: boolean(), max_size :: integer(), + pending_roll = false :: boolean(), pending_delete = false :: boolean(), binary_mode = false :: boolean()}). @@ -135,7 +139,21 @@ cdb_directfetch(Pid, PositionList, Info) -> gen_server:call(Pid, {direct_fetch, PositionList, Info}, infinity). cdb_close(Pid) -> - gen_server:call(Pid, cdb_close, infinity). + cdb_close(Pid, ?PENDING_ROLL_WAIT). + +cdb_close(Pid, WaitsLeft) -> + if + WaitsLeft > 0 -> + case gen_server:call(Pid, cdb_close, infinity) of + pending_roll -> + timer:sleep(1), + cdb_close(Pid, WaitsLeft - 1); + R -> + R + end; + true -> + gen_server:call(Pid, cdb_kill, infinity) + end. cdb_complete(Pid) -> gen_server:call(Pid, cdb_complete, infinity). @@ -143,6 +161,9 @@ cdb_complete(Pid) -> cdb_roll(Pid) -> gen_server:cast(Pid, cdb_roll). +cdb_returnhashtable(Pid, IndexList, HashTreeBin) -> + gen_server:call(Pid, {return_hashtable, IndexList, HashTreeBin}, infinity). + cdb_destroy(Pid) -> gen_server:cast(Pid, destroy). @@ -210,46 +231,36 @@ handle_call({open_reader, Filename}, _From, State) -> writer=false, hash_index=Index}}; handle_call({get_kv, Key}, _From, State) -> - case {State#state.writer, State#state.hash_index} of - {true, _} -> + case State#state.writer of + true -> {reply, get_mem(Key, State#state.handle, State#state.hashtree), State}; - {false, []} -> + false -> {reply, - get(State#state.handle, Key), - State}; - {false, Cache} -> - {reply, - get_withcache(State#state.handle, Key, Cache), + get_withcache(State#state.handle, Key, State#state.hash_index), State} end; handle_call({key_check, Key}, _From, State) -> - case {State#state.writer, State#state.hash_index} of - {true, _} -> + case State#state.writer of + true -> {reply, get_mem(Key, State#state.handle, State#state.hashtree, loose_presence), State}; - {false, []} -> - {reply, - get(State#state.handle, - Key, - loose_presence), - State}; - {false, Cache} -> + false -> {reply, get(State#state.handle, Key, loose_presence, - Cache), + State#state.hash_index), State} end; handle_call({put_kv, Key, Value}, _From, State) -> - case State#state.writer of - true -> + case {State#state.writer, State#state.pending_roll} of + {true, false} -> Result = put(State#state.handle, Key, Value, {State#state.last_position, State#state.hashtree}, @@ -265,7 +276,7 @@ handle_call({put_kv, Key, Value}, _From, State) -> last_key=Key, hashtree=HashTree}} end; - false -> + _ -> {reply, {error, read_only}, State} @@ -334,9 +345,14 @@ handle_call({cdb_scan, FilterFun, Acc, StartPos}, _From, State) -> empty -> {reply, {eof, Acc}, State} end; +handle_call(cdb_close, _From, State=#state{pending_roll=RollPending}) + when RollPending == true -> + {reply, pending_roll, State}; handle_call(cdb_close, _From, State) -> ok = file:close(State#state.handle), {stop, normal, ok, State#state{handle=undefined}}; +handle_call(cdb_kill, _From, State) -> + {stop, killed, ok, State}; handle_call(cdb_complete, _From, State=#state{writer=Writer}) when Writer == true -> NewName = determine_new_filename(State#state.filename), @@ -347,7 +363,25 @@ handle_call(cdb_complete, _From, State=#state{writer=Writer}) {stop, normal, {ok, NewName}, State}; handle_call(cdb_complete, _From, State) -> ok = file:close(State#state.handle), - {stop, normal, {ok, State#state.filename}, State}. + {stop, normal, {ok, State#state.filename}, State}; +handle_call({return_hashtable, IndexList, HashTreeBin}, + _From, + State=#state{pending_roll=RollPending}) when RollPending == true -> + Handle = State#state.handle, + {ok, BasePos} = file:position(Handle, State#state.last_position), + NewName = determine_new_filename(State#state.filename), + ok = perform_write_hash_tables(Handle, HashTreeBin, BasePos), + ok = write_top_index_table(Handle, BasePos, IndexList), + file:close(Handle), + ok = rename_for_read(State#state.filename, NewName), + io:format("Opening file for reading with filename ~s~n", [NewName]), + {NewHandle, Index, LastKey} = open_for_readonly(NewName), + {reply, ok, State#state{handle=NewHandle, + last_key=LastKey, + filename=NewName, + writer=false, + pending_roll=false, + hash_index=Index}}. handle_cast(destroy, State) -> @@ -355,22 +389,12 @@ handle_cast(destroy, State) -> ok = file:delete(State#state.filename), {noreply, State}; handle_cast(delete_pending, State) -> - {noreply, State#state{pending_delete = true}}; + {noreply, State#state{pending_delete=true}}; handle_cast(cdb_roll, State=#state{writer=Writer}) when Writer == true -> - NewName = determine_new_filename(State#state.filename), - ok = close_file(State#state.handle, - State#state.hashtree, - State#state.last_position), - ok = rename_for_read(State#state.filename, NewName), - io:format("Opening file for reading with filename ~s~n", [NewName]), - {Handle, Index, LastKey} = open_for_readonly(NewName), - {noreply, State#state{handle=Handle, - last_key=LastKey, - filename=NewName, - writer=false, - hash_index=Index}}; -handle_cast(_Msg, State) -> - {noreply, State}. + ok = leveled_iclerk:clerk_hashtablecalc(State#state.hashtree, + State#state.last_position, + self()), + {noreply, State#state{pending_roll=true}}. handle_info(_Info, State) -> {noreply, State}. @@ -670,6 +694,17 @@ fold_keys(Handle, FoldFun, Acc0) -> {FirstHashPosition, _} = read_next_2_integers(Handle), fold(Handle, FoldFun, Acc0, {256 * ?DWORD_SIZE, FirstHashPosition}, true). +hashtable_calc(HashTree, StartPos) -> + Seq = lists:seq(0, 255), + SWC = os:timestamp(), + {IndexList, HashTreeBin} = write_hash_tables(Seq, + HashTree, + StartPos, + [], + <<>>), + io:format("HashTree computed in ~w microseconds~n", + [timer:now_diff(os:timestamp(), SWC)]), + {IndexList, HashTreeBin}. %%%%%%%%%%%%%%%%%%%% %% Internal functions @@ -756,18 +791,11 @@ scan_index_returnpositions(Handle, Position, Count, PosList0) -> %% the hash tables close_file(Handle, HashTree, BasePos) -> {ok, BasePos} = file:position(Handle, BasePos), - SW1 = os:timestamp(), - L2 = write_hash_tables(Handle, HashTree), - SW2 = os:timestamp(), - io:format("Hash Table write took ~w microseconds~n", - [timer:now_diff(SW2, SW1)]), - write_top_index_table(Handle, BasePos, L2), - io:format("Top Index Table write took ~w microseconds~n", - [timer:now_diff(os:timestamp(),SW2)]), + IndexList = write_hash_tables(Handle, HashTree), + ok = write_top_index_table(Handle, BasePos, IndexList), file:close(Handle). - %% Fetch a list of positions by passing a key to the HashTree get_hashtree(Key, HashTree) -> Hash = hash(Key), @@ -897,7 +925,6 @@ scan_over_file(Handle, Position, FilterFun, Output, LastKey) -> check_last_key(LastKey) -> case LastKey of - undefined -> error; empty -> empty; _ -> ok end. @@ -1096,23 +1123,20 @@ write_key_value_pairs(Handle, [HeadPair|TailList], Acc) -> %% corresponding to a key and the second word is a file pointer to the %% corresponding {key,value} tuple. write_hash_tables(Handle, HashTree) -> - Seq = lists:seq(0, 255), {ok, StartPos} = file:position(Handle, cur), - SWC = os:timestamp(), - {IndexList, HashTreeBin} = write_hash_tables(Seq, - HashTree, - StartPos, - [], - <<>>), - io:format("HashTree computed in ~w microseconds~n", - [timer:now_diff(os:timestamp(), SWC)]), + {IndexList, HashTreeBin} = hashtable_calc(HashTree, StartPos), + ok = perform_write_hash_tables(Handle, HashTreeBin, StartPos), + IndexList. + +perform_write_hash_tables(Handle, HashTreeBin, StartPos) -> SWW = os:timestamp(), ok = file:write(Handle, HashTreeBin), - io:format("HashTree written in ~w microseconds~n", - [timer:now_diff(os:timestamp(), SWW)]), {ok, EndPos} = file:position(Handle, cur), ok = file:advise(Handle, StartPos, EndPos - StartPos, will_need), - IndexList. + io:format("HashTree written in ~w microseconds~n", + [timer:now_diff(os:timestamp(), SWW)]), + ok. + write_hash_tables([], _HashTree, _CurrPos, IndexList, HashTreeBin) -> {IndexList, HashTreeBin}; @@ -1217,7 +1241,8 @@ write_top_index_table(Handle, BasePos, List) -> CompleteList), {ok, _} = file:position(Handle, 0), ok = file:write(Handle, IndexBin), - ok = file:advise(Handle, 0, ?DWORD_SIZE * 256, will_need). + ok = file:advise(Handle, 0, ?DWORD_SIZE * 256, will_need), + ok. %% To make this compatible with original Bernstein format this endian flip %% and also the use of the standard hash function required. @@ -1622,9 +1647,11 @@ find_lastkey_test() -> ok = cdb_put(P1, "Key2", "Value2"), ?assertMatch("Key2", cdb_lastkey(P1)), ?assertMatch("Key1", cdb_firstkey(P1)), + probably = cdb_keycheck(P1, "Key2"), ok = cdb_close(P1), {ok, P2} = cdb_open_writer("../test/lastkey.pnd"), ?assertMatch("Key2", cdb_lastkey(P2)), + probably = cdb_keycheck(P2, "Key2"), {ok, F2} = cdb_complete(P2), {ok, P3} = cdb_open_reader(F2), ?assertMatch("Key2", cdb_lastkey(P3)), diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 33e5e9b..8b58c4b 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -13,6 +13,7 @@ terminate/2, clerk_new/1, clerk_compact/6, + clerk_hashtablecalc/3, clerk_stop/1, code_change/3]). @@ -56,6 +57,10 @@ clerk_compact(Pid, Checker, InitiateFun, FilterFun, Inker, Timeout) -> Inker, Timeout}). +clerk_hashtablecalc(HashTree, StartPos, CDBpid) -> + {ok, Clerk} = gen_server:start(?MODULE, [#iclerk_options{}], []), + gen_server:cast(Clerk, {hashtable_calc, HashTree, StartPos, CDBpid}). + clerk_stop(Pid) -> gen_server:cast(Pid, stop). @@ -129,6 +134,10 @@ handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, ok = leveled_inker:ink_compactioncomplete(Inker), {noreply, State} end; +handle_cast({hashtable_calc, HashTree, StartPos, CDBpid}, State) -> + {IndexList, HashTreeBin} = leveled_cdb:hashtable_calc(HashTree, StartPos), + ok = leveled_cdb:cdb_returnhashtable(CDBpid, IndexList, HashTreeBin), + {stop, normal, State}; handle_cast(stop, State) -> {stop, normal, State}. From 7eb5a16899498fee926b4ebf2b16ff9dcedb410e Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 14 Oct 2016 18:43:16 +0100 Subject: [PATCH 066/167] Supporting Tags - Improving abstraction between Riak and non-Riak workloads The object tag "o" which was taken from eleveldb has been an extended to allow for specific functions to be triggered for different object types, in particular when extracting metadata for stroing in the Ledger. There is now a riak tag (o_rkv@v1), and in theory other tags can be added and used, as long as their is an appropriate set of functions in the leveled_codec. --- include/leveled.hrl | 4 +- src/leveled_bookie.erl | 97 +++++++++++------- src/leveled_cdb.erl | 5 +- src/leveled_codec.erl | 176 ++++++++++++++++++++++---------- src/leveled_sft.erl | 2 +- test/end_to_end/basic_SUITE.erl | 2 +- 6 files changed, 191 insertions(+), 95 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index cd82d2a..e421500 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -51,8 +51,6 @@ max_run_length :: integer(), cdb_options :: #cdb_options{}}). -%% Temp location for records related to riak - -record(r_content, { metadata, value :: term() @@ -65,4 +63,4 @@ vclock, updatemetadata=dict:store(clean, true, dict:new()), updatevalue :: term()}). - \ No newline at end of file + \ No newline at end of file diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index ecf5f81..28b628d 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -71,11 +71,11 @@ %% The Bookie should generate a series of ledger key changes from this %% information, using a function passed in at startup. For Riak this will be %% of the form: -%% {{o, Bucket, Key, SubKey|null}, +%% {{o_rkv@v1, Bucket, Key, SubKey|null}, %% SQN, %% {Hash, Size, {Riak_Metadata}}, %% {active, TS}|{tomb, TS}} or -%% {{i, Bucket, IndexTerm, IndexField, Key}, +%% {{i, Bucket, {IndexTerm, IndexField}, Key}, %% SQN, %% null, %% {active, TS}|{tomb, TS}} @@ -136,6 +136,9 @@ book_riakput/3, book_riakget/3, book_riakhead/3, + book_put/5, + book_get/3, + book_head/3, book_returnfolder/2, book_snapshotstore/3, book_snapshotledger/3, @@ -167,17 +170,33 @@ book_start(Opts) -> gen_server:start(?MODULE, [Opts], []). -book_riakput(Pid, Object, IndexSpecs) -> - PrimaryKey = {o, Object#r_object.bucket, Object#r_object.key, null}, - gen_server:call(Pid, {put, PrimaryKey, Object, IndexSpecs}, infinity). +book_riakput(Pid, RiakObject, IndexSpecs) -> + {Bucket, Key} = leveled_codec:riakto_keydetails(RiakObject), + book_put(Pid, Bucket, Key, RiakObject, IndexSpecs, o_rkv@v1). + +book_put(Pid, Bucket, Key, Object, IndexSpecs) -> + book_put(Pid, Bucket, Key, Object, IndexSpecs, o). book_riakget(Pid, Bucket, Key) -> - PrimaryKey = {o, Bucket, Key, null}, - gen_server:call(Pid, {get, PrimaryKey}, infinity). + book_get(Pid, Bucket, Key, o_rkv@v1). + +book_get(Pid, Bucket, Key) -> + book_get(Pid, Bucket, Key, o). book_riakhead(Pid, Bucket, Key) -> - PrimaryKey = {o, Bucket, Key, null}, - gen_server:call(Pid, {head, PrimaryKey}, infinity). + book_head(Pid, Bucket, Key, o_rkv@v1). + +book_head(Pid, Bucket, Key) -> + book_head(Pid, Bucket, Key, o). + +book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag) -> + gen_server:call(Pid, {put, Bucket, Key, Object, IndexSpecs, Tag}, infinity). + +book_get(Pid, Bucket, Key, Tag) -> + gen_server:call(Pid, {get, Bucket, Key, Tag}, infinity). + +book_head(Pid, Bucket, Key, Tag) -> + gen_server:call(Pid, {head, Bucket, Key, Tag}, infinity). book_returnfolder(Pid, FolderType) -> gen_server:call(Pid, {return_folder, FolderType}, infinity). @@ -231,12 +250,13 @@ init([Opts]) -> end. -handle_call({put, PrimaryKey, Object, IndexSpecs}, From, State) -> +handle_call({put, Bucket, Key, Object, IndexSpecs, Tag}, From, State) -> + LedgerKey = leveled_codec:to_ledgerkey(Bucket, Key, Tag), {ok, SQN, ObjSize} = leveled_inker:ink_put(State#state.inker, - PrimaryKey, + LedgerKey, Object, IndexSpecs), - Changes = preparefor_ledgercache(PrimaryKey, + Changes = preparefor_ledgercache(LedgerKey, SQN, Object, ObjSize, @@ -251,8 +271,11 @@ handle_call({put, PrimaryKey, Object, IndexSpecs}, From, State) -> {pause, NewCache} -> {noreply, State#state{ledger_cache=NewCache, back_pressure=true}} end; -handle_call({get, Key}, _From, State) -> - case fetch_head(Key, State#state.penciller, State#state.ledger_cache) of +handle_call({get, Bucket, Key, Tag}, _From, State) -> + LedgerKey = leveled_codec:to_ledgerkey(Bucket, Key, Tag), + case fetch_head(LedgerKey, + State#state.penciller, + State#state.ledger_cache) of not_present -> {reply, not_found, State}; Head -> @@ -261,7 +284,7 @@ handle_call({get, Key}, _From, State) -> {tomb, _} -> {reply, not_found, State}; {active, _} -> - case fetch_value(Key, Seqn, State#state.inker) of + case fetch_value(LedgerKey, Seqn, State#state.inker) of not_present -> {reply, not_found, State}; Object -> @@ -269,8 +292,11 @@ handle_call({get, Key}, _From, State) -> end end end; -handle_call({head, Key}, _From, State) -> - case fetch_head(Key, State#state.penciller, State#state.ledger_cache) of +handle_call({head, Bucket, Key, Tag}, _From, State) -> + LedgerKey = leveled_codec:to_ledgerkey(Bucket, Key, Tag), + case fetch_head(LedgerKey, + State#state.penciller, + State#state.ledger_cache) of not_present -> {reply, not_found, State}; Head -> @@ -279,7 +305,7 @@ handle_call({head, Key}, _From, State) -> {tomb, _} -> {reply, not_found, State}; {active, _} -> - OMD = leveled_codec:build_metadata_object(Key, MD), + OMD = leveled_codec:build_metadata_object(LedgerKey, MD), {reply, {ok, OMD}, State} end end; @@ -308,11 +334,12 @@ handle_call({snapshot, _Requestor, SnapType, _Timeout}, _From, State) -> end; handle_call({return_folder, FolderType}, _From, State) -> case FolderType of - {bucket_stats, Bucket} -> + {riakbucket_stats, Bucket} -> {reply, bucket_stats(State#state.penciller, State#state.ledger_cache, - Bucket), + Bucket, + o_rkv@v1), State} end; handle_call({compact_journal, Timeout}, _From, State) -> @@ -351,7 +378,7 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================ -bucket_stats(Penciller, LedgerCache, Bucket) -> +bucket_stats(Penciller, LedgerCache, Bucket, Tag) -> PCLopts = #penciller_options{start_snapshot=true, source_penciller=Penciller}, {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), @@ -361,8 +388,8 @@ bucket_stats(Penciller, LedgerCache, Bucket) -> [length(Increment)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, {infinity, Increment}), - StartKey = {o, Bucket, null, null}, - EndKey = {o, Bucket, null, null}, + StartKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), + EndKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, StartKey, EndKey, @@ -433,21 +460,16 @@ fetch_value(Key, SQN, Inker) -> not_present end. - -accumulate_size(_Key, Value, {Size, Count}) -> - {_, _, MD} = Value, - {_, _, _, ObjSize} = MD, - {Size + ObjSize, Count + 1}. - - +accumulate_size(Key, Value, {Size, Count}) -> + {Size + leveled_codec:get_size(Key, Value), Count + 1}. preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs) -> - PrimaryChange = {PK, - {SQN, - {active, infinity}, - leveled_codec:extract_metadata(Obj, Size)}}, - SecChanges = leveled_codec:convert_indexspecs(IndexSpecs, SQN, PK), - [PrimaryChange] ++ SecChanges. + {Bucket, Key, PrimaryChange} = leveled_codec:generate_ledgerkv(PK, + SQN, + Obj, + Size), + ConvSpecs = leveled_codec:convert_indexspecs(IndexSpecs, Bucket, Key, SQN), + [PrimaryChange] ++ ConvSpecs. addto_ledgercache(Changes, Cache) -> lists:foldl(fun({K, V}, Acc) -> gb_trees:enter(K, V, Acc) end, @@ -511,6 +533,9 @@ reset_filestructure() -> leveled_penciller:clean_testdir(RootPath ++ "/" ++ ?LEDGER_FP), RootPath. + + + generate_multiple_objects(Count, KeyNumber) -> generate_multiple_objects(Count, KeyNumber, []). diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index e49138a..d368896 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -1655,7 +1655,10 @@ find_lastkey_test() -> {ok, F2} = cdb_complete(P2), {ok, P3} = cdb_open_reader(F2), ?assertMatch("Key2", cdb_lastkey(P3)), - ok = cdb_close(P3), + {ok, _FN} = cdb_complete(P3), + {ok, P4} = cdb_open_reader(F2), + ?assertMatch("Key2", cdb_lastkey(P4)), + ok = cdb_close(P4), ok = file:delete("../test/lastkey.cdb"). get_keys_byposition_simple_test() -> diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 6ae0f6a..5a384e8 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -1,15 +1,35 @@ %% -------- Key Codec --------- %% -%% Functions for manipulating keys and values within leveled. These are -%% currently static functions, they cannot be overridden in the store other -%% than by changing them here. The formats are focused on the problem of -%% supporting Riak KV +%% Functions for manipulating keys and values within leveled. +%% +%% +%% Within the LEDGER: +%% Keys are of the form - +%% {Tag, Bucket, Key, SubKey|null} +%% Values are of the form +%% {SQN, Status, MD} +%% +%% Within the JOURNAL: +%% Keys are of the form - +%% {SQN, LedgerKey} +%% Values are of the form +%% {Object, IndexSpecs} (as a binary) +%% +%% IndexSpecs are of the form of a Ledger Key/Value +%% +%% Tags need to be set during PUT operations and each Tag used must be +%% supported in an extract_metadata and a build_metadata_object function clause +%% +%% Currently the only tags supported are: +%% - o (standard objects) +%% - o_rkv@v1 (riak objects) +%% - i (index entries) + -module(leveled_codec). -include("../include/leveled.hrl"). - -include_lib("eunit/include/eunit.hrl"). -export([strip_to_keyonly/1, @@ -21,15 +41,20 @@ endkey_passed/2, key_dominates/2, print_key/1, - extract_metadata/2, + to_ledgerkey/3, build_metadata_object/2, - convert_indexspecs/3]). - + generate_ledgerkv/4, + generate_ledgerkv/5, + get_size/2, + convert_indexspecs/4, + riakto_keydetails/1]). + + strip_to_keyonly({keyonly, K}) -> K; strip_to_keyonly({K, _V}) -> K. -strip_to_keyseqonly({K, {SeqN, _, _}}) -> {K, SeqN}. +strip_to_keyseqonly({K, {SeqN, _, _ }}) -> {K, SeqN}. strip_to_keyseqstatusonly({K, {SeqN, St, _MD}}) -> {K, SeqN, St}. @@ -53,56 +78,19 @@ key_dominates(LeftKey, RightKey) -> right_hand_dominant end. +to_ledgerkey(Bucket, Key, Tag) -> + {Tag, Bucket, Key, null}. - -get_metadatas(#r_object{contents=Contents}) -> - [Content#r_content.metadata || Content <- Contents]. - -set_vclock(Object=#r_object{}, VClock) -> Object#r_object{vclock=VClock}. - -vclock(#r_object{vclock=VClock}) -> VClock. - -to_binary(v0, Obj) -> - term_to_binary(Obj). - -hash(Obj=#r_object{}) -> - Vclock = vclock(Obj), - UpdObj = set_vclock(Obj, lists:sort(Vclock)), - erlang:phash2(to_binary(v0, UpdObj)). - -extract_metadata(Obj, Size) -> - {get_metadatas(Obj), vclock(Obj), hash(Obj), Size}. - - -build_metadata_object(PrimaryKey, Head) -> - {o, Bucket, Key, null} = PrimaryKey, - {MD, VC, _, _} = Head, - Contents = lists:foldl(fun(X, Acc) -> Acc ++ [#r_content{metadata=X}] end, - [], - MD), - #r_object{contents=Contents, bucket=Bucket, key=Key, vclock=VC}. - -convert_indexspecs(IndexSpecs, SQN, PrimaryKey) -> - lists:map(fun({IndexOp, IndexField, IndexValue}) -> - Status = case IndexOp of - add -> - %% TODO: timestamp support - {active, infinity}; - remove -> - %% TODO: timestamps for delayed reaping - {tomb, infinity} - end, - {o, B, K, _SK} = PrimaryKey, - {{i, B, {IndexField, IndexValue}, K}, - {SQN, Status, null}} - end, - IndexSpecs). +hash(Obj) -> + erlang:phash2(term_to_binary(Obj)). % Return a tuple of string to ease the printing of keys to logs print_key(Key) -> case Key of {o, B, K, _SK} -> {"Object", B, K}; + {o_rkv@v1, B, K, _SK} -> + {"RiakObject", B, K}; {i, B, {F, _V}, _K} -> {"Index", B, F} end. @@ -118,6 +106,88 @@ endkey_passed({EK1, EK2, EK3, null}, {CK1, CK2, CK3, _}) -> endkey_passed(EndKey, CheckingKey) -> EndKey < CheckingKey. +convert_indexspecs(IndexSpecs, Bucket, Key, SQN) -> + lists:map(fun({IndexOp, IndexField, IndexValue}) -> + Status = case IndexOp of + add -> + %% TODO: timestamp support + {active, infinity}; + remove -> + %% TODO: timestamps for delayed reaping + {tomb, infinity} + end, + {{i, Bucket, {IndexField, IndexValue}, Key}, + {SQN, Status, null}} + end, + IndexSpecs). + +generate_ledgerkv(PrimaryKey, SQN, Obj, Size) -> + generate_ledgerkv(PrimaryKey, SQN, Obj, Size, infinity). + +generate_ledgerkv(PrimaryKey, SQN, Obj, Size, TS) -> + {Tag, Bucket, Key, _} = PrimaryKey, + {Bucket, + Key, + {PrimaryKey, {SQN, {active, TS}, extract_metadata(Obj, Size, Tag)}}}. + + +extract_metadata(Obj, Size, o_rkv@v1) -> + riak_extract_metadata(Obj, Size); +extract_metadata(Obj, Size, o) -> + {hash(Obj), Size}. + +get_size(PK, Value) -> + {Tag, _Bucket, _Key, _} = PK, + {_, _, MD} = Value, + case Tag of + o_rkv@v1 -> + {_RMD, _VC, _Hash, Size} = MD, + Size; + o -> + {_Hash, Size} = MD, + Size + end. + + +build_metadata_object(PrimaryKey, MD) -> + {Tag, Bucket, Key, null} = PrimaryKey, + case Tag of + o_rkv@v1 -> + riak_metadata_object(Bucket, Key, MD); + o -> + MD + end. + + + + +riak_metadata_object(Bucket, Key, MD) -> + {RMD, VC, _Hash, _Size} = MD, + Contents = lists:foldl(fun(X, Acc) -> Acc ++ [#r_content{metadata=X}] end, + [], + RMD), + #r_object{contents=Contents, bucket=Bucket, key=Key, vclock=VC}. + +riak_extract_metadata(Obj, Size) -> + {get_metadatas(Obj), vclock(Obj), riak_hash(Obj), Size}. + +riak_hash(Obj=#r_object{}) -> + Vclock = vclock(Obj), + UpdObj = set_vclock(Obj, lists:sort(Vclock)), + erlang:phash2(term_to_binary(UpdObj)). + +riakto_keydetails(Object) -> + {Object#r_object.bucket, Object#r_object.key}. + +get_metadatas(#r_object{contents=Contents}) -> + [Content#r_content.metadata || Content <- Contents]. + +set_vclock(Object=#r_object{}, VClock) -> Object#r_object{vclock=VClock}. + +vclock(#r_object{vclock=VClock}) -> VClock. + + + %%%============================================================================ @@ -131,7 +201,7 @@ indexspecs_test() -> IndexSpecs = [{add, "t1_int", 456}, {add, "t1_bin", "adbc123"}, {remove, "t1_bin", "abdc456"}], - Changes = convert_indexspecs(IndexSpecs, 1, {o, "Bucket", "Key2", null}), + Changes = convert_indexspecs(IndexSpecs, "Bucket", "Key2", 1), ?assertMatch({{i, "Bucket", {"t1_int", 456}, "Key2"}, {1, {active, infinity}, null}}, lists:nth(1, Changes)), ?assertMatch({{i, "Bucket", {"t1_bin", "adbc123"}, "Key2"}, diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 6cead92..95a3f63 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -14,7 +14,7 @@ %% %% All keys are not equal in sft files, keys are only expected in a specific %% series of formats -%% - {o, Bucket, Key, SubKey|null} - Object Keys +%% - {Tag, Bucket, Key, SubKey|null} - Object Keys %% - {i, Bucket, {IndexName, IndexTerm}, Key} - Postings %% The {Bucket, Key} part of all types of keys are hashed for segment filters. %% For Postings the {Bucket, IndexName, IndexTerm} is also hashed. This diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 2f30975..40ad320 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -300,7 +300,7 @@ check_bucket_stats(Bookie, Bucket) -> FoldSW1 = os:timestamp(), io:format("Checking bucket size~n"), {async, Folder1} = leveled_bookie:book_returnfolder(Bookie, - {bucket_stats, + {riakbucket_stats, Bucket}), {B1Size, B1Count} = Folder1(), io:format("Bucket fold completed in ~w microseconds~n", From ed17e44f523cd6ceba77deee80937528526d19aa Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 14 Oct 2016 22:58:01 +0100 Subject: [PATCH 067/167] Improve test coverage Some additional tests following previous refactoring for abstraction, primarily to make manifest print safer an dprove co-existence of Riak and non-Riak objects. --- src/leveled_codec.erl | 46 ++++++++++++++++++++++++++------- src/leveled_penciller.erl | 24 ++++++++++++----- test/end_to_end/basic_SUITE.erl | 17 +++++++++++- 3 files changed, 71 insertions(+), 16 deletions(-) diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 5a384e8..9d9a6cf 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -84,16 +84,32 @@ to_ledgerkey(Bucket, Key, Tag) -> hash(Obj) -> erlang:phash2(term_to_binary(Obj)). -% Return a tuple of string to ease the printing of keys to logs +% Return a tuple of strings to ease the printing of keys to logs print_key(Key) -> - case Key of - {o, B, K, _SK} -> - {"Object", B, K}; - {o_rkv@v1, B, K, _SK} -> - {"RiakObject", B, K}; - {i, B, {F, _V}, _K} -> - {"Index", B, F} + {A_STR, B_TERM, C_TERM} = case Key of + {o, B, K, _SK} -> + {"Object", B, K}; + {o_rkv@v1, B, K, _SK} -> + {"RiakObject", B, K}; + {i, B, {F, _V}, _K} -> + {"Index", B, F} + end, + {B_STR, FB} = check_for_string(B_TERM), + {C_STR, FC} = check_for_string(C_TERM), + {A_STR, B_STR, C_STR, FB, FC}. + +check_for_string(Item) -> + if + is_binary(Item) == true -> + {binary_to_list(Item), "~s"}; + is_integer(Item) == true -> + {integer_to_list(Item), "~s"}; + is_list(Item) == true -> + {Item, "~s"}; + true -> + {Item, "~w"} end. + % Compare a key against a query key, only comparing elements that are non-null % in the Query key. This is used for comparing against end keys in queries. @@ -208,5 +224,17 @@ indexspecs_test() -> {1, {active, infinity}, null}}, lists:nth(2, Changes)), ?assertMatch({{i, "Bucket", {"t1_bin", "abdc456"}, "Key2"}, {1, {tomb, infinity}, null}}, lists:nth(3, Changes)). - + +endkey_passed_test() -> + TestKey = {i, null, null, null}, + K1 = {i, 123, {"a", "b"}, <<>>}, + K2 = {o, 123, {"a", "b"}, <<>>}, + ?assertMatch(false, endkey_passed(TestKey, K1)), + ?assertMatch(true, endkey_passed(TestKey, K2)). + +stringcheck_test() -> + ?assertMatch({"Bucket", "~s"}, check_for_string("Bucket")), + ?assertMatch({"Bucket", "~s"}, check_for_string(<<"Bucket">>)), + ?assertMatch({bucket, "~w"}, check_for_string(bucket)). + -endif. \ No newline at end of file diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index b1399ce..0ce0dd0 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -1022,15 +1022,18 @@ print_manifest(Manifest) -> end end, Level) end, - lists:seq(0, ?MAX_LEVELS - 1)). + lists:seq(0, ?MAX_LEVELS - 1)), + ok. print_manifest_entry(Entry) -> - {S1, S2, S3} = leveled_codec:print_key(Entry#manifest_entry.start_key), - {E1, E2, E3} = leveled_codec:print_key(Entry#manifest_entry.end_key), + {S1, S2, S3, + FS2, FS3} = leveled_codec:print_key(Entry#manifest_entry.start_key), + {E1, E2, E3, + FE2, FE3} = leveled_codec:print_key(Entry#manifest_entry.end_key), io:format("Manifest entry of " ++ - "startkey ~s ~s ~s " ++ - "endkey ~s ~s ~s " ++ - "filename=~s~n", + "startkey ~s " ++ FS2 ++ " " ++ FS3 ++ + " endkey ~s " ++ FE2 ++ " " ++ FE3 ++ + " filename=~s~n", [S1, S2, S3, E1, E2, E3, Entry#manifest_entry.filename]). @@ -1667,6 +1670,15 @@ rangequery_manifest_test() -> Man), ?assertMatch([], R3). +print_manifest_test() -> + M1 = #manifest_entry{start_key={i, "Bucket1", {<<"Idx1">>, "Fld1"}, "K8"}, + end_key={i, 4565, {"Idx1", "Fld9"}, "K93"}, + filename="Z1"}, + M2 = #manifest_entry{start_key={i, self(), {null, "Fld1"}, "K8"}, + end_key={i, <<200:32/integer>>, {"Idx1", "Fld9"}, "K93"}, + filename="Z1"}, + ?assertMatch(ok, print_manifest([{1, [M1, M2]}])). + simple_findnextkey_test() -> QueryArray = [ {2, [{{o, "Bucket1", "Key1"}, {5, {active, infinity}, null}}, diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 40ad320..fa0e54d 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -12,7 +12,8 @@ all() -> [simple_put_fetch_head, many_put_fetch_head, journal_compaction, fetchput_snapshot, - load_and_count]. + load_and_count + ]. simple_put_fetch_head(_Config) -> @@ -36,7 +37,21 @@ simple_put_fetch_head(_Config) -> check_bookie_forlist(Bookie2, ChkList1), check_bookie_forobject(Bookie2, TestObject), check_bookie_formissingobject(Bookie2, "Bucket1", "Key2"), + ok = leveled_bookie:book_put(Bookie2, "Bucket1", "Key2", "Value2", + [{add, "Index1", "Term1"}]), + {ok, "Value2"} = leveled_bookie:book_get(Bookie2, "Bucket1", "Key2"), + {ok, {62888926, 43}} = leveled_bookie:book_head(Bookie2, + "Bucket1", + "Key2"), + check_bookie_formissingobject(Bookie2, "Bucket1", "Key2"), + ok = leveled_bookie:book_put(Bookie2, "Bucket1", "Key2", <<"Value2">>, + [{remove, "Index1", "Term1"}, + {add, "Index1", <<"Term2">>}]), + {ok, <<"Value2">>} = leveled_bookie:book_get(Bookie2, "Bucket1", "Key2"), ok = leveled_bookie:book_close(Bookie2), + {ok, Bookie3} = leveled_bookie:book_start(StartOpts2), + {ok, <<"Value2">>} = leveled_bookie:book_get(Bookie3, "Bucket1", "Key2"), + ok = leveled_bookie:book_close(Bookie3), reset_filestructure(). many_put_fetch_head(_Config) -> From e3ce372f31f8ac44eabe90887e29c52dc180dce6 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sun, 16 Oct 2016 15:41:09 +0100 Subject: [PATCH 068/167] Delete Add functionality to delete keys. No tombstone reaping yet. --- include/leveled.hrl | 4 ++ src/leveled_bookie.erl | 35 +++++++++++------ src/leveled_codec.erl | 43 ++++++++++++++------ src/leveled_sft.erl | 2 - test/end_to_end/basic_SUITE.erl | 69 ++++++++++++++++++++++++++++++--- 5 files changed, 122 insertions(+), 31 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index e421500..481b8db 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -1,4 +1,8 @@ +-define(RIAK_TAG, o_rkv). +-define(STD_TAG, o). +-define(IDX_TAG, i). + -record(sft_options, {wait = true :: boolean(), expire_tombstones = false :: boolean()}). diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 28b628d..72a6966 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -71,7 +71,7 @@ %% The Bookie should generate a series of ledger key changes from this %% information, using a function passed in at startup. For Riak this will be %% of the form: -%% {{o_rkv@v1, Bucket, Key, SubKey|null}, +%% {{o_rkv, Bucket, Key, SubKey|null}, %% SQN, %% {Hash, Size, {Riak_Metadata}}, %% {active, TS}|{tomb, TS}} or @@ -134,9 +134,11 @@ code_change/3, book_start/1, book_riakput/3, + book_riakdelete/4, book_riakget/3, book_riakhead/3, book_put/5, + book_delete/4, book_get/3, book_head/3, book_returnfolder/2, @@ -172,22 +174,28 @@ book_start(Opts) -> book_riakput(Pid, RiakObject, IndexSpecs) -> {Bucket, Key} = leveled_codec:riakto_keydetails(RiakObject), - book_put(Pid, Bucket, Key, RiakObject, IndexSpecs, o_rkv@v1). + book_put(Pid, Bucket, Key, RiakObject, IndexSpecs, ?RIAK_TAG). book_put(Pid, Bucket, Key, Object, IndexSpecs) -> - book_put(Pid, Bucket, Key, Object, IndexSpecs, o). + book_put(Pid, Bucket, Key, Object, IndexSpecs, ?STD_TAG). + +book_riakdelete(Pid, Bucket, Key, IndexSpecs) -> + book_put(Pid, Bucket, Key, delete, IndexSpecs, ?RIAK_TAG). + +book_delete(Pid, Bucket, Key, IndexSpecs) -> + book_put(Pid, Bucket, Key, delete, IndexSpecs, ?STD_TAG). book_riakget(Pid, Bucket, Key) -> - book_get(Pid, Bucket, Key, o_rkv@v1). + book_get(Pid, Bucket, Key, ?RIAK_TAG). book_get(Pid, Bucket, Key) -> - book_get(Pid, Bucket, Key, o). + book_get(Pid, Bucket, Key, ?STD_TAG). book_riakhead(Pid, Bucket, Key) -> - book_head(Pid, Bucket, Key, o_rkv@v1). + book_head(Pid, Bucket, Key, ?RIAK_TAG). book_head(Pid, Bucket, Key) -> - book_head(Pid, Bucket, Key, o). + book_head(Pid, Bucket, Key, ?STD_TAG). book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag) -> gen_server:call(Pid, {put, Bucket, Key, Object, IndexSpecs, Tag}, infinity). @@ -281,7 +289,7 @@ handle_call({get, Bucket, Key, Tag}, _From, State) -> Head -> {Seqn, Status, _MD} = leveled_codec:striphead_to_details(Head), case Status of - {tomb, _} -> + tomb -> {reply, not_found, State}; {active, _} -> case fetch_value(LedgerKey, Seqn, State#state.inker) of @@ -302,7 +310,7 @@ handle_call({head, Bucket, Key, Tag}, _From, State) -> Head -> {_Seqn, Status, MD} = leveled_codec:striphead_to_details(Head), case Status of - {tomb, _} -> + tomb -> {reply, not_found, State}; {active, _} -> OMD = leveled_codec:build_metadata_object(LedgerKey, MD), @@ -339,7 +347,7 @@ handle_call({return_folder, FolderType}, _From, State) -> bucket_stats(State#state.penciller, State#state.ledger_cache, Bucket, - o_rkv@v1), + ?RIAK_TAG), State} end; handle_call({compact_journal, Timeout}, _From, State) -> @@ -461,7 +469,12 @@ fetch_value(Key, SQN, Inker) -> end. accumulate_size(Key, Value, {Size, Count}) -> - {Size + leveled_codec:get_size(Key, Value), Count + 1}. + case leveled_codec:is_active(Key, Value) of + true -> + {Size + leveled_codec:get_size(Key, Value), Count + 1}; + false -> + {Size, Count} + end. preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs) -> {Bucket, Key, PrimaryChange} = leveled_codec:generate_ledgerkv(PK, diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 9d9a6cf..b8170c6 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -22,7 +22,7 @@ %% %% Currently the only tags supported are: %% - o (standard objects) -%% - o_rkv@v1 (riak objects) +%% - o_rkv (riak objects) %% - i (index entries) @@ -38,6 +38,7 @@ strip_to_statusonly/1, strip_to_keyseqstatusonly/1, striphead_to_details/1, + is_active/2, endkey_passed/2, key_dominates/2, print_key/1, @@ -77,7 +78,15 @@ key_dominates(LeftKey, RightKey) -> when LK == RK, LSN < RSN -> right_hand_dominant end. - + +is_active(Key, Value) -> + case strip_to_statusonly({Key, Value}) of + {active, infinity} -> + true; + tomb -> + false + end. + to_ledgerkey(Bucket, Key, Tag) -> {Tag, Bucket, Key, null}. @@ -87,11 +96,11 @@ hash(Obj) -> % Return a tuple of strings to ease the printing of keys to logs print_key(Key) -> {A_STR, B_TERM, C_TERM} = case Key of - {o, B, K, _SK} -> + {?STD_TAG, B, K, _SK} -> {"Object", B, K}; - {o_rkv@v1, B, K, _SK} -> + {?RIAK_TAG, B, K, _SK} -> {"RiakObject", B, K}; - {i, B, {F, _V}, _K} -> + {?IDX_TAG, B, {F, _V}, _K} -> {"Index", B, F} end, {B_STR, FB} = check_for_string(B_TERM), @@ -142,24 +151,32 @@ generate_ledgerkv(PrimaryKey, SQN, Obj, Size) -> generate_ledgerkv(PrimaryKey, SQN, Obj, Size, TS) -> {Tag, Bucket, Key, _} = PrimaryKey, + Status = case Obj of + delete -> + tomb; + _ -> + {active, TS} + end, {Bucket, Key, - {PrimaryKey, {SQN, {active, TS}, extract_metadata(Obj, Size, Tag)}}}. + {PrimaryKey, {SQN, Status, extract_metadata(Obj, Size, Tag)}}}. -extract_metadata(Obj, Size, o_rkv@v1) -> + + +extract_metadata(Obj, Size, ?RIAK_TAG) -> riak_extract_metadata(Obj, Size); -extract_metadata(Obj, Size, o) -> +extract_metadata(Obj, Size, ?STD_TAG) -> {hash(Obj), Size}. get_size(PK, Value) -> {Tag, _Bucket, _Key, _} = PK, {_, _, MD} = Value, case Tag of - o_rkv@v1 -> + ?RIAK_TAG -> {_RMD, _VC, _Hash, Size} = MD, Size; - o -> + ?STD_TAG -> {_Hash, Size} = MD, Size end. @@ -168,9 +185,9 @@ get_size(PK, Value) -> build_metadata_object(PrimaryKey, MD) -> {Tag, Bucket, Key, null} = PrimaryKey, case Tag of - o_rkv@v1 -> + ?RIAK_TAG -> riak_metadata_object(Bucket, Key, MD); - o -> + ?STD_TAG -> MD end. @@ -184,6 +201,8 @@ riak_metadata_object(Bucket, Key, MD) -> RMD), #r_object{contents=Contents, bucket=Bucket, key=Key, vclock=VC}. +riak_extract_metadata(delete, Size) -> + {delete, null, null, Size}; riak_extract_metadata(Obj, Size) -> {get_metadatas(Obj), vclock(Obj), riak_hash(Obj), Size}. diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 95a3f63..6df0b02 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -712,8 +712,6 @@ get_nearestkey(KVList, all) -> end; get_nearestkey(KVList, Key) -> case Key of - {first, K} -> - get_firstkeytomatch(KVList, K, not_found); {next, K} -> get_nextkeyaftermatch(KVList, K, not_found); _ -> diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index fa0e54d..8f8f8f1 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -2,21 +2,23 @@ -include_lib("common_test/include/ct.hrl"). -include("../include/leveled.hrl"). -export([all/0]). --export([simple_put_fetch_head/1, +-export([simple_put_fetch_head_delete/1, many_put_fetch_head/1, journal_compaction/1, fetchput_snapshot/1, - load_and_count/1]). + load_and_count/1, + load_and_count_withdelete/1]). -all() -> [simple_put_fetch_head, +all() -> [simple_put_fetch_head_delete, many_put_fetch_head, journal_compaction, fetchput_snapshot, - load_and_count + load_and_count, + load_and_count_withdelete ]. -simple_put_fetch_head(_Config) -> +simple_put_fetch_head_delete(_Config) -> RootPath = reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), @@ -51,7 +53,13 @@ simple_put_fetch_head(_Config) -> ok = leveled_bookie:book_close(Bookie2), {ok, Bookie3} = leveled_bookie:book_start(StartOpts2), {ok, <<"Value2">>} = leveled_bookie:book_get(Bookie3, "Bucket1", "Key2"), + ok = leveled_bookie:book_delete(Bookie3, "Bucket1", "Key2", + [{remove, "Index1", "Term1"}]), + not_found = leveled_bookie:book_get(Bookie3, "Bucket1", "Key2"), ok = leveled_bookie:book_close(Bookie3), + {ok, Bookie4} = leveled_bookie:book_start(StartOpts2), + not_found = leveled_bookie:book_get(Bookie4, "Bucket1", "Key2"), + ok = leveled_bookie:book_close(Bookie4), reset_filestructure(). many_put_fetch_head(_Config) -> @@ -300,6 +308,50 @@ load_and_count(_Config) -> ok = leveled_bookie:book_close(Bookie2), reset_filestructure(). +load_and_count_withdelete(_Config) -> + RootPath = reset_filestructure(), + StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=50000000}, + {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), + {TestObject, TestSpec} = generate_testobject(), + ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), + check_bookie_forobject(Bookie1, TestObject), + io:format("Loading initial small objects~n"), + lists:foldl(fun(_X, Acc) -> + load_objects(5000, [Acc + 2], Bookie1, TestObject, + fun generate_multiple_smallobjects/2), + {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + if + Acc + 5000 == Count -> + ok + end, + Acc + 5000 end, + 0, + lists:seq(1, 20)), + check_bookie_forobject(Bookie1, TestObject), + {BucketD, KeyD} = leveled_codec:riakto_keydetails(TestObject), + {_, 1} = check_bucket_stats(Bookie1, BucketD), + ok = leveled_bookie:book_riakdelete(Bookie1, BucketD, KeyD, []), + not_found = leveled_bookie:book_riakget(Bookie1, BucketD, KeyD), + {_, 0} = check_bucket_stats(Bookie1, BucketD), + io:format("Loading larger compressible objects~n"), + lists:foldl(fun(_X, Acc) -> + load_objects(5000, [Acc + 2], Bookie1, no_check, + fun generate_multiple_compressibleobjects/2), + {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + if + Acc + 5000 == Count -> + ok + end, + Acc + 5000 end, + 100000, + lists:seq(1, 20)), + not_found = leveled_bookie:book_riakget(Bookie1, BucketD, KeyD), + ok = leveled_bookie:book_close(Bookie1), + {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), + check_bookie_formissingobject(Bookie2, BucketD, KeyD), + {_BSize, 0} = check_bucket_stats(Bookie2, BucketD), + ok = leveled_bookie:book_close(Bookie2). + reset_filestructure() -> RootPath = "test", @@ -445,6 +497,11 @@ load_objects(ChunkSize, GenList, Bookie, TestObject, Generator) -> Time = timer:now_diff(os:timestamp(), StartWatchA), io:format("~w objects loaded in ~w seconds~n", [ChunkSize, Time/1000000]), - check_bookie_forobject(Bookie, TestObject), + if + TestObject == no_check -> + ok; + true -> + check_bookie_forobject(Bookie, TestObject) + end, lists:sublist(ObjListA, 1000) end, GenList). From 8653e9d90d4ed47e30a27a1da23440268d21d558 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sun, 16 Oct 2016 16:58:55 +0100 Subject: [PATCH 069/167] Improve inker unit test Change in filename labelling had stopped a unit test from covering stratup correctly. Now offering better coverage --- src/leveled_inker.erl | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 946a118..55bb71b 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -299,10 +299,7 @@ handle_call({close, Force}, _From, State) -> {reply, pause, State}; _ -> {stop, normal, ok, State} - end; -handle_call(Msg, _From, State) -> - io:format("Unexpected message ~w~n", [Msg]), - {reply, error, State}. + end. handle_cast(_Msg, State) -> {noreply, State}. @@ -814,10 +811,10 @@ simple_inker_completeactivejournal_test() -> RootPath = "../test/journal", build_dummy_journal(), CDBopts = #cdb_options{max_size=300000}, - {ok, PidW} = leveled_cdb:cdb_open_writer(filepath(RootPath, - 3, - new_journal)), - {ok, _FN} = leveled_cdb:cdb_complete(PidW), + JournalFP = filepath(RootPath, journal_dir), + F2 = filename:join(JournalFP, "nursery_3.pnd"), + {ok, PidW} = leveled_cdb:cdb_open_writer(F2), + {ok, _F2} = leveled_cdb:cdb_complete(PidW), {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, cdb_options=CDBopts}), Obj1 = ink_get(Ink1, "Key1", 1), From 59ea46120e08675232e08c22abc5f49458cb4980 Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Mon, 17 Oct 2016 14:24:32 +0100 Subject: [PATCH 070/167] Fix include target --- test/end_to_end/basic_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 8f8f8f1..cd5704f 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -1,6 +1,6 @@ -module(basic_SUITE). -include_lib("common_test/include/ct.hrl"). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -export([all/0]). -export([simple_put_fetch_head_delete/1, many_put_fetch_head/1, From 3e475f46e8a5bd9732fa79e6b23caa55383aa007 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 18 Oct 2016 01:59:03 +0100 Subject: [PATCH 071/167] Support for 2i query part1 Added basic support for 2i query. This involved some refactoring of the test code to share functions between suites. There is sill a need for a Part 2 as no tests currently cover removal of index entries. --- src/leveled_bookie.erl | 89 +++++- src/leveled_cdb.erl | 2 +- src/leveled_codec.erl | 26 +- src/leveled_iclerk.erl | 2 +- src/leveled_inker.erl | 15 +- src/leveled_pclerk.erl | 2 +- src/leveled_penciller.erl | 2 +- src/leveled_sft.erl | 2 +- test/end_to_end/basic_SUITE.erl | 416 ++++++++++------------------- test/end_to_end/iterator_SUITE.erl | 182 +++++++++++++ test/end_to_end/testutil.erl | 232 ++++++++++++++++ 11 files changed, 682 insertions(+), 288 deletions(-) create mode 100644 test/end_to_end/iterator_SUITE.erl create mode 100644 test/end_to_end/testutil.erl diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 72a6966..c607606 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -124,7 +124,7 @@ -behaviour(gen_server). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -export([init/1, handle_call/3, @@ -348,6 +348,17 @@ handle_call({return_folder, FolderType}, _From, State) -> State#state.ledger_cache, Bucket, ?RIAK_TAG), + State}; + {index_query, + Bucket, + {IdxField, StartValue, EndValue}, + {ReturnTerms, TermRegex}} -> + {reply, + index_query(State#state.penciller, + State#state.ledger_cache, + Bucket, + {IdxField, StartValue, EndValue}, + {ReturnTerms, TermRegex}), State} end; handle_call({compact_journal, Timeout}, _From, State) -> @@ -408,6 +419,41 @@ bucket_stats(Penciller, LedgerCache, Bucket, Tag) -> end, {async, Folder}. +index_query(Penciller, LedgerCache, + Bucket, + {IdxField, StartValue, EndValue}, + {ReturnTerms, TermRegex}) -> + PCLopts = #penciller_options{start_snapshot=true, + source_penciller=Penciller}, + {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), + Folder = fun() -> + Increment = gb_trees:to_list(LedgerCache), + io:format("Length of increment in snapshot is ~w~n", + [length(Increment)]), + ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, + {infinity, Increment}), + StartKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, + IdxField, StartValue), + EndKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, + IdxField, EndValue), + AddFun = case ReturnTerms of + true -> + fun add_terms/3; + _ -> + fun add_keys/3 + end, + AccFun = accumulate_index(TermRegex, AddFun), + Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, + StartKey, + EndKey, + AccFun, + []), + ok = leveled_penciller:pcl_close(LedgerSnapshot), + Acc + end, + {async, Folder}. + + shutdown_wait([], _Inker) -> false; shutdown_wait([TopPause|Rest], Inker) -> @@ -476,6 +522,47 @@ accumulate_size(Key, Value, {Size, Count}) -> {Size, Count} end. + +add_keys(ObjKey, _IdxValue, Acc) -> + Acc ++ [ObjKey]. + +add_terms(ObjKey, IdxValue, Acc) -> + Acc ++ [{IdxValue, ObjKey}]. + +accumulate_index(TermRe, AddFun) -> + case TermRe of + undefined -> + fun(Key, Value, Acc) -> + case leveled_codec:is_active(Key, Value) of + true -> + {_Bucket, + ObjKey, + IdxValue} = leveled_codec:from_ledgerkey(Key), + AddFun(ObjKey, IdxValue, Acc); + false -> + Acc + end end; + TermRe -> + fun(Key, Value, Acc) -> + case leveled_codec:is_active(Key, Value) of + true -> + {_Bucket, + ObjKey, + IdxValue} = leveled_codec:from_ledgerkey(Key), + case re:run(IdxValue, TermRe) of + nomatch -> + Acc; + _ -> + AddFun(ObjKey, IdxValue, Acc) + end; + false -> + Acc + end end + end. + + + + preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs) -> {Bucket, Key, PrimaryChange} = leveled_codec:generate_ledgerkv(PK, SQN, diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index d368896..94d3a2f 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -44,7 +44,7 @@ -module(leveled_cdb). -behaviour(gen_server). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -export([init/1, handle_call/3, diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index b8170c6..370133c 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -28,7 +28,7 @@ -module(leveled_codec). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -43,14 +43,24 @@ key_dominates/2, print_key/1, to_ledgerkey/3, + to_ledgerkey/5, + from_ledgerkey/1, build_metadata_object/2, generate_ledgerkv/4, generate_ledgerkv/5, get_size/2, convert_indexspecs/4, - riakto_keydetails/1]). + riakto_keydetails/1, + generate_uuid/0]). +%% Credit to +%% https://github.com/afiskon/erlang-uuid-v4/blob/master/src/uuid.erl +generate_uuid() -> + <> = crypto:rand_bytes(16), + io_lib:format("~8.16.0b-~4.16.0b-4~3.16.0b-~4.16.0b-~12.16.0b", + [A, B, C band 16#0fff, D band 16#3fff bor 16#8000, E]). + strip_to_keyonly({keyonly, K}) -> K; strip_to_keyonly({K, _V}) -> K. @@ -87,6 +97,13 @@ is_active(Key, Value) -> false end. +from_ledgerkey({Tag, Bucket, {_IdxField, IdxValue}, Key}) + when Tag == ?IDX_TAG -> + {Bucket, Key, IdxValue}. + +to_ledgerkey(Bucket, Key, Tag, Field, Value) when Tag == ?IDX_TAG -> + {?IDX_TAG, Bucket, {Field, Value}, Key}. + to_ledgerkey(Bucket, Key, Tag) -> {Tag, Bucket, Key, null}. @@ -132,7 +149,7 @@ endkey_passed(EndKey, CheckingKey) -> EndKey < CheckingKey. convert_indexspecs(IndexSpecs, Bucket, Key, SQN) -> - lists:map(fun({IndexOp, IndexField, IndexValue}) -> + lists:map(fun({IndexOp, IdxField, IdxValue}) -> Status = case IndexOp of add -> %% TODO: timestamp support @@ -141,7 +158,8 @@ convert_indexspecs(IndexSpecs, Bucket, Key, SQN) -> %% TODO: timestamps for delayed reaping {tomb, infinity} end, - {{i, Bucket, {IndexField, IndexValue}, Key}, + {to_ledgerkey(Bucket, Key, ?IDX_TAG, + IdxField, IdxValue), {SQN, Status, null}} end, IndexSpecs). diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 8b58c4b..ca1a6f3 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -4,7 +4,7 @@ -behaviour(gen_server). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -export([init/1, handle_call/3, diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 55bb71b..a876fa8 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -76,7 +76,7 @@ -behaviour(gen_server). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -export([init/1, handle_call/3, @@ -669,20 +669,14 @@ filepath(RootPath, journal_compact_dir) -> filepath(RootPath, NewSQN, new_journal) -> filename:join(filepath(RootPath, journal_dir), integer_to_list(NewSQN) ++ "_" - ++ generate_uuid() + ++ leveled_codec:generate_uuid() ++ "." ++ ?PENDING_FILEX); filepath(CompactFilePath, NewSQN, compact_journal) -> filename:join(CompactFilePath, integer_to_list(NewSQN) ++ "_" - ++ generate_uuid() + ++ leveled_codec:generate_uuid() ++ "." ++ ?PENDING_FILEX). -%% Credit to -%% https://github.com/afiskon/erlang-uuid-v4/blob/master/src/uuid.erl -generate_uuid() -> - <> = crypto:rand_bytes(16), - io_lib:format("~8.16.0b-~4.16.0b-4~3.16.0b-~4.16.0b-~12.16.0b", - [A, B, C band 16#0fff, D band 16#3fff bor 16#8000, E]). simple_manifest_reader(SQN, RootPath) -> ManifestPath = filepath(RootPath, manifest_dir), @@ -815,6 +809,9 @@ simple_inker_completeactivejournal_test() -> F2 = filename:join(JournalFP, "nursery_3.pnd"), {ok, PidW} = leveled_cdb:cdb_open_writer(F2), {ok, _F2} = leveled_cdb:cdb_complete(PidW), + F1 = filename:join(JournalFP, "nursery_1.cdb"), + F1r = filename:join(JournalFP, "nursery_1.pnd"), + ok = file:rename(F1, F1r), {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, cdb_options=CDBopts}), Obj1 = ink_get(Ink1, "Key1", 1), diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 32d97ee..7ec6144 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -52,7 +52,7 @@ -behaviour(gen_server). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -export([init/1, handle_call/3, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 0ce0dd0..34a99a4 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -223,7 +223,7 @@ -behaviour(gen_server). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -export([init/1, handle_call/3, diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 6df0b02..000dd45 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -143,7 +143,7 @@ -module(leveled_sft). -behaviour(gen_server). --include("../include/leveled.hrl"). +-include("include/leveled.hrl"). -export([init/1, handle_call/3, diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index cd5704f..bf5a700 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -7,7 +7,8 @@ journal_compaction/1, fetchput_snapshot/1, load_and_count/1, - load_and_count_withdelete/1]). + load_and_count_withdelete/1 + ]). all() -> [simple_put_fetch_head_delete, many_put_fetch_head, @@ -19,33 +20,33 @@ all() -> [simple_put_fetch_head_delete, simple_put_fetch_head_delete(_Config) -> - RootPath = reset_filestructure(), + RootPath = testutil:reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {TestObject, TestSpec} = generate_testobject(), + {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), - check_bookie_forobject(Bookie1, TestObject), - check_bookie_formissingobject(Bookie1, "Bucket1", "Key2"), + testutil:check_forobject(Bookie1, TestObject), + testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"), ok = leveled_bookie:book_close(Bookie1), StartOpts2 = #bookie_options{root_path=RootPath, max_journalsize=3000000}, {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), - check_bookie_forobject(Bookie2, TestObject), - ObjList1 = generate_multiple_objects(5000, 2), + testutil:check_forobject(Bookie2, TestObject), + ObjList1 = testutil:generate_objects(5000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, ObjList1), ChkList1 = lists:sublist(lists:sort(ObjList1), 100), - check_bookie_forlist(Bookie2, ChkList1), - check_bookie_forobject(Bookie2, TestObject), - check_bookie_formissingobject(Bookie2, "Bucket1", "Key2"), + testutil:check_forlist(Bookie2, ChkList1), + testutil:check_forobject(Bookie2, TestObject), + testutil:check_formissingobject(Bookie2, "Bucket1", "Key2"), ok = leveled_bookie:book_put(Bookie2, "Bucket1", "Key2", "Value2", [{add, "Index1", "Term1"}]), {ok, "Value2"} = leveled_bookie:book_get(Bookie2, "Bucket1", "Key2"), {ok, {62888926, 43}} = leveled_bookie:book_head(Bookie2, "Bucket1", "Key2"), - check_bookie_formissingobject(Bookie2, "Bucket1", "Key2"), + testutil:check_formissingobject(Bookie2, "Bucket1", "Key2"), ok = leveled_bookie:book_put(Bookie2, "Bucket1", "Key2", <<"Value2">>, [{remove, "Index1", "Term1"}, {add, "Index1", <<"Term2">>}]), @@ -60,110 +61,111 @@ simple_put_fetch_head_delete(_Config) -> {ok, Bookie4} = leveled_bookie:book_start(StartOpts2), not_found = leveled_bookie:book_get(Bookie4, "Bucket1", "Key2"), ok = leveled_bookie:book_close(Bookie4), - reset_filestructure(). + testutil:reset_filestructure(). many_put_fetch_head(_Config) -> - RootPath = reset_filestructure(), + RootPath = testutil:reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {TestObject, TestSpec} = generate_testobject(), + {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), ok = leveled_bookie:book_close(Bookie1), StartOpts2 = #bookie_options{root_path=RootPath, max_journalsize=1000000000}, {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), - check_bookie_forobject(Bookie2, TestObject), + testutil:check_forobject(Bookie2, TestObject), GenList = [2, 20002, 40002, 60002, 80002, 100002, 120002, 140002, 160002, 180002], - CLs = load_objects(20000, GenList, Bookie2, TestObject, - fun generate_multiple_smallobjects/2), + CLs = testutil:load_objects(20000, GenList, Bookie2, TestObject, + fun testutil:generate_smallobjects/2), CL1A = lists:nth(1, CLs), ChkListFixed = lists:nth(length(CLs), CLs), - check_bookie_forlist(Bookie2, CL1A), - ObjList2A = generate_multiple_objects(5000, 2), + testutil:check_forlist(Bookie2, CL1A), + ObjList2A = testutil:generate_objects(5000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, ObjList2A), ChkList2A = lists:sublist(lists:sort(ObjList2A), 1000), - check_bookie_forlist(Bookie2, ChkList2A), - check_bookie_forlist(Bookie2, ChkListFixed), - check_bookie_forobject(Bookie2, TestObject), - check_bookie_forlist(Bookie2, ChkList2A), - check_bookie_forlist(Bookie2, ChkListFixed), - check_bookie_forobject(Bookie2, TestObject), + testutil:check_forlist(Bookie2, ChkList2A), + testutil:check_forlist(Bookie2, ChkListFixed), + testutil:check_forobject(Bookie2, TestObject), + testutil:check_forlist(Bookie2, ChkList2A), + testutil:check_forlist(Bookie2, ChkListFixed), + testutil:check_forobject(Bookie2, TestObject), ok = leveled_bookie:book_close(Bookie2), {ok, Bookie3} = leveled_bookie:book_start(StartOpts2), - check_bookie_forlist(Bookie3, ChkList2A), - check_bookie_forobject(Bookie3, TestObject), + testutil:check_forlist(Bookie3, ChkList2A), + testutil:check_forobject(Bookie3, TestObject), ok = leveled_bookie:book_close(Bookie3), - reset_filestructure(). + testutil:reset_filestructure(). journal_compaction(_Config) -> - RootPath = reset_filestructure(), + RootPath = testutil:reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=4000000}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {TestObject, TestSpec} = generate_testobject(), + {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), - check_bookie_forobject(Bookie1, TestObject), - ObjList1 = generate_multiple_objects(5000, 2), + testutil:check_forobject(Bookie1, TestObject), + ObjList1 = testutil:generate_objects(5000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjList1), ChkList1 = lists:sublist(lists:sort(ObjList1), 1000), - check_bookie_forlist(Bookie1, ChkList1), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forlist(Bookie1, ChkList1), + testutil:check_forobject(Bookie1, TestObject), {B2, K2, V2, Spec2, MD} = {"Bucket1", "Key1", "Value1", [], {"MDK1", "MDV1"}}, - {TestObject2, TestSpec2} = generate_testobject(B2, K2, V2, Spec2, MD), + {TestObject2, TestSpec2} = testutil:generate_testobject(B2, K2, + V2, Spec2, MD), ok = leveled_bookie:book_riakput(Bookie1, TestObject2, TestSpec2), ok = leveled_bookie:book_compactjournal(Bookie1, 30000), - check_bookie_forlist(Bookie1, ChkList1), - check_bookie_forobject(Bookie1, TestObject), - check_bookie_forobject(Bookie1, TestObject2), + testutil:check_forlist(Bookie1, ChkList1), + testutil:check_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject2), timer:sleep(5000), % Allow for compaction to complete io:format("Has journal completed?~n"), - check_bookie_forlist(Bookie1, ChkList1), - check_bookie_forobject(Bookie1, TestObject), - check_bookie_forobject(Bookie1, TestObject2), + testutil:check_forlist(Bookie1, ChkList1), + testutil:check_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject2), %% Now replace all the objects - ObjList2 = generate_multiple_objects(5000, 2), + ObjList2 = testutil:generate_objects(5000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjList2), ok = leveled_bookie:book_compactjournal(Bookie1, 30000), ChkList3 = lists:sublist(lists:sort(ObjList2), 500), - check_bookie_forlist(Bookie1, ChkList3), + testutil:check_forlist(Bookie1, ChkList3), ok = leveled_bookie:book_close(Bookie1), % Restart {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), - check_bookie_forobject(Bookie2, TestObject), - check_bookie_forlist(Bookie2, ChkList3), + testutil:check_forobject(Bookie2, TestObject), + testutil:check_forlist(Bookie2, ChkList3), ok = leveled_bookie:book_close(Bookie2), - reset_filestructure(). + testutil:reset_filestructure(). fetchput_snapshot(_Config) -> - RootPath = reset_filestructure(), + RootPath = testutil:reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=3000000}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {TestObject, TestSpec} = generate_testobject(), + {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), - ObjList1 = generate_multiple_objects(5000, 2), + ObjList1 = testutil:generate_objects(5000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjList1), SnapOpts1 = #bookie_options{snapshot_bookie=Bookie1}, {ok, SnapBookie1} = leveled_bookie:book_start(SnapOpts1), ChkList1 = lists:sublist(lists:sort(ObjList1), 100), - check_bookie_forlist(Bookie1, ChkList1), - check_bookie_forlist(SnapBookie1, ChkList1), + testutil:check_forlist(Bookie1, ChkList1), + testutil:check_forlist(SnapBookie1, ChkList1), ok = leveled_bookie:book_close(SnapBookie1), - check_bookie_forlist(Bookie1, ChkList1), + testutil:check_forlist(Bookie1, ChkList1), ok = leveled_bookie:book_close(Bookie1), io:format("Closed initial bookies~n"), @@ -172,89 +174,94 @@ fetchput_snapshot(_Config) -> {ok, SnapBookie2} = leveled_bookie:book_start(SnapOpts2), io:format("Bookies restarted~n"), - check_bookie_forlist(Bookie2, ChkList1), + testutil:check_forlist(Bookie2, ChkList1), io:format("Check active bookie still contains original data~n"), - check_bookie_forlist(SnapBookie2, ChkList1), + testutil:check_forlist(SnapBookie2, ChkList1), io:format("Check snapshot still contains original data~n"), - ObjList2 = generate_multiple_objects(5000, 2), + ObjList2 = testutil:generate_objects(5000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, ObjList2), io:format("Replacement objects put~n"), ChkList2 = lists:sublist(lists:sort(ObjList2), 100), - check_bookie_forlist(Bookie2, ChkList2), - check_bookie_forlist(SnapBookie2, ChkList1), + testutil:check_forlist(Bookie2, ChkList2), + testutil:check_forlist(SnapBookie2, ChkList1), io:format("Checked for replacement objects in active bookie" ++ ", old objects in snapshot~n"), {ok, FNsA} = file:list_dir(RootPath ++ "/ledger/ledger_files"), - ObjList3 = generate_multiple_objects(15000, 5002), + ObjList3 = testutil:generate_objects(15000, 5002), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, ObjList3), ChkList3 = lists:sublist(lists:sort(ObjList3), 100), - check_bookie_forlist(Bookie2, ChkList3), - check_bookie_formissinglist(SnapBookie2, ChkList3), + testutil:check_forlist(Bookie2, ChkList3), + testutil:check_formissinglist(SnapBookie2, ChkList3), GenList = [20002, 40002, 60002, 80002, 100002, 120002], - CLs2 = load_objects(20000, GenList, Bookie2, TestObject, - fun generate_multiple_smallobjects/2), + CLs2 = testutil:load_objects(20000, GenList, Bookie2, TestObject, + fun testutil:generate_smallobjects/2), io:format("Loaded significant numbers of new objects~n"), - check_bookie_forlist(Bookie2, lists:nth(length(CLs2), CLs2)), + testutil:check_forlist(Bookie2, lists:nth(length(CLs2), CLs2)), io:format("Checked active bookie has new objects~n"), {ok, SnapBookie3} = leveled_bookie:book_start(SnapOpts2), - check_bookie_forlist(SnapBookie3, lists:nth(length(CLs2), CLs2)), - check_bookie_formissinglist(SnapBookie2, ChkList3), - check_bookie_formissinglist(SnapBookie2, lists:nth(length(CLs2), CLs2)), - check_bookie_forlist(SnapBookie3, ChkList2), - check_bookie_forlist(SnapBookie2, ChkList1), + testutil:check_forlist(SnapBookie3, lists:nth(length(CLs2), CLs2)), + testutil:check_formissinglist(SnapBookie2, ChkList3), + testutil:check_formissinglist(SnapBookie2, lists:nth(length(CLs2), CLs2)), + testutil:check_forlist(SnapBookie3, ChkList2), + testutil:check_forlist(SnapBookie2, ChkList1), io:format("Started new snapshot and check for new objects~n"), - CLs3 = load_objects(20000, GenList, Bookie2, TestObject, - fun generate_multiple_smallobjects/2), - check_bookie_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), - check_bookie_forlist(Bookie2, lists:nth(1, CLs3)), + CLs3 = testutil:load_objects(20000, GenList, Bookie2, TestObject, + fun testutil:generate_smallobjects/2), + testutil:check_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), + testutil:check_forlist(Bookie2, lists:nth(1, CLs3)), {ok, FNsB} = file:list_dir(RootPath ++ "/ledger/ledger_files"), ok = leveled_bookie:book_close(SnapBookie2), - check_bookie_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), + testutil:check_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), ok = leveled_bookie:book_close(SnapBookie3), - check_bookie_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), - check_bookie_forlist(Bookie2, lists:nth(1, CLs3)), + testutil:check_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), + testutil:check_forlist(Bookie2, lists:nth(1, CLs3)), timer:sleep(90000), {ok, FNsC} = file:list_dir(RootPath ++ "/ledger/ledger_files"), true = length(FNsB) > length(FNsA), true = length(FNsB) > length(FNsC), - {B1Size, B1Count} = check_bucket_stats(Bookie2, "Bucket1"), + {B1Size, B1Count} = testutil:check_bucket_stats(Bookie2, "Bucket1"), true = B1Size > 0, true = B1Count == 1, - {B1Size, B1Count} = check_bucket_stats(Bookie2, "Bucket1"), - {BSize, BCount} = check_bucket_stats(Bookie2, "Bucket"), + {B1Size, B1Count} = testutil:check_bucket_stats(Bookie2, "Bucket1"), + {BSize, BCount} = testutil:check_bucket_stats(Bookie2, "Bucket"), true = BSize > 0, true = BCount == 140000, ok = leveled_bookie:book_close(Bookie2), - reset_filestructure(). + testutil:reset_filestructure(). load_and_count(_Config) -> % Use artificially small files, and the load keys, counting they're all % present - RootPath = reset_filestructure(), + RootPath = testutil:reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=50000000}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {TestObject, TestSpec} = generate_testobject(), + {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), io:format("Loading initial small objects~n"), + G1 = fun testutil:generate_smallobjects/2, lists:foldl(fun(_X, Acc) -> - load_objects(5000, [Acc + 2], Bookie1, TestObject, - fun generate_multiple_smallobjects/2), - {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + testutil:load_objects(5000, + [Acc + 2], + Bookie1, + TestObject, + G1), + {_S, Count} = testutil:check_bucket_stats(Bookie1, + "Bucket"), if Acc + 5000 == Count -> ok @@ -262,12 +269,17 @@ load_and_count(_Config) -> Acc + 5000 end, 0, lists:seq(1, 20)), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), io:format("Loading larger compressible objects~n"), + G2 = fun testutil:generate_compressibleobjects/2, lists:foldl(fun(_X, Acc) -> - load_objects(5000, [Acc + 2], Bookie1, TestObject, - fun generate_multiple_compressibleobjects/2), - {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + testutil:load_objects(5000, + [Acc + 2], + Bookie1, + TestObject, + G2), + {_S, Count} = testutil:check_bucket_stats(Bookie1, + "Bucket"), if Acc + 5000 == Count -> ok @@ -275,12 +287,16 @@ load_and_count(_Config) -> Acc + 5000 end, 100000, lists:seq(1, 20)), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), io:format("Replacing small objects~n"), lists:foldl(fun(_X, Acc) -> - load_objects(5000, [Acc + 2], Bookie1, TestObject, - fun generate_multiple_smallobjects/2), - {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + testutil:load_objects(5000, + [Acc + 2], + Bookie1, + TestObject, + G1), + {_S, Count} = testutil:check_bucket_stats(Bookie1, + "Bucket"), if Count == 200000 -> ok @@ -288,12 +304,16 @@ load_and_count(_Config) -> Acc + 5000 end, 0, lists:seq(1, 20)), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), io:format("Loading more small objects~n"), lists:foldl(fun(_X, Acc) -> - load_objects(5000, [Acc + 2], Bookie1, TestObject, - fun generate_multiple_compressibleobjects/2), - {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + testutil:load_objects(5000, + [Acc + 2], + Bookie1, + TestObject, + G2), + {_S, Count} = testutil:check_bucket_stats(Bookie1, + "Bucket"), if Acc + 5000 == Count -> ok @@ -301,25 +321,30 @@ load_and_count(_Config) -> Acc + 5000 end, 200000, lists:seq(1, 20)), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), ok = leveled_bookie:book_close(Bookie1), {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), - {_BSize, 300000} = check_bucket_stats(Bookie2, "Bucket"), + {_, 300000} = testutil:check_bucket_stats(Bookie2, "Bucket"), ok = leveled_bookie:book_close(Bookie2), - reset_filestructure(). + testutil:reset_filestructure(). load_and_count_withdelete(_Config) -> - RootPath = reset_filestructure(), + RootPath = testutil:reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=50000000}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {TestObject, TestSpec} = generate_testobject(), + {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), io:format("Loading initial small objects~n"), + G1 = fun testutil:generate_smallobjects/2, lists:foldl(fun(_X, Acc) -> - load_objects(5000, [Acc + 2], Bookie1, TestObject, - fun generate_multiple_smallobjects/2), - {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + testutil:load_objects(5000, + [Acc + 2], + Bookie1, + TestObject, + G1), + {_S, Count} = testutil:check_bucket_stats(Bookie1, + "Bucket"), if Acc + 5000 == Count -> ok @@ -327,17 +352,22 @@ load_and_count_withdelete(_Config) -> Acc + 5000 end, 0, lists:seq(1, 20)), - check_bookie_forobject(Bookie1, TestObject), + testutil:check_forobject(Bookie1, TestObject), {BucketD, KeyD} = leveled_codec:riakto_keydetails(TestObject), - {_, 1} = check_bucket_stats(Bookie1, BucketD), + {_, 1} = testutil:check_bucket_stats(Bookie1, BucketD), ok = leveled_bookie:book_riakdelete(Bookie1, BucketD, KeyD, []), not_found = leveled_bookie:book_riakget(Bookie1, BucketD, KeyD), - {_, 0} = check_bucket_stats(Bookie1, BucketD), + {_, 0} = testutil:check_bucket_stats(Bookie1, BucketD), io:format("Loading larger compressible objects~n"), + G2 = fun testutil:generate_compressibleobjects/2, lists:foldl(fun(_X, Acc) -> - load_objects(5000, [Acc + 2], Bookie1, no_check, - fun generate_multiple_compressibleobjects/2), - {_Size, Count} = check_bucket_stats(Bookie1, "Bucket"), + testutil:load_objects(5000, + [Acc + 2], + Bookie1, + no_check, + G2), + {_S, Count} = testutil:check_bucket_stats(Bookie1, + "Bucket"), if Acc + 5000 == Count -> ok @@ -348,160 +378,8 @@ load_and_count_withdelete(_Config) -> not_found = leveled_bookie:book_riakget(Bookie1, BucketD, KeyD), ok = leveled_bookie:book_close(Bookie1), {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), - check_bookie_formissingobject(Bookie2, BucketD, KeyD), - {_BSize, 0} = check_bucket_stats(Bookie2, BucketD), - ok = leveled_bookie:book_close(Bookie2). + testutil:check_formissingobject(Bookie2, BucketD, KeyD), + {_BSize, 0} = testutil:check_bucket_stats(Bookie2, BucketD), + ok = leveled_bookie:book_close(Bookie2), + testutil:reset_filestructure(). - -reset_filestructure() -> - RootPath = "test", - filelib:ensure_dir(RootPath ++ "/journal/"), - filelib:ensure_dir(RootPath ++ "/ledger/"), - leveled_inker:clean_testdir(RootPath ++ "/journal"), - leveled_penciller:clean_testdir(RootPath ++ "/ledger"), - RootPath. - - - -check_bucket_stats(Bookie, Bucket) -> - FoldSW1 = os:timestamp(), - io:format("Checking bucket size~n"), - {async, Folder1} = leveled_bookie:book_returnfolder(Bookie, - {riakbucket_stats, - Bucket}), - {B1Size, B1Count} = Folder1(), - io:format("Bucket fold completed in ~w microseconds~n", - [timer:now_diff(os:timestamp(), FoldSW1)]), - io:format("Bucket ~s has size ~w and count ~w~n", - [Bucket, B1Size, B1Count]), - {B1Size, B1Count}. - - -check_bookie_forlist(Bookie, ChkList) -> - check_bookie_forlist(Bookie, ChkList, false). - -check_bookie_forlist(Bookie, ChkList, Log) -> - SW = os:timestamp(), - lists:foreach(fun({_RN, Obj, _Spc}) -> - if - Log == true -> - io:format("Fetching Key ~w~n", [Obj#r_object.key]); - true -> - ok - end, - R = leveled_bookie:book_riakget(Bookie, - Obj#r_object.bucket, - Obj#r_object.key), - R = {ok, Obj} end, - ChkList), - io:format("Fetch check took ~w microseconds checking list of length ~w~n", - [timer:now_diff(os:timestamp(), SW), length(ChkList)]). - -check_bookie_formissinglist(Bookie, ChkList) -> - SW = os:timestamp(), - lists:foreach(fun({_RN, Obj, _Spc}) -> - R = leveled_bookie:book_riakget(Bookie, - Obj#r_object.bucket, - Obj#r_object.key), - R = not_found end, - ChkList), - io:format("Miss check took ~w microseconds checking list of length ~w~n", - [timer:now_diff(os:timestamp(), SW), length(ChkList)]). - -check_bookie_forobject(Bookie, TestObject) -> - {ok, TestObject} = leveled_bookie:book_riakget(Bookie, - TestObject#r_object.bucket, - TestObject#r_object.key), - {ok, HeadObject} = leveled_bookie:book_riakhead(Bookie, - TestObject#r_object.bucket, - TestObject#r_object.key), - ok = case {HeadObject#r_object.bucket, - HeadObject#r_object.key, - HeadObject#r_object.vclock} of - {B1, K1, VC1} when B1 == TestObject#r_object.bucket, - K1 == TestObject#r_object.key, - VC1 == TestObject#r_object.vclock -> - ok - end. - -check_bookie_formissingobject(Bookie, Bucket, Key) -> - not_found = leveled_bookie:book_riakget(Bookie, Bucket, Key), - not_found = leveled_bookie:book_riakhead(Bookie, Bucket, Key). - - -generate_testobject() -> - {B1, K1, V1, Spec1, MD} = {"Bucket1", - "Key1", - "Value1", - [], - {"MDK1", "MDV1"}}, - generate_testobject(B1, K1, V1, Spec1, MD). - -generate_testobject(B, K, V, Spec, MD) -> - Content = #r_content{metadata=MD, value=V}, - {#r_object{bucket=B, key=K, contents=[Content], vclock=[{'a',1}]}, - Spec}. - - -generate_multiple_compressibleobjects(Count, KeyNumber) -> - S1 = "111111111111111", - S2 = "222222222222222", - S3 = "333333333333333", - S4 = "aaaaaaaaaaaaaaa", - S5 = "AAAAAAAAAAAAAAA", - S6 = "GGGGGGGGGGGGGGG", - S7 = "===============", - S8 = "...............", - Selector = [{1, S1}, {2, S2}, {3, S3}, {4, S4}, - {5, S5}, {6, S6}, {7, S7}, {8, S8}], - L = lists:seq(1, 1024), - V = lists:foldl(fun(_X, Acc) -> - {_, Str} = lists:keyfind(random:uniform(8), 1, Selector), - Acc ++ Str end, - "", - L), - generate_multiple_objects(Count, KeyNumber, [], V). - -generate_multiple_smallobjects(Count, KeyNumber) -> - generate_multiple_objects(Count, KeyNumber, [], crypto:rand_bytes(512)). - -generate_multiple_objects(Count, KeyNumber) -> - generate_multiple_objects(Count, KeyNumber, [], crypto:rand_bytes(4096)). - -generate_multiple_objects(0, _KeyNumber, ObjL, _Value) -> - ObjL; -generate_multiple_objects(Count, KeyNumber, ObjL, Value) -> - Obj = {"Bucket", - "Key" ++ integer_to_list(KeyNumber), - Value, - [], - [{"MDK", "MDV" ++ integer_to_list(KeyNumber)}, - {"MDK2", "MDV" ++ integer_to_list(KeyNumber)}]}, - {B1, K1, V1, Spec1, MD} = Obj, - Content = #r_content{metadata=MD, value=V1}, - Obj1 = #r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]}, - generate_multiple_objects(Count - 1, - KeyNumber + 1, - ObjL ++ [{random:uniform(), Obj1, Spec1}], - Value). - - -load_objects(ChunkSize, GenList, Bookie, TestObject, Generator) -> - lists:map(fun(KN) -> - ObjListA = Generator(ChunkSize, KN), - StartWatchA = os:timestamp(), - lists:foreach(fun({_RN, Obj, Spc}) -> - leveled_bookie:book_riakput(Bookie, Obj, Spc) - end, - ObjListA), - Time = timer:now_diff(os:timestamp(), StartWatchA), - io:format("~w objects loaded in ~w seconds~n", - [ChunkSize, Time/1000000]), - if - TestObject == no_check -> - ok; - true -> - check_bookie_forobject(Bookie, TestObject) - end, - lists:sublist(ObjListA, 1000) end, - GenList). diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl new file mode 100644 index 0000000..a7a9fd6 --- /dev/null +++ b/test/end_to_end/iterator_SUITE.erl @@ -0,0 +1,182 @@ +-module(iterator_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include("include/leveled.hrl"). + +-export([all/0]). +-export([simple_load_with2i/1, + simple_querycount/1]). + +all() -> [simple_load_with2i, + simple_querycount]. + + +simple_load_with2i(_Config) -> + RootPath = testutil:reset_filestructure(), + StartOpts1 = #bookie_options{root_path=RootPath, + max_journalsize=50000000}, + {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), + {TestObject, TestSpec} = testutil:generate_testobject(), + ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), + testutil:check_forobject(Bookie1, TestObject), + testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"), + testutil:check_forobject(Bookie1, TestObject), + ObjL1 = testutil:generate_objects(10000, + uuid, + [], + testutil:get_compressiblevalue(), + testutil:get_randomindexes_generator(8)), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, + ObjL1), + ChkList1 = lists:sublist(lists:sort(ObjL1), 100), + testutil:check_forlist(Bookie1, ChkList1), + testutil:check_forobject(Bookie1, TestObject), + ok = leveled_bookie:book_close(Bookie1), + testutil:reset_filestructure(). + + +simple_querycount(_Config) -> + RootPath = testutil:reset_filestructure(), + StartOpts1 = #bookie_options{root_path=RootPath, + max_journalsize=50000000}, + {ok, Book1} = leveled_bookie:book_start(StartOpts1), + {TestObject, TestSpec} = testutil:generate_testobject(), + ok = leveled_bookie:book_riakput(Book1, TestObject, TestSpec), + testutil:check_forobject(Book1, TestObject), + testutil:check_formissingobject(Book1, "Bucket1", "Key2"), + testutil:check_forobject(Book1, TestObject), + lists:foreach(fun(_X) -> + V = testutil:get_compressiblevalue(), + Indexes = testutil:get_randomindexes_generator(8), + SW = os:timestamp(), + ObjL1 = testutil:generate_objects(10000, + uuid, + [], + V, + Indexes), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Book1, + Obj, + Spc) + end, + ObjL1), + io:format("Put of 10000 objects with 8 index entries " + ++ + "each completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(), SW)]) + end, + lists:seq(1, 8)), + testutil:check_forobject(Book1, TestObject), + Total = lists:foldl(fun(X, Acc) -> + IdxF = "idx" ++ integer_to_list(X) ++ "_bin", + T = count_termsonindex("Bucket", + IdxF, + Book1, + {false, undefined}), + io:format("~w terms found on index ~s~n", + [T, IdxF]), + Acc + T + end, + 0, + lists:seq(1, 8)), + ok = case Total of + 640000 -> + ok + end, + Index1Count = count_termsonindex("Bucket", + "idx1_bin", + Book1, + {false, undefined}), + ok = leveled_bookie:book_close(Book1), + {ok, Book2} = leveled_bookie:book_start(StartOpts1), + Index1Count = count_termsonindex("Bucket", + "idx1_bin", + Book2, + {false, undefined}), + NameList = testutil:name_list(), + TotalNameByName = lists:foldl(fun({_X, Name}, Acc) -> + {ok, Regex} = re:compile("[0-9]+" ++ + Name), + SW = os:timestamp(), + T = count_termsonindex("Bucket", + "idx1_bin", + Book2, + {false, + Regex}), + TD = timer:now_diff(os:timestamp(), + SW), + io:format("~w terms found on " ++ + "index idx1 with a " ++ + "regex in ~w " ++ + "microseconds~n", + [T, TD]), + Acc + T + end, + 0, + NameList), + ok = case TotalNameByName of + Index1Count -> + ok + end, + RegMia = re:compile("[0-9]+Mia"), + {async, + Mia2KFolder1} = leveled_bookie:book_returnfolder(Book2, + {index_query, + "Bucket", + {"idx2_bin", + "2000L", + "2000N~"}, + {false, + RegMia}}), + Mia2000Count1 = length(Mia2KFolder1()), + {async, + Mia2KFolder2} = leveled_bookie:book_returnfolder(Book2, + {index_query, + "Bucket", + {"idx2_bin", + "2000Ma", + "2000Mz"}, + {true, + undefined}}), + Mia2000Count2 = lists:foldl(fun({Term, _Key}, Acc) -> + case Term of + "2000Mia" -> + Acc + 1; + _ -> + Acc + end end, + 0, + Mia2KFolder2()), + ok = case Mia2000Count2 of + Mia2000Count1 -> + ok + end, + ok = leveled_bookie:book_close(Book2), + testutil:reset_filestructure(). + + + +count_termsonindex(Bucket, IdxField, Book, QType) -> + lists:foldl(fun(X, Acc) -> + SW = os:timestamp(), + ST = integer_to_list(X), + ET = ST ++ "~", + R = leveled_bookie:book_returnfolder(Book, + {index_query, + Bucket, + {IdxField, + ST, + ET}, + QType}), + {async, Folder} = R, + Items = length(Folder()), + io:format("2i query from term ~s on index ~s took " ++ + "~w microseconds~n", + [ST, + IdxField, + timer:now_diff(os:timestamp(), SW)]), + Acc + Items + end, + 0, + lists:seq(1901, 2218)). diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl new file mode 100644 index 0000000..0d472cc --- /dev/null +++ b/test/end_to_end/testutil.erl @@ -0,0 +1,232 @@ +-module(testutil). + +-include("../include/leveled.hrl"). + +-export([reset_filestructure/0, + check_bucket_stats/2, + check_forlist/2, + check_forlist/3, + check_formissinglist/2, + check_forobject/2, + check_formissingobject/3, + generate_testobject/0, + generate_testobject/5, + generate_compressibleobjects/2, + generate_smallobjects/2, + generate_objects/2, + generate_objects/5, + get_compressiblevalue/0, + get_randomindexes_generator/1, + name_list/0, + load_objects/5]). + + +reset_filestructure() -> + RootPath = "test", + filelib:ensure_dir(RootPath ++ "/journal/"), + filelib:ensure_dir(RootPath ++ "/ledger/"), + leveled_inker:clean_testdir(RootPath ++ "/journal"), + leveled_penciller:clean_testdir(RootPath ++ "/ledger"), + RootPath. + + + +check_bucket_stats(Bookie, Bucket) -> + FoldSW1 = os:timestamp(), + io:format("Checking bucket size~n"), + {async, Folder1} = leveled_bookie:book_returnfolder(Bookie, + {riakbucket_stats, + Bucket}), + {B1Size, B1Count} = Folder1(), + io:format("Bucket fold completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(), FoldSW1)]), + io:format("Bucket ~s has size ~w and count ~w~n", + [Bucket, B1Size, B1Count]), + {B1Size, B1Count}. + + +check_forlist(Bookie, ChkList) -> + check_forlist(Bookie, ChkList, false). + +check_forlist(Bookie, ChkList, Log) -> + SW = os:timestamp(), + lists:foreach(fun({_RN, Obj, _Spc}) -> + if + Log == true -> + io:format("Fetching Key ~w~n", [Obj#r_object.key]); + true -> + ok + end, + R = leveled_bookie:book_riakget(Bookie, + Obj#r_object.bucket, + Obj#r_object.key), + R = {ok, Obj} end, + ChkList), + io:format("Fetch check took ~w microseconds checking list of length ~w~n", + [timer:now_diff(os:timestamp(), SW), length(ChkList)]). + +check_formissinglist(Bookie, ChkList) -> + SW = os:timestamp(), + lists:foreach(fun({_RN, Obj, _Spc}) -> + R = leveled_bookie:book_riakget(Bookie, + Obj#r_object.bucket, + Obj#r_object.key), + R = not_found end, + ChkList), + io:format("Miss check took ~w microseconds checking list of length ~w~n", + [timer:now_diff(os:timestamp(), SW), length(ChkList)]). + +check_forobject(Bookie, TestObject) -> + {ok, TestObject} = leveled_bookie:book_riakget(Bookie, + TestObject#r_object.bucket, + TestObject#r_object.key), + {ok, HeadObject} = leveled_bookie:book_riakhead(Bookie, + TestObject#r_object.bucket, + TestObject#r_object.key), + ok = case {HeadObject#r_object.bucket, + HeadObject#r_object.key, + HeadObject#r_object.vclock} of + {B1, K1, VC1} when B1 == TestObject#r_object.bucket, + K1 == TestObject#r_object.key, + VC1 == TestObject#r_object.vclock -> + ok + end. + +check_formissingobject(Bookie, Bucket, Key) -> + not_found = leveled_bookie:book_riakget(Bookie, Bucket, Key), + not_found = leveled_bookie:book_riakhead(Bookie, Bucket, Key). + + +generate_testobject() -> + {B1, K1, V1, Spec1, MD} = {"Bucket1", + "Key1", + "Value1", + [], + {"MDK1", "MDV1"}}, + generate_testobject(B1, K1, V1, Spec1, MD). + +generate_testobject(B, K, V, Spec, MD) -> + Content = #r_content{metadata=MD, value=V}, + {#r_object{bucket=B, key=K, contents=[Content], vclock=[{'a',1}]}, + Spec}. + + +generate_compressibleobjects(Count, KeyNumber) -> + V = get_compressiblevalue(), + generate_objects(Count, KeyNumber, [], V). + + +get_compressiblevalue() -> + S1 = "111111111111111", + S2 = "222222222222222", + S3 = "333333333333333", + S4 = "aaaaaaaaaaaaaaa", + S5 = "AAAAAAAAAAAAAAA", + S6 = "GGGGGGGGGGGGGGG", + S7 = "===============", + S8 = "...............", + Selector = [{1, S1}, {2, S2}, {3, S3}, {4, S4}, + {5, S5}, {6, S6}, {7, S7}, {8, S8}], + L = lists:seq(1, 1024), + lists:foldl(fun(_X, Acc) -> + {_, Str} = lists:keyfind(random:uniform(8), 1, Selector), + Acc ++ Str end, + "", + L). + +generate_smallobjects(Count, KeyNumber) -> + generate_objects(Count, KeyNumber, [], crypto:rand_bytes(512)). + +generate_objects(Count, KeyNumber) -> + generate_objects(Count, KeyNumber, [], crypto:rand_bytes(4096)). + + +generate_objects(Count, KeyNumber, ObjL, Value) -> + generate_objects(Count, KeyNumber, ObjL, Value, fun() -> [] end). + +generate_objects(0, _KeyNumber, ObjL, _Value, _IndexGen) -> + ObjL; +generate_objects(Count, uuid, ObjL, Value, IndexGen) -> + {Obj1, Spec1} = set_object(leveled_codec:generate_uuid(), + Value, + IndexGen), + generate_objects(Count - 1, + uuid, + ObjL ++ [{random:uniform(), Obj1, Spec1}], + Value, + IndexGen); +generate_objects(Count, KeyNumber, ObjL, Value, IndexGen) -> + {Obj1, Spec1} = set_object("Key" ++ integer_to_list(KeyNumber), + Value, + IndexGen), + generate_objects(Count - 1, + KeyNumber + 1, + ObjL ++ [{random:uniform(), Obj1, Spec1}], + Value, + IndexGen). + +set_object(Key, Value, IndexGen) -> + Obj = {"Bucket", + Key, + Value, + IndexGen(), + [{"MDK", "MDV" ++ Key}, + {"MDK2", "MDV" ++ Key}]}, + {B1, K1, V1, Spec1, MD} = Obj, + Content = #r_content{metadata=MD, value=V1}, + {#r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]}, + Spec1}. + + +load_objects(ChunkSize, GenList, Bookie, TestObject, Generator) -> + lists:map(fun(KN) -> + ObjListA = Generator(ChunkSize, KN), + StartWatchA = os:timestamp(), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie, Obj, Spc) + end, + ObjListA), + Time = timer:now_diff(os:timestamp(), StartWatchA), + io:format("~w objects loaded in ~w seconds~n", + [ChunkSize, Time/1000000]), + if + TestObject == no_check -> + ok; + true -> + check_forobject(Bookie, TestObject) + end, + lists:sublist(ObjListA, 1000) end, + GenList). + + +get_randomindexes_generator(Count) -> + Generator = fun() -> + lists:map(fun(X) -> + {add, + "idx" ++ integer_to_list(X) ++ "_bin", + get_randomdate() ++ get_randomname()} end, + lists:seq(1, Count)) + end, + Generator. + +name_list() -> + [{1, "Sophia"}, {2, "Emma"}, {3, "Olivia"}, {4, "Ava"}, + {5, "Isabella"}, {6, "Mia"}, {7, "Zoe"}, {8, "Lily"}, + {9, "Emily"}, {10, "Madelyn"}, {11, "Madison"}, {12, "Chloe"}, + {13, "Charlotte"}, {14, "Aubrey"}, {15, "Avery"}, + {16, "Abigail"}]. + +get_randomname() -> + NameList = name_list(), + N = random:uniform(16), + {N, Name} = lists:keyfind(N, 1, NameList), + Name. + +get_randomdate() -> + LowTime = 60000000000, + HighTime = 70000000000, + RandPoint = LowTime + random:uniform(HighTime - LowTime), + Date = calendar:gregorian_seconds_to_datetime(RandPoint), + {{Year, Month, Day}, {Hour, Minute, Second}} = Date, + lists:flatten(io_lib:format("~4..0w~2..0w~2..0w~2..0w~2..0w~2..0w", + [Year, Month, Day, Hour, Minute, Second])). \ No newline at end of file From 905b712764e5532e0bf0848af6c614b959077367 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 18 Oct 2016 09:42:33 +0100 Subject: [PATCH 072/167] 2i query test The 2i query test added in the previous commit didn't correctly test regex queries. This has now been improved. --- test/end_to_end/iterator_SUITE.erl | 33 +++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index a7a9fd6..0e47b59 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -119,14 +119,14 @@ simple_querycount(_Config) -> Index1Count -> ok end, - RegMia = re:compile("[0-9]+Mia"), + {ok, RegMia} = re:compile("[0-9]+Mia"), {async, Mia2KFolder1} = leveled_bookie:book_returnfolder(Book2, {index_query, "Bucket", {"idx2_bin", - "2000L", - "2000N~"}, + "2000", + "2000~"}, {false, RegMia}}), Mia2000Count1 = length(Mia2KFolder1()), @@ -135,23 +135,36 @@ simple_querycount(_Config) -> {index_query, "Bucket", {"idx2_bin", - "2000Ma", - "2000Mz"}, + "2000", + "2001"}, {true, undefined}}), Mia2000Count2 = lists:foldl(fun({Term, _Key}, Acc) -> - case Term of - "2000Mia" -> - Acc + 1; + case re:run(Term, RegMia) of + nomatch -> + Acc; _ -> - Acc + Acc + 1 end end, 0, Mia2KFolder2()), ok = case Mia2000Count2 of - Mia2000Count1 -> + Mia2000Count1 when Mia2000Count1 > 0 -> + io:format("Mia2000 counts match at ~w~n", + [Mia2000Count1]), ok end, + {ok, RxMia2K} = re:compile("^2000[0-9]+Mia"), + {async, + Mia2KFolder3} = leveled_bookie:book_returnfolder(Book2, + {index_query, + "Bucket", + {"idx2_bin", + "1980", + "2100"}, + {false, + RxMia2K}}), + Mia2000Count1 = length(Mia2KFolder3()), ok = leveled_bookie:book_close(Book2), testutil:reset_filestructure(). From 8f29a6c40f57b23090cb819832fbab3c85aadd27 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 18 Oct 2016 19:41:33 +0100 Subject: [PATCH 073/167] Complete 2i work - some refactoring The 2i work now has tests for removals as well as regex etc. Some initial refactoring work has also been tried - to try and take some tasks of the critical path of push_mem. The primary change has been to avoid putting index keys into the gb_tree, and building the KeyChanges list in parallel to the gb_tree (now known as ObjectTree) within the Ledger Cache. Some initial experiments done as to changing the ETS table in the Penciller now that it will now be used for iterating - but that has been reverted for now. --- src/leveled_bookie.erl | 51 +++++++++++-------- src/leveled_codec.erl | 10 +++- src/leveled_inker.erl | 5 +- src/leveled_penciller.erl | 39 ++++++++++----- src/leveled_sft.erl | 4 +- test/end_to_end/iterator_SUITE.erl | 80 ++++++++++++++++++++++++++++-- 6 files changed, 144 insertions(+), 45 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index c607606..3794dab 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -149,7 +149,7 @@ -include_lib("eunit/include/eunit.hrl"). --define(CACHE_SIZE, 1000). +-define(CACHE_SIZE, 2000). -define(JOURNAL_FP, "journal"). -define(LEDGER_FP, "ledger"). -define(SHUTDOWN_WAITS, 60). @@ -160,7 +160,7 @@ penciller :: pid(), cache_size :: integer(), back_pressure :: boolean(), - ledger_cache :: gb_trees:tree(), + ledger_cache :: {gb_trees:tree(), list()}, is_snapshot :: boolean()}). @@ -242,7 +242,7 @@ init([Opts]) -> {ok, #state{inker=Inker, penciller=Penciller, cache_size=CacheSize, - ledger_cache=gb_trees:empty(), + ledger_cache={gb_trees:empty(), []}, is_snapshot=false}}; Bookie -> {ok, @@ -397,16 +397,15 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================ -bucket_stats(Penciller, LedgerCache, Bucket, Tag) -> +bucket_stats(Penciller, {_ObjTree, ChangeList}, Bucket, Tag) -> PCLopts = #penciller_options{start_snapshot=true, source_penciller=Penciller}, {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), Folder = fun() -> - Increment = gb_trees:to_list(LedgerCache), io:format("Length of increment in snapshot is ~w~n", - [length(Increment)]), + [length(ChangeList)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, - {infinity, Increment}), + {infinity, ChangeList}), StartKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), EndKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, @@ -419,7 +418,7 @@ bucket_stats(Penciller, LedgerCache, Bucket, Tag) -> end, {async, Folder}. -index_query(Penciller, LedgerCache, +index_query(Penciller, {_ObjTree, ChangeList}, Bucket, {IdxField, StartValue, EndValue}, {ReturnTerms, TermRegex}) -> @@ -427,11 +426,10 @@ index_query(Penciller, LedgerCache, source_penciller=Penciller}, {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), Folder = fun() -> - Increment = gb_trees:to_list(LedgerCache), io:format("Length of increment in snapshot is ~w~n", - [length(Increment)]), + [length(ChangeList)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, - {infinity, Increment}), + {infinity, ChangeList}), StartKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, IdxField, StartValue), EndKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, @@ -493,8 +491,9 @@ startup(InkerOpts, PencillerOpts) -> {Inker, Penciller}. -fetch_head(Key, Penciller, Cache) -> - case gb_trees:lookup(Key, Cache) of +fetch_head(Key, Penciller, {ObjTree, _ChangeList}) -> + + case gb_trees:lookup(Key, ObjTree) of {value, Head} -> Head; none -> @@ -561,8 +560,6 @@ accumulate_index(TermRe, AddFun) -> end. - - preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs) -> {Bucket, Key, PrimaryChange} = leveled_codec:generate_ledgerkv(PK, SQN, @@ -572,20 +569,30 @@ preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs) -> [PrimaryChange] ++ ConvSpecs. addto_ledgercache(Changes, Cache) -> - lists:foldl(fun({K, V}, Acc) -> gb_trees:enter(K, V, Acc) end, - Cache, - Changes). + {ObjectTree, ChangeList} = Cache, + {lists:foldl(fun({K, V}, Acc) -> + case leveled_codec:is_indexkey(K) of + false -> + gb_trees:enter(K, V, Acc); + true -> + Acc + end + end, + ObjectTree, + Changes), + ChangeList ++ Changes}. maybepush_ledgercache(MaxCacheSize, Cache, Penciller) -> - CacheSize = gb_trees:size(Cache), + {_ObjectTree, ChangeList} = Cache, + CacheSize = length(ChangeList), if CacheSize > MaxCacheSize -> case leveled_penciller:pcl_pushmem(Penciller, - gb_trees:to_list(Cache)) of + ChangeList) of ok -> - {ok, gb_trees:empty()}; + {ok, {gb_trees:empty(), []}}; pause -> - {pause, gb_trees:empty()}; + {pause, {gb_trees:empty(), []}}; refused -> {ok, Cache} end; diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 370133c..096e48f 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -39,6 +39,7 @@ strip_to_keyseqstatusonly/1, striphead_to_details/1, is_active/2, + is_indexkey/1, endkey_passed/2, key_dominates/2, print_key/1, @@ -107,6 +108,11 @@ to_ledgerkey(Bucket, Key, Tag, Field, Value) when Tag == ?IDX_TAG -> to_ledgerkey(Bucket, Key, Tag) -> {Tag, Bucket, Key, null}. +is_indexkey({Tag, _, _, _}) when Tag == ?IDX_TAG -> + true; +is_indexkey(_Key) -> + false. + hash(Obj) -> erlang:phash2(term_to_binary(Obj)). @@ -156,7 +162,7 @@ convert_indexspecs(IndexSpecs, Bucket, Key, SQN) -> {active, infinity}; remove -> %% TODO: timestamps for delayed reaping - {tomb, infinity} + tomb end, {to_ledgerkey(Bucket, Key, ?IDX_TAG, IdxField, IdxValue), @@ -260,7 +266,7 @@ indexspecs_test() -> ?assertMatch({{i, "Bucket", {"t1_bin", "adbc123"}, "Key2"}, {1, {active, infinity}, null}}, lists:nth(2, Changes)), ?assertMatch({{i, "Bucket", {"t1_bin", "abdc456"}, "Key2"}, - {1, {tomb, infinity}, null}}, lists:nth(3, Changes)). + {1, tomb, null}}, lists:nth(3, Changes)). endkey_passed_test() -> TestKey = {i, null, null, null}, diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index a876fa8..0ddc6d7 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -724,10 +724,9 @@ manifest_printer(Manifest) -> initiate_penciller_snapshot(Bookie) -> {ok, - {LedgerSnap, LedgerCache}, + {LedgerSnap, {_ObjTree, ChangeList}}, _} = leveled_bookie:book_snapshotledger(Bookie, self(), undefined), - ok = leveled_penciller:pcl_loadsnapshot(LedgerSnap, - gb_trees:to_list(LedgerCache)), + ok = leveled_penciller:pcl_loadsnapshot(LedgerSnap, ChangeList), MaxSQN = leveled_penciller:pcl_getstartupsequencenumber(LedgerSnap), {LedgerSnap, MaxSQN}. diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 34a99a4..32c70a3 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -639,6 +639,14 @@ start_from_file(PCLopts) -> M -> M end, + % Options (not) chosen here: + % - As we pass the ETS table to the sft file when the L0 file is created + % then this cannot be private. + % - There is no use of iterator, so a set could be used, but then the + % output of tab2list would need to be sorted + % TODO: + % - Test switching to [set, private] and sending the L0 snapshots to the + % sft_new cast TID = ets:new(?MEMTABLE, [ordered_set]), {ok, Clerk} = leveled_pclerk:clerk_new(self()), InitState = #state{memtable=TID, @@ -1371,14 +1379,12 @@ confirm_delete(Filename, UnreferencedFiles, RegisteredSnapshots) -> assess_sqn([]) -> empty; -assess_sqn(DumpList) -> - assess_sqn(DumpList, infinity, 0). - -assess_sqn([], MinSQN, MaxSQN) -> - {MinSQN, MaxSQN}; -assess_sqn([HeadKey|Tail], MinSQN, MaxSQN) -> - {_K, SQN} = leveled_codec:strip_to_keyseqonly(HeadKey), - assess_sqn(Tail, min(MinSQN, SQN), max(MaxSQN, SQN)). +assess_sqn([HeadKV|[]]) -> + {leveled_codec:strip_to_seqonly(HeadKV), + leveled_codec:strip_to_seqonly(HeadKV)}; +assess_sqn([HeadKV|DumpList]) -> + {leveled_codec:strip_to_seqonly(HeadKV), + leveled_codec:strip_to_seqonly(lists:last(DumpList))}. %%%============================================================================ @@ -1406,6 +1412,13 @@ clean_subdir(DirPath) -> ok end. +assess_sqn_test() -> + L1 = [{{}, {5, active, {}}}, {{}, {6, active, {}}}], + ?assertMatch({5, 6}, assess_sqn(L1)), + L2 = [{{}, {5, active, {}}}], + ?assertMatch({5, 5}, assess_sqn(L2)), + ?assertMatch(empty, assess_sqn([])). + compaction_work_assessment_test() -> L0 = [{{o, "B1", "K1", null}, {o, "B3", "K3", null}, dummy_pid}], L1 = [{{o, "B1", "K1", null}, {o, "B2", "K2", null}, dummy_pid}, @@ -1462,13 +1475,13 @@ simple_server_test() -> {ok, PCL} = pcl_start(#penciller_options{root_path=RootPath, max_inmemory_tablesize=1000}), Key1 = {{o,"Bucket0001", "Key0001", null}, {1, {active, infinity}, null}}, - KL1 = lists:sort(leveled_sft:generate_randomkeys({1000, 2})), + KL1 = leveled_sft:generate_randomkeys({1000, 2}), Key2 = {{o,"Bucket0002", "Key0002", null}, {1002, {active, infinity}, null}}, - KL2 = lists:sort(leveled_sft:generate_randomkeys({1000, 1002})), + KL2 = leveled_sft:generate_randomkeys({1000, 1002}), Key3 = {{o,"Bucket0003", "Key0003", null}, {2002, {active, infinity}, null}}, - KL3 = lists:sort(leveled_sft:generate_randomkeys({1000, 2002})), + KL3 = leveled_sft:generate_randomkeys({1000, 2002}), Key4 = {{o,"Bucket0004", "Key0004", null}, {3002, {active, infinity}, null}}, - KL4 = lists:sort(leveled_sft:generate_randomkeys({1000, 3002})), + KL4 = leveled_sft:generate_randomkeys({1000, 3002}), ok = pcl_pushmem(PCL, [Key1]), ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), ok = pcl_pushmem(PCL, KL1), @@ -1546,7 +1559,7 @@ simple_server_test() -> % sees the old version in the previous snapshot, but will see the new version % in a new snapshot Key1A = {{o,"Bucket0001", "Key0001", null}, {4002, {active, infinity}, null}}, - KL1A = lists:sort(leveled_sft:generate_randomkeys({4002, 2})), + KL1A = leveled_sft:generate_randomkeys({4002, 2}), maybe_pause_push(pcl_pushmem(PCLr, [Key1A])), maybe_pause_push(pcl_pushmem(PCLr, KL1A)), ?assertMatch(true, pcl_checksequencenumber(PclSnap, diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 000dd45..1e846e2 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -1378,7 +1378,7 @@ generate_randomkeys(Count) -> generate_randomkeys(Count, 0, []). generate_randomkeys(0, _SQN, Acc) -> - Acc; + lists:reverse(Acc); generate_randomkeys(Count, SQN, Acc) -> RandKey = {{o, lists:concat(["Bucket", random:uniform(1024)]), @@ -1651,7 +1651,7 @@ initial_create_file_test() -> big_create_file_test() -> Filename = "../test/bigtest1.sft", {KL1, KL2} = {lists:sort(generate_randomkeys(2000)), - lists:sort(generate_randomkeys(50000))}, + lists:sort(generate_randomkeys(40000))}, {InitHandle, InitFileMD} = create_file(Filename), {Handle, FileMD, {_KL1Rem, _KL2Rem}} = complete_file(InitHandle, InitFileMD, diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 0e47b59..3b8701f 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -3,6 +3,8 @@ -include_lib("common_test/include/ct.hrl"). -include("include/leveled.hrl"). +-define(KEY_ONLY, {false, undefined}). + -export([all/0]). -export([simple_load_with2i/1, simple_querycount/1]). @@ -73,7 +75,7 @@ simple_querycount(_Config) -> T = count_termsonindex("Bucket", IdxF, Book1, - {false, undefined}), + ?KEY_ONLY), io:format("~w terms found on index ~s~n", [T, IdxF]), Acc + T @@ -87,13 +89,13 @@ simple_querycount(_Config) -> Index1Count = count_termsonindex("Bucket", "idx1_bin", Book1, - {false, undefined}), + ?KEY_ONLY), ok = leveled_bookie:book_close(Book1), {ok, Book2} = leveled_bookie:book_start(StartOpts1), Index1Count = count_termsonindex("Bucket", "idx1_bin", Book2, - {false, undefined}), + ?KEY_ONLY), NameList = testutil:name_list(), TotalNameByName = lists:foldl(fun({_X, Name}, Acc) -> {ok, Regex} = re:compile("[0-9]+" ++ @@ -165,7 +167,79 @@ simple_querycount(_Config) -> {false, RxMia2K}}), Mia2000Count1 = length(Mia2KFolder3()), + + V9 = testutil:get_compressiblevalue(), + Indexes9 = testutil:get_randomindexes_generator(8), + [{_RN, Obj9, Spc9}] = testutil:generate_objects(1, uuid, [], V9, Indexes9), + ok = leveled_bookie:book_riakput(Book2, Obj9, Spc9), + R9 = lists:map(fun({add, IdxF, IdxT}) -> + R = leveled_bookie:book_returnfolder(Book2, + {index_query, + "Bucket", + {IdxF, + IdxT, + IdxT}, + ?KEY_ONLY}), + {async, Fldr} = R, + case length(Fldr()) of + X when X > 0 -> + {IdxF, IdxT, X} + end + end, + Spc9), + Spc9Del = lists:map(fun({add, IdxF, IdxT}) -> {remove, IdxF, IdxT} end, + Spc9), + ok = leveled_bookie:book_riakput(Book2, Obj9, Spc9Del), + lists:foreach(fun({IdxF, IdxT, X}) -> + R = leveled_bookie:book_returnfolder(Book2, + {index_query, + "Bucket", + {IdxF, + IdxT, + IdxT}, + ?KEY_ONLY}), + {async, Fldr} = R, + case length(Fldr()) of + Y -> + Y = X - 1 + end + end, + R9), ok = leveled_bookie:book_close(Book2), + {ok, Book3} = leveled_bookie:book_start(StartOpts1), + lists:foreach(fun({IdxF, IdxT, X}) -> + R = leveled_bookie:book_returnfolder(Book3, + {index_query, + "Bucket", + {IdxF, + IdxT, + IdxT}, + ?KEY_ONLY}), + {async, Fldr} = R, + case length(Fldr()) of + Y -> + Y = X - 1 + end + end, + R9), + ok = leveled_bookie:book_riakput(Book3, Obj9, Spc9), + {ok, Book4} = leveled_bookie:book_start(StartOpts1), + lists:foreach(fun({IdxF, IdxT, X}) -> + R = leveled_bookie:book_returnfolder(Book4, + {index_query, + "Bucket", + {IdxF, + IdxT, + IdxT}, + ?KEY_ONLY}), + {async, Fldr} = R, + case length(Fldr()) of + X -> + ok + end + end, + R9), + ok = leveled_bookie:book_close(Book4), testutil:reset_filestructure(). From f16f71ae81c4637fd488829eed585e8e307e8787 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 19 Oct 2016 00:10:48 +0100 Subject: [PATCH 074/167] Revert ominshambles performance refactoring To try and improve performance index entries had been removed from the Ledger Cache, and a shadow list of the LedgerCache (in SQN order) was kept to avoid gb_trees:to_list on push_mem. This did not go well. The issue was that ets does not deal with duplicate keys in the list when inserting (it will only insert one, but it is not clear which one). This has been reverted back out. The ETS parameters have been changed to [set, private]. It is not used as an iterator, and is no longer passed out of the process (the memtable_copy is sent instead). This also avoids the tab2list function being called. --- src/leveled_bookie.erl | 47 +++++++++-------------- src/leveled_inker.erl | 5 ++- src/leveled_penciller.erl | 68 +++++++++++++++++++-------------- src/leveled_sft.erl | 17 ++++++++- test/end_to_end/basic_SUITE.erl | 2 + test/end_to_end/testutil.erl | 12 +++++- 6 files changed, 89 insertions(+), 62 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 3794dab..18b018f 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -160,7 +160,7 @@ penciller :: pid(), cache_size :: integer(), back_pressure :: boolean(), - ledger_cache :: {gb_trees:tree(), list()}, + ledger_cache :: gb_trees:tree(), is_snapshot :: boolean()}). @@ -242,7 +242,7 @@ init([Opts]) -> {ok, #state{inker=Inker, penciller=Penciller, cache_size=CacheSize, - ledger_cache={gb_trees:empty(), []}, + ledger_cache=gb_trees:empty(), is_snapshot=false}}; Bookie -> {ok, @@ -397,15 +397,16 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================ -bucket_stats(Penciller, {_ObjTree, ChangeList}, Bucket, Tag) -> +bucket_stats(Penciller, LedgerCache, Bucket, Tag) -> PCLopts = #penciller_options{start_snapshot=true, source_penciller=Penciller}, {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), Folder = fun() -> + Increment = gb_trees:to_list(LedgerCache), io:format("Length of increment in snapshot is ~w~n", - [length(ChangeList)]), + [length(Increment)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, - {infinity, ChangeList}), + {infinity, Increment}), StartKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), EndKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, @@ -418,7 +419,7 @@ bucket_stats(Penciller, {_ObjTree, ChangeList}, Bucket, Tag) -> end, {async, Folder}. -index_query(Penciller, {_ObjTree, ChangeList}, +index_query(Penciller, LedgerCache, Bucket, {IdxField, StartValue, EndValue}, {ReturnTerms, TermRegex}) -> @@ -426,10 +427,11 @@ index_query(Penciller, {_ObjTree, ChangeList}, source_penciller=Penciller}, {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), Folder = fun() -> + Increment = gb_trees:to_list(LedgerCache), io:format("Length of increment in snapshot is ~w~n", - [length(ChangeList)]), + [length(Increment)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, - {infinity, ChangeList}), + {infinity, Increment}), StartKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, IdxField, StartValue), EndKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, @@ -491,9 +493,8 @@ startup(InkerOpts, PencillerOpts) -> {Inker, Penciller}. -fetch_head(Key, Penciller, {ObjTree, _ChangeList}) -> - - case gb_trees:lookup(Key, ObjTree) of +fetch_head(Key, Penciller, LedgerCache) -> + case gb_trees:lookup(Key, LedgerCache) of {value, Head} -> Head; none -> @@ -569,30 +570,20 @@ preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs) -> [PrimaryChange] ++ ConvSpecs. addto_ledgercache(Changes, Cache) -> - {ObjectTree, ChangeList} = Cache, - {lists:foldl(fun({K, V}, Acc) -> - case leveled_codec:is_indexkey(K) of - false -> - gb_trees:enter(K, V, Acc); - true -> - Acc - end - end, - ObjectTree, - Changes), - ChangeList ++ Changes}. + lists:foldl(fun({K, V}, Acc) -> gb_trees:enter(K, V, Acc) end, + Cache, + Changes). maybepush_ledgercache(MaxCacheSize, Cache, Penciller) -> - {_ObjectTree, ChangeList} = Cache, - CacheSize = length(ChangeList), + CacheSize = gb_trees:size(Cache), if CacheSize > MaxCacheSize -> case leveled_penciller:pcl_pushmem(Penciller, - ChangeList) of + gb_trees:to_list(Cache)) of ok -> - {ok, {gb_trees:empty(), []}}; + {ok, gb_trees:empty()}; pause -> - {pause, {gb_trees:empty(), []}}; + {pause, gb_trees:empty()}; refused -> {ok, Cache} end; diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 0ddc6d7..c346cc1 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -724,9 +724,10 @@ manifest_printer(Manifest) -> initiate_penciller_snapshot(Bookie) -> {ok, - {LedgerSnap, {_ObjTree, ChangeList}}, + {LedgerSnap, LedgerCache}, _} = leveled_bookie:book_snapshotledger(Bookie, self(), undefined), - ok = leveled_penciller:pcl_loadsnapshot(LedgerSnap, ChangeList), + ok = leveled_penciller:pcl_loadsnapshot(LedgerSnap, + gb_trees:to_list(LedgerCache)), MaxSQN = leveled_penciller:pcl_getstartupsequencenumber(LedgerSnap), {LedgerSnap, MaxSQN}. diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 32c70a3..2aa99c9 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -246,6 +246,7 @@ pcl_loadsnapshot/2, pcl_getstartupsequencenumber/1, roll_new_tree/3, + roll_into_list/1, clean_testdir/1]). -include_lib("eunit/include/eunit.hrl"). @@ -377,7 +378,7 @@ handle_call({push_mem, DumpList}, From, State=#state{is_snapshot=Snap}) % then add to memory in the background before updating the loop state % - Push the update into memory (do_pushtomem/3) % - If we haven't got through quickcheck now need to check if there is a - % definite need to write a new L0 file (roll_memory/2). If all clear this + % definite need to write a new L0 file (roll_memory/3). If all clear this % will write the file in the background and allow a response to the user. % If not the change has still been made but the the L0 file will not have % been prompted - so the reply does not indicate failure but returns the @@ -414,7 +415,7 @@ handle_call({push_mem, DumpList}, From, State=#state{is_snapshot=Snap}) State1#state.memtable_copy, MaxSQN), - case roll_memory(State1, MaxTableSize) of + case roll_memory(State1, MaxTableSize, L0Snap) of {ok, L0Pend, ManSN, TableSize2} -> io:format("Push completed in ~w microseconds~n", [timer:now_diff(os:timestamp(), StartWatch)]), @@ -587,9 +588,16 @@ terminate(Reason, State) -> no_change -> State end, - Dump = ets:tab2list(UpdState#state.memtable), + % TODO: + % This next section (to the end of the case clause), appears to be + % pointless. It will persist the in-memory state to a SFT file, but on + % startup that file will be ignored as the manifest has not bene updated + % + % Should we update the manifest, or stop trying to persist on closure? + Dump = roll_into_list(State#state.memtable_copy), case {UpdState#state.levelzero_pending, - get_item(0, UpdState#state.manifest, []), length(Dump)} of + get_item(0, UpdState#state.manifest, []), + length(Dump)} of {?L0PEND_RESET, [], L} when L > 0 -> MSN = UpdState#state.manifest_sqn + 1, FileName = UpdState#state.root_path @@ -616,6 +624,8 @@ terminate(Reason, State) -> ++ " with ~w keys discarded~n", [length(Dump)]) end, + + % Tidy shutdown of individual files ok = close_files(0, UpdState#state.manifest), lists:foreach(fun({_FN, Pid, _SN}) -> leveled_sft:sft_close(Pid) end, @@ -639,15 +649,9 @@ start_from_file(PCLopts) -> M -> M end, - % Options (not) chosen here: - % - As we pass the ETS table to the sft file when the L0 file is created - % then this cannot be private. - % - There is no use of iterator, so a set could be used, but then the - % output of tab2list would need to be sorted - % TODO: - % - Test switching to [set, private] and sending the L0 snapshots to the - % sft_new cast - TID = ets:new(?MEMTABLE, [ordered_set]), + % There is no need to export this ets table (hence private) or iterate + % over it (hence set not ordered_set) + TID = ets:new(?MEMTABLE, [set, private]), {ok, Clerk} = leveled_pclerk:clerk_new(self()), InitState = #state{memtable=TID, clerk=Clerk, @@ -767,12 +771,17 @@ quickcheck_pushtomem(DumpList, TableSize, MaxSize) -> do_pushtomem(DumpList, MemTable, Snapshot, MaxSQN) -> SW = os:timestamp(), UpdSnapshot = add_increment_to_memcopy(Snapshot, MaxSQN, DumpList), + % Note that the DumpList must have been taken from a source which + % naturally de-duplicates the keys. It is not possible just to cache + % changes in a list (in the Bookie for example), as the insert method does + % not apply the list in order, and so it is not clear which of a duplicate + % key will be applied ets:insert(MemTable, DumpList), io:format("Push into memory timed at ~w microseconds~n", [timer:now_diff(os:timestamp(), SW)]), UpdSnapshot. -roll_memory(State, MaxSize) -> +roll_memory(State, MaxSize, MemTableCopy) -> case ets:info(State#state.memtable, size) of Size when Size > MaxSize -> L0 = get_item(0, State#state.manifest, []), @@ -784,7 +793,7 @@ roll_memory(State, MaxSize) -> ++ integer_to_list(MSN) ++ "_0_0", Opts = #sft_options{wait=false}, {ok, L0Pid} = leveled_sft:sft_new(FileName, - State#state.memtable, + MemTableCopy, [], 0, Opts), @@ -938,7 +947,6 @@ return_work(State, From) -> %% This takes the three parts of a memtable copy - the increments, the tree %% and the SQN at which the tree was formed, and outputs a new tree - roll_new_tree(Tree, [], HighSQN) -> {Tree, HighSQN}; roll_new_tree(Tree, [{SQN, KVList}|TailIncs], HighSQN) when SQN >= HighSQN -> @@ -954,6 +962,14 @@ roll_new_tree(Tree, [{SQN, KVList}|TailIncs], HighSQN) when SQN >= HighSQN -> roll_new_tree(Tree, [_H|TailIncs], HighSQN) -> roll_new_tree(Tree, TailIncs, HighSQN). +%% This takes the three parts of a memtable copy - the increments, the tree +%% and the SQN at which the tree was formed, and outputs a sorted list +roll_into_list(MemTableCopy) -> + {Tree, _SQN} = roll_new_tree(MemTableCopy#l0snapshot.tree, + MemTableCopy#l0snapshot.increments, + MemTableCopy#l0snapshot.ledger_sqn), + gb_trees:to_list(Tree). + %% Update the memtable copy if the tree created advances the SQN cache_tree_in_memcopy(MemCopy, Tree, SQN) -> case MemCopy#l0snapshot.ledger_sqn of @@ -1331,7 +1347,7 @@ rename_manifest_files(RootPath, NewMSN) -> filelib:is_file(OldFN), NewFN, filelib:is_file(NewFN)]), - file:rename(OldFN,NewFN). + ok = file:rename(OldFN,NewFN). filepath(RootPath, manifest) -> RootPath ++ "/" ++ ?MANIFEST_FP; @@ -1379,13 +1395,14 @@ confirm_delete(Filename, UnreferencedFiles, RegisteredSnapshots) -> assess_sqn([]) -> empty; -assess_sqn([HeadKV|[]]) -> - {leveled_codec:strip_to_seqonly(HeadKV), - leveled_codec:strip_to_seqonly(HeadKV)}; -assess_sqn([HeadKV|DumpList]) -> - {leveled_codec:strip_to_seqonly(HeadKV), - leveled_codec:strip_to_seqonly(lists:last(DumpList))}. +assess_sqn(DumpList) -> + assess_sqn(DumpList, infinity, 0). +assess_sqn([], MinSQN, MaxSQN) -> + {MinSQN, MaxSQN}; +assess_sqn([HeadKey|Tail], MinSQN, MaxSQN) -> + SQN = leveled_codec:strip_to_seqonly(HeadKey), + assess_sqn(Tail, min(MinSQN, SQN), max(MaxSQN, SQN)). %%%============================================================================ %%% Test @@ -1499,11 +1516,6 @@ simple_server_test() -> max_inmemory_tablesize=1000}), TopSQN = pcl_getstartupsequencenumber(PCLr), Check = case TopSQN of - 2001 -> - %% Last push not persisted - S3a = pcl_pushmem(PCL, [Key3]), - if S3a == pause -> timer:sleep(1000); true -> ok end, - ok; 2002 -> %% everything got persisted ok; diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 1e846e2..af56fa9 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -183,7 +183,7 @@ -define(MERGE_SCANWIDTH, 8). -define(DELETE_TIMEOUT, 60000). -define(MAX_KEYS, ?SLOT_COUNT * ?BLOCK_COUNT * ?BLOCK_SIZE). - +-define(DISCARD_EXT, ".discarded"). -record(state, {version = ?CURRENT_VERSION :: tuple(), slot_index :: list(), @@ -387,7 +387,7 @@ create_levelzero(Inp1, Filename) -> true -> Inp1; false -> - ets:tab2list(Inp1) + leveled_penciller:roll_into_list(Inp1) end, {TmpFilename, PrmFilename} = generate_filenames(Filename), case create_file(TmpFilename) of @@ -510,6 +510,19 @@ complete_file(Handle, FileMD, KL1, KL2, Level, Rename) -> open_file(FileMD); {true, OldName, NewName} -> io:format("Renaming file from ~s to ~s~n", [OldName, NewName]), + case filelib:is_file(NewName) of + true -> + io:format("Filename ~s already exists~n", + [NewName]), + AltName = filename:join(filename:dirname(NewName), + filename:basename(NewName)) + ++ ?DISCARD_EXT, + io:format("Rename rogue filename ~s to ~s~n", + [NewName, AltName]), + ok = file:rename(NewName, AltName); + false -> + ok + end, ok = file:rename(OldName, NewName), open_file(FileMD#state{filename=NewName}) end, diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index bf5a700..dac295f 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -192,6 +192,7 @@ fetchput_snapshot(_Config) -> io:format("Checked for replacement objects in active bookie" ++ ", old objects in snapshot~n"), + ok = filelib:ensure_dir(RootPath ++ "/ledger/ledger_files"), {ok, FNsA} = file:list_dir(RootPath ++ "/ledger/ledger_files"), ObjList3 = testutil:generate_objects(15000, 5002), lists:foreach(fun({_RN, Obj, Spc}) -> @@ -212,6 +213,7 @@ fetchput_snapshot(_Config) -> testutil:check_forlist(SnapBookie3, lists:nth(length(CLs2), CLs2)), testutil:check_formissinglist(SnapBookie2, ChkList3), testutil:check_formissinglist(SnapBookie2, lists:nth(length(CLs2), CLs2)), + testutil:check_forlist(Bookie2, ChkList2), testutil:check_forlist(SnapBookie3, ChkList2), testutil:check_forlist(SnapBookie2, ChkList1), io:format("Started new snapshot and check for new objects~n"), diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index 0d472cc..c938d6a 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -53,14 +53,22 @@ check_forlist(Bookie, ChkList, Log) -> lists:foreach(fun({_RN, Obj, _Spc}) -> if Log == true -> - io:format("Fetching Key ~w~n", [Obj#r_object.key]); + io:format("Fetching Key ~s~n", [Obj#r_object.key]); true -> ok end, R = leveled_bookie:book_riakget(Bookie, Obj#r_object.bucket, Obj#r_object.key), - R = {ok, Obj} end, + ok = case R of + {ok, Obj} -> + ok; + not_found -> + io:format("Object not found for key ~s~n", + [Obj#r_object.key]), + error + end + end, ChkList), io:format("Fetch check took ~w microseconds checking list of length ~w~n", [timer:now_diff(os:timestamp(), SW), length(ChkList)]). From 12fe1d01bd2abace45f82ba9eb3566afbd442866 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 19 Oct 2016 17:34:58 +0100 Subject: [PATCH 075/167] Penciller Manifest and Locking The penciller had the concept of a manifest_lock - but it wasn't clear what the purpose of it was. The updating of the manifest has now been updated to reduce the code and make the process cleaner and more obvious. Now the committed manifest only covers non-L0 levels. A clerk can work concurrently on a manifest change whilst the Penciller is accepting a new L0 file. On startup the manifets is opened as well as any L0 file. There is a possible race condition with killing process where there may be a L0 file which is merged but undeleted - and this is believed to be inert. There is some outstanding work still. Currently the whole store is paused if a push_mem is received by the Penciller, and the writing of a L0 sft file has not been completed. The creation of a L0 file appears to take about 300ms, so if the ledger_cache fills in this period a pause will occurr (perhaps due to objects with lots of index entries). It would be preferable to pause more elegantly in this situation. Perhaps there should be a harsh timeout on the call to check the SFT complete, and catching it should cause a refused response. The next PUT will then wait, but a any queued GETs can progress. --- include/leveled.hrl | 2 - src/leveled_bookie.erl | 6 ++ src/leveled_codec.erl | 6 -- src/leveled_inker.erl | 2 +- src/leveled_penciller.erl | 136 ++++++++++++++++------------- src/leveled_sft.erl | 6 +- test/end_to_end/iterator_SUITE.erl | 10 +-- 7 files changed, 88 insertions(+), 80 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 481b8db..55192ca 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -46,8 +46,6 @@ {root_path :: string(), cache_size :: integer(), max_journalsize :: integer(), - metadata_extractor :: function(), - indexspec_converter :: function(), snapshot_bookie :: pid()}). -record(iclerk_options, diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 18b018f..f95b37c 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -133,6 +133,7 @@ terminate/2, code_change/3, book_start/1, + book_start/3, book_riakput/3, book_riakdelete/4, book_riakget/3, @@ -169,6 +170,11 @@ %%% API %%%============================================================================ +book_start(RootPath, LedgerCacheSize, JournalSize) -> + book_start(#bookie_options{root_path=RootPath, + cache_size=LedgerCacheSize, + max_journalsize=JournalSize}). + book_start(Opts) -> gen_server:start(?MODULE, [Opts], []). diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 096e48f..be0bcf1 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -39,7 +39,6 @@ strip_to_keyseqstatusonly/1, striphead_to_details/1, is_active/2, - is_indexkey/1, endkey_passed/2, key_dominates/2, print_key/1, @@ -108,11 +107,6 @@ to_ledgerkey(Bucket, Key, Tag, Field, Value) when Tag == ?IDX_TAG -> to_ledgerkey(Bucket, Key, Tag) -> {Tag, Bucket, Key, null}. -is_indexkey({Tag, _, _, _}) when Tag == ?IDX_TAG -> - true; -is_indexkey(_Key) -> - false. - hash(Obj) -> erlang:phash2(term_to_binary(Obj)). diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index c346cc1..5be590b 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -695,7 +695,7 @@ simple_manifest_writer(Manifest, ManSQN, RootPath) -> TmpFN = filename:join(ManPath, integer_to_list(ManSQN) ++ "." ++ ?PENDING_FILEX), MBin = term_to_binary(lists:map(fun({SQN, FN, _PID}) -> {SQN, FN} end, - Manifest)), + Manifest), [compressed]), case filelib:is_file(NewFN) of true -> io:format("Error - trying to write manifest for" diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 2aa99c9..13833f2 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -139,7 +139,10 @@ %% - nonzero_.crr %% %% On startup, the Penciller should look for the nonzero_*.crr file with the -%% highest such manifest sequence number. +%% highest such manifest sequence number. This will be started as the +%% manifest, together with any _0_0.sft file found at that Manifest SQN. +%% Level zero files are not kept in the persisted manifest, and adding a L0 +%% file does not advanced the Manifest SQN. %% %% The pace at which the store can accept updates will be dependent on the %% speed at which the Penciller's Clerk can merge files at lower levels plus @@ -157,6 +160,27 @@ %% table and build a new table starting with the remainder, and the keys from %% the latest push. %% +%% Only a single L0 file may exist at any one moment in time. If pushes are +%% received when memory is over the maximum size, the pushes must be kept into +%% memory. +%% +%% 1 - A L0 file is prompted to be created at ManifestSQN n +%% 2 - The next push to memory will be stalled until the L0 write is reported +%% as completed (as the memory needs to be flushed) +%% 3 - The completion of the L0 file will cause a prompt to be cast to the +%% clerk for them to look for work +%% 4 - On completion of the merge (of the L0 file into L1, as this will be the +%% highest priority work), the clerk will create a new manifest file at +%% manifest SQN n+1 +%% 5 - The clerk will prompt the penciller about the change, and the Penciller +%% will then commit the change (by renaming the manifest file to be active, and +%% advancing th ein-memory state of the manifest and manifest SQN) +%% 6 - The Penciller having committed the change will cast back to the Clerk +%% to inform the Clerk that the chnage has been committed, and so it can carry +%% on requetsing new work +%% 7 - If the Penciller now receives a Push to over the max size, a new L0 file +%% can now be created with the ManifestSQN of n+1 +%% %% ---------- NOTES ON THE USE OF ETS ---------- %% %% Insertion into ETS is very fast, and so using ETS does not slow the PUT @@ -177,7 +201,7 @@ %% they need to iterate, or iterate through map functions scanning all the %% keys. The conversion may not be expensive, as we know loading into an ETS %% table is fast - but there may be some hidden overheads with creating and -%5 destroying many ETS tables. +%% destroying many ETS tables. %% %% A3 - keep a parallel list of lists of things that have gone in the ETS %% table in the format they arrived in @@ -252,7 +276,8 @@ -include_lib("eunit/include/eunit.hrl"). -define(LEVEL_SCALEFACTOR, [{0, 0}, {1, 8}, {2, 64}, {3, 512}, - {4, 4096}, {5, 32768}, {6, 262144}, {7, infinity}]). + {4, 4096}, {5, 32768}, {6, 262144}, + {7, infinity}]). -define(MAX_LEVELS, 8). -define(MAX_WORK_WAIT, 300). -define(MANIFEST_FP, "ledger_manifest"). @@ -383,9 +408,6 @@ handle_call({push_mem, DumpList}, From, State=#state{is_snapshot=Snap}) % If not the change has still been made but the the L0 file will not have % been prompted - so the reply does not indicate failure but returns the % atom 'pause' to signal a loose desire for back-pressure to be applied. - % The only reason in this case why there should be a pause is if the - % manifest is locked pending completion of a manifest change - so reacting - % to the pause signal may not be sensible StartWatch = os:timestamp(), case assess_sqn(DumpList) of {MinSQN, MaxSQN} when MaxSQN >= MinSQN, @@ -494,8 +516,6 @@ handle_call({fetch_keys, StartKey, EndKey, AccFun, InitAcc}, SFTiter = initiate_rangequery_frommanifest(StartKey, EndKey, State#state.manifest), - io:format("Manifest for iterator of:~n"), - print_manifest(SFTiter), Acc = keyfolder(L0iter, SFTiter, StartKey, EndKey, {AccFun, InitAcc}), {reply, Acc, State}; handle_call(work_for_clerk, From, State) -> @@ -588,18 +608,12 @@ terminate(Reason, State) -> no_change -> State end, - % TODO: - % This next section (to the end of the case clause), appears to be - % pointless. It will persist the in-memory state to a SFT file, but on - % startup that file will be ignored as the manifest has not bene updated - % - % Should we update the manifest, or stop trying to persist on closure? Dump = roll_into_list(State#state.memtable_copy), case {UpdState#state.levelzero_pending, get_item(0, UpdState#state.manifest, []), length(Dump)} of {?L0PEND_RESET, [], L} when L > 0 -> - MSN = UpdState#state.manifest_sqn + 1, + MSN = UpdState#state.manifest_sqn, FileName = UpdState#state.root_path ++ "/" ++ ?FILES_FP ++ "/" ++ integer_to_list(MSN) ++ "_0_0", @@ -701,7 +715,7 @@ start_from_file(PCLopts) -> %% Find any L0 files L0FN = filepath(RootPath, - TopManSQN + 1, + TopManSQN, new_merge_files) ++ "_0_0.sft", case filelib:is_file(L0FN) of true -> @@ -785,9 +799,9 @@ roll_memory(State, MaxSize, MemTableCopy) -> case ets:info(State#state.memtable, size) of Size when Size > MaxSize -> L0 = get_item(0, State#state.manifest, []), - case {L0, manifest_locked(State)} of - {[], false} -> - MSN = State#state.manifest_sqn + 1, + case L0 of + [] -> + MSN = State#state.manifest_sqn, FileName = State#state.root_path ++ "/" ++ ?FILES_FP ++ "/" ++ integer_to_list(MSN) ++ "_0_0", @@ -798,10 +812,6 @@ roll_memory(State, MaxSize, MemTableCopy) -> 0, Opts), {ok, {true, L0Pid, os:timestamp()}, MSN, Size}; - {[], true} -> - {pause, - "L0 file write blocked by change at sqn=~w~n", - [State#state.manifest_sqn]}; _ -> {pause, "L0 file write blocked by L0 file in manifest~n", @@ -870,23 +880,6 @@ compare_to_sqn(Obj, SQN) -> end. -%% Manifest lock - don't have two changes to the manifest happening -%% concurrently -% TODO: Is this necessary now? - -manifest_locked(State) -> - if - length(State#state.ongoing_work) > 0 -> - true; - true -> - case State#state.levelzero_pending of - {true, _Pid, _TS} -> - true; - _ -> - false - end - end. - %% Work out what the current work queue should be %% %% The work queue should have a lower level work at the front, and no work @@ -905,8 +898,24 @@ return_work(State, From) -> io:format("Work at Level ~w to be scheduled for ~w with ~w " ++ "queue items outstanding~n", [SrcLevel, From, length(OtherWork)]), - case {manifest_locked(State), State#state.ongoing_work} of - {false, _} -> + case {element(1, State#state.levelzero_pending), + State#state.ongoing_work} of + {true, _} -> + % Once the L0 file is completed there will be more work + % - so don't be busy doing other work now + io:format("Allocation of work blocked as L0 pending~n"), + {State, none}; + {_, [OutstandingWork]} -> + % Still awaiting a response + io:format("Ongoing work requested by ~w " ++ + "but work outstanding from Level ~w " ++ + "and Clerk ~w at sequence number ~w~n", + [From, + OutstandingWork#penciller_work.src_level, + OutstandingWork#penciller_work.clerk, + OutstandingWork#penciller_work.next_sqn]), + {State, none}; + _ -> %% No work currently outstanding %% Can allocate work NextSQN = State#state.manifest_sqn + 1, @@ -923,22 +932,7 @@ return_work(State, From) -> start_time = os:timestamp(), ledger_filepath = FP, manifest_file = ManFile}, - {State#state{ongoing_work=[WI]}, WI}; - {true, [OutstandingWork]} -> - %% Still awaiting a response - io:format("Ongoing work requested by ~w " ++ - "but work outstanding from Level ~w " ++ - "and Clerk ~w at sequence number ~w~n", - [From, - OutstandingWork#penciller_work.src_level, - OutstandingWork#penciller_work.clerk, - OutstandingWork#penciller_work.next_sqn]), - {State, none}; - {true, _} -> - %% Manifest locked - io:format("Manifest locked but no work outstanding " ++ - "with clerk~n"), - {State, none} + {State#state{ongoing_work=[WI]}, WI} end; _ -> {State, none} @@ -1313,11 +1307,12 @@ commit_manifest_change(ReturnedWorkItem, State) -> {NewMSN, _From} -> MTime = timer:now_diff(os:timestamp(), SentWorkItem#penciller_work.start_time), + WISrcLevel = SentWorkItem#penciller_work.src_level, io:format("Merge to sqn ~w completed in ~w microseconds " ++ - "at Level ~w~n", + "from Level ~w~n", [SentWorkItem#penciller_work.next_sqn, MTime, - SentWorkItem#penciller_work.src_level]), + WISrcLevel]), ok = rename_manifest_files(RootPath, NewMSN), FilesToDelete = ReturnedWorkItem#penciller_work.unreferenced_files, UnreferencedFilesUpd = update_deletions(FilesToDelete, @@ -1326,10 +1321,26 @@ commit_manifest_change(ReturnedWorkItem, State) -> io:format("Merge has been commmitted at sequence number ~w~n", [NewMSN]), NewManifest = ReturnedWorkItem#penciller_work.new_manifest, - print_manifest(NewManifest), + + CurrL0 = get_item(0, State#state.manifest, []), + % If the work isn't L0 work, then we may have an uncommitted + % manifest change at L0 - so add this back into the Manifest loop + % state + RevisedManifest = case {WISrcLevel, CurrL0} of + {0, _} -> + NewManifest; + {_, []} -> + NewManifest; + {_, [L0ManEntry]} -> + lists:keystore(0, + 1, + NewManifest, + {0, [L0ManEntry]}) + end, + print_manifest(RevisedManifest), {ok, State#state{ongoing_work=[], manifest_sqn=NewMSN, - manifest=NewManifest, + manifest=RevisedManifest, unreferenced_files=UnreferencedFilesUpd}}; {MaybeWrongMSN, From} -> io:format("Merge commit at sqn ~w not matched to expected" ++ @@ -1508,6 +1519,7 @@ simple_server_test() -> ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), maybe_pause_push(pcl_pushmem(PCL, KL2)), maybe_pause_push(pcl_pushmem(PCL, [Key3])), + ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), ?assertMatch(Key3, pcl_fetch(PCL, {o,"Bucket0003", "Key0003", null})), diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index af56fa9..7669660 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -282,9 +282,6 @@ handle_call({sft_new, Filename, KL1, KL2, Level}, _From, State) -> FileMD, KL1, KL2, Level), - {KL1Rem, KL2Rem} = KeyRemainders, - io:format("File created with remainders of size ~w ~w~n", - [length(KL1Rem), length(KL2Rem)]), {reply, {KeyRemainders, UpdFileMD#state.smallest_key, UpdFileMD#state.highest_key}, @@ -334,7 +331,10 @@ handle_call(get_maxsqn, _From, State) -> {reply, State#state.highest_sqn, State}. handle_cast({sft_new, Filename, Inp1, [], 0}, _State) -> + SW = os:timestamp(), {ok, State} = create_levelzero(Inp1, Filename), + io:format("File creation of L0 file ~s took ~w microseconds~n", + [Filename, timer:now_diff(os:timestamp(), SW)]), {noreply, State}; handle_cast(_Msg, State) -> {noreply, State}. diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 3b8701f..00a34af 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -40,9 +40,7 @@ simple_load_with2i(_Config) -> simple_querycount(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, - max_journalsize=50000000}, - {ok, Book1} = leveled_bookie:book_start(StartOpts1), + {ok, Book1} = leveled_bookie:book_start(RootPath, 4000, 50000000), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Book1, TestObject, TestSpec), testutil:check_forobject(Book1, TestObject), @@ -91,7 +89,7 @@ simple_querycount(_Config) -> Book1, ?KEY_ONLY), ok = leveled_bookie:book_close(Book1), - {ok, Book2} = leveled_bookie:book_start(StartOpts1), + {ok, Book2} = leveled_bookie:book_start(RootPath, 2000, 50000000), Index1Count = count_termsonindex("Bucket", "idx1_bin", Book2, @@ -206,7 +204,7 @@ simple_querycount(_Config) -> end, R9), ok = leveled_bookie:book_close(Book2), - {ok, Book3} = leveled_bookie:book_start(StartOpts1), + {ok, Book3} = leveled_bookie:book_start(RootPath, 2000, 50000000), lists:foreach(fun({IdxF, IdxT, X}) -> R = leveled_bookie:book_returnfolder(Book3, {index_query, @@ -223,7 +221,7 @@ simple_querycount(_Config) -> end, R9), ok = leveled_bookie:book_riakput(Book3, Obj9, Spc9), - {ok, Book4} = leveled_bookie:book_start(StartOpts1), + {ok, Book4} = leveled_bookie:book_start(RootPath, 2000, 50000000), lists:foreach(fun({IdxF, IdxT, X}) -> R = leveled_bookie:book_returnfolder(Book4, {index_query, From 7319b8f41529c9b9e0262003f810784ff0e21494 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 19 Oct 2016 20:51:30 +0100 Subject: [PATCH 076/167] Redundant clauses Remove some redundant clauses, and fix up some logging --- src/leveled_codec.erl | 3 --- src/leveled_pclerk.erl | 22 +++++----------------- src/leveled_penciller.erl | 30 ++++-------------------------- test/end_to_end/iterator_SUITE.erl | 7 ++++++- 4 files changed, 15 insertions(+), 47 deletions(-) diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index be0bcf1..875fb88 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -33,7 +33,6 @@ -include_lib("eunit/include/eunit.hrl"). -export([strip_to_keyonly/1, - strip_to_keyseqonly/1, strip_to_seqonly/1, strip_to_statusonly/1, strip_to_keyseqstatusonly/1, @@ -65,8 +64,6 @@ generate_uuid() -> strip_to_keyonly({keyonly, K}) -> K; strip_to_keyonly({K, _V}) -> K. -strip_to_keyseqonly({K, {SeqN, _, _ }}) -> {K, SeqN}. - strip_to_keyseqstatusonly({K, {SeqN, St, _MD}}) -> {K, SeqN, St}. strip_to_statusonly({_, {_, St, _}}) -> St. diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 7ec6144..cac5d6e 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -129,9 +129,7 @@ handle_call({manifest_change, confirm, Closing}, From, State) -> handle_cast(prompt, State) -> io:format("Clerk reducing timeout due to prompt~n"), - {noreply, State, ?QUICK_TIMEOUT}; -handle_cast(_Msg, State) -> - {noreply, State}. + {noreply, State, ?QUICK_TIMEOUT}. handle_info(timeout, State=#state{change_pending=Pnd}) when Pnd == false -> case requestandhandle_work(State) of @@ -142,9 +140,7 @@ handle_info(timeout, State=#state{change_pending=Pnd}) when Pnd == false -> % change {noreply, State#state{change_pending=true, work_item=WI}} - end; -handle_info(_Info, State) -> - {noreply, State}. + end. terminate(_Reason, _State) -> ok. @@ -373,17 +369,9 @@ find_randomkeys(FList, Count, Source) -> K1 = leveled_codec:strip_to_keyonly(KV1), P1 = choose_pid_toquery(FList, K1), FoundKV = leveled_sft:sft_get(P1, K1), - Check = case FoundKV of - not_present -> - io:format("Failed to find ~w in ~w~n", [K1, P1]), - error; - _ -> - Found = leveled_codec:strip_to_keyonly(FoundKV), - io:format("success finding ~w in ~w~n", [K1, P1]), - ?assertMatch(K1, Found), - ok - end, - ?assertMatch(Check, ok), + Found = leveled_codec:strip_to_keyonly(FoundKV), + io:format("success finding ~w in ~w~n", [K1, P1]), + ?assertMatch(K1, Found), find_randomkeys(FList, Count - 1, Source). diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 13833f2..0ea83c8 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -751,8 +751,7 @@ start_from_file(PCLopts) -> checkready_pushtomem(State) -> {TableSize, UpdState} = case State#state.levelzero_pending of {true, Pid, _TS} -> - %% Need to handle error scenarios? - %% N.B. Sync call - so will be ready + % N.B. Sync call - so will be ready {ok, SrcFN, StartKey, EndKey} = leveled_sft:sft_checkready(Pid), true = ets:delete_all_objects(State#state.memtable), ManifestEntry = #manifest_entry{start_key=StartKey, @@ -898,23 +897,12 @@ return_work(State, From) -> io:format("Work at Level ~w to be scheduled for ~w with ~w " ++ "queue items outstanding~n", [SrcLevel, From, length(OtherWork)]), - case {element(1, State#state.levelzero_pending), - State#state.ongoing_work} of - {true, _} -> + case element(1, State#state.levelzero_pending) of + true -> % Once the L0 file is completed there will be more work % - so don't be busy doing other work now io:format("Allocation of work blocked as L0 pending~n"), {State, none}; - {_, [OutstandingWork]} -> - % Still awaiting a response - io:format("Ongoing work requested by ~w " ++ - "but work outstanding from Level ~w " ++ - "and Clerk ~w at sequence number ~w~n", - [From, - OutstandingWork#penciller_work.src_level, - OutstandingWork#penciller_work.clerk, - OutstandingWork#penciller_work.next_sqn]), - {State, none}; _ -> %% No work currently outstanding %% Can allocate work @@ -1526,17 +1514,7 @@ simple_server_test() -> ok = pcl_close(PCL), {ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath, max_inmemory_tablesize=1000}), - TopSQN = pcl_getstartupsequencenumber(PCLr), - Check = case TopSQN of - 2002 -> - %% everything got persisted - ok; - _ -> - io:format("Unexpected sequence number on restart ~w~n", - [TopSQN]), - error - end, - ?assertMatch(ok, Check), + ?assertMatch(2002, pcl_getstartupsequencenumber(PCLr)), ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002", null})), ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003", null})), diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 00a34af..30cc711 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -41,7 +41,11 @@ simple_load_with2i(_Config) -> simple_querycount(_Config) -> RootPath = testutil:reset_filestructure(), {ok, Book1} = leveled_bookie:book_start(RootPath, 4000, 50000000), - {TestObject, TestSpec} = testutil:generate_testobject(), + {TestObject, TestSpec} = testutil:generate_testobject("Bucket", + "Key1", + "Value1", + [], + {"MDK1", "MDV1"}), ok = leveled_bookie:book_riakput(Book1, TestObject, TestSpec), testutil:check_forobject(Book1, TestObject), testutil:check_formissingobject(Book1, "Bucket1", "Key2"), @@ -237,6 +241,7 @@ simple_querycount(_Config) -> end end, R9), + testutil:check_forobject(Book4, TestObject), ok = leveled_bookie:book_close(Book4), testutil:reset_filestructure(). From cf66431c8e18b9bb23116fd2ae58483dad029aa0 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 20 Oct 2016 02:23:45 +0100 Subject: [PATCH 077/167] Smoother handling of back-pressure The Penciller had two problems in previous commits: - If it had a push_mem soon after a L0 file had been created, the push_mem would stall waiting for the L0 file to complete - and this count take 100-200ms - The penciller's clerk favoured L0 work, but was lazy about asking for other work in-between, so often the L1 layer was bursting over capacity and the clerk was doing nothing but merging more L0 files in (with those merges getting more and more expensive as they had to cover more and more files) There are some partial resolutions to this. There is now an aggressive timeout when checking whther the L0 file is ready on a push_mem, and if the timeout is breached the error is caught and a 'returned' message goes back to the Bookie. the Bookie doesn't now empty its cache, it carrie son filling it, but on some probability it will keep trying to push_mem on future pushes. This increases Jitter around the expensive operation and split out the L0 delay into defined chunks. The penciller's clerk is now more aggressive in asking for work. There is also some simplification of the relationship between clerk timeouts and penciller back-pressure. Also resolved is an issue of inconcistency between the loader and the on startup (replaying the transaction log) and the standard push_mem process. The loader was not correctly de-duplicating by adding first (in order) to a tree before outputting the list from the tree. Some thought will be given later as to whether non-L0 work can be safely prioritised if the merge process still keeps getting behind. --- src/leveled_bookie.erl | 35 ++++++-- src/leveled_inker.erl | 12 ++- src/leveled_pclerk.erl | 23 ++---- src/leveled_penciller.erl | 167 ++++++++++++++++++++++---------------- src/leveled_sft.erl | 2 +- 5 files changed, 142 insertions(+), 97 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index f95b37c..9d61661 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -156,6 +156,7 @@ -define(SHUTDOWN_WAITS, 60). -define(SHUTDOWN_PAUSE, 10000). -define(SNAPSHOT_TIMEOUT, 300000). +-define(JITTER_PROBABILITY, 0.1). -record(state, {inker :: pid(), penciller :: pid(), @@ -582,23 +583,41 @@ addto_ledgercache(Changes, Cache) -> maybepush_ledgercache(MaxCacheSize, Cache, Penciller) -> CacheSize = gb_trees:size(Cache), + TimeToPush = maybe_withjitter(CacheSize, MaxCacheSize), if - CacheSize > MaxCacheSize -> - case leveled_penciller:pcl_pushmem(Penciller, - gb_trees:to_list(Cache)) of + TimeToPush -> + Dump = gb_trees:to_list(Cache), + case leveled_penciller:pcl_pushmem(Penciller, Dump) of ok -> {ok, gb_trees:empty()}; pause -> {pause, gb_trees:empty()}; - refused -> + returned -> {ok, Cache} end; true -> - {ok, Cache} + {ok, Cache} + end. + + +maybe_withjitter(CacheSize, MaxCacheSize) -> + if + CacheSize > 2 * MaxCacheSize -> + true; + CacheSize > MaxCacheSize -> + R = random:uniform(), + if + R < ?JITTER_PROBABILITY -> + true; + true -> + false + end; + true -> + false end. load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> - {MinSQN, MaxSQN, Output} = Acc0, + {MinSQN, MaxSQN, OutputTree} = Acc0, {SQN, PK} = KeyInLedger, % VBin may already be a term {VBin, VSize} = ExtractFun(ValueInLedger), @@ -613,11 +632,11 @@ load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> {loop, Acc0}; SQN when SQN < MaxSQN -> Changes = preparefor_ledgercache(PK, SQN, Obj, VSize, IndexSpecs), - {loop, {MinSQN, MaxSQN, Output ++ Changes}}; + {loop, {MinSQN, MaxSQN, addto_ledgercache(Changes, OutputTree)}}; MaxSQN -> io:format("Reached end of load batch with SQN ~w~n", [SQN]), Changes = preparefor_ledgercache(PK, SQN, Obj, VSize, IndexSpecs), - {stop, {MinSQN, MaxSQN, Output ++ Changes}}; + {stop, {MinSQN, MaxSQN, addto_ledgercache(Changes, OutputTree)}}; SQN when SQN > MaxSQN -> io:format("Skipping as exceeded MaxSQN ~w with SQN ~w~n", [MaxSQN, SQN]), diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 5be590b..79a0cd8 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -609,7 +609,7 @@ load_from_sequence(MinSQN, FilterFun, Penciller, [{_LowSQN, FN, Pid}|Rest]) -> load_between_sequence(MinSQN, MaxSQN, FilterFun, Penciller, CDBpid, StartPos, FN, Rest) -> io:format("Loading from filename ~s from SQN ~w~n", [FN, MinSQN]), - InitAcc = {MinSQN, MaxSQN, []}, + InitAcc = {MinSQN, MaxSQN, gb_trees:empty()}, Res = case leveled_cdb:cdb_scan(CDBpid, FilterFun, InitAcc, StartPos) of {eof, {AccMinSQN, _AccMaxSQN, AccKL}} -> ok = push_to_penciller(Penciller, AccKL), @@ -633,12 +633,18 @@ load_between_sequence(MinSQN, MaxSQN, FilterFun, Penciller, ok end. -push_to_penciller(Penciller, KeyList) -> +push_to_penciller(Penciller, KeyTree) -> + % The push to penciller must start as a tree to correctly de-duplicate + % the list by order before becoming a de-duplicated list for loading + KeyList = gb_trees:to_list(KeyTree), R = leveled_penciller:pcl_pushmem(Penciller, KeyList), if R == pause -> timer:sleep(?LOADING_PAUSE); - true -> + R == returned -> + timer:sleep(?LOADING_PAUSE), + push_to_penciller(Penciller, KeyTree); + R == ok -> ok end. diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index cac5d6e..2a915ae 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -67,9 +67,8 @@ -include_lib("eunit/include/eunit.hrl"). --define(INACTIVITY_TIMEOUT, 5000). --define(QUICK_TIMEOUT, 500). --define(HAPPYTIME_MULTIPLIER, 2). +-define(MAX_TIMEOUT, 2000). +-define(MIN_TIMEOUT, 200). -record(state, {owner :: pid(), change_pending=false :: boolean(), @@ -98,7 +97,7 @@ init([]) -> {ok, #state{}}. handle_call({register, Owner}, _From, State) -> - {reply, ok, State#state{owner=Owner}, ?INACTIVITY_TIMEOUT}; + {reply, ok, State#state{owner=Owner}, ?MIN_TIMEOUT}; handle_call({manifest_change, return, true}, _From, State) -> io:format("Request for manifest change from clerk on closing~n"), case State#state.change_pending of @@ -124,12 +123,11 @@ handle_call({manifest_change, confirm, Closing}, From, State) -> State#state.owner), {noreply, State#state{work_item=null, change_pending=false}, - ?QUICK_TIMEOUT} + ?MIN_TIMEOUT} end. handle_cast(prompt, State) -> - io:format("Clerk reducing timeout due to prompt~n"), - {noreply, State, ?QUICK_TIMEOUT}. + {noreply, State, ?MIN_TIMEOUT}. handle_info(timeout, State=#state{change_pending=Pnd}) when Pnd == false -> case requestandhandle_work(State) of @@ -155,15 +153,10 @@ code_change(_OldVsn, State, _Extra) -> requestandhandle_work(State) -> case leveled_penciller:pcl_workforclerk(State#state.owner) of - {none, Backlog} -> + none -> io:format("Work prompted but none needed~n"), - case Backlog of - false -> - {false, ?INACTIVITY_TIMEOUT * ?HAPPYTIME_MULTIPLIER}; - _ -> - {false, ?INACTIVITY_TIMEOUT} - end; - {WI, _} -> + {false, ?MAX_TIMEOUT}; + WI -> {NewManifest, FilesToDelete} = merge(WI), UpdWI = WI#penciller_work{new_manifest=NewManifest, unreferenced_files=FilesToDelete}, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 0ea83c8..39c2626 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -306,7 +306,6 @@ memtable_copy = #l0snapshot{} :: #l0snapshot{}, levelzero_snapshot = gb_trees:empty() :: gb_trees:tree(), memtable, - backlog = false :: boolean(), memtable_maxsize :: integer(), is_snapshot = false :: boolean(), snapshot_fully_loaded = false :: boolean(), @@ -412,58 +411,25 @@ handle_call({push_mem, DumpList}, From, State=#state{is_snapshot=Snap}) case assess_sqn(DumpList) of {MinSQN, MaxSQN} when MaxSQN >= MinSQN, MinSQN >= State#state.ledger_sqn -> - MaxTableSize = State#state.memtable_maxsize, - {TableSize0, State1} = checkready_pushtomem(State), - case quickcheck_pushtomem(DumpList, - TableSize0, - MaxTableSize) of - {twist, TableSize1} -> - gen_server:reply(From, ok), - io:format("Reply made on push in ~w microseconds~n", + case checkready_pushtomem(State) of + {ok, TableSize0, State1} -> + push_and_roll(DumpList, + TableSize0, + State#state.memtable_maxsize, + MaxSQN, + StartWatch, + From, + State1); + timeout -> + io:format("Timeout of ~w microseconds awaiting " ++ + "L0 SFT write~n", [timer:now_diff(os:timestamp(), StartWatch)]), - L0Snap = do_pushtomem(DumpList, - State1#state.memtable, - State1#state.memtable_copy, - MaxSQN), - io:format("Push completed in ~w microseconds~n", - [timer:now_diff(os:timestamp(), StartWatch)]), - {noreply, - State1#state{memtable_copy=L0Snap, - table_size=TableSize1, - ledger_sqn=MaxSQN}}; - {maybe_roll, TableSize1} -> - L0Snap = do_pushtomem(DumpList, - State1#state.memtable, - State1#state.memtable_copy, - MaxSQN), - - case roll_memory(State1, MaxTableSize, L0Snap) of - {ok, L0Pend, ManSN, TableSize2} -> - io:format("Push completed in ~w microseconds~n", - [timer:now_diff(os:timestamp(), StartWatch)]), - {reply, - ok, - State1#state{levelzero_pending=L0Pend, - table_size=TableSize2, - manifest_sqn=ManSN, - memtable_copy=L0Snap, - ledger_sqn=MaxSQN, - backlog=false}}; - {pause, Reason, Details} -> - io:format("Excess work due to - " ++ Reason, - Details), - {reply, - pause, - State1#state{backlog=true, - memtable_copy=L0Snap, - table_size=TableSize1, - ledger_sqn=MaxSQN}} - end + {reply, returned, State} end; {MinSQN, MaxSQN} -> io:format("Mismatch of sequence number expectations with push " - ++ "having sequence numbers between ~w and ~w " - ++ "but current sequence number is ~w~n", + ++ "having sequence numbers between ~w and ~w " + ++ "but current sequence number is ~w~n", [MinSQN, MaxSQN, State#state.ledger_sqn]), {reply, refused, State}; empty -> @@ -520,7 +486,7 @@ handle_call({fetch_keys, StartKey, EndKey, AccFun, InitAcc}, {reply, Acc, State}; handle_call(work_for_clerk, From, State) -> {UpdState, Work} = return_work(State, From), - {reply, {Work, UpdState#state.backlog}, UpdState}; + {reply, Work, UpdState}; handle_call(get_startup_sqn, _From, State) -> {reply, State#state.ledger_sqn, State}; handle_call({register_snapshot, Snapshot}, _From, State) -> @@ -568,7 +534,8 @@ handle_cast({release_snapshot, Snapshot}, State) -> io:format("Penciller snapshot ~w released~n", [Snapshot]), {noreply, State#state{registered_snapshots=Rs}}. -handle_info(_Info, State) -> +handle_info({_Ref, {ok, SrcFN, _StartKey, _EndKey}}, State) -> + io:format("Orphaned reply after timeout on L0 file write ~s~n", [SrcFN]), {noreply, State}. terminate(Reason, State=#state{is_snapshot=Snap}) when Snap == true -> @@ -749,28 +716,87 @@ start_from_file(PCLopts) -> checkready_pushtomem(State) -> - {TableSize, UpdState} = case State#state.levelzero_pending of + case State#state.levelzero_pending of {true, Pid, _TS} -> - % N.B. Sync call - so will be ready - {ok, SrcFN, StartKey, EndKey} = leveled_sft:sft_checkready(Pid), - true = ets:delete_all_objects(State#state.memtable), - ManifestEntry = #manifest_entry{start_key=StartKey, - end_key=EndKey, - owner=Pid, - filename=SrcFN}, - % Prompt clerk to ask about work - do this for every L0 roll - ok = leveled_pclerk:clerk_prompt(State#state.clerk), - {0, - State#state{manifest=lists:keystore(0, + case checkready(Pid) of + timeout -> + timeout; + {ok, SrcFN, StartKey, EndKey} -> + true = ets:delete_all_objects(State#state.memtable), + ManifestEntry = #manifest_entry{start_key=StartKey, + end_key=EndKey, + owner=Pid, + filename=SrcFN}, + % Prompt clerk to ask about work - do this for every + % L0 roll + ok = leveled_pclerk:clerk_prompt(State#state.clerk), + UpdManifest = lists:keystore(0, 1, State#state.manifest, {0, [ManifestEntry]}), - levelzero_pending=?L0PEND_RESET, - memtable_copy=#l0snapshot{}}}; + {ok, + 0, + State#state{manifest=UpdManifest, + levelzero_pending=?L0PEND_RESET, + memtable_copy=#l0snapshot{}}} + end; ?L0PEND_RESET -> - {State#state.table_size, State} - end, - {TableSize, UpdState}. + {ok, State#state.table_size, State} + end. + + +checkready(Pid) -> + try + leveled_sft:sft_checkready(Pid) + catch + exit:{timeout, _} -> + timeout + end. + + +push_and_roll(DumpList, TableSize, MaxTableSize, MaxSQN, StartWatch, From, State) -> + case quickcheck_pushtomem(DumpList, TableSize, MaxTableSize) of + {twist, TableSize1} -> + gen_server:reply(From, ok), + io:format("Reply made on push in ~w microseconds~n", + [timer:now_diff(os:timestamp(), StartWatch)]), + L0Snap = do_pushtomem(DumpList, + State#state.memtable, + State#state.memtable_copy, + MaxSQN), + io:format("Push completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(), StartWatch)]), + {noreply, + State#state{memtable_copy=L0Snap, + table_size=TableSize1, + ledger_sqn=MaxSQN}}; + {maybe_roll, TableSize1} -> + L0Snap = do_pushtomem(DumpList, + State#state.memtable, + State#state.memtable_copy, + MaxSQN), + + case roll_memory(State, MaxTableSize, L0Snap) of + {ok, L0Pend, ManSN, TableSize2} -> + io:format("Push completed in ~w microseconds~n", + [timer:now_diff(os:timestamp(), StartWatch)]), + {reply, + ok, + State#state{levelzero_pending=L0Pend, + table_size=TableSize2, + manifest_sqn=ManSN, + memtable_copy=L0Snap, + ledger_sqn=MaxSQN}}; + {pause, Reason, Details} -> + io:format("Excess work due to - " ++ Reason, + Details), + {reply, + pause, + State#state{memtable_copy=L0Snap, + table_size=TableSize1, + ledger_sqn=MaxSQN}} + end + end. quickcheck_pushtomem(DumpList, TableSize, MaxSize) -> case TableSize + length(DumpList) of @@ -894,9 +920,10 @@ return_work(State, From) -> case length(WorkQueue) of L when L > 0 -> [{SrcLevel, Manifest}|OtherWork] = WorkQueue, + Backlog = length(OtherWork), io:format("Work at Level ~w to be scheduled for ~w with ~w " ++ "queue items outstanding~n", - [SrcLevel, From, length(OtherWork)]), + [SrcLevel, From, Backlog]), case element(1, State#state.levelzero_pending) of true -> % Once the L0 file is completed there will be more work @@ -1557,7 +1584,7 @@ simple_server_test() -> "Key0004", null}, 3002)), - % Add some more keys and confirm that chekc sequence number still + % Add some more keys and confirm that check sequence number still % sees the old version in the previous snapshot, but will see the new version % in a new snapshot Key1A = {{o,"Bucket0001", "Key0001", null}, {4002, {active, infinity}, null}}, diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 7669660..f292339 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -252,7 +252,7 @@ sft_close(Pid) -> gen_server:call(Pid, close, infinity). sft_checkready(Pid) -> - gen_server:call(Pid, background_complete, infinity). + gen_server:call(Pid, background_complete, 50). sft_getmaxsequencenumber(Pid) -> gen_server:call(Pid, get_maxsqn, infinity). From 0324edd6f694d0a5e984c3354825ff80ad08c793 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 20 Oct 2016 12:16:17 +0100 Subject: [PATCH 078/167] Rotating object tests Recent fixes have been made to problems associated with rapidly changing objexts especially on re-opening of the bookie. Test of rotating objects from both an index query and a fetch perspective added to better detect such issues in the future. --- test/end_to_end/iterator_SUITE.erl | 107 ++++++++++++++++++++++++++++- test/end_to_end/testutil.erl | 42 ++++++++--- 2 files changed, 137 insertions(+), 12 deletions(-) diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 30cc711..8fb7505 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -4,13 +4,16 @@ -include("include/leveled.hrl"). -define(KEY_ONLY, {false, undefined}). +-define(RETURN_TERMS, {true, undefined}). -export([all/0]). -export([simple_load_with2i/1, - simple_querycount/1]). + simple_querycount/1, + rotating_objects/1]). all() -> [simple_load_with2i, - simple_querycount]. + simple_querycount, + rotating_objects]. simple_load_with2i(_Config) -> @@ -270,3 +273,103 @@ count_termsonindex(Bucket, IdxField, Book, QType) -> end, 0, lists:seq(1901, 2218)). + + +rotating_objects(_Config) -> + RootPath = testutil:reset_filestructure(), + ok = rotating_object_check(RootPath, "Bucket1", 10), + ok = rotating_object_check(RootPath, "Bucket2", 200), + ok = rotating_object_check(RootPath, "Bucket3", 800), + ok = rotating_object_check(RootPath, "Bucket4", 1600), + ok = rotating_object_check(RootPath, "Bucket5", 3200), + ok = rotating_object_check(RootPath, "Bucket6", 9600), + testutil:reset_filestructure(). + + +rotating_object_check(RootPath, Bucket, NumberOfObjects) -> + {ok, Book1} = leveled_bookie:book_start(RootPath, 2000, 5000000), + {KSpcL1, V1} = put_indexed_objects(Book1, Bucket, NumberOfObjects), + ok = check_indexed_objects(Book1, Bucket, KSpcL1, V1), + {KSpcL2, V2} = put_altered_indexed_objects(Book1, Bucket, KSpcL1), + ok = check_indexed_objects(Book1, Bucket, KSpcL2, V2), + {KSpcL3, V3} = put_altered_indexed_objects(Book1, Bucket, KSpcL2), + ok = leveled_bookie:book_close(Book1), + {ok, Book2} = leveled_bookie:book_start(RootPath, 1000, 5000000), + ok = check_indexed_objects(Book2, Bucket, KSpcL3, V3), + {KSpcL4, V4} = put_altered_indexed_objects(Book2, Bucket, KSpcL3), + ok = check_indexed_objects(Book2, Bucket, KSpcL4, V4), + ok = leveled_bookie:book_close(Book2), + ok. + + + +check_indexed_objects(Book, B, KSpecL, V) -> + % Check all objects match, return what should eb the results of an all + % index query + IdxR = lists:map(fun({K, Spc}) -> + {ok, O} = leveled_bookie:book_riakget(Book, B, K), + V = testutil:get_value(O), + {add, + "idx1_bin", + IdxVal} = lists:keyfind(add, 1, Spc), + {IdxVal, K} end, + KSpecL), + % Check the all index query matxhes expectations + R = leveled_bookie:book_returnfolder(Book, + {index_query, + B, + {"idx1_bin", + "0", + "~"}, + ?RETURN_TERMS}), + {async, Fldr} = R, + QR = lists:sort(Fldr()), + ER = lists:sort(IdxR), + ok = if + ER == QR -> + ok + end, + ok. + + +put_indexed_objects(Book, Bucket, Count) -> + V = testutil:get_compressiblevalue(), + IndexGen = testutil:get_randomindexes_generator(1), + SW = os:timestamp(), + ObjL1 = testutil:generate_objects(Count, + uuid, + [], + V, + IndexGen, + Bucket), + KSpecL = lists:map(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Book, + Obj, + Spc), + {testutil:get_key(Obj), Spc} + end, + ObjL1), + io:format("Put of ~w objects with ~w index entries " + ++ + "each completed in ~w microseconds~n", + [Count, 1, timer:now_diff(os:timestamp(), SW)]), + {KSpecL, V}. + +put_altered_indexed_objects(Book, Bucket, KSpecL) -> + IndexGen = testutil:get_randomindexes_generator(1), + V = testutil:get_compressiblevalue(), + RplKSpecL = lists:map(fun({K, Spc}) -> + AddSpc = lists:keyfind(add, 1, Spc), + {O, AltSpc} = testutil:set_object(Bucket, + K, + V, + IndexGen, + [AddSpc]), + ok = leveled_bookie:book_riakput(Book, + O, + AltSpc), + {K, AltSpc} end, + KSpecL), + {RplKSpecL, V}. + + diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index c938d6a..e3bf77d 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -15,6 +15,10 @@ generate_smallobjects/2, generate_objects/2, generate_objects/5, + generate_objects/6, + set_object/5, + get_key/1, + get_value/1, get_compressiblevalue/0, get_randomindexes_generator/1, name_list/0, @@ -152,32 +156,44 @@ generate_objects(Count, KeyNumber) -> generate_objects(Count, KeyNumber, ObjL, Value) -> generate_objects(Count, KeyNumber, ObjL, Value, fun() -> [] end). -generate_objects(0, _KeyNumber, ObjL, _Value, _IndexGen) -> +generate_objects(Count, KeyNumber, ObjL, Value, IndexGen) -> + generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, "Bucket"). + +generate_objects(0, _KeyNumber, ObjL, _Value, _IndexGen, _Bucket) -> ObjL; -generate_objects(Count, uuid, ObjL, Value, IndexGen) -> - {Obj1, Spec1} = set_object(leveled_codec:generate_uuid(), +generate_objects(Count, uuid, ObjL, Value, IndexGen, Bucket) -> + {Obj1, Spec1} = set_object(Bucket, + leveled_codec:generate_uuid(), Value, IndexGen), generate_objects(Count - 1, uuid, ObjL ++ [{random:uniform(), Obj1, Spec1}], Value, - IndexGen); -generate_objects(Count, KeyNumber, ObjL, Value, IndexGen) -> - {Obj1, Spec1} = set_object("Key" ++ integer_to_list(KeyNumber), + IndexGen, + Bucket); +generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, Bucket) -> + {Obj1, Spec1} = set_object(Bucket, + "Key" ++ integer_to_list(KeyNumber), Value, IndexGen), generate_objects(Count - 1, KeyNumber + 1, ObjL ++ [{random:uniform(), Obj1, Spec1}], Value, - IndexGen). + IndexGen, + Bucket). -set_object(Key, Value, IndexGen) -> - Obj = {"Bucket", +set_object(Bucket, Key, Value, IndexGen) -> + set_object(Bucket, Key, Value, IndexGen, []). + +set_object(Bucket, Key, Value, IndexGen, Indexes2Remove) -> + Obj = {Bucket, Key, Value, - IndexGen(), + IndexGen() ++ lists:map(fun({add, IdxF, IdxV}) -> + {remove, IdxF, IdxV} end, + Indexes2Remove), [{"MDK", "MDV" ++ Key}, {"MDK2", "MDV" ++ Key}]}, {B1, K1, V1, Spec1, MD} = Obj, @@ -185,6 +201,12 @@ set_object(Key, Value, IndexGen) -> {#r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]}, Spec1}. +get_key(Object) -> + Object#r_object.key. + +get_value(Object) -> + [Content] = Object#r_object.contents, + Content#r_content.value. load_objects(ChunkSize, GenList, Bookie, TestObject, Generator) -> lists:map(fun(KN) -> From 5c2029668dc8d7d3e2637d7ffd015e92808101b9 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 20 Oct 2016 16:00:08 +0100 Subject: [PATCH 079/167] Tombstone preperation Some initial code changes preparing for the test and implementation of tombstones and tombstone reaping --- include/leveled.hrl | 3 ++- src/leveled_pclerk.erl | 33 ++++++++++++++++------------ src/leveled_penciller.erl | 45 ++++++++++++++++++++++++--------------- src/leveled_sft.erl | 39 +++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 32 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 55192ca..3b06a40 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -16,7 +16,8 @@ ledger_filepath :: string(), manifest_file :: string(), new_manifest :: list(), - unreferenced_files :: list()}). + unreferenced_files :: list(), + target_is_basement = false ::boolean()}). -record(manifest_entry, {start_key :: tuple(), diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 2a915ae..5a321fa 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -62,8 +62,7 @@ clerk_new/1, clerk_prompt/1, clerk_manifestchange/3, - code_change/3, - perform_merge/4]). + code_change/3]). -include_lib("eunit/include/eunit.hrl"). @@ -193,7 +192,7 @@ merge(WI) -> perform_merge({SrcF#manifest_entry.owner, SrcF#manifest_entry.filename}, Candidates, - SrcLevel, + {SrcLevel, WI#penciller_work.target_is_basement}, {WI#penciller_work.ledger_filepath, WI#penciller_work.next_sqn}) end, @@ -283,28 +282,32 @@ select_filetomerge(SrcLevel, Manifest) -> %% %% The level is the level which the new files should be created at. -perform_merge({UpperSFTPid, Filename}, CandidateList, Level, {Filepath, MSN}) -> +perform_merge({SrcPid, SrcFN}, CandidateList, LevelInfo, {Filepath, MSN}) -> io:format("Merge to be commenced for FileToMerge=~s with MSN=~w~n", - [Filename, MSN]), + [SrcFN, MSN]), PointerList = lists:map(fun(P) -> {next, P#manifest_entry.owner, all} end, CandidateList), - do_merge([{next, UpperSFTPid, all}], - PointerList, Level, {Filepath, MSN}, 0, []). + do_merge([{next, SrcPid, all}], + PointerList, + LevelInfo, + {Filepath, MSN}, + 0, + []). -do_merge([], [], Level, {_Filepath, MSN}, FileCounter, OutList) -> +do_merge([], [], {SrcLevel, _IsB}, {_Filepath, MSN}, FileCounter, OutList) -> io:format("Merge completed with MSN=~w Level=~w and FileCounter=~w~n", - [MSN, Level, FileCounter]), + [MSN, SrcLevel, FileCounter]), OutList; -do_merge(KL1, KL2, Level, {Filepath, MSN}, FileCounter, OutList) -> +do_merge(KL1, KL2, {SrcLevel, IsB}, {Filepath, MSN}, FileCounter, OutList) -> FileName = lists:flatten(io_lib:format(Filepath ++ "_~w_~w.sft", - [Level + 1, FileCounter])), + [SrcLevel + 1, FileCounter])), io:format("File to be created as part of MSN=~w Filename=~s~n", [MSN, FileName]), % Attempt to trace intermittent eaccess failures false = filelib:is_file(FileName), TS1 = os:timestamp(), - {ok, Pid, Reply} = leveled_sft:sft_new(FileName, KL1, KL2, Level + 1), + {ok, Pid, Reply} = leveled_sft:sft_new(FileName, KL1, KL2, SrcLevel + 1), {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, ExtMan = lists:append(OutList, [#manifest_entry{start_key=SmallestKey, @@ -313,7 +316,9 @@ do_merge(KL1, KL2, Level, {Filepath, MSN}, FileCounter, OutList) -> filename=FileName}]), MTime = timer:now_diff(os:timestamp(), TS1), io:format("File creation took ~w microseconds ~n", [MTime]), - do_merge(KL1Rem, KL2Rem, Level, {Filepath, MSN}, FileCounter + 1, ExtMan). + do_merge(KL1Rem, KL2Rem, + {SrcLevel, IsB}, {Filepath, MSN}, + FileCounter + 1, ExtMan). get_item(Index, List, Default) -> @@ -389,7 +394,7 @@ merge_file_test() -> #manifest_entry{owner=PidL2_2}, #manifest_entry{owner=PidL2_3}, #manifest_entry{owner=PidL2_4}], - 2, {"../test/", 99}), + {2, false}, {"../test/", 99}), lists:foreach(fun(ManEntry) -> {o, B1, K1} = ManEntry#manifest_entry.start_key, {o, B2, K2} = ManEntry#manifest_entry.end_key, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 39c2626..1c69eee 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -914,16 +914,20 @@ compare_to_sqn(Obj, SQN) -> %% The full queue is calculated for logging purposes only return_work(State, From) -> - WorkQueue = assess_workqueue([], - 0, - State#state.manifest), - case length(WorkQueue) of + {WorkQ, BasementL} = assess_workqueue([], 0, State#state.manifest, 0), + case length(WorkQ) of L when L > 0 -> - [{SrcLevel, Manifest}|OtherWork] = WorkQueue, + [{SrcLevel, Manifest}|OtherWork] = WorkQ, Backlog = length(OtherWork), io:format("Work at Level ~w to be scheduled for ~w with ~w " ++ "queue items outstanding~n", [SrcLevel, From, Backlog]), + IsBasement = if + SrcLevel + 1 == BasementL -> + true; + true -> + false + end, case element(1, State#state.levelzero_pending) of true -> % Once the L0 file is completed there will be more work @@ -946,7 +950,8 @@ return_work(State, From) -> manifest=Manifest, start_time = os:timestamp(), ledger_filepath = FP, - manifest_file = ManFile}, + manifest_file = ManFile, + target_is_basement = IsBasement}, {State#state{ongoing_work=[WI]}, WI} end; _ -> @@ -1161,7 +1166,7 @@ find_nextkey(QueryArray, LCnt, {BestKeyLevel, BestKV}, QueryFunT) -> QueryFunT); {{Key, Val}, null, null} -> % No best key set - so can assume that this key is the best key, - % and check the higher levels + % and check the lower levels find_nextkey(QueryArray, LCnt + 1, {LCnt, {Key, Val}}, @@ -1270,14 +1275,21 @@ keyfolder(IMMiterator, SFTiterator, StartKey, EndKey, {AccFun, Acc}) -> end. -assess_workqueue(WorkQ, ?MAX_LEVELS - 1, _Manifest) -> - WorkQ; -assess_workqueue(WorkQ, LevelToAssess, Manifest)-> +assess_workqueue(WorkQ, ?MAX_LEVELS - 1, _Man, BasementLevel) -> + {WorkQ, BasementLevel}; +assess_workqueue(WorkQ, LevelToAssess, Man, BasementLevel) -> MaxFiles = get_item(LevelToAssess, ?LEVEL_SCALEFACTOR, 0), - FileCount = length(get_item(LevelToAssess, Manifest, [])), - NewWQ = maybe_append_work(WorkQ, LevelToAssess, Manifest, MaxFiles, - FileCount), - assess_workqueue(NewWQ, LevelToAssess + 1, Manifest). + case length(get_item(LevelToAssess, Man, [])) of + FileCount when FileCount > 0 -> + NewWQ = maybe_append_work(WorkQ, + LevelToAssess, + Man, + MaxFiles, + FileCount), + assess_workqueue(NewWQ, LevelToAssess + 1, Man, LevelToAssess); + 0 -> + assess_workqueue(WorkQ, LevelToAssess + 1, Man, BasementLevel) + end. maybe_append_work(WorkQ, Level, Manifest, @@ -1418,7 +1430,6 @@ confirm_delete(Filename, UnreferencedFiles, RegisteredSnapshots) -> end. - assess_sqn([]) -> empty; assess_sqn(DumpList) -> @@ -1467,7 +1478,7 @@ compaction_work_assessment_test() -> L1 = [{{o, "B1", "K1", null}, {o, "B2", "K2", null}, dummy_pid}, {{o, "B2", "K3", null}, {o, "B4", "K4", null}, dummy_pid}], Manifest = [{0, L0}, {1, L1}], - WorkQ1 = assess_workqueue([], 0, Manifest), + {WorkQ1, 1} = assess_workqueue([], 0, Manifest, 0), ?assertMatch(WorkQ1, [{0, Manifest}]), L1Alt = lists:append(L1, [{{o, "B5", "K0001", null}, {o, "B5", "K9999", null}, @@ -1485,7 +1496,7 @@ compaction_work_assessment_test() -> {{o, "BB", "K0001", null}, {o, "BB", "K9999", null}, dummy_pid}]), Manifest3 = [{0, []}, {1, L1Alt}], - WorkQ3 = assess_workqueue([], 0, Manifest3), + {WorkQ3, 1} = assess_workqueue([], 0, Manifest3, 0), ?assertMatch(WorkQ3, [{1, Manifest3}]). confirm_delete_test() -> diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index f292339..5399e01 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -1744,6 +1744,45 @@ initial_iterator_test() -> ok = file:close(UpdHandle), ok = file:delete(Filename). +key_dominates_test() -> + KV1 = {{o, "Bucket", "Key1", null}, {5, {active, infinity}, []}}, + KV2 = {{o, "Bucket", "Key3", null}, {6, {active, infinity}, []}}, + KV3 = {{o, "Bucket", "Key2", null}, {3, {active, infinity}, []}}, + KV4 = {{o, "Bucket", "Key4", null}, {7, {active, infinity}, []}}, + KV5 = {{o, "Bucket", "Key1", null}, {4, {active, infinity}, []}}, + KV6 = {{o, "Bucket", "Key1", null}, {99, {tomb, 999}, []}}, + KL1 = [KV1, KV2], + KL2 = [KV3, KV4], + ?assertMatch({{next_key, KV1}, [KV2], KL2}, + key_dominates(KL1, KL2, 1)), + ?assertMatch({{next_key, KV1}, KL2, [KV2]}, + key_dominates(KL2, KL1, 1)), + ?assertMatch({skipped_key, KL2, KL1}, + key_dominates([KV5|KL2], KL1, 1)), + ?assertMatch({{next_key, KV1}, [KV2], []}, + key_dominates(KL1, [], 1)), + ?assertMatch({skipped_key, [KV6|KL2], [KV2]}, + key_dominates([KV6|KL2], KL1, 1)), + ?assertMatch({{next_key, KV6}, KL2, [KV2]}, + key_dominates([KV6|KL2], [KV2], 1)), + ?assertMatch({skipped_key, [KV6|KL2], [KV2]}, + key_dominates([KV6|KL2], KL1, {basement, 1})), + ?assertMatch({skipped_key, [KV6|KL2], [KV2]}, + key_dominates([KV6|KL2], KL1, {basement, 1000})), + ?assertMatch({{next_key, KV6}, KL2, [KV2]}, + key_dominates([KV6|KL2], [KV2], {basement, 1})), + ?assertMatch({skipped_key, KL2, [KV2]}, + key_dominates([KV6|KL2], [KV2], {basement, 1000})), + ?assertMatch({skipped_key, [], []}, + key_dominates([KV6], [], {basement, 1000})), + ?assertMatch({skipped_key, [], []}, + key_dominates([], [KV6], {basement, 1000})), + ?assertMatch({{next_key, KV6}, [], []}, + key_dominates([KV6], [], {basement, 1})), + ?assertMatch({{next_key, KV6}, [], []}, + key_dominates([], [KV6], {basement, 1})). + + big_iterator_test() -> Filename = "../test/bigtest1.sft", {KL1, KL2} = {lists:sort(generate_randomkeys(10000)), []}, From caa8d26e3e920bbb977501099e4a687b1fcb1d5e Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 20 Oct 2016 19:18:49 +0100 Subject: [PATCH 080/167] Stop file check File check now covered by measure in the sft_new path, whihc will backup any existing file before moving. This gets triggered by incomplete changes on shutdown. --- src/leveled_pclerk.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 5a321fa..e008ea0 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -304,8 +304,6 @@ do_merge(KL1, KL2, {SrcLevel, IsB}, {Filepath, MSN}, FileCounter, OutList) -> [SrcLevel + 1, FileCounter])), io:format("File to be created as part of MSN=~w Filename=~s~n", [MSN, FileName]), - % Attempt to trace intermittent eaccess failures - false = filelib:is_file(FileName), TS1 = os:timestamp(), {ok, Pid, Reply} = leveled_sft:sft_new(FileName, KL1, KL2, SrcLevel + 1), {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, From c431bf3b0a740a8cec93a8ad721af48ede171e29 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 21 Oct 2016 11:38:30 +0100 Subject: [PATCH 081/167] Broken snapshot test The test confirming that deleting sft files wer eheld open whilst snapshots were registered was actually broken. This test has now been fixed, as well as the logic in registring snapshots which had used ledger_sqn mistakenly rather than manifest_sqn. --- src/leveled_pclerk.erl | 4 ++-- src/leveled_penciller.erl | 4 ++-- src/leveled_sft.erl | 24 ++++++++++++++++-------- test/end_to_end/basic_SUITE.erl | 30 ++++++++++++++++++++---------- 4 files changed, 40 insertions(+), 22 deletions(-) diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index e008ea0..b9d7164 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -118,8 +118,8 @@ handle_call({manifest_change, confirm, Closing}, From, State) -> io:format("Prompted confirmation of manifest change~n"), gen_server:reply(From, ok), WI = State#state.work_item, - mark_for_delete(WI#penciller_work.unreferenced_files, - State#state.owner), + ok = mark_for_delete(WI#penciller_work.unreferenced_files, + State#state.owner), {noreply, State#state{work_item=null, change_pending=false}, ?MIN_TIMEOUT} diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 1c69eee..3267421 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -490,7 +490,7 @@ handle_call(work_for_clerk, From, State) -> handle_call(get_startup_sqn, _From, State) -> {reply, State#state.ledger_sqn, State}; handle_call({register_snapshot, Snapshot}, _From, State) -> - Rs = [{Snapshot, State#state.ledger_sqn}|State#state.registered_snapshots], + Rs = [{Snapshot, State#state.manifest_sqn}|State#state.registered_snapshots], {reply, {ok, State#state.ledger_sqn, @@ -540,7 +540,7 @@ handle_info({_Ref, {ok, SrcFN, _StartKey, _EndKey}}, State) -> terminate(Reason, State=#state{is_snapshot=Snap}) when Snap == true -> ok = pcl_releasesnapshot(State#state.source_penciller, self()), - io:format("Sent release message for snapshot following close for " + io:format("Sent release message for cloned Penciller following close for " ++ "reason ~w~n", [Reason]), ok; terminate(Reason, State) -> diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 5399e01..58226de 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -181,7 +181,7 @@ -define(HEADER_LEN, 56). -define(ITERATOR_SCANWIDTH, 1). -define(MERGE_SCANWIDTH, 8). --define(DELETE_TIMEOUT, 60000). +-define(DELETE_TIMEOUT, 10000). -define(MAX_KEYS, ?SLOT_COUNT * ?BLOCK_COUNT * ?BLOCK_SIZE). -define(DISCARD_EXT, ".discarded"). @@ -296,7 +296,7 @@ handle_call({sft_open, Filename}, _From, _State) -> FileMD}; handle_call({get_kv, Key}, _From, State) -> Reply = fetch_keyvalue(State#state.handle, State, Key), - {reply, Reply, State}; + statecheck_onreply(Reply, State); handle_call({get_kvrange, StartKey, EndKey, ScanWidth}, _From, State) -> Reply = pointer_append_queryresults(fetch_range_kv(State#state.handle, State, @@ -304,7 +304,7 @@ handle_call({get_kvrange, StartKey, EndKey, ScanWidth}, _From, State) -> EndKey, ScanWidth), self()), - {reply, Reply, State}; + statecheck_onreply(Reply, State); handle_call(close, _From, State) -> {stop, normal, ok, State}; handle_call(clear, _From, State) -> @@ -322,34 +322,33 @@ handle_call(background_complete, _From, State) -> {reply, {error, State#state.background_failure}, State} end; handle_call({set_for_delete, Penciller}, _From, State) -> + io:format("File ~s has been set for delete~n", [State#state.filename]), {reply, ok, State#state{ready_for_delete=true, penciller=Penciller}, ?DELETE_TIMEOUT}; handle_call(get_maxsqn, _From, State) -> - {reply, State#state.highest_sqn, State}. + statecheck_onreply(State#state.highest_sqn, State). handle_cast({sft_new, Filename, Inp1, [], 0}, _State) -> SW = os:timestamp(), {ok, State} = create_levelzero(Inp1, Filename), io:format("File creation of L0 file ~s took ~w microseconds~n", [Filename, timer:now_diff(os:timestamp(), SW)]), - {noreply, State}; -handle_cast(_Msg, State) -> {noreply, State}. handle_info(timeout, State) -> case State#state.ready_for_delete of true -> + io:format("File ~s prompting for delete status check~n", + [State#state.filename]), case leveled_penciller:pcl_confirmdelete(State#state.penciller, State#state.filename) of true -> {stop, shutdown, State}; false -> - io:format("Polled for deletion but ~s not ready~n", - [State#state.filename]), {noreply, State, ?DELETE_TIMEOUT} end; false -> @@ -377,6 +376,15 @@ terminate(Reason, State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. + +statecheck_onreply(Reply, State) -> + case State#state.ready_for_delete of + true -> + {reply, Reply, State, ?DELETE_TIMEOUT}; + false -> + {reply, Reply, State} + end. + %%%============================================================================ %%% Internal functions %%%============================================================================ diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index dac295f..cfc4b0c 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -10,12 +10,13 @@ load_and_count_withdelete/1 ]). -all() -> [simple_put_fetch_head_delete, - many_put_fetch_head, - journal_compaction, - fetchput_snapshot, - load_and_count, - load_and_count_withdelete +all() -> [ + % simple_put_fetch_head_delete, + % many_put_fetch_head, + % journal_compaction, + fetchput_snapshot %, + % load_and_count, + % load_and_count_withdelete ]. @@ -151,7 +152,7 @@ journal_compaction(_Config) -> fetchput_snapshot(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=3000000}, + StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=30000000}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), @@ -201,7 +202,7 @@ fetchput_snapshot(_Config) -> ChkList3 = lists:sublist(lists:sort(ObjList3), 100), testutil:check_forlist(Bookie2, ChkList3), testutil:check_formissinglist(SnapBookie2, ChkList3), - GenList = [20002, 40002, 60002, 80002, 100002, 120002], + GenList = [20002, 40002, 60002, 80002], CLs2 = testutil:load_objects(20000, GenList, Bookie2, TestObject, fun testutil:generate_smallobjects/2), io:format("Loaded significant numbers of new objects~n"), @@ -222,13 +223,22 @@ fetchput_snapshot(_Config) -> fun testutil:generate_smallobjects/2), testutil:check_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), testutil:check_forlist(Bookie2, lists:nth(1, CLs3)), + + io:format("Starting 15s sleep in which snap2 should block deletion~n"), + timer:sleep(15000), {ok, FNsB} = file:list_dir(RootPath ++ "/ledger/ledger_files"), ok = leveled_bookie:book_close(SnapBookie2), + io:format("Starting 15s sleep as snap2 close should unblock deletion~n"), + timer:sleep(15000), + io:format("Pause for deletion has ended~n"), + testutil:check_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), ok = leveled_bookie:book_close(SnapBookie3), + io:format("Starting 15s sleep as snap3 close should unblock deletion~n"), + timer:sleep(15000), + io:format("Pause for deletion has ended~n"), testutil:check_forlist(Bookie2, lists:nth(length(CLs3), CLs3)), testutil:check_forlist(Bookie2, lists:nth(1, CLs3)), - timer:sleep(90000), {ok, FNsC} = file:list_dir(RootPath ++ "/ledger/ledger_files"), true = length(FNsB) > length(FNsA), true = length(FNsB) > length(FNsC), @@ -239,7 +249,7 @@ fetchput_snapshot(_Config) -> {B1Size, B1Count} = testutil:check_bucket_stats(Bookie2, "Bucket1"), {BSize, BCount} = testutil:check_bucket_stats(Bookie2, "Bucket"), true = BSize > 0, - true = BCount == 140000, + true = BCount == 100000, ok = leveled_bookie:book_close(Bookie2), testutil:reset_filestructure(). From 3ad9e42b614e5446e09cef2ec73d45dac6960a89 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 21 Oct 2016 12:18:06 +0100 Subject: [PATCH 082/167] Changed SFT shutdown to cast-based The SFT shutdown process ahs become a series of casts to-and-from between Penciller and SFT to stop the two processes syncronously making requests on each other --- src/leveled_penciller.erl | 35 ++++++++++++++++++----------------- src/leveled_sft.erl | 19 ++++++++++--------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 3267421..d6bea4e 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -342,7 +342,7 @@ pcl_promptmanifestchange(Pid, WI) -> gen_server:cast(Pid, {manifest_change, WI}). pcl_confirmdelete(Pid, FileName) -> - gen_server:call(Pid, {confirm_delete, FileName}, infinity). + gen_server:cast(Pid, {confirm_delete, FileName}). pcl_getstartupsequencenumber(Pid) -> gen_server:call(Pid, get_startup_sqn, infinity). @@ -436,18 +436,6 @@ handle_call({push_mem, DumpList}, From, State=#state{is_snapshot=Snap}) io:format("Empty request pushed to Penciller~n"), {reply, ok, State} end; -handle_call({confirm_delete, FileName}, _From, State=#state{is_snapshot=Snap}) - when Snap == false -> - Reply = confirm_delete(FileName, - State#state.unreferenced_files, - State#state.registered_snapshots), - case Reply of - true -> - UF1 = lists:keydelete(FileName, 1, State#state.unreferenced_files), - {reply, true, State#state{unreferenced_files=UF1}}; - _ -> - {reply, Reply, State} - end; handle_call({fetch, Key}, _From, State=#state{is_snapshot=Snap}) when Snap == false -> {reply, @@ -532,7 +520,20 @@ handle_cast({manifest_change, WI}, State) -> handle_cast({release_snapshot, Snapshot}, State) -> Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots), io:format("Penciller snapshot ~w released~n", [Snapshot]), - {noreply, State#state{registered_snapshots=Rs}}. + {noreply, State#state{registered_snapshots=Rs}}; +handle_cast({confirm_delete, FileName}, State=#state{is_snapshot=Snap}) + when Snap == false -> + Reply = confirm_delete(FileName, + State#state.unreferenced_files, + State#state.registered_snapshots), + case Reply of + {true, Pid} -> + UF1 = lists:keydelete(FileName, 1, State#state.unreferenced_files), + ok = leveled_sft:sft_deleteconfirmed(Pid), + {noreply, State#state{unreferenced_files=UF1}}; + _ -> + {noreply, State} + end. handle_info({_Ref, {ok, SrcFN, _StartKey, _EndKey}}, State) -> io:format("Orphaned reply after timeout on L0 file write ~s~n", [SrcFN]), @@ -1417,7 +1418,7 @@ confirm_delete(Filename, UnreferencedFiles, RegisteredSnapshots) -> case lists:keyfind(Filename, 1, UnreferencedFiles) of false -> false; - {Filename, _Pid, MSN} -> + {Filename, Pid, MSN} -> LowSQN = lists:foldl(fun({_, SQN}, MinSQN) -> min(SQN, MinSQN) end, infinity, RegisteredSnapshots), @@ -1425,7 +1426,7 @@ confirm_delete(Filename, UnreferencedFiles, RegisteredSnapshots) -> MSN >= LowSQN -> false; true -> - true + {true, Pid} end end. @@ -1505,7 +1506,7 @@ confirm_delete_test() -> {Filename, dummy_owner, 10}], RegisteredIterators1 = [{dummy_pid, 16}, {dummy_pid, 12}], R1 = confirm_delete(Filename, UnreferencedFiles, RegisteredIterators1), - ?assertMatch(R1, true), + ?assertMatch(R1, {true, dummy_owner}), RegisteredIterators2 = [{dummy_pid, 10}, {dummy_pid, 12}], R2 = confirm_delete(Filename, UnreferencedFiles, RegisteredIterators2), ?assertMatch(R2, false), diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 58226de..93d22d8 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -160,6 +160,7 @@ sft_clear/1, sft_checkready/1, sft_setfordelete/2, + sft_deleteconfirmed/1, sft_getmaxsequencenumber/1, generate_randomkeys/1]). @@ -251,6 +252,9 @@ sft_clear(Pid) -> sft_close(Pid) -> gen_server:call(Pid, close, infinity). +sft_deleteconfirmed(Pid) -> + gen_server:cast(Pid, close). + sft_checkready(Pid) -> gen_server:call(Pid, background_complete, 50). @@ -336,21 +340,18 @@ handle_cast({sft_new, Filename, Inp1, [], 0}, _State) -> {ok, State} = create_levelzero(Inp1, Filename), io:format("File creation of L0 file ~s took ~w microseconds~n", [Filename, timer:now_diff(os:timestamp(), SW)]), - {noreply, State}. + {noreply, State}; +handle_cast(close, State) -> + {stop, normal, State}. handle_info(timeout, State) -> case State#state.ready_for_delete of true -> io:format("File ~s prompting for delete status check~n", [State#state.filename]), - case leveled_penciller:pcl_confirmdelete(State#state.penciller, - State#state.filename) - of - true -> - {stop, shutdown, State}; - false -> - {noreply, State, ?DELETE_TIMEOUT} - end; + ok = leveled_penciller:pcl_confirmdelete(State#state.penciller, + State#state.filename), + {noreply, State, ?DELETE_TIMEOUT}; false -> {noreply, State} end. From b2089baa1e413db6750729349cf20a211190f2ed Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 21 Oct 2016 15:21:37 +0100 Subject: [PATCH 083/167] Correct tombstone handling Prepare SFT files for handling tombstones correctly (without expiry dates). Also some work as it can be seen from tests that some SFT files ar enot be cleared out correctly. Pausing before trying t clear out the fles to experiment and trial the possibility that there is a timing issue. --- src/leveled_sft.erl | 17 ++++++++++++++++- test/end_to_end/basic_SUITE.erl | 12 ++++++------ test/end_to_end/testutil.erl | 2 ++ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 93d22d8..14ba7df 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -1101,6 +1101,8 @@ maybe_reap_expiredkey({_, infinity}, _) -> false; % key is not set to expire maybe_reap_expiredkey({_, TS}, {basement, CurrTS}) when CurrTS > TS -> true; % basement and ready to expire +maybe_reap_expiredkey(tomb, {basement, _CurrTS}) -> + true; % always expire in basement maybe_reap_expiredkey(_, _) -> false. @@ -1760,6 +1762,7 @@ key_dominates_test() -> KV4 = {{o, "Bucket", "Key4", null}, {7, {active, infinity}, []}}, KV5 = {{o, "Bucket", "Key1", null}, {4, {active, infinity}, []}}, KV6 = {{o, "Bucket", "Key1", null}, {99, {tomb, 999}, []}}, + KV7 = {{o, "Bucket", "Key1", null}, {99, tomb, []}}, KL1 = [KV1, KV2], KL2 = [KV3, KV4], ?assertMatch({{next_key, KV1}, [KV2], KL2}, @@ -1789,7 +1792,19 @@ key_dominates_test() -> ?assertMatch({{next_key, KV6}, [], []}, key_dominates([KV6], [], {basement, 1})), ?assertMatch({{next_key, KV6}, [], []}, - key_dominates([], [KV6], {basement, 1})). + key_dominates([], [KV6], {basement, 1})), + ?assertMatch({skipped_key, [], []}, + key_dominates([KV7], [], {basement, 1})), + ?assertMatch({skipped_key, [], []}, + key_dominates([], [KV7], {basement, 1})), + ?assertMatch({skipped_key, [KV7|KL2], [KV2]}, + key_dominates([KV7|KL2], KL1, 1)), + ?assertMatch({{next_key, KV7}, KL2, [KV2]}, + key_dominates([KV7|KL2], [KV2], 1)), + ?assertMatch({skipped_key, [KV7|KL2], [KV2]}, + key_dominates([KV7|KL2], KL1, {basement, 1})), + ?assertMatch({skipped_key, KL2, [KV2]}, + key_dominates([KV7|KL2], [KV2], {basement, 1})). big_iterator_test() -> diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index cfc4b0c..718587c 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -11,12 +11,12 @@ ]). all() -> [ - % simple_put_fetch_head_delete, - % many_put_fetch_head, - % journal_compaction, - fetchput_snapshot %, - % load_and_count, - % load_and_count_withdelete + simple_put_fetch_head_delete, + many_put_fetch_head, + journal_compaction, + fetchput_snapshot, + load_and_count, + load_and_count_withdelete ]. diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index e3bf77d..9043bf9 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -26,6 +26,8 @@ reset_filestructure() -> + io:format("Waiting 2s to give a chance for all file closes to complete~n"), + timer:sleep(2000), RootPath = "test", filelib:ensure_dir(RootPath ++ "/journal/"), filelib:ensure_dir(RootPath ++ "/ledger/"), From 3710d09fbf85b7484a57b0a1cc9dfc85dee89a5c Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 21 Oct 2016 15:30:53 +0100 Subject: [PATCH 084/167] Reuse codec key comparison There was duplication of key comparison logic between leveled_codec and leveled_sft. Now both use the leveled_codec key_dominates function --- src/leveled_sft.erl | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 14ba7df..60054a1 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -1069,25 +1069,21 @@ key_dominates_expanded([], [H2|T2], Level) -> {{next_key, H2}, [], maybe_expand_pointer(T2)} end; key_dominates_expanded([H1|T1], [H2|T2], Level) -> - {{K1, V1}, {K2, V2}} = {H1, H2}, - {Sq1, St1, _MD1} = leveled_codec:striphead_to_details(V1), - {Sq2, St2, _MD2} = leveled_codec:striphead_to_details(V2), - case K1 of - K2 -> - case Sq1 > Sq2 of - true -> - {skipped_key, [H1|T1], maybe_expand_pointer(T2)}; - false -> - {skipped_key, maybe_expand_pointer(T1), [H2|T2]} - end; - K1 when K1 < K2 -> + case leveled_codec:key_dominates(H1, H2) of + left_hand_first -> + St1 = leveled_codec:strip_to_statusonly(H1), case maybe_reap_expiredkey(St1, Level) of true -> {skipped_key, maybe_expand_pointer(T1), [H2|T2]}; false -> {{next_key, H1}, maybe_expand_pointer(T1), [H2|T2]} end; - _ -> + left_hand_dominant -> + {skipped_key, [H1|T1], maybe_expand_pointer(T2)}; + right_hand_dominant -> + {skipped_key, maybe_expand_pointer(T1), [H2|T2]}; + right_hand_first -> + St2 = leveled_codec:strip_to_statusonly(H2), case maybe_reap_expiredkey(St2, Level) of true -> {skipped_key, [H1|T1], maybe_expand_pointer(T2)}; From 0a2053b557707c36c18358ea6603f708a2500ab3 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 21 Oct 2016 16:08:41 +0100 Subject: [PATCH 085/167] Improved unit test of CRC chekcing in bloom filter Confirm the impact of bit-flipping in the bloom filter --- src/leveled_codec.erl | 15 ++++++ src/leveled_sft.erl | 76 ++++++++++++++++++++---------- test/end_to_end/iterator_SUITE.erl | 2 +- 3 files changed, 68 insertions(+), 25 deletions(-) diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 875fb88..535ab78 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -40,6 +40,7 @@ is_active/2, endkey_passed/2, key_dominates/2, + maybe_reap_expiredkey/2, print_key/1, to_ledgerkey/3, to_ledgerkey/5, @@ -86,6 +87,20 @@ key_dominates(LeftKey, RightKey) -> right_hand_dominant end. + +maybe_reap_expiredkey(KV, IsBasement) -> + Status = strip_to_statusonly(KV), + maybe_reap(Status, IsBasement). + +maybe_reap({_, infinity}, _) -> + false; % key is not set to expire +maybe_reap({_, TS}, {basement, CurrTS}) when CurrTS > TS -> + true; % basement and ready to expire +maybe_reap(tomb, {basement, _CurrTS}) -> + true; % always expire in basement +maybe_reap(_, _) -> + false. + is_active(Key, Value) -> case strip_to_statusonly({Key, Value}) of {active, infinity} -> diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 60054a1..b88b254 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -1053,16 +1053,14 @@ key_dominates(KL1, KL2, Level) -> Level). key_dominates_expanded([H1|T1], [], Level) -> - St1 = leveled_codec:strip_to_statusonly(H1), - case maybe_reap_expiredkey(St1, Level) of + case leveled_codec:maybe_reap_expiredkey(H1, Level) of true -> {skipped_key, maybe_expand_pointer(T1), []}; false -> {{next_key, H1}, maybe_expand_pointer(T1), []} end; key_dominates_expanded([], [H2|T2], Level) -> - St2 = leveled_codec:strip_to_statusonly(H2), - case maybe_reap_expiredkey(St2, Level) of + case leveled_codec:maybe_reap_expiredkey(H2, Level) of true -> {skipped_key, [], maybe_expand_pointer(T2)}; false -> @@ -1071,37 +1069,26 @@ key_dominates_expanded([], [H2|T2], Level) -> key_dominates_expanded([H1|T1], [H2|T2], Level) -> case leveled_codec:key_dominates(H1, H2) of left_hand_first -> - St1 = leveled_codec:strip_to_statusonly(H1), - case maybe_reap_expiredkey(St1, Level) of + case leveled_codec:maybe_reap_expiredkey(H1, Level) of true -> {skipped_key, maybe_expand_pointer(T1), [H2|T2]}; false -> {{next_key, H1}, maybe_expand_pointer(T1), [H2|T2]} end; - left_hand_dominant -> - {skipped_key, [H1|T1], maybe_expand_pointer(T2)}; - right_hand_dominant -> - {skipped_key, maybe_expand_pointer(T1), [H2|T2]}; right_hand_first -> - St2 = leveled_codec:strip_to_statusonly(H2), - case maybe_reap_expiredkey(St2, Level) of + case leveled_codec:maybe_reap_expiredkey(H2, Level) of true -> {skipped_key, [H1|T1], maybe_expand_pointer(T2)}; false -> {{next_key, H2}, [H1|T1], maybe_expand_pointer(T2)} - end + end; + left_hand_dominant -> + {skipped_key, [H1|T1], maybe_expand_pointer(T2)}; + right_hand_dominant -> + {skipped_key, maybe_expand_pointer(T1), [H2|T2]} end. -maybe_reap_expiredkey({_, infinity}, _) -> - false; % key is not set to expire -maybe_reap_expiredkey({_, TS}, {basement, CurrTS}) when CurrTS > TS -> - true; % basement and ready to expire -maybe_reap_expiredkey(tomb, {basement, _CurrTS}) -> - true; % always expire in basement -maybe_reap_expiredkey(_, _) -> - false. - %% When a list is provided it may include a pointer to gain another batch of %% entries from the same file, or a new batch of entries from another file %% @@ -1550,8 +1537,49 @@ merge_seglists_test() -> R8 = check_for_segments(SegBin, [0,900], false), ?assertMatch(R8, {maybe_present, [0]}), R9 = check_for_segments(SegBin, [1024*1024 - 1], false), - ?assertMatch(R9, not_present). - + ?assertMatch(R9, not_present), + io:format("Try corrupted bloom filter with flipped bit in " ++ + "penultimate delta~n"), + ExpectedDeltasFlippedBit = <<0:1, 0:13, 0:2, + 0:1, 50:13, 1:2, + 0:1, 25:13, 2:2, + 0:1, 25:13, 0:2, + 0:1, 100:13, 0:2, + 0:1, 0:13, 1:2, + 2:2, 1709:13, 2:2>>, + SegBin1 = <>, + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin1, [900], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin1, [200], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin1, [0,900], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin1, [1024*1024 - 1], true)), + % This match is before the flipped bit, so still works without CRC check + ?assertMatch({maybe_present, [0]}, + check_for_segments(SegBin1, [0,900], false)), + io:format("Try corrupted bloom filter with flipped bit in " ++ + "final block's top hash~n"), + ExpectedTopHashesFlippedBit = <<200:20, 200:20, 10000:20, 1:20>>, + SegBin2 = <>, + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin2, [900], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin2, [200], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin2, [0,900], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin2, [1024*1024 - 1], true)), + % This match is before the flipped bit, so still works without CRC check + ?assertMatch({maybe_present, [0]}, + check_for_segments(SegBin2, [0,900], false)). createslot_stage1_test() -> {KeyList1, KeyList2} = sample_keylist(), diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 8fb7505..f00571e 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -43,7 +43,7 @@ simple_load_with2i(_Config) -> simple_querycount(_Config) -> RootPath = testutil:reset_filestructure(), - {ok, Book1} = leveled_bookie:book_start(RootPath, 4000, 50000000), + {ok, Book1} = leveled_bookie:book_start(RootPath, 2500, 50000000), {TestObject, TestSpec} = testutil:generate_testobject("Bucket", "Key1", "Value1", From e9c568a8b3c05e2ce1cdb9a0dd13ea08ae1e2e6e Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 21 Oct 2016 21:26:28 +0100 Subject: [PATCH 086/167] Test fix-up There was a test that failed to close down a bookie and that caused some issues. The issues are double-reoslved, the close down was tidied as well as the forgotten close being added back in. There is some generla tidy around in anticipation of TTL support. --- include/leveled.hrl | 5 + src/leveled_codec.erl | 8 +- src/leveled_pclerk.erl | 25 +++- src/leveled_penciller.erl | 9 +- src/leveled_sft.erl | 214 +++++++++++++++++------------ test/end_to_end/iterator_SUITE.erl | 1 + test/end_to_end/testutil.erl | 6 +- 7 files changed, 167 insertions(+), 101 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 3b06a40..f1ca86a 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -19,6 +19,11 @@ unreferenced_files :: list(), target_is_basement = false ::boolean()}). +-record(level, + {level :: integer(), + is_basement = false :: boolean(), + timestamp :: integer()}). + -record(manifest_entry, {start_key :: tuple(), end_key :: tuple(), diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 535ab78..a1c20b4 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -88,15 +88,15 @@ key_dominates(LeftKey, RightKey) -> end. -maybe_reap_expiredkey(KV, IsBasement) -> +maybe_reap_expiredkey(KV, LevelD) -> Status = strip_to_statusonly(KV), - maybe_reap(Status, IsBasement). + maybe_reap(Status, LevelD). maybe_reap({_, infinity}, _) -> false; % key is not set to expire -maybe_reap({_, TS}, {basement, CurrTS}) when CurrTS > TS -> +maybe_reap({_, TS}, {true, CurrTS}) when CurrTS > TS -> true; % basement and ready to expire -maybe_reap(tomb, {basement, _CurrTS}) -> +maybe_reap(tomb, {true, _CurrTS}) -> true; % always expire in basement maybe_reap(_, _) -> false. diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index b9d7164..bb8978c 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -80,6 +80,7 @@ clerk_new(Owner) -> {ok, Pid} = gen_server:start(?MODULE, [], []), ok = gen_server:call(Pid, {register, Owner}, infinity), + io:format("Penciller's clerk ~w started with owner ~w~n", [Pid, Owner]), {ok, Pid}. clerk_manifestchange(Pid, Action, Closing) -> @@ -104,7 +105,7 @@ handle_call({manifest_change, return, true}, _From, State) -> WI = State#state.work_item, {reply, {ok, WI}, State}; false -> - {reply, no_change, State} + {stop, normal, no_change, State} end; handle_call({manifest_change, confirm, Closing}, From, State) -> case Closing of @@ -139,8 +140,9 @@ handle_info(timeout, State=#state{change_pending=Pnd}) when Pnd == false -> State#state{change_pending=true, work_item=WI}} end. -terminate(_Reason, _State) -> - ok. +terminate(Reason, _State) -> + io:format("Penciller's Clerk ~w shutdown now complete for reason ~w~n", + [self(), Reason]). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -153,7 +155,7 @@ code_change(_OldVsn, State, _Extra) -> requestandhandle_work(State) -> case leveled_penciller:pcl_workforclerk(State#state.owner) of none -> - io:format("Work prompted but none needed~n"), + io:format("Work prompted but none needed ~w~n", [self()]), {false, ?MAX_TIMEOUT}; WI -> {NewManifest, FilesToDelete} = merge(WI), @@ -219,7 +221,7 @@ merge(WI) -> mark_for_delete([], _Penciller) -> ok; mark_for_delete([Head|Tail], Penciller) -> - leveled_sft:sft_setfordelete(Head#manifest_entry.owner, Penciller), + ok = leveled_sft:sft_setfordelete(Head#manifest_entry.owner, Penciller), mark_for_delete(Tail, Penciller). @@ -305,7 +307,18 @@ do_merge(KL1, KL2, {SrcLevel, IsB}, {Filepath, MSN}, FileCounter, OutList) -> io:format("File to be created as part of MSN=~w Filename=~s~n", [MSN, FileName]), TS1 = os:timestamp(), - {ok, Pid, Reply} = leveled_sft:sft_new(FileName, KL1, KL2, SrcLevel + 1), + LevelR = case IsB of + true -> + #level{level = SrcLevel + 1, + is_basement = true, + timestamp = os:timestamp()}; + false -> + SrcLevel + 1 + end, + {ok, Pid, Reply} = leveled_sft:sft_new(FileName, + KL1, + KL2, + LevelR), {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, ExtMan = lists:append(OutList, [#manifest_entry{start_key=SmallestKey, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index d6bea4e..bfc2124 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -529,6 +529,9 @@ handle_cast({confirm_delete, FileName}, State=#state{is_snapshot=Snap}) case Reply of {true, Pid} -> UF1 = lists:keydelete(FileName, 1, State#state.unreferenced_files), + io:format("Filename ~s removed from unreferenced files as delete " + ++ "is confirmed - file should now close~n", + [FileName]), ok = leveled_sft:sft_deleteconfirmed(Pid), {noreply, State#state{unreferenced_files=UF1}}; _ -> @@ -610,8 +613,9 @@ terminate(Reason, State) -> % Tidy shutdown of individual files ok = close_files(0, UpdState#state.manifest), lists:foreach(fun({_FN, Pid, _SN}) -> - leveled_sft:sft_close(Pid) end, + ok = leveled_sft:sft_close(Pid) end, UpdState#state.unreferenced_files), + io:format("Shutdown complete for Penciller~n"), ok. @@ -1015,7 +1019,8 @@ close_files(?MAX_LEVELS - 1, _Manifest) -> ok; close_files(Level, Manifest) -> LevelList = get_item(Level, Manifest, []), - lists:foreach(fun(F) -> leveled_sft:sft_close(F#manifest_entry.owner) end, + lists:foreach(fun(F) -> + ok = leveled_sft:sft_close(F#manifest_entry.owner) end, LevelList), close_files(Level + 1, Manifest). diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index b88b254..f85ebcc 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -211,20 +211,29 @@ %%% API %%%============================================================================ -sft_new(Filename, KL1, KL2, Level) -> - sft_new(Filename, KL1, KL2, Level, #sft_options{}). +sft_new(Filename, KL1, KL2, LevelInfo) -> + sft_new(Filename, KL1, KL2, LevelInfo, #sft_options{}). -sft_new(Filename, KL1, KL2, Level, Options) -> +sft_new(Filename, KL1, KL2, LevelInfo, Options) -> + LevelR = case is_integer(LevelInfo) of + true -> + #level{level=LevelInfo}; + _ -> + if + is_record(LevelInfo, level) -> + LevelInfo + end + end, {ok, Pid} = gen_server:start(?MODULE, [], []), case Options#sft_options.wait of true -> Reply = gen_server:call(Pid, - {sft_new, Filename, KL1, KL2, Level}, + {sft_new, Filename, KL1, KL2, LevelR}, infinity), {ok, Pid, Reply}; false -> gen_server:cast(Pid, - {sft_new, Filename, KL1, KL2, Level}), + {sft_new, Filename, KL1, KL2, LevelR}), {ok, Pid} end. @@ -270,22 +279,22 @@ sft_getmaxsequencenumber(Pid) -> init([]) -> {ok, #state{}}. -handle_call({sft_new, Filename, KL1, [], 0}, _From, _State) -> +handle_call({sft_new, Filename, KL1, [], _LevelR=#level{level=L}}, + _From, + _State) when L == 0 -> {ok, State} = create_levelzero(KL1, Filename), {reply, {{[], []}, State#state.smallest_key, State#state.highest_key}, State}; -handle_call({sft_new, Filename, KL1, KL2, Level}, _From, State) -> +handle_call({sft_new, Filename, KL1, KL2, LevelR}, _From, _State) -> case create_file(Filename) of - {error, Reason} -> - {reply, {error, Reason}, State}; {Handle, FileMD} -> {ReadHandle, UpdFileMD, KeyRemainders} = complete_file(Handle, FileMD, KL1, KL2, - Level), + LevelR), {reply, {KeyRemainders, UpdFileMD#state.smallest_key, UpdFileMD#state.highest_key}, @@ -335,7 +344,8 @@ handle_call({set_for_delete, Penciller}, _From, State) -> handle_call(get_maxsqn, _From, State) -> statecheck_onreply(State#state.highest_sqn, State). -handle_cast({sft_new, Filename, Inp1, [], 0}, _State) -> +handle_cast({sft_new, Filename, Inp1, [], _LevelR=#level{level=L}}, _State) + when L == 0-> SW = os:timestamp(), {ok, State} = create_levelzero(Inp1, Filename), io:format("File creation of L0 file ~s took ~w microseconds~n", @@ -364,7 +374,12 @@ terminate(Reason, State) -> io:format("Exit called and now clearing ~s~n", [State#state.filename]), ok = file:close(State#state.handle), - ok = file:delete(State#state.filename); + ok = case filelib:is_file(State#state.filename) of + true -> + file:delete(State#state.filename); + false -> + ok + end; _ -> case State#state.handle of undefined -> @@ -407,12 +422,11 @@ create_levelzero(Inp1, Filename) -> InputSize = length(ListForFile), io:format("Creating file with input of size ~w~n", [InputSize]), Rename = {true, TmpFilename, PrmFilename}, - {ReadHandle, UpdFileMD, {[], []}} = complete_file(Handle, - FileMD, - ListForFile, - [], - 0, - Rename), + {ReadHandle, + UpdFileMD, + {[], []}} = complete_file(Handle, FileMD, + ListForFile, [], + #level{level=0}, Rename), {ok, UpdFileMD#state{handle=ReadHandle, filename=PrmFilename, @@ -504,15 +518,15 @@ open_file(FileMD) -> %% Take a file handle with a previously created header and complete it based on %% the two key lists KL1 and KL2 -complete_file(Handle, FileMD, KL1, KL2, Level) -> - complete_file(Handle, FileMD, KL1, KL2, Level, false). +complete_file(Handle, FileMD, KL1, KL2, LevelR) -> + complete_file(Handle, FileMD, KL1, KL2, LevelR, false). -complete_file(Handle, FileMD, KL1, KL2, Level, Rename) -> +complete_file(Handle, FileMD, KL1, KL2, LevelR, Rename) -> {ok, KeyRemainders} = write_keys(Handle, maybe_expand_pointer(KL1), maybe_expand_pointer(KL2), [], <<>>, - Level, + LevelR, fun sftwrite_function/2), {ReadHandle, UpdFileMD} = case Rename of false -> @@ -773,18 +787,27 @@ get_nextkeyaftermatch([_KTuple|T], KeyToFind, PrevV) -> %% Slots are created then written in bulk to impove I/O efficiency. Slots will %% be written in groups of 32 -write_keys(Handle, KL1, KL2, SlotIndex, SerialisedSlots, Level, WriteFun) -> - write_keys(Handle, KL1, KL2, {0, 0}, +write_keys(Handle, + KL1, KL2, + SlotIndex, SerialisedSlots, + LevelR, WriteFun) -> + write_keys(Handle, + KL1, KL2, + {0, 0}, SlotIndex, SerialisedSlots, - {infinity, 0}, null, {last, null}, Level, WriteFun). + {infinity, 0}, null, {last, null}, + LevelR, WriteFun). -write_keys(Handle, KL1, KL2, {SlotCount, SlotTotal}, - SlotIndex, SerialisedSlots, - {LSN, HSN}, LowKey, LastKey, Level, WriteFun) +write_keys(Handle, + KL1, KL2, + {SlotCount, SlotTotal}, + SlotIndex, SerialisedSlots, + {LSN, HSN}, LowKey, LastKey, + LevelR, WriteFun) when SlotCount =:= ?SLOT_GROUPWRITE_COUNT -> UpdHandle = WriteFun(slots , {Handle, SerialisedSlots}), - case maxslots_bylevel(SlotTotal, Level) of + case maxslots_bylevel(SlotTotal, LevelR#level.level) of reached -> {complete_keywrite(UpdHandle, SlotIndex, @@ -792,14 +815,20 @@ write_keys(Handle, KL1, KL2, {SlotCount, SlotTotal}, WriteFun), {KL1, KL2}}; continue -> - write_keys(UpdHandle, KL1, KL2, {0, SlotTotal}, + write_keys(UpdHandle, + KL1, KL2, + {0, SlotTotal}, SlotIndex, <<>>, - {LSN, HSN}, LowKey, LastKey, Level, WriteFun) + {LSN, HSN}, LowKey, LastKey, + LevelR, WriteFun) end; -write_keys(Handle, KL1, KL2, {SlotCount, SlotTotal}, - SlotIndex, SerialisedSlots, - {LSN, HSN}, LowKey, LastKey, Level, WriteFun) -> - SlotOutput = create_slot(KL1, KL2, Level), +write_keys(Handle, + KL1, KL2, + {SlotCount, SlotTotal}, + SlotIndex, SerialisedSlots, + {LSN, HSN}, LowKey, LastKey, + LevelR, WriteFun) -> + SlotOutput = create_slot(KL1, KL2, LevelR), {{LowKey_Slot, SegFilter, SerialisedSlot, LengthList}, {{LSN_Slot, HSN_Slot}, LastKey_Slot, Status}, KL1rem, KL2rem} = SlotOutput, @@ -818,9 +847,12 @@ write_keys(Handle, KL1, KL2, {SlotCount, SlotTotal}, WriteFun), {KL1rem, KL2rem}}; full -> - write_keys(Handle, KL1rem, KL2rem, {SlotCount + 1, SlotTotal + 1}, + write_keys(Handle, + KL1rem, KL2rem, + {SlotCount + 1, SlotTotal + 1}, UpdSlotIndex, UpdSlots, - SNExtremes, FirstKey, FinalKey, Level, WriteFun); + SNExtremes, FirstKey, FinalKey, + LevelR, WriteFun); complete -> UpdHandle = WriteFun(slots , {Handle, UpdSlots}), {complete_keywrite(UpdHandle, @@ -929,11 +961,11 @@ maxslots_bylevel(SlotTotal, _Level) -> %% Also this should return a partial block if the KeyLists have been exhausted %% but the block is full -create_block(KeyList1, KeyList2, Level) -> - create_block(KeyList1, KeyList2, [], {infinity, 0}, [], Level). +create_block(KeyList1, KeyList2, LevelR) -> + create_block(KeyList1, KeyList2, [], {infinity, 0}, [], LevelR). create_block(KeyList1, KeyList2, - BlockKeyList, {LSN, HSN}, SegmentList, _) + BlockKeyList, {LSN, HSN}, SegmentList, _LevelR) when length(BlockKeyList)==?BLOCK_SIZE -> case {KeyList1, KeyList2} of {[], []} -> @@ -942,11 +974,13 @@ create_block(KeyList1, KeyList2, {BlockKeyList, full, {LSN, HSN}, SegmentList, KeyList1, KeyList2} end; create_block([], [], - BlockKeyList, {LSN, HSN}, SegmentList, _) -> + BlockKeyList, {LSN, HSN}, SegmentList, _LevelR) -> {BlockKeyList, partial, {LSN, HSN}, SegmentList, [], []}; create_block(KeyList1, KeyList2, - BlockKeyList, {LSN, HSN}, SegmentList, Level) -> - case key_dominates(KeyList1, KeyList2, Level) of + BlockKeyList, {LSN, HSN}, SegmentList, LevelR) -> + case key_dominates(KeyList1, + KeyList2, + {LevelR#level.is_basement, LevelR#level.timestamp}) of {{next_key, TopKey}, Rem1, Rem2} -> {UpdLSN, UpdHSN} = update_sequencenumbers(TopKey, LSN, HSN), NewBlockKeyList = lists:append(BlockKeyList, @@ -955,11 +989,11 @@ create_block(KeyList1, KeyList2, [hash_for_segmentid(TopKey)]), create_block(Rem1, Rem2, NewBlockKeyList, {UpdLSN, UpdHSN}, - NewSegmentList, Level); + NewSegmentList, LevelR); {skipped_key, Rem1, Rem2} -> create_block(Rem1, Rem2, BlockKeyList, {LSN, HSN}, - SegmentList, Level) + SegmentList, LevelR) end. @@ -998,11 +1032,11 @@ create_slot(KL1, KL2, _, _, SegLists, SerialisedSlot, LengthList, {{LowKey, generate_segment_filter(SegLists), SerialisedSlot, LengthList}, {{LSN, HSN}, LastKey, partial}, KL1, KL2}; -create_slot(KL1, KL2, Level, BlockCount, SegLists, SerialisedSlot, LengthList, +create_slot(KL1, KL2, LevelR, BlockCount, SegLists, SerialisedSlot, LengthList, {LowKey, LSN, HSN, LastKey, _Status}) -> {BlockKeyList, Status, {LSNb, HSNb}, - SegmentList, KL1b, KL2b} = create_block(KL1, KL2, Level), + SegmentList, KL1b, KL2b} = create_block(KL1, KL2, LevelR), TrackingMetadata = case LowKey of null -> [NewLowKeyV|_] = BlockKeyList, @@ -1021,7 +1055,7 @@ create_slot(KL1, KL2, Level, BlockCount, SegLists, SerialisedSlot, LengthList, SerialisedBlock = serialise_block(BlockKeyList), BlockLength = byte_size(SerialisedBlock), SerialisedSlot2 = <>, - create_slot(KL1b, KL2b, Level, BlockCount - 1, SegLists ++ [SegmentList], + create_slot(KL1b, KL2b, LevelR, BlockCount - 1, SegLists ++ [SegmentList], SerialisedSlot2, LengthList ++ [BlockLength], TrackingMetadata). @@ -1416,7 +1450,7 @@ simple_create_block_test() -> KeyList2 = [{{o, "Bucket1", "Key2", null}, {3, {active, infinity}, null}}], {MergedKeyList, ListStatus, SN, _, _, _} = create_block(KeyList1, KeyList2, - 1), + #level{level=1}), ?assertMatch(partial, ListStatus), [H1|T1] = MergedKeyList, ?assertMatch(H1, {{o, "Bucket1", "Key1", null}, {1, {active, infinity}, null}}), @@ -1431,7 +1465,7 @@ dominate_create_block_test() -> KeyList2 = [{{o, "Bucket1", "Key2", null}, {3, {tomb, infinity}, null}}], {MergedKeyList, ListStatus, SN, _, _, _} = create_block(KeyList1, KeyList2, - 1), + #level{level=1}), ?assertMatch(partial, ListStatus), [K1, K2] = MergedKeyList, ?assertMatch(K1, {{o, "Bucket1", "Key1", null}, {1, {active, infinity}, null}}), @@ -1476,8 +1510,8 @@ sample_keylist() -> alternating_create_block_test() -> {KeyList1, KeyList2} = sample_keylist(), {MergedKeyList, ListStatus, _, _, _, _} = create_block(KeyList1, - KeyList2, - 1), + KeyList2, + #level{level=1}), BlockSize = length(MergedKeyList), ?assertMatch(BlockSize, 32), ?assertMatch(ListStatus, complete), @@ -1488,7 +1522,9 @@ alternating_create_block_test() -> K32 = lists:nth(32, MergedKeyList), ?assertMatch(K32, {{o, "Bucket4", "Key1", null}, {1, {active, infinity}, null}}), HKey = {{o, "Bucket1", "Key0", null}, {1, {active, infinity}, null}}, - {_, ListStatus2, _, _, _, _} = create_block([HKey|KeyList1], KeyList2, 1), + {_, ListStatus2, _, _, _, _} = create_block([HKey|KeyList1], + KeyList2, + #level{level=1}), ?assertMatch(ListStatus2, full). @@ -1583,7 +1619,7 @@ merge_seglists_test() -> createslot_stage1_test() -> {KeyList1, KeyList2} = sample_keylist(), - Out = create_slot(KeyList1, KeyList2, 1), + Out = create_slot(KeyList1, KeyList2, #level{level=1}), {{LowKey, SegFilter, _SerialisedSlot, _LengthList}, {{LSN, HSN}, LastKey, Status}, KL1, KL2} = Out, @@ -1606,7 +1642,7 @@ createslot_stage1_test() -> createslot_stage2_test() -> Out = create_slot(lists:sort(generate_randomkeys(100)), lists:sort(generate_randomkeys(100)), - 1), + #level{level=1}), {{_LowKey, _SegFilter, SerialisedSlot, LengthList}, {{_LSN, _HSN}, _LastKey, Status}, _KL1, _KL2} = Out, @@ -1619,7 +1655,7 @@ createslot_stage2_test() -> createslot_stage3_test() -> Out = create_slot(lists:sort(generate_sequentialkeys(100, 1)), lists:sort(generate_sequentialkeys(100, 101)), - 1), + #level{level=1}), {{LowKey, SegFilter, SerialisedSlot, LengthList}, {{_LSN, _HSN}, LastKey, Status}, KL1, KL2} = Out, @@ -1662,7 +1698,10 @@ testwrite_function(finalise, {Handle, C_SlotIndex, SNExtremes, KeyExtremes}) -> writekeys_stage1_test() -> {KL1, KL2} = sample_keylist(), - {FunOut, {_KL1Rem, _KL2Rem}} = write_keys([], KL1, KL2, [], <<>>, 1, + {FunOut, {_KL1Rem, _KL2Rem}} = write_keys([], + KL1, KL2, + [], <<>>, + #level{level=1}, fun testwrite_function/2), {Handle, {_, PointerIndex}, SNExtremes, KeyExtremes} = FunOut, ?assertMatch(SNExtremes, {1,3}), @@ -1685,7 +1724,9 @@ initial_create_file_test() -> Filename = "../test/test1.sft", {KL1, KL2} = sample_keylist(), {Handle, FileMD} = create_file(Filename), - {UpdHandle, UpdFileMD, {[], []}} = complete_file(Handle, FileMD, KL1, KL2, 1), + {UpdHandle, UpdFileMD, {[], []}} = complete_file(Handle, FileMD, + KL1, KL2, + #level{level=1}), Result1 = fetch_keyvalue(UpdHandle, UpdFileMD, {o, "Bucket1", "Key8", null}), io:format("Result is ~w~n", [Result1]), ?assertMatch(Result1, {{o, "Bucket1", "Key8", null}, @@ -1703,7 +1744,8 @@ big_create_file_test() -> {InitHandle, InitFileMD} = create_file(Filename), {Handle, FileMD, {_KL1Rem, _KL2Rem}} = complete_file(InitHandle, InitFileMD, - KL1, KL2, 1), + KL1, KL2, + #level{level=1}), [{K1, {Sq1, St1, V1}}|_] = KL1, [{K2, {Sq2, St2, V2}}|_] = KL2, Result1 = fetch_keyvalue(Handle, FileMD, K1), @@ -1736,11 +1778,9 @@ initial_iterator_test() -> Filename = "../test/test2.sft", {KL1, KL2} = sample_keylist(), {Handle, FileMD} = create_file(Filename), - {UpdHandle, UpdFileMD, {[], []}} = complete_file(Handle, - FileMD, - KL1, - KL2, - 1), + {UpdHandle, UpdFileMD, {[], []}} = complete_file(Handle, FileMD, + KL1, KL2, + #level{level=1}), Result1 = fetch_range_keysonly(UpdHandle, UpdFileMD, {o, "Bucket1", "Key8", null}, {o, "Bucket1", "Key9d", null}), @@ -1790,54 +1830,54 @@ key_dominates_test() -> KL1 = [KV1, KV2], KL2 = [KV3, KV4], ?assertMatch({{next_key, KV1}, [KV2], KL2}, - key_dominates(KL1, KL2, 1)), + key_dominates(KL1, KL2, {undefined, 1})), ?assertMatch({{next_key, KV1}, KL2, [KV2]}, - key_dominates(KL2, KL1, 1)), + key_dominates(KL2, KL1, {undefined, 1})), ?assertMatch({skipped_key, KL2, KL1}, - key_dominates([KV5|KL2], KL1, 1)), + key_dominates([KV5|KL2], KL1, {undefined, 1})), ?assertMatch({{next_key, KV1}, [KV2], []}, - key_dominates(KL1, [], 1)), + key_dominates(KL1, [], {undefined, 1})), ?assertMatch({skipped_key, [KV6|KL2], [KV2]}, - key_dominates([KV6|KL2], KL1, 1)), + key_dominates([KV6|KL2], KL1, {undefined, 1})), ?assertMatch({{next_key, KV6}, KL2, [KV2]}, - key_dominates([KV6|KL2], [KV2], 1)), + key_dominates([KV6|KL2], [KV2], {undefined, 1})), ?assertMatch({skipped_key, [KV6|KL2], [KV2]}, - key_dominates([KV6|KL2], KL1, {basement, 1})), + key_dominates([KV6|KL2], KL1, {true, 1})), ?assertMatch({skipped_key, [KV6|KL2], [KV2]}, - key_dominates([KV6|KL2], KL1, {basement, 1000})), + key_dominates([KV6|KL2], KL1, {true, 1000})), ?assertMatch({{next_key, KV6}, KL2, [KV2]}, - key_dominates([KV6|KL2], [KV2], {basement, 1})), + key_dominates([KV6|KL2], [KV2], {true, 1})), ?assertMatch({skipped_key, KL2, [KV2]}, - key_dominates([KV6|KL2], [KV2], {basement, 1000})), + key_dominates([KV6|KL2], [KV2], {true, 1000})), ?assertMatch({skipped_key, [], []}, - key_dominates([KV6], [], {basement, 1000})), + key_dominates([KV6], [], {true, 1000})), ?assertMatch({skipped_key, [], []}, - key_dominates([], [KV6], {basement, 1000})), + key_dominates([], [KV6], {true, 1000})), ?assertMatch({{next_key, KV6}, [], []}, - key_dominates([KV6], [], {basement, 1})), + key_dominates([KV6], [], {true, 1})), ?assertMatch({{next_key, KV6}, [], []}, - key_dominates([], [KV6], {basement, 1})), + key_dominates([], [KV6], {true, 1})), ?assertMatch({skipped_key, [], []}, - key_dominates([KV7], [], {basement, 1})), + key_dominates([KV7], [], {true, 1})), ?assertMatch({skipped_key, [], []}, - key_dominates([], [KV7], {basement, 1})), + key_dominates([], [KV7], {true, 1})), ?assertMatch({skipped_key, [KV7|KL2], [KV2]}, - key_dominates([KV7|KL2], KL1, 1)), + key_dominates([KV7|KL2], KL1, {undefined, 1})), ?assertMatch({{next_key, KV7}, KL2, [KV2]}, - key_dominates([KV7|KL2], [KV2], 1)), + key_dominates([KV7|KL2], [KV2], {undefined, 1})), ?assertMatch({skipped_key, [KV7|KL2], [KV2]}, - key_dominates([KV7|KL2], KL1, {basement, 1})), + key_dominates([KV7|KL2], KL1, {true, 1})), ?assertMatch({skipped_key, KL2, [KV2]}, - key_dominates([KV7|KL2], [KV2], {basement, 1})). + key_dominates([KV7|KL2], [KV2], {true, 1})). big_iterator_test() -> Filename = "../test/bigtest1.sft", {KL1, KL2} = {lists:sort(generate_randomkeys(10000)), []}, {InitHandle, InitFileMD} = create_file(Filename), - {Handle, FileMD, {KL1Rem, KL2Rem}} = complete_file(InitHandle, - InitFileMD, - KL1, KL2, 1), + {Handle, FileMD, {KL1Rem, KL2Rem}} = complete_file(InitHandle, InitFileMD, + KL1, KL2, + #level{level=1}), io:format("Remainder lengths are ~w and ~w ~n", [length(KL1Rem), length(KL2Rem)]), {complete, Result1} = fetch_range_keysonly(Handle, FileMD, diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index f00571e..d1e3967 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -228,6 +228,7 @@ simple_querycount(_Config) -> end, R9), ok = leveled_bookie:book_riakput(Book3, Obj9, Spc9), + ok = leveled_bookie:book_close(Book3), {ok, Book4} = leveled_bookie:book_start(RootPath, 2000, 50000000), lists:foreach(fun({IdxF, IdxT, X}) -> R = leveled_bookie:book_returnfolder(Book4, diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index 9043bf9..c836f65 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -25,9 +25,11 @@ load_objects/5]). + reset_filestructure() -> - io:format("Waiting 2s to give a chance for all file closes to complete~n"), - timer:sleep(2000), + % io:format("Waiting ~w ms to give a chance for all file closes " ++ + "to complete~n", [Wait]), + % timer:sleep(Wait), RootPath = "test", filelib:ensure_dir(RootPath ++ "/journal/"), filelib:ensure_dir(RootPath ++ "/ledger/"), From c78b5bca7d4cb30ba7f8ac8bb566867534b3a521 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sun, 23 Oct 2016 22:45:43 +0100 Subject: [PATCH 087/167] Basement Tombstones Further progress towards the tidying up of basement tombstones in the Ledger, with support added for key-listing to help with testing (and as a potentially required feature). The test is incomplete, but committing at this stage as the last commit broke some tests (within the test code). There are some outstanding questions about the handling of tombstones in the Journal during compaction. There exists a condition whereby values could return if a recent journal is compacted and tombstones are removed (as they are no longer present), but older journals have not been compacted. Now on stop/start - if the Ledger is wiped the removal of the keys will be forgotten but the original PUTs would still remain. The safest thing maybe to have rule that tombstones are never deleted from the Inker's Journal - and accept the build-up of garbage. Or there could be an addition to the compaction process that checks back through all the inker files to check that the Key of a tombstone is not present in the past, before it is removed in the compaction. --- src/leveled_bookie.erl | 38 +++++++++++++++- src/leveled_codec.erl | 4 +- src/leveled_iclerk.erl | 3 +- src/leveled_inker.erl | 12 ++++- test/end_to_end/basic_SUITE.erl | 78 ++++++++++++++++++++++++++++++++- test/end_to_end/testutil.erl | 2 +- 6 files changed, 130 insertions(+), 7 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 9d61661..acd0d9e 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -366,7 +366,13 @@ handle_call({return_folder, FolderType}, _From, State) -> Bucket, {IdxField, StartValue, EndValue}, {ReturnTerms, TermRegex}), - State} + State}; + {keylist, Tag} -> + {reply, + allkey_query(State#state.penciller, + State#state.ledger_cache, + Tag), + State} end; handle_call({compact_journal, Timeout}, _From, State) -> ok = leveled_inker:ink_compactjournal(State#state.inker, @@ -460,6 +466,28 @@ index_query(Penciller, LedgerCache, end, {async, Folder}. +allkey_query(Penciller, LedgerCache, Tag) -> + PCLopts = #penciller_options{start_snapshot=true, + source_penciller=Penciller}, + {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), + Folder = fun() -> + Increment = gb_trees:to_list(LedgerCache), + io:format("Length of increment in snapshot is ~w~n", + [length(Increment)]), + ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, + {infinity, Increment}), + SK = leveled_codec:to_ledgerkey(null, null, Tag), + EK = leveled_codec:to_ledgerkey(null, null, Tag), + Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, + SK, + EK, + fun accumulate_keys/3, + []), + ok = leveled_penciller:pcl_close(LedgerSnapshot), + lists:reverse(Acc) + end, + {async, Folder}. + shutdown_wait([], _Inker) -> false; @@ -529,6 +557,14 @@ accumulate_size(Key, Value, {Size, Count}) -> {Size, Count} end. +accumulate_keys(Key, Value, KeyList) -> + case leveled_codec:is_active(Key, Value) of + true -> + [leveled_codec:from_ledgerkey(Key)|KeyList]; + false -> + KeyList + end. + add_keys(ObjKey, _IdxValue, Acc) -> Acc ++ [ObjKey]. diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index a1c20b4..a5474dc 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -111,7 +111,9 @@ is_active(Key, Value) -> from_ledgerkey({Tag, Bucket, {_IdxField, IdxValue}, Key}) when Tag == ?IDX_TAG -> - {Bucket, Key, IdxValue}. + {Bucket, Key, IdxValue}; +from_ledgerkey({_Tag, Bucket, Key, null}) -> + {Bucket, Key}. to_ledgerkey(Bucket, Key, Tag, Field, Value) when Tag == ?IDX_TAG -> {?IDX_TAG, Bucket, {Field, Value}, Key}. diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index ca1a6f3..8fb0579 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -405,7 +405,8 @@ write_values([KVC|Rest], CDBopts, Journal0, ManSlice0) -> _ -> {ok, Journal0} end, - R = leveled_cdb:cdb_put(Journal1, {SQN, PK}, V), + ValueToStore = leveled_inker:create_value_for_cdb(V), + R = leveled_cdb:cdb_put(Journal1, {SQN, PK}, ValueToStore), case R of ok -> write_values(Rest, CDBopts, Journal1, ManSlice0); diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 79a0cd8..e46ddfa 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -97,6 +97,7 @@ ink_print_manifest/1, ink_close/1, ink_forceclose/1, + create_value_for_cdb/1, build_dummy_journal/0, simple_manifest_reader/2, clean_testdir/1, @@ -375,7 +376,7 @@ put_object(PrimaryKey, Object, KeyChanges, State) -> %% as the CDB will also do the same conversion %% Perhaps have CDB started up in apure binary mode, when it doesn't %5 receive terms? - Bin1 = term_to_binary({Object, KeyChanges}, [compressed]), + Bin1 = create_value_for_cdb({Object, KeyChanges}), ObjSize = byte_size(Bin1), case leveled_cdb:cdb_put(State#state.active_journaldb, {NewSQN, PrimaryKey}, @@ -406,6 +407,15 @@ put_object(PrimaryKey, Object, KeyChanges, State) -> end. +create_value_for_cdb(Value) -> + case Value of + {Object, KeyChanges} -> + term_to_binary({Object, KeyChanges}, [compressed]); + Value when is_binary(Value) -> + Value + end. + + get_object(PrimaryKey, SQN, Manifest) -> JournalP = find_in_manifest(SQN, Manifest), Obj = leveled_cdb:cdb_get(JournalP, {SQN, PrimaryKey}), diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 718587c..6887556 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -7,7 +7,8 @@ journal_compaction/1, fetchput_snapshot/1, load_and_count/1, - load_and_count_withdelete/1 + load_and_count_withdelete/1, + space_clear_ondelete_test/1 ]). all() -> [ @@ -16,7 +17,8 @@ all() -> [ journal_compaction, fetchput_snapshot, load_and_count, - load_and_count_withdelete + load_and_count_withdelete, + space_clear_ondelete_test ]. @@ -395,3 +397,75 @@ load_and_count_withdelete(_Config) -> ok = leveled_bookie:book_close(Bookie2), testutil:reset_filestructure(). + +space_clear_ondelete_test(_Config) -> + % Test is a work in progress + RootPath = testutil:reset_filestructure(), + StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=20000000}, + {ok, Book1} = leveled_bookie:book_start(StartOpts1), + G2 = fun testutil:generate_compressibleobjects/2, + testutil:load_objects(20000, + [uuid, uuid, uuid, uuid], + Book1, + no_check, + G2), + + {async, F1} = leveled_bookie:book_returnfolder(Book1, {keylist, o_rkv}), + SW1 = os:timestamp(), + KL1 = F1(), + ok = case length(KL1) of + 80000 -> + io:format("Key list took ~w microseconds for 80K keys~n", + [timer:now_diff(os:timestamp(), SW1)]), + ok + end, + timer:sleep(10000), % Allow for any L0 file to be rolled + {ok, FNsA_L} = file:list_dir(RootPath ++ "/ledger/ledger_files"), + {ok, FNsA_J} = file:list_dir(RootPath ++ "/journal/journal_files"), + io:format("Bookie created ~w journal files and ~w ledger files~n", + [length(FNsA_J), length(FNsA_L)]), + SW2 = os:timestamp(), + lists:foreach(fun({Bucket, Key}) -> + ok = leveled_bookie:book_riakdelete(Book1, + Bucket, + Key, + []) + end, + KL1), + io:format("Deletion took ~w microseconds for 80K keys~n", + [timer:now_diff(os:timestamp(), SW2)]), + ok = leveled_bookie:book_compactjournal(Book1, 30000), + timer:sleep(30000), % Allow for any L0 file to be rolled + {ok, FNsB_L} = file:list_dir(RootPath ++ "/ledger/ledger_files"), + {ok, FNsB_J} = file:list_dir(RootPath ++ "/journal/journal_files"), + io:format("Bookie has ~w journal files and ~w ledger files " ++ + "after deletes~n", + [length(FNsB_J), length(FNsB_L)]), + + {async, F2} = leveled_bookie:book_returnfolder(Book1, {keylist, o_rkv}), + SW3 = os:timestamp(), + KL2 = F2(), + ok = case length(KL2) of + 0 -> + io:format("Key list took ~w microseconds for no keys~n", + [timer:now_diff(os:timestamp(), SW3)]), + ok + end, + ok = leveled_bookie:book_close(Book1), + + {ok, Book2} = leveled_bookie:book_start(StartOpts1), + {async, F3} = leveled_bookie:book_returnfolder(Book2, {keylist, o_rkv}), + SW4 = os:timestamp(), + KL3 = F3(), + ok = case length(KL3) of + 0 -> + io:format("Key list took ~w microseconds for no keys~n", + [timer:now_diff(os:timestamp(), SW4)]), + ok + end, + ok = leveled_bookie:book_close(Book2), + {ok, FNsC_L} = file:list_dir(RootPath ++ "/ledger/ledger_files"), + {ok, FNsC_J} = file:list_dir(RootPath ++ "/journal/journal_files"), + io:format("Bookie has ~w journal files and ~w ledger files " ++ + "after deletes~n", + [length(FNsC_J), length(FNsC_L)]). diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index c836f65..b0a4707 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -28,7 +28,7 @@ reset_filestructure() -> % io:format("Waiting ~w ms to give a chance for all file closes " ++ - "to complete~n", [Wait]), + % "to complete~n", [Wait]), % timer:sleep(Wait), RootPath = "test", filelib:ensure_dir(RootPath ++ "/journal/"), From d988c66ac618b674964ef077f53ee662dedeaca0 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 24 Oct 2016 11:44:28 +0100 Subject: [PATCH 088/167] Enhance unit tests for corruped segment filters --- src/leveled_bookie.erl | 9 +++++++++ src/leveled_sft.erl | 45 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index acd0d9e..5305eaf 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -186,6 +186,15 @@ book_riakput(Pid, RiakObject, IndexSpecs) -> book_put(Pid, Bucket, Key, Object, IndexSpecs) -> book_put(Pid, Bucket, Key, Object, IndexSpecs, ?STD_TAG). +%% TODO: +%% It is not enough simply to change the value to delete, as the journal +%% needs to know the key is a tombstone at compaction time, and currently at +%% compaction time the clerk only knows the Key and not the Value. +%% +%% The tombstone cannot be removed from the Journal on compaction, as the +%% journal entry the tombstone deletes may not have been reaped - and so if the +%% ledger got erased, the value would be resurrected. + book_riakdelete(Pid, Bucket, Key, IndexSpecs) -> book_put(Pid, Bucket, Key, delete, IndexSpecs, ?RIAK_TAG). diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index f85ebcc..52809f3 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -1615,7 +1615,50 @@ merge_seglists_test() -> check_for_segments(SegBin2, [1024*1024 - 1], true)), % This match is before the flipped bit, so still works without CRC check ?assertMatch({maybe_present, [0]}, - check_for_segments(SegBin2, [0,900], false)). + check_for_segments(SegBin2, [0,900], false)), + + ExpectedDeltasAll1s = <<4294967295:32/integer>>, + SegBin3 = <>, + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin3, [900], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin3, [200], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin3, [0,900], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin3, [1024*1024 - 1], true)), + % This is so badly mangled, the error gets detected event without CRC + % checking being enforced + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin3, [900], false)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin3, [200], false)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin3, [0,900], false)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin3, [1024*1024 - 1], false)), + + ExpectedDeltasNearlyAll1s = <<4294967287:32/integer>>, + SegBin4 = <>, + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin4, [900], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin4, [200], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin4, [0,900], true)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin4, [1024*1024 - 1], true)), + % This is so badly mangled, the error gets detected event without CRC + % checking being enforced + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin4, [900], false)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin4, [200], false)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin4, [0,900], false)), + ?assertMatch(error_so_maybe_present, + check_for_segments(SegBin4, [1024*1024 - 1], false)). createslot_stage1_test() -> {KeyList1, KeyList2} = sample_keylist(), From 102cfe7f6f43675b51f9e3bf02547b994b86d27f Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 25 Oct 2016 01:57:12 +0100 Subject: [PATCH 089/167] Move towards Inker Key Types The current mechanism of re-loading data from the Journla to the Ledger from any potential SQN is not safe when combined with Journla compaction. This commit doesn't resolve thes eproblems, but starts the groundwork for resolving by introducing Inker Key Types. These types would differentiate between objects which are standard Key/Value pairs, objects which are tombstones for keys, and objects whihc represent Key Changes only. The idea is that there will be flexible reload strategies based on object tags - retain (retain a key change object when compacting a standard object) - recalc (allow key changes to be recalculated from objects and ledger state when loading the Ledger from the journal - recover (allow for the potential loss of data on loss within the perisste dpart of the ledger, potentially due to recovery through externla anti-entropy operations). --- src/leveled_bookie.erl | 2 +- src/leveled_cdb.erl | 21 +++++++--- src/leveled_iclerk.erl | 95 +++++++++++++++++++++++++++++++++--------- src/leveled_inker.erl | 57 ++++++++++++++++--------- 4 files changed, 129 insertions(+), 46 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 5305eaf..e116b8f 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -663,7 +663,7 @@ maybe_withjitter(CacheSize, MaxCacheSize) -> load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> {MinSQN, MaxSQN, OutputTree} = Acc0, - {SQN, PK} = KeyInLedger, + {SQN, _Type, PK} = KeyInLedger, % VBin may already be a term {VBin, VSize} = ExtractFun(ValueInLedger), {Obj, IndexSpecs} = case is_binary(VBin) of diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 94d3a2f..f3eedd9 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -57,6 +57,7 @@ cdb_open_reader/1, cdb_get/2, cdb_put/3, + cdb_put/4, cdb_getpositions/2, cdb_directfetch/3, cdb_lastkey/1, @@ -126,7 +127,10 @@ cdb_get(Pid, Key) -> gen_server:call(Pid, {get_kv, Key}, infinity). cdb_put(Pid, Key, Value) -> - gen_server:call(Pid, {put_kv, Key, Value}, infinity). + cdb_put(Pid, Key, Value, hash). + +cdb_put(Pid, Key, Value, HashOpt) -> + gen_server:call(Pid, {put_kv, Key, Value, HashOpt}, infinity). %% SampleSize can be an integer or the atom all cdb_getpositions(Pid, SampleSize) -> @@ -258,7 +262,7 @@ handle_call({key_check, Key}, _From, State) -> State#state.hash_index), State} end; -handle_call({put_kv, Key, Value}, _From, State) -> +handle_call({put_kv, Key, Value, HashOpt}, _From, State) -> case {State#state.writer, State#state.pending_roll} of {true, false} -> Result = put(State#state.handle, @@ -266,15 +270,20 @@ handle_call({put_kv, Key, Value}, _From, State) -> {State#state.last_position, State#state.hashtree}, State#state.binary_mode, State#state.max_size), - case Result of - roll -> + case {Result, HashOpt} of + {roll, _} -> %% Key and value could not be written {reply, roll, State}; - {UpdHandle, NewPosition, HashTree} -> + {{UpdHandle, NewPosition, HashTree}, hash} -> {reply, ok, State#state{handle=UpdHandle, last_position=NewPosition, last_key=Key, - hashtree=HashTree}} + hashtree=HashTree}}; + {{UpdHandle, NewPosition, _HashTree}, no_hash} -> + %% Don't update the hashtree + {reply, ok, State#state{handle=UpdHandle, + last_position=NewPosition, + last_key=Key}} end; _ -> {reply, diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 8fb0579..e8eeb9c 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -1,4 +1,61 @@ - +%% -------- Inker's Clerk --------- +%% +%% The Inker's clerk runs compaction jobs on behalf of the Inker, informing the +%% Inker of any manifest changes when complete. +%% +%% -------- Value Compaction --------- +%% +%% Compaction requires the Inker to have four different types of keys +%% * stnd - A standard key of the form {SQN, stnd, LedgerKey} which maps to a +%% value of {Object, KeyDeltas} +%% * tomb - A tombstone for a LedgerKey {SQN, tomb, LedgerKey} +%% * keyd - An object containing key deltas only of the form +%% {SQN, keyd, LedgerKey} which maps to a value of {KeyDeltas} +%% +%% Each LedgerKey has a Tag, and for each Tag there should be a compaction +%% strategy, which will be set to one of the following: +%% * retain - KeyDeltas must be retained permanently, only values can be +%% compacted (if replaced or not_present in the ledger) +%% * recalc - The full object can be removed through comapction (if replaced or +%% not_present in the ledger), as each object with that tag can have the Key +%% Deltas recreated by passing into an assigned recalc function {LedgerKey, +%% SQN, Object, KeyDeltas, PencillerSnapshot} +%% * recovr - At compaction time this is equivalent to recalc, only KeyDeltas +%% are lost when reloading the Ledger from the Journal, and it is assumed that +%% those deltas will be resolved through external anti-entropy (e.g. read +%% repair or AAE) - or alternatively the risk of loss of persisted data from +%% the ledger is accepted for this data type +%% +%% During the compaction process for the Journal, the file chosen for +%% compaction is scanned in SQN order, and a FilterFun is passed (which will +%% normally perform a check against a snapshot of the persisted part of the +%% Ledger). If the given key is of type stnd, and this object is no longer the +%% active object under the LedgerKey, then the object can be compacted out of +%% the journal. This will lead to either its removal (if the strategy for the +%% Tag is recovr or recalc), or its replacement with a KeyDelta object. +%% +%% Tombstones cannot be reaped through this compaction process. +%% +%% Currently, KeyDeltas are also reaped if the LedgerKey has been updated and +%% the Tag has a recovr strategy. This may be the case when KeyDeltas are used +%% as a way of directly representing a change, and where anti-entropy can +%% recover from a loss. +%% +%% -------- Tombstone Reaping --------- +%% +%% Value compaction does not remove tombstones from the database, and so a +%% separate compaction job is required for this. +%% +%% Tombstones can only be reaped for Tags set to recovr or recalc. +%% +%% The tombstone reaping process should select a file to compact, and then +%% take that file and discover the LedgerKeys of all reapable tombstones. +%% The lesger should then be scanned from SQN 0 looking for unreaped objects +%% before the tombstone. If no ushc objects exist for that tombstone, it can +%% now be reaped as part of the compaction job. +%% +%% Other tombstones cannot be reaped, as otherwis eon laoding a ledger an old +%% version of the object may re-emerge. -module(leveled_iclerk). @@ -161,7 +218,7 @@ check_single_file(CDB, FilterFun, FilterServer, MaxSQN, SampleSize, BatchSize) - PositionList = leveled_cdb:cdb_getpositions(CDB, SampleSize), KeySizeList = fetch_inbatches(PositionList, BatchSize, CDB, []), R0 = lists:foldl(fun(KS, {ActSize, RplSize}) -> - {{SQN, PK}, Size} = KS, + {{SQN, _Type, PK}, Size} = KS, Check = FilterFun(FilterServer, PK, SQN), case {Check, SQN > MaxSQN} of {true, _} -> @@ -368,7 +425,7 @@ split_positions_into_batches(Positions, Journal, Batches) -> filter_output(KVCs, FilterFun, FilterServer, MaxSQN) -> lists:foldl(fun(KVC, {Acc, PromptDelete}) -> - {{SQN, PK}, _V, CrcCheck} = KVC, + {{SQN, _Type, PK}, _V, CrcCheck} = KVC, KeyValid = FilterFun(FilterServer, PK, SQN), case {KeyValid, CrcCheck, SQN > MaxSQN} of {true, true, _} -> @@ -390,7 +447,7 @@ filter_output(KVCs, FilterFun, FilterServer, MaxSQN) -> write_values([], _CDBopts, Journal0, ManSlice0) -> {Journal0, ManSlice0}; write_values([KVC|Rest], CDBopts, Journal0, ManSlice0) -> - {{SQN, PK}, V, _CrcCheck} = KVC, + {{SQN, Type, PK}, V, _CrcCheck} = KVC, {ok, Journal1} = case Journal0 of null -> FP = CDBopts#cdb_options.file_path, @@ -406,7 +463,7 @@ write_values([KVC|Rest], CDBopts, Journal0, ManSlice0) -> {ok, Journal0} end, ValueToStore = leveled_inker:create_value_for_cdb(V), - R = leveled_cdb:cdb_put(Journal1, {SQN, PK}, ValueToStore), + R = leveled_cdb:cdb_put(Journal1, {SQN, Type, PK}, ValueToStore), case R of ok -> write_values(Rest, CDBopts, Journal1, ManSlice0); @@ -419,7 +476,7 @@ write_values([KVC|Rest], CDBopts, Journal0, ManSlice0) -> generate_manifest_entry(ActiveJournal) -> {ok, NewFN} = leveled_cdb:cdb_complete(ActiveJournal), {ok, PidR} = leveled_cdb:cdb_open_reader(NewFN), - {StartSQN, _PK} = leveled_cdb:cdb_firstkey(PidR), + {StartSQN, _Type, _PK} = leveled_cdb:cdb_firstkey(PidR), [{StartSQN, NewFN, PidR}]. @@ -514,14 +571,14 @@ find_bestrun_test() -> fetch_testcdb(RP) -> FN1 = leveled_inker:filepath(RP, 1, new_journal), {ok, CDB1} = leveled_cdb:cdb_open_writer(FN1, #cdb_options{}), - {K1, V1} = {{1, "Key1"}, term_to_binary("Value1")}, - {K2, V2} = {{2, "Key2"}, term_to_binary("Value2")}, - {K3, V3} = {{3, "Key3"}, term_to_binary("Value3")}, - {K4, V4} = {{4, "Key1"}, term_to_binary("Value4")}, - {K5, V5} = {{5, "Key1"}, term_to_binary("Value5")}, - {K6, V6} = {{6, "Key1"}, term_to_binary("Value6")}, - {K7, V7} = {{7, "Key1"}, term_to_binary("Value7")}, - {K8, V8} = {{8, "Key1"}, term_to_binary("Value8")}, + {K1, V1} = {{1, stnd, "Key1"}, term_to_binary("Value1")}, + {K2, V2} = {{2, stnd, "Key2"}, term_to_binary("Value2")}, + {K3, V3} = {{3, stnd, "Key3"}, term_to_binary("Value3")}, + {K4, V4} = {{4, stnd, "Key1"}, term_to_binary("Value4")}, + {K5, V5} = {{5, stnd, "Key1"}, term_to_binary("Value5")}, + {K6, V6} = {{6, stnd, "Key1"}, term_to_binary("Value6")}, + {K7, V7} = {{7, stnd, "Key1"}, term_to_binary("Value7")}, + {K8, V8} = {{8, stnd, "Key1"}, term_to_binary("Value8")}, ok = leveled_cdb:cdb_put(CDB1, K1, V1), ok = leveled_cdb:cdb_put(CDB1, K2, V2), ok = leveled_cdb:cdb_put(CDB1, K3, V3), @@ -583,10 +640,10 @@ compact_single_file_test() -> [{LowSQN, FN, PidR}] = ManSlice1, io:format("FN of ~s~n", [FN]), ?assertMatch(2, LowSQN), - ?assertMatch(probably, leveled_cdb:cdb_keycheck(PidR, {8, "Key1"})), - ?assertMatch(missing, leveled_cdb:cdb_get(PidR, {7, "Key1"})), - ?assertMatch(missing, leveled_cdb:cdb_get(PidR, {1, "Key1"})), - {_RK1, RV1} = leveled_cdb:cdb_get(PidR, {2, "Key2"}), + ?assertMatch(probably, leveled_cdb:cdb_keycheck(PidR, {8, stnd, "Key1"})), + ?assertMatch(missing, leveled_cdb:cdb_get(PidR, {7, stnd, "Key1"})), + ?assertMatch(missing, leveled_cdb:cdb_get(PidR, {1, stnd, "Key1"})), + {_RK1, RV1} = leveled_cdb:cdb_get(PidR, {2, stnd, "Key2"}), ?assertMatch("Value2", binary_to_term(RV1)), ok = leveled_cdb:cdb_destroy(CDB). @@ -596,7 +653,7 @@ compact_empty_file_test() -> FN1 = leveled_inker:filepath(RP, 1, new_journal), CDBopts = #cdb_options{binary_mode=true}, {ok, CDB1} = leveled_cdb:cdb_open_writer(FN1, CDBopts), - ok = leveled_cdb:cdb_put(CDB1, {1, "Key1"}, <<>>), + ok = leveled_cdb:cdb_put(CDB1, {1, stnd, "Key1"}, <<>>), {ok, FN2} = leveled_cdb:cdb_complete(CDB1), {ok, CDB2} = leveled_cdb:cdb_open_reader(FN2), LedgerSrv1 = [{8, "Key1"}, {2, "Key2"}, {3, "Key3"}], diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index e46ddfa..b7f26aa 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -50,10 +50,19 @@ %% -------- Objects --------- %% %% From the perspective of the Inker, objects to store are made up of: -%% - A Primary Key (as an Erlang term) -%% - A sequence number (assigned by the Inker) -%% - An object (an Erlang term) -%% - A set of Key Deltas associated with the change +%% - An Inker Key formed from +%% - A sequence number (assigned by the Inker) +%% - An Inker key type (stnd, tomb or keyd) +%% - A Ledger Key (as an Erlang term) +%% - A value formed from +%% - An object (an Erlang term) which should be null for tomb types, and +%% maybe null for keyd types +%% - A set of Key Deltas associated with the change (which may be an +%% empty list ) +%% +%% Note that only the Inker key type of stnd is directly fetchable, other +%% key types are to be found only in scans and so can be added without being +%% entered into the hashtree %% %% -------- Compaction --------- %% @@ -372,15 +381,20 @@ start_from_file(InkerOpts) -> put_object(PrimaryKey, Object, KeyChanges, State) -> NewSQN = State#state.journal_sqn + 1, - %% TODO: The term goes through a double binary_to_term conversion - %% as the CDB will also do the same conversion - %% Perhaps have CDB started up in apure binary mode, when it doesn't - %5 receive terms? + {InkerType, HashOpt} = case Object of + delete -> + {tomb, no_hash}; + %delta -> + % {keyd, no_hash + _ -> + {stnd, hash} + end, Bin1 = create_value_for_cdb({Object, KeyChanges}), ObjSize = byte_size(Bin1), case leveled_cdb:cdb_put(State#state.active_journaldb, - {NewSQN, PrimaryKey}, - Bin1) of + {NewSQN, InkerType, PrimaryKey}, + Bin1, + HashOpt) of ok -> {ok, State#state{journal_sqn=NewSQN}, ObjSize}; roll -> @@ -394,7 +408,10 @@ put_object(PrimaryKey, Object, KeyChanges, State) -> ok = simple_manifest_writer(NewManifest, State#state.manifest_sqn + 1, State#state.root_path), - ok = leveled_cdb:cdb_put(NewJournalP, {NewSQN, PrimaryKey}, Bin1), + ok = leveled_cdb:cdb_put(NewJournalP, + {NewSQN, InkerType, PrimaryKey}, + Bin1, + HashOpt), io:format("Put to new active journal " ++ "with manifest write took ~w microseconds~n", [timer:now_diff(os:timestamp(),SW)]), @@ -418,11 +435,11 @@ create_value_for_cdb(Value) -> get_object(PrimaryKey, SQN, Manifest) -> JournalP = find_in_manifest(SQN, Manifest), - Obj = leveled_cdb:cdb_get(JournalP, {SQN, PrimaryKey}), + Obj = leveled_cdb:cdb_get(JournalP, {SQN, stnd, PrimaryKey}), case Obj of - {{SQN, PK}, Bin} when is_binary(Bin) -> + {{SQN, stnd, PK}, Bin} when is_binary(Bin) -> {{SQN, PK}, binary_to_term(Bin)}; - {{SQN, PK}, Term} -> + {{SQN, stnd, PK}, Term} -> {{SQN, PK}, Term}; _ -> Obj @@ -456,7 +473,7 @@ build_manifest(ManifestFilenames, JournalSQN = case leveled_cdb:cdb_lastkey(ActiveJournal) of empty -> ActiveLowSQN; - {JSQN, _LastKey} -> + {JSQN, _Type, _LastKey} -> JSQN end, @@ -499,7 +516,7 @@ open_all_manifest(Man0, RootPath, CDBOpts) -> io:format("Head manifest entry ~s is complete~n", [HeadFN]), {ok, HeadR} = leveled_cdb:cdb_open_reader(CompleteHeadFN), - {LastSQN, _LastPK} = leveled_cdb:cdb_lastkey(HeadR), + {LastSQN, _Type, _PK} = leveled_cdb:cdb_lastkey(HeadR), add_to_manifest(add_to_manifest(ManifestTail, {HeadSQN, HeadFN, HeadR}), start_new_activejournal(LastSQN + 1, @@ -765,8 +782,8 @@ build_dummy_journal() -> {ok, J1} = leveled_cdb:cdb_open_writer(F1), {K1, V1} = {"Key1", "TestValue1"}, {K2, V2} = {"Key2", "TestValue2"}, - ok = leveled_cdb:cdb_put(J1, {1, K1}, term_to_binary({V1, []})), - ok = leveled_cdb:cdb_put(J1, {2, K2}, term_to_binary({V2, []})), + ok = leveled_cdb:cdb_put(J1, {1, stnd, K1}, term_to_binary({V1, []})), + ok = leveled_cdb:cdb_put(J1, {2, stnd, K2}, term_to_binary({V2, []})), ok = leveled_cdb:cdb_roll(J1), _LK = leveled_cdb:cdb_lastkey(J1), ok = leveled_cdb:cdb_close(J1), @@ -774,8 +791,8 @@ build_dummy_journal() -> {ok, J2} = leveled_cdb:cdb_open_writer(F2), {K1, V3} = {"Key1", "TestValue3"}, {K4, V4} = {"Key4", "TestValue4"}, - ok = leveled_cdb:cdb_put(J2, {3, K1}, term_to_binary({V3, []})), - ok = leveled_cdb:cdb_put(J2, {4, K4}, term_to_binary({V4, []})), + ok = leveled_cdb:cdb_put(J2, {3, stnd, K1}, term_to_binary({V3, []})), + ok = leveled_cdb:cdb_put(J2, {4, stnd, K4}, term_to_binary({V4, []})), ok = leveled_cdb:cdb_close(J2), Manifest = [{1, "../test/journal/journal_files/nursery_1"}, {3, "../test/journal/journal_files/nursery_3"}], From 97087a6b2b22b21e91b4381dca44d0cb2c4eae08 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 25 Oct 2016 23:13:14 +0100 Subject: [PATCH 090/167] Work on reload strategies Further work on variable reload srategies wiht some unit test coverage. Also work on potentially supporting no_hash on PUT to journal files for objects which will never be directly fetched. --- include/leveled.hrl | 21 ++++- src/leveled_bookie.erl | 42 +++++---- src/leveled_cdb.erl | 81 ++++++++++++++--- src/leveled_codec.erl | 87 ++++++++++++++++++- src/leveled_iclerk.erl | 192 ++++++++++++++++++++++++++++++----------- src/leveled_inker.erl | 103 +++++++++++----------- 6 files changed, 387 insertions(+), 139 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index f1ca86a..93e13e3 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -1,8 +1,20 @@ +%% Tag to be used on standard Riak KV objects -define(RIAK_TAG, o_rkv). +%% Tag to be used on K/V objects for non-Riak purposes -define(STD_TAG, o). +%% Tag used for secondary index keys -define(IDX_TAG, i). +%% Inker key type used for 'normal' objects +-define(INKT_STND, stnd). +%% Inker key type used for objects which contain no value, only key changes +%% This is used currently for objects formed under a 'retain' strategy on Inker +%% compaction, but could be used for special set-type objects +-define(INKT_KEYD, keyd). +%% Inker key type used for tombstones +-define(INKT_TOMB, tomb). + -record(sft_options, {wait = true :: boolean(), expire_tombstones = false :: boolean()}). @@ -40,7 +52,8 @@ root_path :: string(), cdb_options :: #cdb_options{}, start_snapshot = false :: boolean(), - source_inker :: pid()}). + source_inker :: pid(), + reload_strategy = [] :: list()}). -record(penciller_options, {root_path :: string(), @@ -52,12 +65,14 @@ {root_path :: string(), cache_size :: integer(), max_journalsize :: integer(), - snapshot_bookie :: pid()}). + snapshot_bookie :: pid(), + reload_strategy = [] :: list()}). -record(iclerk_options, {inker :: pid(), max_run_length :: integer(), - cdb_options :: #cdb_options{}}). + cdb_options :: #cdb_options{}, + reload_strategy = [] :: list()}). -record(r_content, { metadata, diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index e116b8f..9c8e507 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -280,7 +280,8 @@ handle_call({put, Bucket, Key, Object, IndexSpecs, Tag}, From, State) -> LedgerKey, Object, IndexSpecs), - Changes = preparefor_ledgercache(LedgerKey, + Changes = preparefor_ledgercache(no_type_assigned, + LedgerKey, SQN, Object, ObjSize, @@ -505,7 +506,8 @@ shutdown_wait([TopPause|Rest], Inker) -> ok -> true; pause -> - io:format("Inker shutdown stil waiting process to complete~n"), + io:format("Inker shutdown stil waiting for process to complete" ++ + " with further wait of ~w~n", [lists:sum(Rest)]), ok = timer:sleep(TopPause), shutdown_wait(Rest, Inker) end. @@ -518,12 +520,17 @@ set_options(Opts) -> MS -> MS end, - {#inker_options{root_path = Opts#bookie_options.root_path ++ - "/" ++ ?JOURNAL_FP, + + AltStrategy = Opts#bookie_options.reload_strategy, + ReloadStrategy = leveled_codec:inker_reload_strategy(AltStrategy), + + JournalFP = Opts#bookie_options.root_path ++ "/" ++ ?JOURNAL_FP, + LedgerFP = Opts#bookie_options.root_path ++ "/" ++ ?LEDGER_FP, + {#inker_options{root_path = JournalFP, + reload_strategy = ReloadStrategy, cdb_options = #cdb_options{max_size=MaxJournalSize, binary_mode=true}}, - #penciller_options{root_path=Opts#bookie_options.root_path ++ - "/" ++ ?LEDGER_FP}}. + #penciller_options{root_path = LedgerFP}}. startup(InkerOpts, PencillerOpts) -> {ok, Inker} = leveled_inker:ink_start(InkerOpts), @@ -613,14 +620,18 @@ accumulate_index(TermRe, AddFun) -> end. -preparefor_ledgercache(PK, SQN, Obj, Size, IndexSpecs) -> - {Bucket, Key, PrimaryChange} = leveled_codec:generate_ledgerkv(PK, +preparefor_ledgercache(?INKT_KEYD, LedgerKey, SQN, _Obj, _Size, IndexSpecs) -> + {Bucket, Key} = leveled_codec:from_ledgerkey(LedgerKey), + leveled_codec:convert_indexspecs(IndexSpecs, Bucket, Key, SQN); +preparefor_ledgercache(_Type, LedgerKey, SQN, Obj, Size, IndexSpecs) -> + {Bucket, Key, PrimaryChange} = leveled_codec:generate_ledgerkv(LedgerKey, SQN, Obj, Size), ConvSpecs = leveled_codec:convert_indexspecs(IndexSpecs, Bucket, Key, SQN), [PrimaryChange] ++ ConvSpecs. + addto_ledgercache(Changes, Cache) -> lists:foldl(fun({K, V}, Acc) -> gb_trees:enter(K, V, Acc) end, Cache, @@ -663,24 +674,21 @@ maybe_withjitter(CacheSize, MaxCacheSize) -> load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> {MinSQN, MaxSQN, OutputTree} = Acc0, - {SQN, _Type, PK} = KeyInLedger, + {SQN, Type, PK} = KeyInLedger, % VBin may already be a term {VBin, VSize} = ExtractFun(ValueInLedger), - {Obj, IndexSpecs} = case is_binary(VBin) of - true -> - binary_to_term(VBin); - false -> - VBin - end, + {Obj, IndexSpecs} = leveled_codec:split_inkvalue(VBin), case SQN of SQN when SQN < MinSQN -> {loop, Acc0}; SQN when SQN < MaxSQN -> - Changes = preparefor_ledgercache(PK, SQN, Obj, VSize, IndexSpecs), + Changes = preparefor_ledgercache(Type, PK, SQN, + Obj, VSize, IndexSpecs), {loop, {MinSQN, MaxSQN, addto_ledgercache(Changes, OutputTree)}}; MaxSQN -> io:format("Reached end of load batch with SQN ~w~n", [SQN]), - Changes = preparefor_ledgercache(PK, SQN, Obj, VSize, IndexSpecs), + Changes = preparefor_ledgercache(Type, PK, SQN, + Obj, VSize, IndexSpecs), {stop, {MinSQN, MaxSQN, addto_ledgercache(Changes, OutputTree)}}; SQN when SQN > MaxSQN -> io:format("Skipping as exceeded MaxSQN ~w with SQN ~w~n", diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index f3eedd9..d967315 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -293,7 +293,14 @@ handle_call({put_kv, Key, Value, HashOpt}, _From, State) -> handle_call(cdb_lastkey, _From, State) -> {reply, State#state.last_key, State}; handle_call(cdb_firstkey, _From, State) -> - {reply, extract_key(State#state.handle, ?BASE_POSITION), State}; + {ok, EOFPos} = file:position(State#state.handle, eof), + FirstKey = case EOFPos of + ?BASE_POSITION -> + empty; + _ -> + extract_key(State#state.handle, ?BASE_POSITION) + end, + {reply, FirstKey, State}; handle_call(cdb_filename, _From, State) -> {reply, State#state.filename, State}; handle_call({get_positions, SampleSize}, _From, State) -> @@ -746,12 +753,31 @@ load_index(Handle) -> %% Function to find the LastKey in the file find_lastkey(Handle, IndexCache) -> - LastPosition = scan_index(Handle, - IndexCache, - {fun scan_index_findlast/4, 0}), - {ok, _} = file:position(Handle, LastPosition), - {KeyLength, _ValueLength} = read_next_2_integers(Handle), - read_next_term(Handle, KeyLength). + {LastPosition, TotalKeys} = scan_index(Handle, + IndexCache, + {fun scan_index_findlast/4, + {0, 0}}), + {ok, EOFPos} = file:position(Handle, eof), + io:format("TotalKeys ~w in file~n", [TotalKeys]), + case TotalKeys of + 0 -> + scan_keys_forlast(Handle, EOFPos, ?BASE_POSITION, empty); + _ -> + {ok, _} = file:position(Handle, LastPosition), + {KeyLength, _ValueLength} = read_next_2_integers(Handle), + read_next_term(Handle, KeyLength) + end. + +scan_keys_forlast(_Handle, EOFPos, NextPos, LastKey) when EOFPos == NextPos -> + LastKey; +scan_keys_forlast(Handle, EOFPos, NextPos, _LastKey) -> + {ok, _} = file:position(Handle, NextPos), + {KeyLength, ValueLength} = read_next_2_integers(Handle), + scan_keys_forlast(Handle, + EOFPos, + NextPos + KeyLength + ValueLength + ?DWORD_SIZE, + read_next_term(Handle, KeyLength)). + scan_index(Handle, IndexCache, {ScanFun, InitAcc}) -> lists:foldl(fun({_X, {Pos, Count}}, Acc) -> @@ -776,11 +802,12 @@ scan_index_forsample(Handle, [CacheEntry|Tail], ScanFun, Acc, SampleSize) -> end. -scan_index_findlast(Handle, Position, Count, LastPosition) -> +scan_index_findlast(Handle, Position, Count, {LastPosition, TotalKeys}) -> {ok, _} = file:position(Handle, Position), - lists:foldl(fun({_Hash, HPos}, MaxPos) -> max(HPos, MaxPos) end, - LastPosition, - read_next_n_integerpairs(Handle, Count)). + MaxPos = lists:foldl(fun({_Hash, HPos}, MaxPos) -> max(HPos, MaxPos) end, + LastPosition, + read_next_n_integerpairs(Handle, Count)), + {MaxPos, TotalKeys + Count}. scan_index_returnpositions(Handle, Position, Count, PosList0) -> {ok, _} = file:position(Handle, Position), @@ -1705,7 +1732,7 @@ get_keys_byposition_simple_test() -> ok = file:delete(F2). generate_sequentialkeys(0, KVList) -> - KVList; + lists:reverse(KVList); generate_sequentialkeys(Count, KVList) -> KV = {"Key" ++ integer_to_list(Count), "Value" ++ integer_to_list(Count)}, generate_sequentialkeys(Count - 1, KVList ++ [KV]). @@ -1741,4 +1768,34 @@ get_keys_byposition_manykeys_test() -> ok = file:delete(F2). +manykeys_but_nohash_test() -> + KeyCount = 1024, + {ok, P1} = cdb_open_writer("../test/nohash_keysinfile.pnd"), + KVList = generate_sequentialkeys(KeyCount, []), + lists:foreach(fun({K, V}) -> cdb_put(P1, K, V, no_hash) end, KVList), + SW1 = os:timestamp(), + {ok, F2} = cdb_complete(P1), + SW2 = os:timestamp(), + io:format("CDB completed in ~w microseconds~n", + [timer:now_diff(SW2, SW1)]), + {ok, P2} = cdb_open_reader(F2), + io:format("FirstKey is ~s~n", [cdb_firstkey(P2)]), + io:format("LastKey is ~s~n", [cdb_lastkey(P2)]), + ?assertMatch("Key1", cdb_firstkey(P2)), + ?assertMatch("Key1024", cdb_lastkey(P2)), + ?assertMatch([], cdb_getpositions(P2, 100)), + ok = cdb_close(P2), + ok = file:delete(F2). + +nokeys_test() -> + {ok, P1} = cdb_open_writer("../test/nohash_emptyfile.pnd"), + {ok, F2} = cdb_complete(P1), + {ok, P2} = cdb_open_reader(F2), + io:format("FirstKey is ~s~n", [cdb_firstkey(P2)]), + io:format("LastKey is ~s~n", [cdb_lastkey(P2)]), + ?assertMatch(empty, cdb_firstkey(P2)), + ?assertMatch(empty, cdb_lastkey(P2)), + ok = cdb_close(P2), + ok = file:delete(F2). + -endif. diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index a5474dc..c9b04c8 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -32,7 +32,9 @@ -include_lib("eunit/include/eunit.hrl"). --export([strip_to_keyonly/1, +-export([ + inker_reload_strategy/1, + strip_to_keyonly/1, strip_to_seqonly/1, strip_to_statusonly/1, strip_to_keyseqstatusonly/1, @@ -45,6 +47,13 @@ to_ledgerkey/3, to_ledgerkey/5, from_ledgerkey/1, + to_inkerkv/4, + from_inkerkv/1, + from_journalkey/1, + compact_inkerkvc/2, + split_inkvalue/1, + check_forinkertype/2, + create_value_for_journal/1, build_metadata_object/2, generate_ledgerkv/4, generate_ledgerkv/5, @@ -61,6 +70,13 @@ generate_uuid() -> io_lib:format("~8.16.0b-~4.16.0b-4~3.16.0b-~4.16.0b-~12.16.0b", [A, B, C band 16#0fff, D band 16#3fff bor 16#8000, E]). +inker_reload_strategy(AltList) -> + ReloadStrategy0 = [{?RIAK_TAG, retain}, {?STD_TAG, retain}], + lists:foldl(fun({X, Y}, SList) -> + lists:keyreplace(X, 1, Y, SList) + end, + ReloadStrategy0, + AltList). strip_to_keyonly({keyonly, K}) -> K; strip_to_keyonly({K, _V}) -> K. @@ -121,6 +137,75 @@ to_ledgerkey(Bucket, Key, Tag, Field, Value) when Tag == ?IDX_TAG -> to_ledgerkey(Bucket, Key, Tag) -> {Tag, Bucket, Key, null}. +%% Return the Key, Value and Hash Option for this object. The hash option +%% indicates whether the key would ever be looked up directly, and so if it +%% requires an entry in the hash table +to_inkerkv(LedgerKey, SQN, to_fetch, null) -> + {{SQN, ?INKT_STND, LedgerKey}, null, true}; +to_inkerkv(LedgerKey, SQN, Object, KeyChanges) -> + {InkerType, HashOpt} = check_forinkertype(LedgerKey, Object), + Value = create_value_for_journal({Object, KeyChanges}), + {{SQN, InkerType, LedgerKey}, Value, HashOpt}. + +%% Used when fetching objects, so only handles standard, hashable entries +from_inkerkv(Object) -> + case Object of + {{SQN, ?INKT_STND, PK}, Bin} when is_binary(Bin) -> + {{SQN, PK}, binary_to_term(Bin)}; + {{SQN, ?INKT_STND, PK}, Term} -> + {{SQN, PK}, Term}; + _ -> + Object + end. + +from_journalkey({SQN, _Type, LedgerKey}) -> + {SQN, LedgerKey}. + +compact_inkerkvc({{_SQN, ?INKT_TOMB, _LK}, _V, _CrcCheck}, _Strategy) -> + skip; +compact_inkerkvc({{_SQN, ?INKT_KEYD, LK}, _V, _CrcCheck}, Strategy) -> + {Tag, _, _, _} = LK, + {Tag, TagStrat} = lists:keyfind(Tag, 1, Strategy), + case TagStrat of + retain -> + skip; + TagStrat -> + {TagStrat, null} + end; +compact_inkerkvc({{SQN, ?INKT_STND, LK}, V, CrcCheck}, Strategy) -> + {Tag, _, _, _} = LK, + {Tag, TagStrat} = lists:keyfind(Tag, 1, Strategy), + case TagStrat of + retain -> + {_V, KeyDeltas} = split_inkvalue(V), + {TagStrat, {{SQN, ?INKT_KEYD, LK}, {null, KeyDeltas}, CrcCheck}}; + TagStrat -> + {TagStrat, null} + end. + +split_inkvalue(VBin) -> + case is_binary(VBin) of + true -> + binary_to_term(VBin); + false -> + VBin + end. + +check_forinkertype(_LedgerKey, delete) -> + {?INKT_TOMB, no_hash}; +check_forinkertype(_LedgerKey, _Object) -> + {?INKT_STND, hash}. + +create_value_for_journal(Value) -> + case Value of + {Object, KeyChanges} -> + term_to_binary({Object, KeyChanges}, [compressed]); + Value when is_binary(Value) -> + Value + end. + + + hash(Obj) -> erlang:phash2(term_to_binary(Obj)). diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index e8eeb9c..5165c94 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -87,10 +87,12 @@ -define(SINGLEFILE_COMPACTION_TARGET, 60.0). -define(MAXRUN_COMPACTION_TARGET, 80.0). -define(CRC_SIZE, 4). +-define(DEFAULT_RELOAD_STRATEGY, leveled_codec:inker_reload_strategy([])). -record(state, {inker :: pid(), max_run_length :: integer(), - cdb_options}). + cdb_options, + reload_strategy = ?DEFAULT_RELOAD_STRATEGY :: list()}). -record(candidate, {low_sqn :: integer(), filename :: string(), @@ -126,15 +128,18 @@ clerk_stop(Pid) -> %%%============================================================================ init([IClerkOpts]) -> + ReloadStrategy = IClerkOpts#iclerk_options.reload_strategy, case IClerkOpts#iclerk_options.max_run_length of undefined -> {ok, #state{max_run_length = ?MAX_COMPACTION_RUN, inker = IClerkOpts#iclerk_options.inker, - cdb_options = IClerkOpts#iclerk_options.cdb_options}}; + cdb_options = IClerkOpts#iclerk_options.cdb_options, + reload_strategy = ReloadStrategy}}; MRL -> {ok, #state{max_run_length = MRL, inker = IClerkOpts#iclerk_options.inker, - cdb_options = IClerkOpts#iclerk_options.cdb_options}} + cdb_options = IClerkOpts#iclerk_options.cdb_options, + reload_strategy = ReloadStrategy}} end. handle_call(_Msg, _From, State) -> @@ -166,7 +171,8 @@ handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, CDBopts, FilterFun, FilterServer, - MaxSQN), + MaxSQN, + State#state.reload_strategy), FilesToDelete = lists:map(fun(C) -> {C#candidate.low_sqn, C#candidate.filename, @@ -296,7 +302,6 @@ assess_candidates(AllCandidates, MaxRunLength) -> end. assess_candidates([], _MaxRunLength, _CurrentRun0, BestAssessment) -> - io:format("Best run of ~w~n", [BestAssessment]), BestAssessment; assess_candidates([HeadC|Tail], MaxRunLength, CurrentRun0, BestAssessment) -> CurrentRun1 = choose_best_assessment(CurrentRun0 ++ [HeadC], @@ -349,13 +354,14 @@ print_compaction_run(BestRun, MaxRunLength) -> [length(BestRun), score_run(BestRun, MaxRunLength)]), lists:foreach(fun(File) -> io:format("Filename ~s is part of compaction run~n", - [File#candidate.filename]) + [File#candidate.filename]) + end, BestRun). -compact_files([], _CDBopts, _FilterFun, _FilterServer, _MaxSQN) -> +compact_files([], _CDBopts, _FilterFun, _FilterServer, _MaxSQN, _RStrategy) -> {[], 0}; -compact_files(BestRun, CDBopts, FilterFun, FilterServer, MaxSQN) -> +compact_files(BestRun, CDBopts, FilterFun, FilterServer, MaxSQN, RStrategy) -> BatchesOfPositions = get_all_positions(BestRun, []), compact_files(BatchesOfPositions, CDBopts, @@ -363,20 +369,21 @@ compact_files(BestRun, CDBopts, FilterFun, FilterServer, MaxSQN) -> FilterFun, FilterServer, MaxSQN, + RStrategy, [], true). compact_files([], _CDBopts, null, _FilterFun, _FilterServer, _MaxSQN, - ManSlice0, PromptDelete0) -> + _RStrategy, ManSlice0, PromptDelete0) -> {ManSlice0, PromptDelete0}; compact_files([], _CDBopts, ActiveJournal0, _FilterFun, _FilterServer, _MaxSQN, - ManSlice0, PromptDelete0) -> + _RStrategy, ManSlice0, PromptDelete0) -> ManSlice1 = ManSlice0 ++ generate_manifest_entry(ActiveJournal0), {ManSlice1, PromptDelete0}; compact_files([Batch|T], CDBopts, ActiveJournal0, FilterFun, FilterServer, MaxSQN, - ManSlice0, PromptDelete0) -> + RStrategy, ManSlice0, PromptDelete0) -> {SrcJournal, PositionList} = Batch, KVCs0 = leveled_cdb:cdb_directfetch(SrcJournal, PositionList, @@ -384,7 +391,8 @@ compact_files([Batch|T], CDBopts, ActiveJournal0, R0 = filter_output(KVCs0, FilterFun, FilterServer, - MaxSQN), + MaxSQN, + RStrategy), {KVCs1, PromptDelete1} = R0, PromptDelete2 = case {PromptDelete0, PromptDelete1} of {true, true} -> @@ -397,7 +405,7 @@ compact_files([Batch|T], CDBopts, ActiveJournal0, ActiveJournal0, ManSlice0), compact_files(T, CDBopts, ActiveJournal1, FilterFun, FilterServer, MaxSQN, - ManSlice1, PromptDelete2). + RStrategy, ManSlice1, PromptDelete2). get_all_positions([], PositionBatches) -> PositionBatches; @@ -423,25 +431,34 @@ split_positions_into_batches(Positions, Journal, Batches) -> Batches ++ [{Journal, ThisBatch}]). -filter_output(KVCs, FilterFun, FilterServer, MaxSQN) -> - lists:foldl(fun(KVC, {Acc, PromptDelete}) -> - {{SQN, _Type, PK}, _V, CrcCheck} = KVC, - KeyValid = FilterFun(FilterServer, PK, SQN), - case {KeyValid, CrcCheck, SQN > MaxSQN} of - {true, true, _} -> - {Acc ++ [KVC], PromptDelete}; - {false, true, true} -> - {Acc ++ [KVC], PromptDelete}; - {false, true, false} -> - {Acc, PromptDelete}; - {_, false, _} -> - io:format("Corrupted value found for " ++ " - Key ~w at SQN ~w~n", [PK, SQN]), - {Acc, false} - end - end, - {[], true}, - KVCs). +filter_output(KVCs, FilterFun, FilterServer, MaxSQN, ReloadStrategy) -> + lists:foldl(fun(KVC0, {Acc, PromptDelete}) -> + R = leveled_codec:compact_inkerkvc(KVC0, ReloadStrategy), + case R of + skip -> + {Acc, PromptDelete}; + {TStrat, KVC1} -> + {K, _V, CrcCheck} = KVC0, + {SQN, LedgerKey} = leveled_codec:from_journalkey(K), + KeyValid = FilterFun(FilterServer, LedgerKey, SQN), + case {KeyValid, CrcCheck, SQN > MaxSQN, TStrat} of + {true, true, _, _} -> + {Acc ++ [KVC0], PromptDelete}; + {false, true, true, _} -> + {Acc ++ [KVC0], PromptDelete}; + {false, true, false, retain} -> + {Acc ++ [KVC1], PromptDelete}; + {false, true, false, _} -> + {Acc, PromptDelete}; + {_, false, _, _} -> + io:format("Corrupted value found for " + ++ "Journal Key ~w~n", [K]), + {Acc, false} + end + end + end, + {[], true}, + KVCs). write_values([], _CDBopts, Journal0, ManSlice0) -> @@ -462,7 +479,7 @@ write_values([KVC|Rest], CDBopts, Journal0, ManSlice0) -> _ -> {ok, Journal0} end, - ValueToStore = leveled_inker:create_value_for_cdb(V), + ValueToStore = leveled_codec:create_value_for_journal(V), R = leveled_cdb:cdb_put(Journal1, {SQN, Type, PK}, ValueToStore), case R of ok -> @@ -568,17 +585,23 @@ find_bestrun_test() -> #candidate{compaction_perc = 65.0}], assess_candidates(CList0, 6)). +test_ledgerkey(Key) -> + {o, "Bucket", Key, null}. + +test_inkerkv(SQN, Key, V, IdxSpecs) -> + {{SQN, ?INKT_STND, test_ledgerkey(Key)}, term_to_binary({V, IdxSpecs})}. + fetch_testcdb(RP) -> FN1 = leveled_inker:filepath(RP, 1, new_journal), {ok, CDB1} = leveled_cdb:cdb_open_writer(FN1, #cdb_options{}), - {K1, V1} = {{1, stnd, "Key1"}, term_to_binary("Value1")}, - {K2, V2} = {{2, stnd, "Key2"}, term_to_binary("Value2")}, - {K3, V3} = {{3, stnd, "Key3"}, term_to_binary("Value3")}, - {K4, V4} = {{4, stnd, "Key1"}, term_to_binary("Value4")}, - {K5, V5} = {{5, stnd, "Key1"}, term_to_binary("Value5")}, - {K6, V6} = {{6, stnd, "Key1"}, term_to_binary("Value6")}, - {K7, V7} = {{7, stnd, "Key1"}, term_to_binary("Value7")}, - {K8, V8} = {{8, stnd, "Key1"}, term_to_binary("Value8")}, + {K1, V1} = test_inkerkv(1, "Key1", "Value1", []), + {K2, V2} = test_inkerkv(2, "Key2", "Value2", []), + {K3, V3} = test_inkerkv(3, "Key3", "Value3", []), + {K4, V4} = test_inkerkv(4, "Key1", "Value4", []), + {K5, V5} = test_inkerkv(5, "Key1", "Value5", []), + {K6, V6} = test_inkerkv(6, "Key1", "Value6", []), + {K7, V7} = test_inkerkv(7, "Key1", "Value7", []), + {K8, V8} = test_inkerkv(8, "Key1", "Value8", []), ok = leveled_cdb:cdb_put(CDB1, K1, V1), ok = leveled_cdb:cdb_put(CDB1, K2, V2), ok = leveled_cdb:cdb_put(CDB1, K3, V3), @@ -593,7 +616,9 @@ fetch_testcdb(RP) -> check_single_file_test() -> RP = "../test/journal", {ok, CDB} = fetch_testcdb(RP), - LedgerSrv1 = [{8, "Key1"}, {2, "Key2"}, {3, "Key3"}], + LedgerSrv1 = [{8, {o, "Bucket", "Key1", null}}, + {2, {o, "Bucket", "Key2", null}}, + {3, {o, "Bucket", "Key3", null}}], LedgerFun1 = fun(Srv, Key, ObjSQN) -> case lists:keyfind(ObjSQN, 1, Srv) of {ObjSQN, Key} -> @@ -613,14 +638,16 @@ check_single_file_test() -> ok = leveled_cdb:cdb_destroy(CDB). -compact_single_file_test() -> +compact_single_file_setup() -> RP = "../test/journal", {ok, CDB} = fetch_testcdb(RP), Candidate = #candidate{journal = CDB, low_sqn = 1, filename = "test", compaction_perc = 37.5}, - LedgerSrv1 = [{8, "Key1"}, {2, "Key2"}, {3, "Key3"}], + LedgerSrv1 = [{8, {o, "Bucket", "Key1", null}}, + {2, {o, "Bucket", "Key2", null}}, + {3, {o, "Bucket", "Key3", null}}], LedgerFun1 = fun(Srv, Key, ObjSQN) -> case lists:keyfind(ObjSQN, 1, Srv) of {ObjSQN, Key} -> @@ -630,33 +657,94 @@ compact_single_file_test() -> end end, CompactFP = leveled_inker:filepath(RP, journal_compact_dir), ok = filelib:ensure_dir(CompactFP), + {Candidate, LedgerSrv1, LedgerFun1, CompactFP, CDB}. + +compact_single_file_recovr_test() -> + {Candidate, + LedgerSrv1, + LedgerFun1, + CompactFP, + CDB} = compact_single_file_setup(), R1 = compact_files([Candidate], #cdb_options{file_path=CompactFP}, LedgerFun1, LedgerSrv1, - 9), + 9, + [{?STD_TAG, recovr}]), {ManSlice1, PromptDelete1} = R1, ?assertMatch(true, PromptDelete1), [{LowSQN, FN, PidR}] = ManSlice1, io:format("FN of ~s~n", [FN]), ?assertMatch(2, LowSQN), - ?assertMatch(probably, leveled_cdb:cdb_keycheck(PidR, {8, stnd, "Key1"})), - ?assertMatch(missing, leveled_cdb:cdb_get(PidR, {7, stnd, "Key1"})), - ?assertMatch(missing, leveled_cdb:cdb_get(PidR, {1, stnd, "Key1"})), - {_RK1, RV1} = leveled_cdb:cdb_get(PidR, {2, stnd, "Key2"}), - ?assertMatch("Value2", binary_to_term(RV1)), + ?assertMatch(probably, + leveled_cdb:cdb_keycheck(PidR, + {8, + stnd, + test_ledgerkey("Key1")})), + ?assertMatch(missing, leveled_cdb:cdb_get(PidR, + {7, + stnd, + test_ledgerkey("Key1")})), + ?assertMatch(missing, leveled_cdb:cdb_get(PidR, + {1, + stnd, + test_ledgerkey("Key1")})), + {_RK1, RV1} = leveled_cdb:cdb_get(PidR, + {2, + stnd, + test_ledgerkey("Key2")}), + ?assertMatch({"Value2", []}, binary_to_term(RV1)), ok = leveled_cdb:cdb_destroy(CDB). +compact_single_file_retain_test() -> + {Candidate, + LedgerSrv1, + LedgerFun1, + CompactFP, + CDB} = compact_single_file_setup(), + R1 = compact_files([Candidate], + #cdb_options{file_path=CompactFP}, + LedgerFun1, + LedgerSrv1, + 9, + [{?STD_TAG, retain}]), + {ManSlice1, PromptDelete1} = R1, + ?assertMatch(true, PromptDelete1), + [{LowSQN, FN, PidR}] = ManSlice1, + io:format("FN of ~s~n", [FN]), + ?assertMatch(1, LowSQN), + ?assertMatch(probably, + leveled_cdb:cdb_keycheck(PidR, + {8, + stnd, + test_ledgerkey("Key1")})), + ?assertMatch(missing, leveled_cdb:cdb_get(PidR, + {7, + stnd, + test_ledgerkey("Key1")})), + ?assertMatch(missing, leveled_cdb:cdb_get(PidR, + {1, + stnd, + test_ledgerkey("Key1")})), + {_RK1, RV1} = leveled_cdb:cdb_get(PidR, + {2, + stnd, + test_ledgerkey("Key2")}), + ?assertMatch({"Value2", []}, binary_to_term(RV1)), + ok = leveled_cdb:cdb_destroy(CDB). + compact_empty_file_test() -> RP = "../test/journal", FN1 = leveled_inker:filepath(RP, 1, new_journal), CDBopts = #cdb_options{binary_mode=true}, {ok, CDB1} = leveled_cdb:cdb_open_writer(FN1, CDBopts), - ok = leveled_cdb:cdb_put(CDB1, {1, stnd, "Key1"}, <<>>), + ok = leveled_cdb:cdb_put(CDB1, {1, stnd, test_ledgerkey("Key1")}, <<>>), {ok, FN2} = leveled_cdb:cdb_complete(CDB1), {ok, CDB2} = leveled_cdb:cdb_open_reader(FN2), - LedgerSrv1 = [{8, "Key1"}, {2, "Key2"}, {3, "Key3"}], + LedgerSrv1 = [{8, {o, "Bucket", "Key1", null}}, + {2, {o, "Bucket", "Key2", null}}, + {3, {o, "Bucket", "Key3", null}}], LedgerFun1 = fun(Srv, Key, ObjSQN) -> case lists:keyfind(ObjSQN, 1, Srv) of {ObjSQN, Key} -> diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index b7f26aa..8b83f49 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -106,7 +106,6 @@ ink_print_manifest/1, ink_close/1, ink_forceclose/1, - create_value_for_cdb/1, build_dummy_journal/0, simple_manifest_reader/2, clean_testdir/1, @@ -342,9 +341,9 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================ -start_from_file(InkerOpts) -> - RootPath = InkerOpts#inker_options.root_path, - CDBopts = InkerOpts#inker_options.cdb_options, +start_from_file(InkOpts) -> + RootPath = InkOpts#inker_options.root_path, + CDBopts = InkOpts#inker_options.cdb_options, JournalFP = filepath(RootPath, journal_dir), filelib:ensure_dir(JournalFP), CompactFP = filepath(RootPath, journal_compact_dir), @@ -360,8 +359,10 @@ start_from_file(InkerOpts) -> end, IClerkCDBOpts = CDBopts#cdb_options{file_path = CompactFP}, + ReloadStrategy = InkOpts#inker_options.reload_strategy, IClerkOpts = #iclerk_options{inker = self(), - cdb_options=IClerkCDBOpts}, + cdb_options=IClerkCDBOpts, + reload_strategy = ReloadStrategy}, {ok, Clerk} = leveled_iclerk:clerk_new(IClerkOpts), {Manifest, @@ -379,24 +380,18 @@ start_from_file(InkerOpts) -> clerk = Clerk}}. -put_object(PrimaryKey, Object, KeyChanges, State) -> +put_object(LedgerKey, Object, KeyChanges, State) -> NewSQN = State#state.journal_sqn + 1, - {InkerType, HashOpt} = case Object of - delete -> - {tomb, no_hash}; - %delta -> - % {keyd, no_hash - _ -> - {stnd, hash} - end, - Bin1 = create_value_for_cdb({Object, KeyChanges}), - ObjSize = byte_size(Bin1), + {JournalKey, JournalBin, HashOpt} = leveled_codec:to_inkerkv(LedgerKey, + NewSQN, + Object, + KeyChanges), case leveled_cdb:cdb_put(State#state.active_journaldb, - {NewSQN, InkerType, PrimaryKey}, - Bin1, + JournalKey, + JournalBin, HashOpt) of ok -> - {ok, State#state{journal_sqn=NewSQN}, ObjSize}; + {ok, State#state{journal_sqn=NewSQN}, byte_size(JournalBin)}; roll -> SW = os:timestamp(), CDBopts = State#state.cdb_options, @@ -409,8 +404,8 @@ put_object(PrimaryKey, Object, KeyChanges, State) -> State#state.manifest_sqn + 1, State#state.root_path), ok = leveled_cdb:cdb_put(NewJournalP, - {NewSQN, InkerType, PrimaryKey}, - Bin1, + JournalKey, + JournalBin, HashOpt), io:format("Put to new active journal " ++ "with manifest write took ~w microseconds~n", @@ -420,30 +415,18 @@ put_object(PrimaryKey, Object, KeyChanges, State) -> manifest=NewManifest, manifest_sqn = State#state.manifest_sqn + 1, active_journaldb=NewJournalP}, - ObjSize} + byte_size(JournalBin)} end. -create_value_for_cdb(Value) -> - case Value of - {Object, KeyChanges} -> - term_to_binary({Object, KeyChanges}, [compressed]); - Value when is_binary(Value) -> - Value - end. - - -get_object(PrimaryKey, SQN, Manifest) -> +get_object(LedgerKey, SQN, Manifest) -> JournalP = find_in_manifest(SQN, Manifest), - Obj = leveled_cdb:cdb_get(JournalP, {SQN, stnd, PrimaryKey}), - case Obj of - {{SQN, stnd, PK}, Bin} when is_binary(Bin) -> - {{SQN, PK}, binary_to_term(Bin)}; - {{SQN, stnd, PK}, Term} -> - {{SQN, PK}, Term}; - _ -> - Obj - end. + {InkerKey, _V, true} = leveled_codec:to_inkerkv(LedgerKey, + SQN, + to_fetch, + null), + Obj = leveled_cdb:cdb_get(JournalP, InkerKey), + leveled_codec:from_inkerkv(Obj). build_manifest(ManifestFilenames, @@ -771,6 +754,10 @@ initiate_penciller_snapshot(Bookie) -> -ifdef(TEST). build_dummy_journal() -> + F = fun(X) -> X end, + build_dummy_journal(F). + +build_dummy_journal(KeyConvertF) -> RootPath = "../test/journal", clean_testdir(RootPath), JournalFP = filepath(RootPath, journal_dir), @@ -780,8 +767,8 @@ build_dummy_journal() -> ok = filelib:ensure_dir(ManifestFP), F1 = filename:join(JournalFP, "nursery_1.pnd"), {ok, J1} = leveled_cdb:cdb_open_writer(F1), - {K1, V1} = {"Key1", "TestValue1"}, - {K2, V2} = {"Key2", "TestValue2"}, + {K1, V1} = {KeyConvertF("Key1"), "TestValue1"}, + {K2, V2} = {KeyConvertF("Key2"), "TestValue2"}, ok = leveled_cdb:cdb_put(J1, {1, stnd, K1}, term_to_binary({V1, []})), ok = leveled_cdb:cdb_put(J1, {2, stnd, K2}, term_to_binary({V2, []})), ok = leveled_cdb:cdb_roll(J1), @@ -789,8 +776,8 @@ build_dummy_journal() -> ok = leveled_cdb:cdb_close(J1), F2 = filename:join(JournalFP, "nursery_3.pnd"), {ok, J2} = leveled_cdb:cdb_open_writer(F2), - {K1, V3} = {"Key1", "TestValue3"}, - {K4, V4} = {"Key4", "TestValue4"}, + {K1, V3} = {KeyConvertF("Key1"), "TestValue3"}, + {K4, V4} = {KeyConvertF("Key4"), "TestValue4"}, ok = leveled_cdb:cdb_put(J2, {3, stnd, K1}, term_to_binary({V3, []})), ok = leveled_cdb:cdb_put(J2, {4, stnd, K4}, term_to_binary({V4, []})), ok = leveled_cdb:cdb_close(J2), @@ -854,29 +841,37 @@ simple_inker_completeactivejournal_test() -> ink_close(Ink1), clean_testdir(RootPath). - +test_ledgerkey(Key) -> + {o, "Bucket", Key, null}. + compact_journal_test() -> RootPath = "../test/journal", - build_dummy_journal(), + build_dummy_journal(fun test_ledgerkey/1), CDBopts = #cdb_options{max_size=300000}, + RStrategy = [{?STD_TAG, recovr}], {ok, Ink1} = ink_start(#inker_options{root_path=RootPath, - cdb_options=CDBopts}), - {ok, NewSQN1, _ObjSize} = ink_put(Ink1, "KeyAA", "TestValueAA", []), + cdb_options=CDBopts, + reload_strategy=RStrategy}), + {ok, NewSQN1, _ObjSize} = ink_put(Ink1, + test_ledgerkey("KeyAA"), + "TestValueAA", []), ?assertMatch(NewSQN1, 5), ok = ink_print_manifest(Ink1), - R0 = ink_get(Ink1, "KeyAA", 5), - ?assertMatch(R0, {{5, "KeyAA"}, {"TestValueAA", []}}), + R0 = ink_get(Ink1, test_ledgerkey("KeyAA"), 5), + ?assertMatch(R0, {{5, test_ledgerkey("KeyAA")}, {"TestValueAA", []}}), FunnyLoop = lists:seq(1, 48), Checker = lists:map(fun(X) -> PK = "KeyZ" ++ integer_to_list(X), {ok, SQN, _} = ink_put(Ink1, - PK, + test_ledgerkey(PK), crypto:rand_bytes(10000), []), - {SQN, PK} + {SQN, test_ledgerkey(PK)} end, FunnyLoop), - {ok, NewSQN2, _ObjSize} = ink_put(Ink1, "KeyBB", "TestValueBB", []), + {ok, NewSQN2, _ObjSize} = ink_put(Ink1, + test_ledgerkey("KeyBB"), + "TestValueBB", []), ?assertMatch(NewSQN2, 54), ActualManifest = ink_getmanifest(Ink1), ok = ink_print_manifest(Ink1), From 2a47acc758d0ca69a9ee1a29197a6d13ad2d01ff Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 26 Oct 2016 11:39:27 +0100 Subject: [PATCH 091/167] Rolback hash|no_hash and batch journal compaction The no_hash option in CDB files became too hard to manage, in particular the need to scan the whole file to find the last_key rather than cheat and use the index. It has been removed for now. The writing to the journal during journal compaction has now been enhanced by a mput option on the CDB file write - so it can write each batch as one pwrite operation. --- src/leveled_cdb.erl | 125 ++++++++++++++++++++------------ src/leveled_codec.erl | 8 +- src/leveled_iclerk.erl | 16 ++-- src/leveled_inker.erl | 14 ++-- test/end_to_end/basic_SUITE.erl | 8 +- 5 files changed, 103 insertions(+), 68 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index d967315..e814779 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -57,7 +57,7 @@ cdb_open_reader/1, cdb_get/2, cdb_put/3, - cdb_put/4, + cdb_mput/2, cdb_getpositions/2, cdb_directfetch/3, cdb_lastkey/1, @@ -127,10 +127,10 @@ cdb_get(Pid, Key) -> gen_server:call(Pid, {get_kv, Key}, infinity). cdb_put(Pid, Key, Value) -> - cdb_put(Pid, Key, Value, hash). + gen_server:call(Pid, {put_kv, Key, Value}, infinity). -cdb_put(Pid, Key, Value, HashOpt) -> - gen_server:call(Pid, {put_kv, Key, Value, HashOpt}, infinity). +cdb_mput(Pid, KVList) -> + gen_server:call(Pid, {mput_kv, KVList}, infinity). %% SampleSize can be an integer or the atom all cdb_getpositions(Pid, SampleSize) -> @@ -262,7 +262,7 @@ handle_call({key_check, Key}, _From, State) -> State#state.hash_index), State} end; -handle_call({put_kv, Key, Value, HashOpt}, _From, State) -> +handle_call({put_kv, Key, Value}, _From, State) -> case {State#state.writer, State#state.pending_roll} of {true, false} -> Result = put(State#state.handle, @@ -270,21 +270,39 @@ handle_call({put_kv, Key, Value, HashOpt}, _From, State) -> {State#state.last_position, State#state.hashtree}, State#state.binary_mode, State#state.max_size), - case {Result, HashOpt} of - {roll, _} -> + case Result of + roll -> %% Key and value could not be written {reply, roll, State}; - {{UpdHandle, NewPosition, HashTree}, hash} -> + {UpdHandle, NewPosition, HashTree} -> {reply, ok, State#state{handle=UpdHandle, last_position=NewPosition, last_key=Key, - hashtree=HashTree}}; - {{UpdHandle, NewPosition, _HashTree}, no_hash} -> - %% Don't update the hashtree + hashtree=HashTree}} + end; + _ -> + {reply, + {error, read_only}, + State} + end; +handle_call({mput_kv, KVList}, _From, State) -> + case {State#state.writer, State#state.pending_roll} of + {true, false} -> + Result = mput(State#state.handle, + KVList, + {State#state.last_position, State#state.hashtree}, + State#state.binary_mode, + State#state.max_size), + case Result of + roll -> + %% Keys and values could not be written + {reply, roll, State}; + {UpdHandle, NewPosition, HashTree, LastKey} -> {reply, ok, State#state{handle=UpdHandle, last_position=NewPosition, - last_key=Key}} - end; + last_key=LastKey, + hashtree=HashTree}} + end; _ -> {reply, {error, read_only}, @@ -542,13 +560,32 @@ put(Handle, Key, Value, {LastPosition, HashTree}, BinaryMode, MaxSize) -> put_hashtree(Key, LastPosition, HashTree)} end. +mput(Handle, [], {LastPosition, HashTree0}, _BinaryMode, _MaxSize) -> + {Handle, LastPosition, HashTree0}; +mput(Handle, KVList, {LastPosition, HashTree0}, BinaryMode, MaxSize) -> + {KPList, Bin, LastKey} = multi_key_value_to_record(KVList, + BinaryMode, + LastPosition), + PotentialNewSize = LastPosition + byte_size(Bin), + if + PotentialNewSize > MaxSize -> + roll; + true -> + ok = file:pwrite(Handle, LastPosition, Bin), + HashTree1 = lists:foldl(fun({K, P}, Acc) -> + put_hashtree(K, P, Acc) + end, + HashTree0, + KPList), + {Handle, PotentialNewSize, HashTree1, LastKey} + end. + %% Should not be used for non-test PUTs by the inker - as the Max File Size %% should be taken from the startup options not the default put(FileName, Key, Value, {LastPosition, HashTree}) -> put(FileName, Key, Value, {LastPosition, HashTree}, ?BINARY_MODE, ?MAX_FILE_SIZE). - %% %% get(FileName,Key) -> {key,value} %% Given a filename and a key, returns a key and value tuple. @@ -757,27 +794,15 @@ find_lastkey(Handle, IndexCache) -> IndexCache, {fun scan_index_findlast/4, {0, 0}}), - {ok, EOFPos} = file:position(Handle, eof), - io:format("TotalKeys ~w in file~n", [TotalKeys]), case TotalKeys of 0 -> - scan_keys_forlast(Handle, EOFPos, ?BASE_POSITION, empty); + empty; _ -> {ok, _} = file:position(Handle, LastPosition), {KeyLength, _ValueLength} = read_next_2_integers(Handle), read_next_term(Handle, KeyLength) end. -scan_keys_forlast(_Handle, EOFPos, NextPos, LastKey) when EOFPos == NextPos -> - LastKey; -scan_keys_forlast(Handle, EOFPos, NextPos, _LastKey) -> - {ok, _} = file:position(Handle, NextPos), - {KeyLength, ValueLength} = read_next_2_integers(Handle), - scan_keys_forlast(Handle, - EOFPos, - NextPos + KeyLength + ValueLength + ?DWORD_SIZE, - read_next_term(Handle, KeyLength)). - scan_index(Handle, IndexCache, {ScanFun, InitAcc}) -> lists:foldl(fun({_X, {Pos, Count}}, Acc) -> @@ -1329,6 +1354,16 @@ key_value_to_record({Key, Value}, BinaryMode) -> <>. +multi_key_value_to_record(KVList, BinaryMode, LastPosition) -> + lists:foldl(fun({K, V}, {KPosL, Bin, _LK}) -> + Bin0 = key_value_to_record({K, V}, BinaryMode), + {[{K, byte_size(Bin) + LastPosition}|KPosL], + <>, + K} end, + {[], <<>>, empty}, + KVList). + + %%%%%%%%%%%%%%%% % T E S T %%%%%%%%%%%%%%% @@ -1768,25 +1803,6 @@ get_keys_byposition_manykeys_test() -> ok = file:delete(F2). -manykeys_but_nohash_test() -> - KeyCount = 1024, - {ok, P1} = cdb_open_writer("../test/nohash_keysinfile.pnd"), - KVList = generate_sequentialkeys(KeyCount, []), - lists:foreach(fun({K, V}) -> cdb_put(P1, K, V, no_hash) end, KVList), - SW1 = os:timestamp(), - {ok, F2} = cdb_complete(P1), - SW2 = os:timestamp(), - io:format("CDB completed in ~w microseconds~n", - [timer:now_diff(SW2, SW1)]), - {ok, P2} = cdb_open_reader(F2), - io:format("FirstKey is ~s~n", [cdb_firstkey(P2)]), - io:format("LastKey is ~s~n", [cdb_lastkey(P2)]), - ?assertMatch("Key1", cdb_firstkey(P2)), - ?assertMatch("Key1024", cdb_lastkey(P2)), - ?assertMatch([], cdb_getpositions(P2, 100)), - ok = cdb_close(P2), - ok = file:delete(F2). - nokeys_test() -> {ok, P1} = cdb_open_writer("../test/nohash_emptyfile.pnd"), {ok, F2} = cdb_complete(P1), @@ -1798,4 +1814,21 @@ nokeys_test() -> ok = cdb_close(P2), ok = file:delete(F2). +mput_test() -> + KeyCount = 1024, + {ok, P1} = cdb_open_writer("../test/nohash_keysinfile.pnd"), + KVList = generate_sequentialkeys(KeyCount, []), + ok = cdb_mput(P1, KVList), + {ok, F2} = cdb_complete(P1), + {ok, P2} = cdb_open_reader(F2), + ?assertMatch("Key1", cdb_firstkey(P2)), + ?assertMatch("Key1024", cdb_lastkey(P2)), + ?assertMatch({"Key1", "Value1"}, cdb_get(P2, "Key1")), + ?assertMatch({"Key1024", "Value1024"}, cdb_get(P2, "Key1024")), + ?assertMatch(missing, cdb_get(P2, "Key1025")), + ?assertMatch(missing, cdb_get(P2, "Key1026")), + ok = cdb_close(P2), + ok = file:delete(F2). + + -endif. diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index c9b04c8..c251489 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -143,9 +143,9 @@ to_ledgerkey(Bucket, Key, Tag) -> to_inkerkv(LedgerKey, SQN, to_fetch, null) -> {{SQN, ?INKT_STND, LedgerKey}, null, true}; to_inkerkv(LedgerKey, SQN, Object, KeyChanges) -> - {InkerType, HashOpt} = check_forinkertype(LedgerKey, Object), + InkerType = check_forinkertype(LedgerKey, Object), Value = create_value_for_journal({Object, KeyChanges}), - {{SQN, InkerType, LedgerKey}, Value, HashOpt}. + {{SQN, InkerType, LedgerKey}, Value}. %% Used when fetching objects, so only handles standard, hashable entries from_inkerkv(Object) -> @@ -192,9 +192,9 @@ split_inkvalue(VBin) -> end. check_forinkertype(_LedgerKey, delete) -> - {?INKT_TOMB, no_hash}; + ?INKT_TOMB; check_forinkertype(_LedgerKey, _Object) -> - {?INKT_STND, hash}. + ?INKT_STND. create_value_for_journal(Value) -> case Value of diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 5165c94..cff5182 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -463,10 +463,15 @@ filter_output(KVCs, FilterFun, FilterServer, MaxSQN, ReloadStrategy) -> write_values([], _CDBopts, Journal0, ManSlice0) -> {Journal0, ManSlice0}; -write_values([KVC|Rest], CDBopts, Journal0, ManSlice0) -> - {{SQN, Type, PK}, V, _CrcCheck} = KVC, +write_values(KVCList, CDBopts, Journal0, ManSlice0) -> + KVList = lists:map(fun({K, V, _C}) -> + {K, leveled_codec:create_value_for_journal(V)} + end, + KVCList), {ok, Journal1} = case Journal0 of null -> + {TK, _TV} = lists:nth(1, KVList), + {SQN, _LK} = leveled_codec:from_journalkey(TK), FP = CDBopts#cdb_options.file_path, FN = leveled_inker:filepath(FP, SQN, @@ -479,14 +484,13 @@ write_values([KVC|Rest], CDBopts, Journal0, ManSlice0) -> _ -> {ok, Journal0} end, - ValueToStore = leveled_codec:create_value_for_journal(V), - R = leveled_cdb:cdb_put(Journal1, {SQN, Type, PK}, ValueToStore), + R = leveled_cdb:cdb_mput(Journal1, KVList), case R of ok -> - write_values(Rest, CDBopts, Journal1, ManSlice0); + {Journal1, ManSlice0}; roll -> ManSlice1 = ManSlice0 ++ generate_manifest_entry(Journal1), - write_values(Rest, CDBopts, null, ManSlice1) + write_values(KVCList, CDBopts, null, ManSlice1) end. diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 8b83f49..0fd512a 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -382,14 +382,13 @@ start_from_file(InkOpts) -> put_object(LedgerKey, Object, KeyChanges, State) -> NewSQN = State#state.journal_sqn + 1, - {JournalKey, JournalBin, HashOpt} = leveled_codec:to_inkerkv(LedgerKey, - NewSQN, - Object, - KeyChanges), + {JournalKey, JournalBin} = leveled_codec:to_inkerkv(LedgerKey, + NewSQN, + Object, + KeyChanges), case leveled_cdb:cdb_put(State#state.active_journaldb, JournalKey, - JournalBin, - HashOpt) of + JournalBin) of ok -> {ok, State#state{journal_sqn=NewSQN}, byte_size(JournalBin)}; roll -> @@ -405,8 +404,7 @@ put_object(LedgerKey, Object, KeyChanges, State) -> State#state.root_path), ok = leveled_cdb:cdb_put(NewJournalP, JournalKey, - JournalBin, - HashOpt), + JournalBin), io:format("Put to new active journal " ++ "with manifest write took ~w microseconds~n", [timer:now_diff(os:timestamp(),SW)]), diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 6887556..61a81ef 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -13,11 +13,11 @@ all() -> [ simple_put_fetch_head_delete, - many_put_fetch_head, + % many_put_fetch_head, journal_compaction, - fetchput_snapshot, - load_and_count, - load_and_count_withdelete, + % fetchput_snapshot, + % load_and_count, + % load_and_count_withdelete, space_clear_ondelete_test ]. From 0c331b9c30b67f10dd35e164ead860c1e584d12a Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 26 Oct 2016 11:45:35 +0100 Subject: [PATCH 092/167] Tests uncommented Accidentally commented tests it pervious commit --- test/end_to_end/basic_SUITE.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 61a81ef..6887556 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -13,11 +13,11 @@ all() -> [ simple_put_fetch_head_delete, - % many_put_fetch_head, + many_put_fetch_head, journal_compaction, - % fetchput_snapshot, - % load_and_count, - % load_and_count_withdelete, + fetchput_snapshot, + load_and_count, + load_and_count_withdelete, space_clear_ondelete_test ]. From 6f40869070a82d2a471bf8e2bee5298580db87bd Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 26 Oct 2016 11:50:59 +0100 Subject: [PATCH 093/167] Parameter Experiment Try some different default parameters --- src/leveled_cdb.erl | 4 ++++ src/leveled_iclerk.erl | 2 +- src/leveled_sft.erl | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index e814779..8d6c6b3 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -1819,6 +1819,10 @@ mput_test() -> {ok, P1} = cdb_open_writer("../test/nohash_keysinfile.pnd"), KVList = generate_sequentialkeys(KeyCount, []), ok = cdb_mput(P1, KVList), + ?assertMatch({"Key1", "Value1"}, cdb_get(P1, "Key1")), + ?assertMatch({"Key1024", "Value1024"}, cdb_get(P1, "Key1024")), + ?assertMatch(missing, cdb_get(P1, "Key1025")), + ?assertMatch(missing, cdb_get(P1, "Key1026")), {ok, F2} = cdb_complete(P1), {ok, P2} = cdb_open_reader(F2), ?assertMatch("Key1", cdb_firstkey(P2)), diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index cff5182..95cb69b 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -79,7 +79,7 @@ -define(JOURNAL_FILEX, "cdb"). -define(PENDING_FILEX, "pnd"). -define(SAMPLE_SIZE, 200). --define(BATCH_SIZE, 16). +-define(BATCH_SIZE, 32). -define(BATCHES_TO_CHECK, 8). %% How many consecutive files to compact in one run -define(MAX_COMPACTION_RUN, 4). diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 52809f3..85392a7 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -265,7 +265,7 @@ sft_deleteconfirmed(Pid) -> gen_server:cast(Pid, close). sft_checkready(Pid) -> - gen_server:call(Pid, background_complete, 50). + gen_server:call(Pid, background_complete, 20). sft_getmaxsequencenumber(Pid) -> gen_server:call(Pid, get_maxsqn, infinity). From 254183369e6644a5ba07a9b4056d3d525fefbd36 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 26 Oct 2016 20:39:16 +0100 Subject: [PATCH 094/167] CDB - switch to gen_fsm The CDB file management server has distinct states, and was growing case logic to prevent certain messages from being handled in ceratin states, and to handle different messages differently. So this has now been converted to a gen_fsm. As part of resolving this, the space_clear_ondelete test has been completed, and completing this revealed that the Penciller could not cope with a change which emptied the ledger. So a series of changes has been handled to allow it to smoothly progress to an empty manifest. --- src/leveled_bookie.erl | 6 + src/leveled_cdb.erl | 503 ++++++++++++++++++-------------- src/leveled_iclerk.erl | 17 +- src/leveled_inker.erl | 52 ++-- src/leveled_pclerk.erl | 29 +- src/leveled_sft.erl | 10 +- test/end_to_end/basic_SUITE.erl | 45 ++- test/end_to_end/testutil.erl | 10 +- 8 files changed, 402 insertions(+), 270 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 9c8e507..c01281a 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -146,6 +146,7 @@ book_snapshotstore/3, book_snapshotledger/3, book_compactjournal/2, + book_islastcompactionpending/1, book_close/1]). -include_lib("eunit/include/eunit.hrl"). @@ -234,6 +235,9 @@ book_snapshotledger(Pid, Requestor, Timeout) -> book_compactjournal(Pid, Timeout) -> gen_server:call(Pid, {compact_journal, Timeout}, infinity). +book_islastcompactionpending(Pid) -> + gen_server:call(Pid, confirm_compact, infinity). + book_close(Pid) -> gen_server:call(Pid, close, infinity). @@ -389,6 +393,8 @@ handle_call({compact_journal, Timeout}, _From, State) -> self(), Timeout), {reply, ok, State}; +handle_call(confirm_compact, _From, State) -> + {reply, leveled_inker:ink_compactionpending(State#state.inker), State}; handle_call(close, _From, State) -> {stop, normal, ok, State}. diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 8d6c6b3..63e48d3 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -40,38 +40,50 @@ %% The first word is the corresponding hash value and the second word is a %% file pointer to the actual {key,value} tuple higher in the file. %% +%% + -module(leveled_cdb). --behaviour(gen_server). +-behaviour(gen_fsm). -include("include/leveled.hrl"). -export([init/1, - handle_call/3, - handle_cast/2, - handle_info/2, - terminate/2, - code_change/3, - cdb_open_writer/1, - cdb_open_writer/2, - cdb_open_reader/1, - cdb_get/2, - cdb_put/3, - cdb_mput/2, - cdb_getpositions/2, - cdb_directfetch/3, - cdb_lastkey/1, - cdb_firstkey/1, - cdb_filename/1, - cdb_keycheck/2, - cdb_scan/4, - cdb_close/1, - cdb_complete/1, - cdb_roll/1, - cdb_returnhashtable/3, - cdb_destroy/1, - cdb_deletepending/1, - hashtable_calc/2]). + handle_sync_event/4, + handle_event/3, + handle_info/3, + terminate/3, + code_change/4, + starting/3, + writer/3, + writer/2, + rolling/3, + reader/3, + reader/2, + delete_pending/3, + delete_pending/2]). + +-export([cdb_open_writer/1, + cdb_open_writer/2, + cdb_open_reader/1, + cdb_get/2, + cdb_put/3, + cdb_mput/2, + cdb_getpositions/2, + cdb_directfetch/3, + cdb_lastkey/1, + cdb_firstkey/1, + cdb_filename/1, + cdb_keycheck/2, + cdb_scan/4, + cdb_close/1, + cdb_complete/1, + cdb_roll/1, + cdb_returnhashtable/3, + cdb_destroy/1, + cdb_deletepending/1, + cdb_deletepending/3, + hashtable_calc/2]). -include_lib("eunit/include/eunit.hrl"). @@ -83,6 +95,7 @@ -define(BASE_POSITION, 2048). -define(WRITE_OPS, [binary, raw, read, write]). -define(PENDING_ROLL_WAIT, 30). +-define(DELETE_TIMEOUT, 10000). -record(state, {hashtree, last_position :: integer(), @@ -90,11 +103,10 @@ hash_index = [] :: list(), filename :: string(), handle :: file:fd(), - writer :: boolean(), max_size :: integer(), - pending_roll = false :: boolean(), - pending_delete = false :: boolean(), - binary_mode = false :: boolean()}). + binary_mode = false :: boolean(), + delete_point = 0 :: integer(), + inker :: pid()}). %%%============================================================================ @@ -106,8 +118,8 @@ cdb_open_writer(Filename) -> cdb_open_writer(Filename, #cdb_options{}). cdb_open_writer(Filename, Opts) -> - {ok, Pid} = gen_server:start(?MODULE, [Opts], []), - case gen_server:call(Pid, {open_writer, Filename}, infinity) of + {ok, Pid} = gen_fsm:start(?MODULE, [Opts], []), + case gen_fsm:sync_send_event(Pid, {open_writer, Filename}, infinity) of ok -> {ok, Pid}; Error -> @@ -115,8 +127,8 @@ cdb_open_writer(Filename, Opts) -> end. cdb_open_reader(Filename) -> - {ok, Pid} = gen_server:start(?MODULE, [#cdb_options{}], []), - case gen_server:call(Pid, {open_reader, Filename}, infinity) of + {ok, Pid} = gen_fsm:start(?MODULE, [#cdb_options{}], []), + case gen_fsm:sync_send_event(Pid, {open_reader, Filename}, infinity) of ok -> {ok, Pid}; Error -> @@ -124,23 +136,23 @@ cdb_open_reader(Filename) -> end. cdb_get(Pid, Key) -> - gen_server:call(Pid, {get_kv, Key}, infinity). + gen_fsm:sync_send_event(Pid, {get_kv, Key}, infinity). cdb_put(Pid, Key, Value) -> - gen_server:call(Pid, {put_kv, Key, Value}, infinity). + gen_fsm:sync_send_event(Pid, {put_kv, Key, Value}, infinity). cdb_mput(Pid, KVList) -> - gen_server:call(Pid, {mput_kv, KVList}, infinity). + gen_fsm:sync_send_event(Pid, {mput_kv, KVList}, infinity). %% SampleSize can be an integer or the atom all cdb_getpositions(Pid, SampleSize) -> - gen_server:call(Pid, {get_positions, SampleSize}, infinity). + gen_fsm:sync_send_event(Pid, {get_positions, SampleSize}, infinity). %% Info can be key_only, key_size (size being the size of the value) or %% key_value_check (with the check part indicating if the CRC is correct for %% the value) cdb_directfetch(Pid, PositionList, Info) -> - gen_server:call(Pid, {direct_fetch, PositionList, Info}, infinity). + gen_fsm:sync_send_event(Pid, {direct_fetch, PositionList, Info}, infinity). cdb_close(Pid) -> cdb_close(Pid, ?PENDING_ROLL_WAIT). @@ -148,7 +160,7 @@ cdb_close(Pid) -> cdb_close(Pid, WaitsLeft) -> if WaitsLeft > 0 -> - case gen_server:call(Pid, cdb_close, infinity) of + case gen_fsm:sync_send_all_state_event(Pid, cdb_close, infinity) of pending_roll -> timer:sleep(1), cdb_close(Pid, WaitsLeft - 1); @@ -156,23 +168,26 @@ cdb_close(Pid, WaitsLeft) -> R end; true -> - gen_server:call(Pid, cdb_kill, infinity) + gen_fsm:sync_send_event(Pid, cdb_kill, infinity) end. cdb_complete(Pid) -> - gen_server:call(Pid, cdb_complete, infinity). + gen_fsm:sync_send_event(Pid, cdb_complete, infinity). cdb_roll(Pid) -> - gen_server:cast(Pid, cdb_roll). + gen_fsm:send_event(Pid, cdb_roll). cdb_returnhashtable(Pid, IndexList, HashTreeBin) -> - gen_server:call(Pid, {return_hashtable, IndexList, HashTreeBin}, infinity). + gen_fsm:sync_send_event(Pid, {return_hashtable, IndexList, HashTreeBin}, infinity). cdb_destroy(Pid) -> - gen_server:cast(Pid, destroy). + gen_fsm:send_event(Pid, destroy). cdb_deletepending(Pid) -> - gen_server:cast(Pid, delete_pending). + cdb_deletepending(Pid, 0, no_poll). + +cdb_deletepending(Pid, ManSQN, Inker) -> + gen_fsm:send_event(Pid, {delete_pending, ManSQN, Inker}). %% cdb_scan returns {LastPosition, Acc}. Use LastPosition as StartPosiiton to %% continue from that point (calling function has to protect against) double @@ -182,26 +197,29 @@ cdb_deletepending(Pid) -> %% the end of the file. last_key must be defined in LoopState. cdb_scan(Pid, FilterFun, InitAcc, StartPosition) -> - gen_server:call(Pid, - {cdb_scan, FilterFun, InitAcc, StartPosition}, - infinity). + gen_fsm:sync_send_all_state_event(Pid, + {cdb_scan, + FilterFun, + InitAcc, + StartPosition}, + infinity). %% Get the last key to be added to the file (which will have the highest %% sequence number) cdb_lastkey(Pid) -> - gen_server:call(Pid, cdb_lastkey, infinity). + gen_fsm:sync_send_all_state_event(Pid, cdb_lastkey, infinity). cdb_firstkey(Pid) -> - gen_server:call(Pid, cdb_firstkey, infinity). + gen_fsm:sync_send_all_state_event(Pid, cdb_firstkey, infinity). %% Get the filename of the database cdb_filename(Pid) -> - gen_server:call(Pid, cdb_filename, infinity). + gen_fsm:sync_send_all_state_event(Pid, cdb_filename, infinity). %% Check to see if the key is probably present, will return either %% probably or missing. Does not do a definitive check cdb_keycheck(Pid, Key) -> - gen_server:call(Pid, {key_check, Key}, infinity). + gen_fsm:sync_send_event(Pid, {key_check, Key}, infinity). %%%============================================================================ %%% gen_server callbacks @@ -214,120 +232,136 @@ init([Opts]) -> M -> M end, - {ok, #state{max_size=MaxSize, binary_mode=Opts#cdb_options.binary_mode}}. + {ok, + starting, + #state{max_size=MaxSize, binary_mode=Opts#cdb_options.binary_mode}}. -handle_call({open_writer, Filename}, _From, State) -> +starting({open_writer, Filename}, _From, State) -> io:format("Opening file for writing with filename ~s~n", [Filename]), {LastPosition, HashTree, LastKey} = open_active_file(Filename), {ok, Handle} = file:open(Filename, [sync | ?WRITE_OPS]), - {reply, ok, State#state{handle=Handle, - last_position=LastPosition, - last_key=LastKey, - filename=Filename, - hashtree=HashTree, - writer=true}}; -handle_call({open_reader, Filename}, _From, State) -> + {reply, ok, writer, State#state{handle=Handle, + last_position=LastPosition, + last_key=LastKey, + filename=Filename, + hashtree=HashTree}}; +starting({open_reader, Filename}, _From, State) -> io:format("Opening file for reading with filename ~s~n", [Filename]), {Handle, Index, LastKey} = open_for_readonly(Filename), - {reply, ok, State#state{handle=Handle, - last_key=LastKey, - filename=Filename, - writer=false, - hash_index=Index}}; -handle_call({get_kv, Key}, _From, State) -> - case State#state.writer of - true -> - {reply, - get_mem(Key, State#state.handle, State#state.hashtree), - State}; - false -> - {reply, - get_withcache(State#state.handle, Key, State#state.hash_index), - State} - end; -handle_call({key_check, Key}, _From, State) -> - case State#state.writer of - true -> - {reply, - get_mem(Key, - State#state.handle, - State#state.hashtree, - loose_presence), - State}; - false -> - {reply, - get(State#state.handle, + {reply, ok, reader, State#state{handle=Handle, + last_key=LastKey, + filename=Filename, + hash_index=Index}}. + +writer({get_kv, Key}, _From, State) -> + {reply, + get_mem(Key, State#state.handle, State#state.hashtree), + writer, + State}; +writer({key_check, Key}, _From, State) -> + {reply, + get_mem(Key, State#state.handle, State#state.hashtree, loose_presence), + writer, + State}; +writer({put_kv, Key, Value}, _From, State) -> + Result = put(State#state.handle, Key, - loose_presence, - State#state.hash_index), - State} - end; -handle_call({put_kv, Key, Value}, _From, State) -> - case {State#state.writer, State#state.pending_roll} of - {true, false} -> - Result = put(State#state.handle, - Key, Value, - {State#state.last_position, State#state.hashtree}, - State#state.binary_mode, - State#state.max_size), - case Result of - roll -> - %% Key and value could not be written - {reply, roll, State}; - {UpdHandle, NewPosition, HashTree} -> - {reply, ok, State#state{handle=UpdHandle, + Value, + {State#state.last_position, State#state.hashtree}, + State#state.binary_mode, + State#state.max_size), + case Result of + roll -> + %% Key and value could not be written + {reply, roll, writer, State}; + {UpdHandle, NewPosition, HashTree} -> + {reply, ok, writer, State#state{handle=UpdHandle, last_position=NewPosition, last_key=Key, hashtree=HashTree}} - end; - _ -> - {reply, - {error, read_only}, - State} end; -handle_call({mput_kv, KVList}, _From, State) -> - case {State#state.writer, State#state.pending_roll} of - {true, false} -> - Result = mput(State#state.handle, - KVList, - {State#state.last_position, State#state.hashtree}, - State#state.binary_mode, - State#state.max_size), - case Result of - roll -> - %% Keys and values could not be written - {reply, roll, State}; - {UpdHandle, NewPosition, HashTree, LastKey} -> - {reply, ok, State#state{handle=UpdHandle, +writer({mput_kv, KVList}, _From, State) -> + Result = mput(State#state.handle, + KVList, + {State#state.last_position, State#state.hashtree}, + State#state.binary_mode, + State#state.max_size), + case Result of + roll -> + %% Keys and values could not be written + {reply, roll, writer, State}; + {UpdHandle, NewPosition, HashTree, LastKey} -> + {reply, ok, writer, State#state{handle=UpdHandle, last_position=NewPosition, last_key=LastKey, hashtree=HashTree}} - end; - _ -> - {reply, - {error, read_only}, - State} end; -handle_call(cdb_lastkey, _From, State) -> - {reply, State#state.last_key, State}; -handle_call(cdb_firstkey, _From, State) -> - {ok, EOFPos} = file:position(State#state.handle, eof), - FirstKey = case EOFPos of - ?BASE_POSITION -> - empty; - _ -> - extract_key(State#state.handle, ?BASE_POSITION) - end, - {reply, FirstKey, State}; -handle_call(cdb_filename, _From, State) -> - {reply, State#state.filename, State}; -handle_call({get_positions, SampleSize}, _From, State) -> +writer(cdb_complete, _From, State) -> + NewName = determine_new_filename(State#state.filename), + ok = close_file(State#state.handle, + State#state.hashtree, + State#state.last_position), + ok = rename_for_read(State#state.filename, NewName), + {stop, normal, {ok, NewName}, State}. + +writer(cdb_roll, State) -> + ok = leveled_iclerk:clerk_hashtablecalc(State#state.hashtree, + State#state.last_position, + self()), + {next_state, rolling, State}. + + +rolling({get_kv, Key}, _From, State) -> + {reply, + get_mem(Key, State#state.handle, State#state.hashtree), + rolling, + State}; +rolling({key_check, Key}, _From, State) -> + {reply, + get_mem(Key, State#state.handle, State#state.hashtree, loose_presence), + rolling, + State}; +rolling(cdb_filename, _From, State) -> + {reply, State#state.filename, rolling, State}; +rolling({return_hashtable, IndexList, HashTreeBin}, _From, State) -> + Handle = State#state.handle, + {ok, BasePos} = file:position(Handle, State#state.last_position), + NewName = determine_new_filename(State#state.filename), + ok = perform_write_hash_tables(Handle, HashTreeBin, BasePos), + ok = write_top_index_table(Handle, BasePos, IndexList), + file:close(Handle), + ok = rename_for_read(State#state.filename, NewName), + io:format("Opening file for reading with filename ~s~n", [NewName]), + {NewHandle, Index, LastKey} = open_for_readonly(NewName), + {reply, ok, reader, State#state{handle=NewHandle, + last_key=LastKey, + filename=NewName, + hash_index=Index}}; +rolling(cdb_kill, _From, State) -> + {stop, killed, ok, State}. + +reader({get_kv, Key}, _From, State) -> + {reply, + get_withcache(State#state.handle, Key, State#state.hash_index), + reader, + State}; +reader({key_check, Key}, _From, State) -> + {reply, + get(State#state.handle, + Key, + loose_presence, + State#state.hash_index), + reader, + State}; +reader({get_positions, SampleSize}, _From, State) -> case SampleSize of all -> - {reply, scan_index(State#state.handle, - State#state.hash_index, - {fun scan_index_returnpositions/4, []}), - State}; + {reply, + scan_index(State#state.handle, + State#state.hash_index, + {fun scan_index_returnpositions/4, []}), + reader, + State}; _ -> SeededL = lists:map(fun(X) -> {random:uniform(), X} end, State#state.hash_index), @@ -339,28 +373,94 @@ handle_call({get_positions, SampleSize}, _From, State) -> fun scan_index_returnpositions/4, [], SampleSize), + reader, State} end; -handle_call({direct_fetch, PositionList, Info}, _From, State) -> +reader({direct_fetch, PositionList, Info}, _From, State) -> H = State#state.handle, case Info of key_only -> KeyList = lists:map(fun(P) -> extract_key(H, P) end, PositionList), - {reply, KeyList, State}; + {reply, KeyList, reader, State}; key_size -> KeySizeList = lists:map(fun(P) -> extract_key_size(H, P) end, PositionList), - {reply, KeySizeList, State}; + {reply, KeySizeList, reader, State}; key_value_check -> KVCList = lists:map(fun(P) -> extract_key_value_check(H, P) end, PositionList), - {reply, KVCList, State} + {reply, KVCList, reader, State} end; -handle_call({cdb_scan, FilterFun, Acc, StartPos}, _From, State) -> +reader(cdb_complete, _From, State) -> + ok = file:close(State#state.handle), + {stop, normal, {ok, State#state.filename}, State#state{handle=undefined}}. + + +reader({delete_pending, 0, no_poll}, State) -> + {next_state, + delete_pending, + State#state{delete_point=0}}; +reader({delete_pending, ManSQN, Inker}, State) -> + {next_state, + delete_pending, + State#state{delete_point=ManSQN, inker=Inker}, + ?DELETE_TIMEOUT}. + + +delete_pending({get_kv, Key}, _From, State) -> + {reply, + get_withcache(State#state.handle, Key, State#state.hash_index), + delete_pending, + State, + ?DELETE_TIMEOUT}; +delete_pending({key_check, Key}, _From, State) -> + {reply, + get(State#state.handle, + Key, + loose_presence, + State#state.hash_index), + delete_pending, + State, + ?DELETE_TIMEOUT}. + +delete_pending(timeout, State) -> + case State#state.delete_point of + 0 -> + {next_state, delete_pending, State}; + ManSQN -> + case is_process_alive(State#state.inker) of + true -> + case leveled_inker:ink_confirmdelete(State#state.inker, + ManSQN) of + true -> + io:format("Deletion confirmed for file ~s " + ++ "at ManifestSQN ~w~n", + [State#state.filename, ManSQN]), + {stop, normal, State}; + false -> + {next_state, + delete_pending, + State, + ?DELETE_TIMEOUT} + end; + false -> + {stop, normal, State} + end + end; +delete_pending(destroy, State) -> + ok = file:close(State#state.handle), + ok = file:delete(State#state.filename), + {stop, normal, State}. + + +handle_sync_event({cdb_scan, FilterFun, Acc, StartPos}, + _From, + StateName, + State) -> {ok, StartPos0} = case StartPos of undefined -> file:position(State#state.handle, @@ -375,77 +475,50 @@ handle_call({cdb_scan, FilterFun, Acc, StartPos}, _From, State) -> FilterFun, Acc, State#state.last_key), - {reply, {LastPosition, Acc2}, State}; + {reply, {LastPosition, Acc2}, StateName, State}; empty -> - {reply, {eof, Acc}, State} + {reply, {eof, Acc}, StateName, State} end; -handle_call(cdb_close, _From, State=#state{pending_roll=RollPending}) - when RollPending == true -> - {reply, pending_roll, State}; -handle_call(cdb_close, _From, State) -> +handle_sync_event(cdb_lastkey, _From, StateName, State) -> + {reply, State#state.last_key, StateName, State}; +handle_sync_event(cdb_firstkey, _From, StateName, State) -> + {ok, EOFPos} = file:position(State#state.handle, eof), + FirstKey = case EOFPos of + ?BASE_POSITION -> + empty; + _ -> + extract_key(State#state.handle, ?BASE_POSITION) + end, + {reply, FirstKey, StateName, State}; +handle_sync_event(cdb_filename, _From, StateName, State) -> + {reply, State#state.filename, StateName, State}; +handle_sync_event(cdb_close, _From, rolling, State) -> + {reply, pending_roll, rolling, State}; +handle_sync_event(cdb_close, _From, _StateName, State) -> ok = file:close(State#state.handle), - {stop, normal, ok, State#state{handle=undefined}}; -handle_call(cdb_kill, _From, State) -> - {stop, killed, ok, State}; -handle_call(cdb_complete, _From, State=#state{writer=Writer}) - when Writer == true -> - NewName = determine_new_filename(State#state.filename), - ok = close_file(State#state.handle, - State#state.hashtree, - State#state.last_position), - ok = rename_for_read(State#state.filename, NewName), - {stop, normal, {ok, NewName}, State}; -handle_call(cdb_complete, _From, State) -> - ok = file:close(State#state.handle), - {stop, normal, {ok, State#state.filename}, State}; -handle_call({return_hashtable, IndexList, HashTreeBin}, - _From, - State=#state{pending_roll=RollPending}) when RollPending == true -> - Handle = State#state.handle, - {ok, BasePos} = file:position(Handle, State#state.last_position), - NewName = determine_new_filename(State#state.filename), - ok = perform_write_hash_tables(Handle, HashTreeBin, BasePos), - ok = write_top_index_table(Handle, BasePos, IndexList), - file:close(Handle), - ok = rename_for_read(State#state.filename, NewName), - io:format("Opening file for reading with filename ~s~n", [NewName]), - {NewHandle, Index, LastKey} = open_for_readonly(NewName), - {reply, ok, State#state{handle=NewHandle, - last_key=LastKey, - filename=NewName, - writer=false, - pending_roll=false, - hash_index=Index}}. + {stop, normal, ok, State#state{handle=undefined}}. +handle_event(_Msg, StateName, State) -> + {next_State, StateName, State}. -handle_cast(destroy, State) -> - ok = file:close(State#state.handle), - ok = file:delete(State#state.filename), - {noreply, State}; -handle_cast(delete_pending, State) -> - {noreply, State#state{pending_delete=true}}; -handle_cast(cdb_roll, State=#state{writer=Writer}) when Writer == true -> - ok = leveled_iclerk:clerk_hashtablecalc(State#state.hashtree, - State#state.last_position, - self()), - {noreply, State#state{pending_roll=true}}. +handle_info(_Msg, StateName, State) -> + {next_State, StateName, State}. -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, State) -> - case {State#state.handle, State#state.pending_delete} of +terminate(Reason, StateName, State) -> + io:format("Closing of filename ~s for Reason ~w~n", + [State#state.filename, Reason]), + case {State#state.handle, StateName} of {undefined, _} -> ok; - {Handle, false} -> - file:close(Handle); - {Handle, true} -> + {Handle, delete_pending} -> file:close(Handle), - file:delete(State#state.filename) + file:delete(State#state.filename); + {Handle, _} -> + file:close(Handle) end. -code_change(_OldVsn, State, _Extra) -> - {ok, State}. +code_change(_OldVsn, StateName, State, _Extra) -> + {ok, StateName, State}. %%%============================================================================ %%% Internal functions diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 95cb69b..807eaf3 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -179,14 +179,20 @@ handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, C#candidate.journal} end, BestRun), - ok = leveled_inker:ink_updatemanifest(Inker, - ManifestSlice, - FilesToDelete), + io:format("Clerk updating Inker as compaction complete of " ++ + "~w files~n", [length(FilesToDelete)]), + {ok, ManSQN} = leveled_inker:ink_updatemanifest(Inker, + ManifestSlice, + FilesToDelete), ok = leveled_inker:ink_compactioncomplete(Inker), + io:format("Clerk has completed compaction process~n"), case PromptDelete of true -> lists:foreach(fun({_SQN, _FN, J2D}) -> - leveled_cdb:cdb_deletepending(J2D) end, + leveled_cdb:cdb_deletepending(J2D, + ManSQN, + Inker) + end, FilesToDelete), {noreply, State}; false -> @@ -639,6 +645,7 @@ check_single_file_test() -> ?assertMatch(37.5, Score3), Score4 = check_single_file(CDB, LedgerFun1, LedgerSrv1, 4, 8, 4), ?assertMatch(75.0, Score4), + ok = leveled_cdb:cdb_deletepending(CDB), ok = leveled_cdb:cdb_destroy(CDB). @@ -698,6 +705,7 @@ compact_single_file_recovr_test() -> stnd, test_ledgerkey("Key2")}), ?assertMatch({"Value2", []}, binary_to_term(RV1)), + ok = leveled_cdb:cdb_deletepending(CDB), ok = leveled_cdb:cdb_destroy(CDB). @@ -736,6 +744,7 @@ compact_single_file_retain_test() -> stnd, test_ledgerkey("Key2")}), ?assertMatch({"Value2", []}, binary_to_term(RV1)), + ok = leveled_cdb:cdb_deletepending(CDB), ok = leveled_cdb:cdb_destroy(CDB). compact_empty_file_test() -> diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 0fd512a..7ef5cf4 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -99,8 +99,10 @@ ink_fetch/3, ink_loadpcl/4, ink_registersnapshot/2, + ink_confirmdelete/2, ink_compactjournal/3, ink_compactioncomplete/1, + ink_compactionpending/1, ink_getmanifest/1, ink_updatemanifest/3, ink_print_manifest/1, @@ -159,6 +161,9 @@ ink_registersnapshot(Pid, Requestor) -> ink_releasesnapshot(Pid, Snapshot) -> gen_server:call(Pid, {release_snapshot, Snapshot}, infinity). +ink_confirmdelete(Pid, ManSQN) -> + gen_server:call(Pid, {confirm_delete, ManSQN}, 1000). + ink_close(Pid) -> gen_server:call(Pid, {close, false}, infinity). @@ -193,6 +198,9 @@ ink_compactjournal(Pid, Checker, InitiateFun, FilterFun, Timeout) -> ink_compactioncomplete(Pid) -> gen_server:call(Pid, compaction_complete, infinity). +ink_compactionpending(Pid) -> + gen_server:call(Pid, compaction_pending, infinity). + ink_getmanifest(Pid) -> gen_server:call(Pid, get_manifest, infinity). @@ -263,6 +271,17 @@ handle_call({release_snapshot, Snapshot}, _From , State) -> io:format("Ledger snapshot ~w released~n", [Snapshot]), io:format("Remaining ledger snapshots are ~w~n", [Rs]), {reply, ok, State#state{registered_snapshots=Rs}}; +handle_call({confirm_delete, ManSQN}, _From, State) -> + Reply = lists:foldl(fun({_R, SnapSQN}, Bool) -> + case SnapSQN < ManSQN of + true -> + Bool; + false -> + false + end end, + true, + State#state.registered_snapshots), + {reply, Reply, State}; handle_call(get_manifest, _From, State) -> {reply, State#state.manifest, State}; handle_call({update_manifest, @@ -280,10 +299,11 @@ handle_call({update_manifest, NewManifestSQN = State#state.manifest_sqn + 1, manifest_printer(Man1), ok = simple_manifest_writer(Man1, NewManifestSQN, State#state.root_path), - PendingRemovals = [{NewManifestSQN, DeletedFiles}], - {reply, ok, State#state{manifest=Man1, - manifest_sqn=NewManifestSQN, - pending_removals=PendingRemovals}}; + {reply, + {ok, NewManifestSQN}, + State#state{manifest=Man1, + manifest_sqn=NewManifestSQN, + pending_removals=DeletedFiles}}; handle_call(print_manifest, _From, State) -> manifest_printer(State#state.manifest), {reply, ok, State}; @@ -302,6 +322,8 @@ handle_call({compact, {reply, ok, State#state{compaction_pending=true}}; handle_call(compaction_complete, _From, State) -> {reply, ok, State#state{compaction_pending=false}}; +handle_call(compaction_pending, _From, State) -> + {reply, State#state.compaction_pending, State}; handle_call({close, Force}, _From, State) -> case {State#state.compaction_pending, Force} of {true, false} -> @@ -329,8 +351,7 @@ terminate(Reason, State) -> lists:foreach(fun({Snap, _SQN}) -> ok = ink_close(Snap) end, State#state.registered_snapshots), manifest_printer(State#state.manifest), - ok = close_allmanifest(State#state.manifest), - ok = close_allremovals(State#state.pending_removals) + ok = close_allmanifest(State#state.manifest) end. code_change(_OldVsn, State, _Extra) -> @@ -552,25 +573,6 @@ find_in_manifest(SQN, [_Head|Tail]) -> find_in_manifest(SQN, Tail). -close_allremovals([]) -> - ok; -close_allremovals([{ManifestSQN, Removals}|Tail]) -> - io:format("Closing removals at ManifestSQN=~w~n", [ManifestSQN]), - lists:foreach(fun({LowSQN, FN, Handle}) -> - io:format("Closing removed file with LowSQN=~w" ++ - " and filename ~s~n", - [LowSQN, FN]), - if - is_pid(Handle) == true -> - ok = leveled_cdb:cdb_close(Handle); - true -> - io:format("Non pid in removal ~w - test~n", - [Handle]) - end - end, - Removals), - close_allremovals(Tail). - %% Scan between sequence numbers applying FilterFun to each entry where %% FilterFun{K, V, Acc} -> Penciller Key List diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index bb8978c..7ccab3f 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -319,17 +319,24 @@ do_merge(KL1, KL2, {SrcLevel, IsB}, {Filepath, MSN}, FileCounter, OutList) -> KL1, KL2, LevelR), - {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, - ExtMan = lists:append(OutList, - [#manifest_entry{start_key=SmallestKey, - end_key=HighestKey, - owner=Pid, - filename=FileName}]), - MTime = timer:now_diff(os:timestamp(), TS1), - io:format("File creation took ~w microseconds ~n", [MTime]), - do_merge(KL1Rem, KL2Rem, - {SrcLevel, IsB}, {Filepath, MSN}, - FileCounter + 1, ExtMan). + case Reply of + {{[], []}, null, _} -> + io:format("Merge resulted in empty file ~s~n", [FileName]), + io:format("Empty file ~s to be cleared~n", [FileName]), + ok = leveled_sft:sft_clear(Pid), + OutList; + {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} -> + ExtMan = lists:append(OutList, + [#manifest_entry{start_key=SmallestKey, + end_key=HighestKey, + owner=Pid, + filename=FileName}]), + MTime = timer:now_diff(os:timestamp(), TS1), + io:format("File creation took ~w microseconds ~n", [MTime]), + do_merge(KL1Rem, KL2Rem, + {SrcLevel, IsB}, {Filepath, MSN}, + FileCounter + 1, ExtMan) + end. get_item(Index, List, Default) -> diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 85392a7..4649f5d 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -467,7 +467,7 @@ create_file(FileName) when is_list(FileName) -> {error, Reason} -> io:format("Error opening filename ~s with reason ~w", [FileName, Reason]), - {error, Reason} + error end. @@ -1037,15 +1037,17 @@ create_slot(KL1, KL2, LevelR, BlockCount, SegLists, SerialisedSlot, LengthList, {BlockKeyList, Status, {LSNb, HSNb}, SegmentList, KL1b, KL2b} = create_block(KL1, KL2, LevelR), - TrackingMetadata = case LowKey of - null -> + TrackingMetadata = case {LowKey, BlockKeyList} of + {null, []} -> + {null, LSN, HSN, LastKey, Status}; + {null, _} -> [NewLowKeyV|_] = BlockKeyList, {leveled_codec:strip_to_keyonly(NewLowKeyV), min(LSN, LSNb), max(HSN, HSNb), leveled_codec:strip_to_keyonly(last(BlockKeyList, {last, LastKey})), Status}; - _ -> + {_, _} -> {LowKey, min(LSN, LSNb), max(HSN, HSNb), leveled_codec:strip_to_keyonly(last(BlockKeyList, diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 6887556..327d421 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -16,7 +16,7 @@ all() -> [ many_put_fetch_head, journal_compaction, fetchput_snapshot, - load_and_count, + load_and_count , load_and_count_withdelete, space_clear_ondelete_test ]. @@ -149,7 +149,7 @@ journal_compaction(_Config) -> testutil:check_forobject(Bookie2, TestObject), testutil:check_forlist(Bookie2, ChkList3), ok = leveled_bookie:book_close(Bookie2), - testutil:reset_filestructure(). + testutil:reset_filestructure(10000). fetchput_snapshot(_Config) -> @@ -435,12 +435,28 @@ space_clear_ondelete_test(_Config) -> io:format("Deletion took ~w microseconds for 80K keys~n", [timer:now_diff(os:timestamp(), SW2)]), ok = leveled_bookie:book_compactjournal(Book1, 30000), - timer:sleep(30000), % Allow for any L0 file to be rolled + F = fun leveled_bookie:book_islastcompactionpending/1, + lists:foldl(fun(X, Pending) -> + case Pending of + false -> + false; + true -> + io:format("Loop ~w waiting for journal " + ++ "compaction to complete~n", [X]), + timer:sleep(20000), + F(Book1) + end end, + true, + lists:seq(1, 15)), + io:format("Waiting for journal deletes~n"), + timer:sleep(20000), {ok, FNsB_L} = file:list_dir(RootPath ++ "/ledger/ledger_files"), {ok, FNsB_J} = file:list_dir(RootPath ++ "/journal/journal_files"), + {ok, FNsB_PC} = file:list_dir(RootPath ++ "/journal/journal_files/post_compact"), + PointB_Journals = length(FNsB_J) + length(FNsB_PC), io:format("Bookie has ~w journal files and ~w ledger files " ++ "after deletes~n", - [length(FNsB_J), length(FNsB_L)]), + [PointB_Journals, length(FNsB_L)]), {async, F2} = leveled_bookie:book_returnfolder(Book1, {keylist, o_rkv}), SW3 = os:timestamp(), @@ -465,7 +481,20 @@ space_clear_ondelete_test(_Config) -> end, ok = leveled_bookie:book_close(Book2), {ok, FNsC_L} = file:list_dir(RootPath ++ "/ledger/ledger_files"), - {ok, FNsC_J} = file:list_dir(RootPath ++ "/journal/journal_files"), - io:format("Bookie has ~w journal files and ~w ledger files " ++ - "after deletes~n", - [length(FNsC_J), length(FNsC_L)]). + io:format("Bookie has ~w ledger files " ++ + "after close~n", [length(FNsC_L)]), + + {ok, Book3} = leveled_bookie:book_start(StartOpts1), + io:format("This should cause a final ledger merge event~n"), + io:format("Will require the penciller to resolve the issue of creating" ++ + " an empty file as all keys compact on merge~n"), + timer:sleep(5000), + ok = leveled_bookie:book_close(Book3), + {ok, FNsD_L} = file:list_dir(RootPath ++ "/ledger/ledger_files"), + io:format("Bookie has ~w ledger files " ++ + "after second close~n", [length(FNsD_L)]), + true = PointB_Journals < length(FNsA_J), + true = length(FNsB_L) =< length(FNsA_L), + true = length(FNsC_L) =< length(FNsB_L), + true = length(FNsD_L) =< length(FNsB_L), + true = length(FNsD_L) == 0. \ No newline at end of file diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index b0a4707..ec002e7 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -3,6 +3,7 @@ -include("../include/leveled.hrl"). -export([reset_filestructure/0, + reset_filestructure/1, check_bucket_stats/2, check_forlist/2, check_forlist/3, @@ -27,9 +28,12 @@ reset_filestructure() -> - % io:format("Waiting ~w ms to give a chance for all file closes " ++ - % "to complete~n", [Wait]), - % timer:sleep(Wait), + reset_filestructure(0). + +reset_filestructure(Wait) -> + io:format("Waiting ~w ms to give a chance for all file closes " ++ + "to complete~n", [Wait]), + timer:sleep(Wait), RootPath = "test", filelib:ensure_dir(RootPath ++ "/journal/"), filelib:ensure_dir(RootPath ++ "/ledger/"), From 4cdc6211a048fe868b402fe60d3ec95415ff0a05 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 26 Oct 2016 21:03:50 +0100 Subject: [PATCH 095/167] Handling 'returned' in penciller unit tests The unit tests for the Penciller couldn't cope with the returned status - and so would intermittently fail (after tightening the timeout on sft check_ready. --- src/leveled_bookie.erl | 2 +- src/leveled_penciller.erl | 24 ++++++++++++++---------- test/end_to_end/iterator_SUITE.erl | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index c01281a..5808690 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -151,7 +151,7 @@ -include_lib("eunit/include/eunit.hrl"). --define(CACHE_SIZE, 2000). +-define(CACHE_SIZE, 1600). -define(JOURNAL_FP, "journal"). -define(LEDGER_FP, "ledger"). -define(SHUTDOWN_WAITS, 60). diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index bfc2124..3611de7 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -1520,11 +1520,15 @@ confirm_delete_test() -> ?assertMatch(R3, false). -maybe_pause_push(R) -> +maybe_pause_push(PCL, KL) -> + R = pcl_pushmem(PCL, KL), if R == pause -> io:format("Pausing push~n"), - timer:sleep(1000); + timer:sleep(500), + ok; + R == returned -> + maybe_pause_push(PCL, KL); true -> ok end. @@ -1546,11 +1550,11 @@ simple_server_test() -> ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), ok = pcl_pushmem(PCL, KL1), ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), - maybe_pause_push(pcl_pushmem(PCL, [Key2])), + ok = maybe_pause_push(PCL, [Key2]), ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), - maybe_pause_push(pcl_pushmem(PCL, KL2)), - maybe_pause_push(pcl_pushmem(PCL, [Key3])), + ok = maybe_pause_push(PCL, KL2), + ok = maybe_pause_push(PCL, [Key3]), ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), @@ -1562,9 +1566,9 @@ simple_server_test() -> ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002", null})), ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003", null})), - maybe_pause_push(pcl_pushmem(PCLr, KL3)), - maybe_pause_push(pcl_pushmem(PCLr, [Key4])), - maybe_pause_push(pcl_pushmem(PCLr, KL4)), + ok = maybe_pause_push(PCLr, KL3), + ok = maybe_pause_push(PCLr, [Key4]), + ok = maybe_pause_push(PCLr, KL4), ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002", null})), ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003", null})), @@ -1606,8 +1610,8 @@ simple_server_test() -> % in a new snapshot Key1A = {{o,"Bucket0001", "Key0001", null}, {4002, {active, infinity}, null}}, KL1A = leveled_sft:generate_randomkeys({4002, 2}), - maybe_pause_push(pcl_pushmem(PCLr, [Key1A])), - maybe_pause_push(pcl_pushmem(PCLr, KL1A)), + ok = maybe_pause_push(PCLr, [Key1A]), + ok = maybe_pause_push(PCLr, KL1A), ?assertMatch(true, pcl_checksequencenumber(PclSnap, {o, "Bucket0001", diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index d1e3967..f119496 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -96,7 +96,7 @@ simple_querycount(_Config) -> Book1, ?KEY_ONLY), ok = leveled_bookie:book_close(Book1), - {ok, Book2} = leveled_bookie:book_start(RootPath, 2000, 50000000), + {ok, Book2} = leveled_bookie:book_start(RootPath, 1000, 50000000), Index1Count = count_termsonindex("Bucket", "idx1_bin", Book2, From a00a1238172485d2c26de9b6c57ca48b1b713a91 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 27 Oct 2016 00:57:19 +0100 Subject: [PATCH 096/167] Recovery strategy testing Test added for the "retain" recovery strategy. This strategy makes sure a full history of index changes is made so that if the Ledger is wiped out, the Ledger cna be fully rebuilt from the Journal. This exposed two journal compaction problems - The BestRun selected did not have the source files correctly sorted in order before compaction - The compaction process incorrectly dealt with the KeyDelta object left after a compaction - i.e. compacting twice the same key caused that key history to be lost. These issues have now been corrected. --- src/leveled_codec.erl | 6 +- src/leveled_iclerk.erl | 27 +++++-- test/end_to_end/basic_SUITE.erl | 7 +- test/end_to_end/iterator_SUITE.erl | 103 +++---------------------- test/end_to_end/restart_SUITE.erl | 107 ++++++++++++++++++++++++++ test/end_to_end/testutil.erl | 116 ++++++++++++++++++++++++++++- 6 files changed, 259 insertions(+), 107 deletions(-) create mode 100644 test/end_to_end/restart_SUITE.erl diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index c251489..afb6792 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -73,7 +73,7 @@ generate_uuid() -> inker_reload_strategy(AltList) -> ReloadStrategy0 = [{?RIAK_TAG, retain}, {?STD_TAG, retain}], lists:foldl(fun({X, Y}, SList) -> - lists:keyreplace(X, 1, Y, SList) + lists:keyreplace(X, 1, SList, {X, Y}) end, ReloadStrategy0, AltList). @@ -163,12 +163,12 @@ from_journalkey({SQN, _Type, LedgerKey}) -> compact_inkerkvc({{_SQN, ?INKT_TOMB, _LK}, _V, _CrcCheck}, _Strategy) -> skip; -compact_inkerkvc({{_SQN, ?INKT_KEYD, LK}, _V, _CrcCheck}, Strategy) -> +compact_inkerkvc({{SQN, ?INKT_KEYD, LK}, V, CrcCheck}, Strategy) -> {Tag, _, _, _} = LK, {Tag, TagStrat} = lists:keyfind(Tag, 1, Strategy), case TagStrat of retain -> - skip; + {retain, {{SQN, ?INKT_KEYD, LK}, V, CrcCheck}}; TagStrat -> {TagStrat, null} end; diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 807eaf3..3a9ecc4 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -162,12 +162,13 @@ handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, ok = filelib:ensure_dir(FP), Candidates = scan_all_files(Manifest, FilterFun, FilterServer, MaxSQN), - BestRun = assess_candidates(Candidates, MaxRunLength), - case score_run(BestRun, MaxRunLength) of + BestRun0 = assess_candidates(Candidates, MaxRunLength), + case score_run(BestRun0, MaxRunLength) of Score when Score > 0 -> - print_compaction_run(BestRun, MaxRunLength), + BestRun1 = sort_run(BestRun0), + print_compaction_run(BestRun1, MaxRunLength), {ManifestSlice, - PromptDelete} = compact_files(BestRun, + PromptDelete} = compact_files(BestRun1, CDBopts, FilterFun, FilterServer, @@ -178,7 +179,7 @@ handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, C#candidate.filename, C#candidate.journal} end, - BestRun), + BestRun1), io:format("Clerk updating Inker as compaction complete of " ++ "~w files~n", [length(FilesToDelete)]), {ok, ManSQN} = leveled_inker:ink_updatemanifest(Inker, @@ -365,6 +366,12 @@ print_compaction_run(BestRun, MaxRunLength) -> end, BestRun). +sort_run(RunOfFiles) -> + CompareFun = fun(Cand1, Cand2) -> + Cand1#candidate.low_sqn =< Cand2#candidate.low_sqn end, + lists:sort(CompareFun, RunOfFiles). + + compact_files([], _CDBopts, _FilterFun, _FilterServer, _MaxSQN, _RStrategy) -> {[], 0}; compact_files(BestRun, CDBopts, FilterFun, FilterServer, MaxSQN, RStrategy) -> @@ -418,6 +425,8 @@ get_all_positions([], PositionBatches) -> get_all_positions([HeadRef|RestOfBest], PositionBatches) -> SrcJournal = HeadRef#candidate.journal, Positions = leveled_cdb:cdb_getpositions(SrcJournal, all), + io:format("Compaction source ~s has yielded ~w positions~n", + [HeadRef#candidate.filename, length(Positions)]), Batches = split_positions_into_batches(lists:sort(Positions), SrcJournal, []), @@ -768,4 +777,12 @@ compact_empty_file_test() -> Score1 = check_single_file(CDB2, LedgerFun1, LedgerSrv1, 9, 8, 4), ?assertMatch(100.0, Score1). +compare_candidate_test() -> + Candidate1 = #candidate{low_sqn=1}, + Candidate2 = #candidate{low_sqn=2}, + Candidate3 = #candidate{low_sqn=3}, + Candidate4 = #candidate{low_sqn=4}, + ?assertMatch([Candidate1, Candidate2, Candidate3, Candidate4], + sort_run([Candidate3, Candidate2, Candidate4, Candidate1])). + -endif. \ No newline at end of file diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 327d421..9096bda 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -8,7 +8,7 @@ fetchput_snapshot/1, load_and_count/1, load_and_count_withdelete/1, - space_clear_ondelete_test/1 + space_clear_ondelete/1 ]). all() -> [ @@ -18,7 +18,7 @@ all() -> [ fetchput_snapshot, load_and_count , load_and_count_withdelete, - space_clear_ondelete_test + space_clear_ondelete ]. @@ -398,8 +398,7 @@ load_and_count_withdelete(_Config) -> testutil:reset_filestructure(). -space_clear_ondelete_test(_Config) -> - % Test is a work in progress +space_clear_ondelete(_Config) -> RootPath = testutil:reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=20000000}, {ok, Book1} = leveled_bookie:book_start(StartOpts1), diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index f119496..5d34786 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -4,14 +4,14 @@ -include("include/leveled.hrl"). -define(KEY_ONLY, {false, undefined}). --define(RETURN_TERMS, {true, undefined}). -export([all/0]). -export([simple_load_with2i/1, simple_querycount/1, rotating_objects/1]). -all() -> [simple_load_with2i, +all() -> [ + simple_load_with2i, simple_querycount, rotating_objects]. @@ -278,99 +278,16 @@ count_termsonindex(Bucket, IdxField, Book, QType) -> rotating_objects(_Config) -> RootPath = testutil:reset_filestructure(), - ok = rotating_object_check(RootPath, "Bucket1", 10), - ok = rotating_object_check(RootPath, "Bucket2", 200), - ok = rotating_object_check(RootPath, "Bucket3", 800), - ok = rotating_object_check(RootPath, "Bucket4", 1600), - ok = rotating_object_check(RootPath, "Bucket5", 3200), - ok = rotating_object_check(RootPath, "Bucket6", 9600), + ok = testutil:rotating_object_check(RootPath, "Bucket1", 10), + ok = testutil:rotating_object_check(RootPath, "Bucket2", 200), + ok = testutil:rotating_object_check(RootPath, "Bucket3", 800), + ok = testutil:rotating_object_check(RootPath, "Bucket4", 1600), + ok = testutil:rotating_object_check(RootPath, "Bucket5", 3200), + ok = testutil:rotating_object_check(RootPath, "Bucket6", 9600), testutil:reset_filestructure(). -rotating_object_check(RootPath, Bucket, NumberOfObjects) -> - {ok, Book1} = leveled_bookie:book_start(RootPath, 2000, 5000000), - {KSpcL1, V1} = put_indexed_objects(Book1, Bucket, NumberOfObjects), - ok = check_indexed_objects(Book1, Bucket, KSpcL1, V1), - {KSpcL2, V2} = put_altered_indexed_objects(Book1, Bucket, KSpcL1), - ok = check_indexed_objects(Book1, Bucket, KSpcL2, V2), - {KSpcL3, V3} = put_altered_indexed_objects(Book1, Bucket, KSpcL2), - ok = leveled_bookie:book_close(Book1), - {ok, Book2} = leveled_bookie:book_start(RootPath, 1000, 5000000), - ok = check_indexed_objects(Book2, Bucket, KSpcL3, V3), - {KSpcL4, V4} = put_altered_indexed_objects(Book2, Bucket, KSpcL3), - ok = check_indexed_objects(Book2, Bucket, KSpcL4, V4), - ok = leveled_bookie:book_close(Book2), - ok. - - - -check_indexed_objects(Book, B, KSpecL, V) -> - % Check all objects match, return what should eb the results of an all - % index query - IdxR = lists:map(fun({K, Spc}) -> - {ok, O} = leveled_bookie:book_riakget(Book, B, K), - V = testutil:get_value(O), - {add, - "idx1_bin", - IdxVal} = lists:keyfind(add, 1, Spc), - {IdxVal, K} end, - KSpecL), - % Check the all index query matxhes expectations - R = leveled_bookie:book_returnfolder(Book, - {index_query, - B, - {"idx1_bin", - "0", - "~"}, - ?RETURN_TERMS}), - {async, Fldr} = R, - QR = lists:sort(Fldr()), - ER = lists:sort(IdxR), - ok = if - ER == QR -> - ok - end, - ok. - - -put_indexed_objects(Book, Bucket, Count) -> - V = testutil:get_compressiblevalue(), - IndexGen = testutil:get_randomindexes_generator(1), - SW = os:timestamp(), - ObjL1 = testutil:generate_objects(Count, - uuid, - [], - V, - IndexGen, - Bucket), - KSpecL = lists:map(fun({_RN, Obj, Spc}) -> - leveled_bookie:book_riakput(Book, - Obj, - Spc), - {testutil:get_key(Obj), Spc} - end, - ObjL1), - io:format("Put of ~w objects with ~w index entries " - ++ - "each completed in ~w microseconds~n", - [Count, 1, timer:now_diff(os:timestamp(), SW)]), - {KSpecL, V}. - -put_altered_indexed_objects(Book, Bucket, KSpecL) -> - IndexGen = testutil:get_randomindexes_generator(1), - V = testutil:get_compressiblevalue(), - RplKSpecL = lists:map(fun({K, Spc}) -> - AddSpc = lists:keyfind(add, 1, Spc), - {O, AltSpc} = testutil:set_object(Bucket, - K, - V, - IndexGen, - [AddSpc]), - ok = leveled_bookie:book_riakput(Book, - O, - AltSpc), - {K, AltSpc} end, - KSpecL), - {RplKSpecL, V}. + + diff --git a/test/end_to_end/restart_SUITE.erl b/test/end_to_end/restart_SUITE.erl new file mode 100644 index 0000000..0017035 --- /dev/null +++ b/test/end_to_end/restart_SUITE.erl @@ -0,0 +1,107 @@ +-module(restart_SUITE). +-include_lib("common_test/include/ct.hrl"). +-include("include/leveled.hrl"). +-export([all/0]). +-export([retain_strategy/1 + ]). + +all() -> [ + retain_strategy + ]. + +retain_strategy(_Config) -> + RootPath = testutil:reset_filestructure(), + BookOpts = #bookie_options{root_path=RootPath, + cache_size=1000, + max_journalsize=5000000, + reload_strategy=[{?RIAK_TAG, retain}]}, + {ok, Spcl3, LastV3} = rotating_object_check(BookOpts, "Bucket3", 800), + ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3}]), + {ok, Spcl4, LastV4} = rotating_object_check(BookOpts, "Bucket4", 1600), + ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3}, + {"Bucket4", Spcl4, LastV4}]), + {ok, Spcl5, LastV5} = rotating_object_check(BookOpts, "Bucket5", 3200), + ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3}, + {"Bucket5", Spcl5, LastV5}]), + {ok, Spcl6, LastV6} = rotating_object_check(BookOpts, "Bucket6", 6400), + ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3}, + {"Bucket4", Spcl4, LastV4}, + {"Bucket5", Spcl5, LastV5}, + {"Bucket6", Spcl6, LastV6}]), + testutil:reset_filestructure(). + + + +rotating_object_check(BookOpts, B, NumberOfObjects) -> + {ok, Book1} = leveled_bookie:book_start(BookOpts), + {KSpcL1, V1} = testutil:put_indexed_objects(Book1, B, NumberOfObjects), + ok = testutil:check_indexed_objects(Book1, + B, + KSpcL1, + V1), + {KSpcL2, V2} = testutil:put_altered_indexed_objects(Book1, + B, + KSpcL1, + false), + ok = testutil:check_indexed_objects(Book1, + B, + KSpcL1 ++ KSpcL2, + V2), + {KSpcL3, V3} = testutil:put_altered_indexed_objects(Book1, + B, + KSpcL2, + false), + ok = leveled_bookie:book_close(Book1), + {ok, Book2} = leveled_bookie:book_start(BookOpts), + ok = testutil:check_indexed_objects(Book2, + B, + KSpcL1 ++ KSpcL2 ++ KSpcL3, + V3), + {KSpcL4, V4} = testutil:put_altered_indexed_objects(Book2, + B, + KSpcL3, + false), + io:format("Bucket complete - checking index before compaction~n"), + ok = testutil:check_indexed_objects(Book2, + B, + KSpcL1 ++ KSpcL2 ++ KSpcL3 ++ KSpcL4, + V4), + + ok = leveled_bookie:book_compactjournal(Book2, 30000), + F = fun leveled_bookie:book_islastcompactionpending/1, + lists:foldl(fun(X, Pending) -> + case Pending of + false -> + false; + true -> + io:format("Loop ~w waiting for journal " + ++ "compaction to complete~n", [X]), + timer:sleep(20000), + F(Book2) + end end, + true, + lists:seq(1, 15)), + io:format("Waiting for journal deletes~n"), + timer:sleep(20000), + + io:format("Checking index following compaction~n"), + ok = testutil:check_indexed_objects(Book2, + B, + KSpcL1 ++ KSpcL2 ++ KSpcL3 ++ KSpcL4, + V4), + + ok = leveled_bookie:book_close(Book2), + {ok, KSpcL1 ++ KSpcL2 ++ KSpcL3 ++ KSpcL4, V4}. + + +restart_from_blankledger(BookOpts, B_SpcL) -> + leveled_penciller:clean_testdir(BookOpts#bookie_options.root_path ++ + "/ledger"), + {ok, Book1} = leveled_bookie:book_start(BookOpts), + io:format("Checking index following restart~n"), + lists:foreach(fun({B, SpcL, V}) -> + ok = testutil:check_indexed_objects(Book1, B, SpcL, V) + end, + B_SpcL), + ok = leveled_bookie:book_close(Book1), + ok. \ No newline at end of file diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index ec002e7..f88e900 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -23,8 +23,14 @@ get_compressiblevalue/0, get_randomindexes_generator/1, name_list/0, - load_objects/5]). + load_objects/5, + put_indexed_objects/3, + put_altered_indexed_objects/3, + put_altered_indexed_objects/4, + check_indexed_objects/4, + rotating_object_check/3]). +-define(RETURN_TERMS, {true, undefined}). reset_filestructure() -> @@ -267,4 +273,110 @@ get_randomdate() -> Date = calendar:gregorian_seconds_to_datetime(RandPoint), {{Year, Month, Day}, {Hour, Minute, Second}} = Date, lists:flatten(io_lib:format("~4..0w~2..0w~2..0w~2..0w~2..0w~2..0w", - [Year, Month, Day, Hour, Minute, Second])). \ No newline at end of file + [Year, Month, Day, Hour, Minute, Second])). + + +check_indexed_objects(Book, B, KSpecL, V) -> + % Check all objects match, return what should be the results of an all + % index query + IdxR = lists:map(fun({K, Spc}) -> + {ok, O} = leveled_bookie:book_riakget(Book, B, K), + V = testutil:get_value(O), + {add, + "idx1_bin", + IdxVal} = lists:keyfind(add, 1, Spc), + {IdxVal, K} end, + KSpecL), + % Check the all index query matches expectations + R = leveled_bookie:book_returnfolder(Book, + {index_query, + B, + {"idx1_bin", + "0", + "~"}, + ?RETURN_TERMS}), + SW = os:timestamp(), + {async, Fldr} = R, + QR0 = Fldr(), + io:format("Query match found of length ~w in ~w microseconds " ++ + "expected ~w ~n", + [length(QR0), + timer:now_diff(os:timestamp(), SW), + length(IdxR)]), + QR = lists:sort(QR0), + ER = lists:sort(IdxR), + + ok = if + ER == QR -> + ok + end, + ok. + + +put_indexed_objects(Book, Bucket, Count) -> + V = testutil:get_compressiblevalue(), + IndexGen = testutil:get_randomindexes_generator(1), + SW = os:timestamp(), + ObjL1 = testutil:generate_objects(Count, + uuid, + [], + V, + IndexGen, + Bucket), + KSpecL = lists:map(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Book, + Obj, + Spc), + {testutil:get_key(Obj), Spc} + end, + ObjL1), + io:format("Put of ~w objects with ~w index entries " + ++ + "each completed in ~w microseconds~n", + [Count, 1, timer:now_diff(os:timestamp(), SW)]), + {KSpecL, V}. + + +put_altered_indexed_objects(Book, Bucket, KSpecL) -> + put_altered_indexed_objects(Book, Bucket, KSpecL, true). + +put_altered_indexed_objects(Book, Bucket, KSpecL, RemoveOld2i) -> + IndexGen = testutil:get_randomindexes_generator(1), + V = testutil:get_compressiblevalue(), + RplKSpecL = lists:map(fun({K, Spc}) -> + AddSpc = if + RemoveOld2i == true -> + [lists:keyfind(add, 1, Spc)]; + RemoveOld2i == false -> + [] + end, + {O, AltSpc} = testutil:set_object(Bucket, + K, + V, + IndexGen, + AddSpc), + ok = leveled_bookie:book_riakput(Book, + O, + AltSpc), + {K, AltSpc} end, + KSpecL), + {RplKSpecL, V}. + +rotating_object_check(RootPath, B, NumberOfObjects) -> + BookOpts = #bookie_options{root_path=RootPath, + cache_size=1000, + max_journalsize=5000000}, + {ok, Book1} = leveled_bookie:book_start(BookOpts), + {KSpcL1, V1} = testutil:put_indexed_objects(Book1, B, NumberOfObjects), + ok = testutil:check_indexed_objects(Book1, B, KSpcL1, V1), + {KSpcL2, V2} = testutil:put_altered_indexed_objects(Book1, B, KSpcL1), + ok = testutil:check_indexed_objects(Book1, B, KSpcL2, V2), + {KSpcL3, V3} = testutil:put_altered_indexed_objects(Book1, B, KSpcL2), + ok = leveled_bookie:book_close(Book1), + {ok, Book2} = leveled_bookie:book_start(BookOpts), + ok = testutil:check_indexed_objects(Book2, B, KSpcL3, V3), + {KSpcL4, V4} = testutil:put_altered_indexed_objects(Book2, B, KSpcL3), + ok = testutil:check_indexed_objects(Book2, B, KSpcL4, V4), + ok = leveled_bookie:book_close(Book2), + ok. + From 30f4f2edf6c90e73c51cf072a0caee0583fc12f6 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 27 Oct 2016 09:45:05 +0100 Subject: [PATCH 097/167] Comment change on stall behaviour --- src/leveled_penciller.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 3611de7..b30aa25 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -73,7 +73,8 @@ %% %% The Penciller MUST NOT accept a new PUSH if the Clerk has commenced the %% conversion of the current ETS table into a SFT file, but not completed this -%% change. This should prompt a stall. +%% change. The Penciller in this case returns the push, and the Bookie should +%% continue to gorw the cache before trying again. %% %% ---------- FETCH ---------- %% From 20cc17f9160711a30c7f0481f0fc491e81c3541d Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 27 Oct 2016 20:56:18 +0100 Subject: [PATCH 098/167] Penciller Refactor Removed o(100) lines of code by refactoring the Penciller to no longer use ETS tables. The code is less confusing, and probably not an awful lot slower. --- src/leveled_bookie.erl | 27 +- src/leveled_inker.erl | 20 +- src/leveled_pclerk.erl | 75 ++- src/leveled_penciller.erl | 929 +++++++++++++++----------------------- src/leveled_sft.erl | 36 +- 5 files changed, 465 insertions(+), 622 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 5808690..f89e399 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -268,7 +268,8 @@ init([Opts]) -> {ok, {Penciller, LedgerCache}, Inker} = book_snapshotstore(Bookie, self(), ?SNAPSHOT_TIMEOUT), - ok = leveled_penciller:pcl_loadsnapshot(Penciller, []), + ok = leveled_penciller:pcl_loadsnapshot(Penciller, + gb_trees:empty()), io:format("Snapshot starting with Pcl ~w Ink ~w~n", [Penciller, Inker]), {ok, #state{penciller=Penciller, @@ -431,11 +432,10 @@ bucket_stats(Penciller, LedgerCache, Bucket, Tag) -> source_penciller=Penciller}, {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), Folder = fun() -> - Increment = gb_trees:to_list(LedgerCache), io:format("Length of increment in snapshot is ~w~n", - [length(Increment)]), + [gb_trees:size(LedgerCache)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, - {infinity, Increment}), + LedgerCache), StartKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), EndKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, @@ -456,11 +456,10 @@ index_query(Penciller, LedgerCache, source_penciller=Penciller}, {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), Folder = fun() -> - Increment = gb_trees:to_list(LedgerCache), io:format("Length of increment in snapshot is ~w~n", - [length(Increment)]), + [gb_trees:size(LedgerCache)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, - {infinity, Increment}), + LedgerCache), StartKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, IdxField, StartValue), EndKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, @@ -487,11 +486,10 @@ allkey_query(Penciller, LedgerCache, Tag) -> source_penciller=Penciller}, {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), Folder = fun() -> - Increment = gb_trees:to_list(LedgerCache), io:format("Length of increment in snapshot is ~w~n", - [length(Increment)]), + [gb_trees:size(LedgerCache)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, - {infinity, Increment}), + LedgerCache), SK = leveled_codec:to_ledgerkey(null, null, Tag), EK = leveled_codec:to_ledgerkey(null, null, Tag), Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, @@ -648,13 +646,10 @@ maybepush_ledgercache(MaxCacheSize, Cache, Penciller) -> TimeToPush = maybe_withjitter(CacheSize, MaxCacheSize), if TimeToPush -> - Dump = gb_trees:to_list(Cache), - case leveled_penciller:pcl_pushmem(Penciller, Dump) of + case leveled_penciller:pcl_pushmem(Penciller, Cache) of ok -> {ok, gb_trees:empty()}; - pause -> - {pause, gb_trees:empty()}; - returned -> + {returned, _Reason} -> {ok, Cache} end; true -> @@ -794,7 +789,7 @@ multi_key_test() -> {ok, F2B} = book_riakget(Bookie1, B2, K2), ?assertMatch(F2B, Obj2), ok = book_close(Bookie1), - %% Now reopen the file, and confirm that a fetch is still possible + % Now reopen the file, and confirm that a fetch is still possible {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath}), {ok, F1C} = book_riakget(Bookie2, B1, K1), ?assertMatch(F1C, Obj1), diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 7ef5cf4..b929c91 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -261,15 +261,15 @@ handle_call({load_pcl, StartSQN, FilterFun, Penciller}, _From, State) -> handle_call({register_snapshot, Requestor}, _From , State) -> Rs = [{Requestor, State#state.manifest_sqn}|State#state.registered_snapshots], - io:format("Inker snapshot ~w registered at SQN ~w~n", + io:format("Journal snapshot ~w registered at SQN ~w~n", [Requestor, State#state.manifest_sqn]), {reply, {State#state.manifest, State#state.active_journaldb}, State#state{registered_snapshots=Rs}}; handle_call({release_snapshot, Snapshot}, _From , State) -> Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots), - io:format("Ledger snapshot ~w released~n", [Snapshot]), - io:format("Remaining ledger snapshots are ~w~n", [Rs]), + io:format("Journal snapshot ~w released~n", [Snapshot]), + io:format("Remaining journal snapshots are ~w~n", [Rs]), {reply, ok, State#state{registered_snapshots=Rs}}; handle_call({confirm_delete, ManSQN}, _From, State) -> Reply = lists:foldl(fun({_R, SnapSQN}, Bool) -> @@ -646,15 +646,12 @@ load_between_sequence(MinSQN, MaxSQN, FilterFun, Penciller, push_to_penciller(Penciller, KeyTree) -> % The push to penciller must start as a tree to correctly de-duplicate % the list by order before becoming a de-duplicated list for loading - KeyList = gb_trees:to_list(KeyTree), - R = leveled_penciller:pcl_pushmem(Penciller, KeyList), - if - R == pause -> - timer:sleep(?LOADING_PAUSE); - R == returned -> + R = leveled_penciller:pcl_pushmem(Penciller, KeyTree), + case R of + {returned, _Reason} -> timer:sleep(?LOADING_PAUSE), push_to_penciller(Penciller, KeyTree); - R == ok -> + ok -> ok end. @@ -742,8 +739,7 @@ initiate_penciller_snapshot(Bookie) -> {ok, {LedgerSnap, LedgerCache}, _} = leveled_bookie:book_snapshotledger(Bookie, self(), undefined), - ok = leveled_penciller:pcl_loadsnapshot(LedgerSnap, - gb_trees:to_list(LedgerCache)), + ok = leveled_penciller:pcl_loadsnapshot(LedgerSnap, LedgerCache), MaxSQN = leveled_penciller:pcl_getstartupsequencenumber(LedgerSnap), {LedgerSnap, MaxSQN}. diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 7ccab3f..792acb4 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -59,9 +59,11 @@ handle_cast/2, handle_info/2, terminate/2, - clerk_new/1, - clerk_prompt/1, - clerk_manifestchange/3, + clerk_new/2, + mergeclerk_prompt/1, + mergeclerk_manifestchange/3, + rollclerk_levelzero/5, + rollclerk_close/1, code_change/3]). -include_lib("eunit/include/eunit.hrl"). @@ -71,24 +73,33 @@ -record(state, {owner :: pid(), change_pending=false :: boolean(), - work_item :: #penciller_work{}|null}). + work_item :: #penciller_work{}|null, + merge_clerk = false :: boolean(), + roll_clerk = false ::boolean()}). %%%============================================================================ %%% API %%%============================================================================ -clerk_new(Owner) -> +clerk_new(Owner, Type) -> {ok, Pid} = gen_server:start(?MODULE, [], []), - ok = gen_server:call(Pid, {register, Owner}, infinity), + ok = gen_server:call(Pid, {register, Owner, Type}, infinity), io:format("Penciller's clerk ~w started with owner ~w~n", [Pid, Owner]), {ok, Pid}. -clerk_manifestchange(Pid, Action, Closing) -> +mergeclerk_manifestchange(Pid, Action, Closing) -> gen_server:call(Pid, {manifest_change, Action, Closing}, infinity). -clerk_prompt(Pid) -> +mergeclerk_prompt(Pid) -> gen_server:cast(Pid, prompt). +rollclerk_levelzero(Pid, LevelZero, LevelMinus1, LedgerSQN, PCL) -> + gen_server:cast(Pid, + {roll_levelzero, LevelZero, LevelMinus1, LedgerSQN, PCL}). + +rollclerk_close(Pid) -> + gen_server:call(Pid, close, infinity). + %%%============================================================================ %%% gen_server callbacks %%%============================================================================ @@ -96,8 +107,18 @@ clerk_prompt(Pid) -> init([]) -> {ok, #state{}}. -handle_call({register, Owner}, _From, State) -> - {reply, ok, State#state{owner=Owner}, ?MIN_TIMEOUT}; +handle_call({register, Owner, Type}, _From, State) -> + case Type of + merge -> + {reply, + ok, + State#state{owner=Owner, merge_clerk = true}, + ?MIN_TIMEOUT}; + roll -> + {reply, + ok, + State#state{owner=Owner, roll_clerk = true}} + end; handle_call({manifest_change, return, true}, _From, State) -> io:format("Request for manifest change from clerk on closing~n"), case State#state.change_pending of @@ -124,20 +145,34 @@ handle_call({manifest_change, confirm, Closing}, From, State) -> {noreply, State#state{work_item=null, change_pending=false}, ?MIN_TIMEOUT} - end. + end; +handle_call(close, _From, State) -> + {stop, normal, ok, State}. handle_cast(prompt, State) -> - {noreply, State, ?MIN_TIMEOUT}. + {noreply, State, ?MIN_TIMEOUT}; +handle_cast({roll_levelzero, LevelZero, LevelMinus1, LedgerSQN, PCL}, State) -> + SW = os:timestamp(), + {NewL0, Size, MaxSQN} = leveled_penciller:roll_new_tree(LevelZero, + LevelMinus1, + LedgerSQN), + ok = leveled_penciller:pcl_updatelevelzero(PCL, NewL0, Size, MaxSQN), + io:format("Rolled tree to size ~w in ~w microseconds~n", + [Size, timer:now_diff(os:timestamp(), SW)]), + {noreply, State}. handle_info(timeout, State=#state{change_pending=Pnd}) when Pnd == false -> - case requestandhandle_work(State) of - {false, Timeout} -> - {noreply, State, Timeout}; - {true, WI} -> - % No timeout now as will wait for call to return manifest - % change - {noreply, - State#state{change_pending=true, work_item=WI}} + if + State#state.merge_clerk -> + case requestandhandle_work(State) of + {false, Timeout} -> + {noreply, State, Timeout}; + {true, WI} -> + % No timeout now as will wait for call to return manifest + % change + {noreply, + State#state{change_pending=true, work_item=WI}} + end end. terminate(Reason, _State) -> diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index b30aa25..2509ad2 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -9,7 +9,7 @@ %% the Penciller's Clerk %% - The Penciller can be cloned and maintains a register of clones who have %% requested snapshots of the Ledger -%% - The accepts new dumps (in the form of lists of keys) from the Bookie, and +%% - The accepts new dumps (in the form of a gb_tree) from the Bookie, and %% calls the Bookie once the process of pencilling this data in the Ledger is %% complete - and the Bookie is free to forget about the data %% - The Penciller's persistence of the ledger may not be reliable, in that it @@ -21,20 +21,24 @@ %% -------- LEDGER --------- %% %% The Ledger is divided into many levels -%% - L0: New keys are received from the Bookie and merged into a single ETS -%% table, until that table is the size of a SFT file, and it is then persisted +%% - L0: New keys are received from the Bookie and merged into a single +%% gb_tree, until that tree is the size of a SFT file, and it is then persisted %% as a SFT file at this level. L0 SFT files can be larger than the normal %% maximum size - so we don't have to consider problems of either having more %% than one L0 file (and handling what happens on a crash between writing the %% files when the second may have overlapping sequence numbers), or having a %% remainder with overlapping in sequence numbers in memory after the file is -%% written. Once the persistence is completed, the ETS table can be erased. +%% written. Once the persistence is completed, the L0 tree can be erased. %% There can be only one SFT file at Level 0, so the work to merge that file %% to the lower level must be the highest priority, as otherwise writes to the %% ledger will stall, when there is next a need to persist. %% - L1 TO L7: May contain multiple processes managing non-overlapping sft %% files. Compaction work should be sheduled if the number of files exceeds %% the target size of the level, where the target size is 8 ^ n. +%% - L Minus 1: Used to cache the last ledger cache push for use in queries +%% whilst the Penciller awaits a callback from the roll_clerk with the new +%% merged L0 file containing the L-1 updates. +%% %% %% The most recent revision of a Key can be found by checking each level until %% the key is found. To check a level the correct file must be sought from the @@ -69,7 +73,8 @@ %% %% The Penciller must support the PUSH of a dump of keys from the Bookie. The %% call to PUSH should be immediately acknowledged, and then work should be -%% completed to merge the ETS table into the L0 ETS table. +%% completed to merge the tree into the L0 tree (with the tree being cached as +%% a Level -1 tree so as not to block reads whilst it waits. %% %% The Penciller MUST NOT accept a new PUSH if the Clerk has commenced the %% conversion of the current ETS table into a SFT file, but not completed this @@ -85,15 +90,8 @@ %% ---------- SNAPSHOT ---------- %% %% Iterators may request a snapshot of the database. A snapshot is a cloned -%% Penciller seeded not from disk, but by the in-memory ETS table and the -%% in-memory manifest. - -%% To provide a snapshot the Penciller must snapshot the ETS table. The -%% snapshot of the ETS table is managed by the Penciller storing a list of the -%% batches of Keys which have been pushed to the Penciller, and it is expected -%% that this will be converted by the clone into a gb_tree. The clone may -%% then update the master Penciller with the gb_tree to be cached and used by -%% other cloned processes. +%% Penciller seeded not from disk, but by the in-memory L0 gb_tree and the +%% in-memory manifest, allowing for direct reference for the SFT file processes. %% %% Clones formed to support snapshots are registered by the Penciller, so that %% SFT files valid at the point of the snapshot until either the iterator is @@ -154,12 +152,11 @@ %% allowed to again reach capacity %% %% The writing of L0 files do not require the involvement of the clerk. -%% The L0 files are prompted directly by the penciller when the in-memory ets -%% table has reached capacity. When there is a next push into memory the -%% penciller calls to check that the file is now active (which may pause if the -%% write is ongoing the acceptence of the push), and if so it can clear the ets -%% table and build a new table starting with the remainder, and the keys from -%% the latest push. +%% The L0 files are prompted directly by the penciller when the in-memory tree +%% has reached capacity. When there is a next push into memory the Penciller +%% calls to check that the file is now active (which may pause if the write is +%% ongoing the acceptence of the push), and if so it can clear the L0 tree +%% and build a new tree from an empty tree and the keys from the latest push. %% %% Only a single L0 file may exist at any one moment in time. If pushes are %% received when memory is over the maximum size, the pushes must be kept into @@ -175,73 +172,33 @@ %% manifest SQN n+1 %% 5 - The clerk will prompt the penciller about the change, and the Penciller %% will then commit the change (by renaming the manifest file to be active, and -%% advancing th ein-memory state of the manifest and manifest SQN) +%% advancing the in-memory state of the manifest and manifest SQN) %% 6 - The Penciller having committed the change will cast back to the Clerk %% to inform the Clerk that the chnage has been committed, and so it can carry %% on requetsing new work %% 7 - If the Penciller now receives a Push to over the max size, a new L0 file %% can now be created with the ManifestSQN of n+1 %% -%% ---------- NOTES ON THE USE OF ETS ---------- +%% ---------- NOTES ON THE (NON) USE OF ETS ---------- %% %% Insertion into ETS is very fast, and so using ETS does not slow the PUT %% path. However, an ETS table is mutable, so it does complicate the %% snapshotting of the Ledger. %% -%% Some alternatives have been considered: +%% Originally the solution had used an ETS tbale for insertion speed as the L0 +%% cache. Insertion speed was an order or magnitude faster than gb_trees. To +%% resolving issues of trying to have fast start-up snapshots though led to +%% keeping a seperate set of trees alongside the ETS table to be used by +%% snapshots. %% -%% A1 - Use gb_trees not ETS table -%% * Speed of inserts are too slow especially as the Bookie is blocked until -%% the insert is complete. Inserting 32K very simple keys takes 250ms. Only -%% the naive commands can be used, as Keys may be present - so not easy to -%% optimise. There is a lack of bulk operations +%% The current strategy is to perform the expensive operation (merging the +%% Ledger cache into the Level0 cache), within a dedicated Penciller's clerk, +%% known as the roll_clerk. This may take 30-40ms, but during this period +%% the Penciller will keep a Level -1 cache of the unmerged elements which +%% it will wipe once the roll_clerk returns with an updated L0 cache. %% -%% A2 - Use some other structure other than gb_trees or ETS tables -%% * There is nothing else that will support iterators, so snapshots would -%% either need to do a conversion when they request the snapshot if -%% they need to iterate, or iterate through map functions scanning all the -%% keys. The conversion may not be expensive, as we know loading into an ETS -%% table is fast - but there may be some hidden overheads with creating and -%% destroying many ETS tables. -%% -%% A3 - keep a parallel list of lists of things that have gone in the ETS -%% table in the format they arrived in -%% * There is doubling up of memory, and the snapshot must do some work to -%% make use of these lists. This combines the continued use of fast ETS -%% with the solution of A2 at a memory cost. -%% -%% A4 - Try and cache the conversion to be shared between snapshots registered -%% at the same Ledger SQN -%% * This is a rif on A2/A3, but if generally there is o(10) or o(100) seconds -%% between memory pushes, but much more frequent snapshots this may be more -%% efficient -%% -%% A5 - Produce a specific snapshot of the ETS table via an iterator on demand -%% for each snapshot -%% * So if a snapshot was required for na iterator, the Penciller would block -%% whilst it iterated over the ETS table first to produce a snapshot-specific -%% immutbale view. If the snapshot was required for a long-lived complete view -%% of the database the Penciller would block for a tab2list. -%% -%% A6 - Have snapshots incrementally create and share immutable trees, from a -%% parallel cache of changes -%% * This is a variance on A3. As changes are pushed to the Penciller in the -%% form of lists the Penciller updates a cache of the lists that are contained -%% in the current ETS table. These lists are returned to the snapshot when -%% the snapshot is registered. All snapshots it is assumed will convert these -%% lists into a gb_tree to use, but following that conversion they can cast -%% to the Penciller to refine the cache, so that the cache will become a -%% gb_tree up the ledger SQN at which the snapshot is registered, and now only -%% store new lists for subsequent updates. Future snapshot requests (before -%% the ets table is flushed) will now receive the array (if no changes have) -%% been made, or the array and only the lists needed to incrementally change -%% the array. If changes are infrequent, each snapshot request will pay the -%% full 20ms to 250ms cost of producing the array (although perhaps the clerk -%% could also update periodiclaly to avoid this). If changes are frequent, -%% the snapshot will generally not require to do a conversion, or will only -%% be required to do a small conversion -%% -%% A6 is the preferred option +%% This means that the in-memory cache is now using immutable objects, and +%% so can be shared with snapshots. -module(leveled_penciller). @@ -267,7 +224,7 @@ pcl_close/1, pcl_registersnapshot/2, pcl_releasesnapshot/2, - pcl_updatesnapshotcache/3, + pcl_updatelevelzero/4, pcl_loadsnapshot/2, pcl_getstartupsequencenumber/1, roll_new_tree/3, @@ -288,29 +245,32 @@ -define(MEMTABLE, mem). -define(MAX_TABLESIZE, 32000). -define(PROMPT_WAIT_ONL0, 5). --define(L0PEND_RESET, {false, null, null}). --record(l0snapshot, {increments = [] :: list(), - tree = gb_trees:empty() :: gb_trees:tree(), - ledger_sqn = 0 :: integer()}). -record(state, {manifest = [] :: list(), - ongoing_work = [] :: list(), manifest_sqn = 0 :: integer(), ledger_sqn = 0 :: integer(), registered_snapshots = [] :: list(), unreferenced_files = [] :: list(), root_path = "../test" :: string(), - table_size = 0 :: integer(), - clerk :: pid(), - levelzero_pending = ?L0PEND_RESET :: tuple(), - memtable_copy = #l0snapshot{} :: #l0snapshot{}, - levelzero_snapshot = gb_trees:empty() :: gb_trees:tree(), - memtable, - memtable_maxsize :: integer(), + + merge_clerk :: pid(), + roll_clerk :: pid(), + + levelzero_pending = false :: boolean(), + levelzero_constructor :: pid(), + levelzero_cache = gb_trees:empty() :: gb_trees:tree(), + levelzero_cachesize = 0 :: integer(), + levelzero_maxcachesize :: integer(), + + levelminus1_active = false :: boolean(), + levelminus1_cache = gb_trees:empty() :: gb_trees:tree(), + is_snapshot = false :: boolean(), snapshot_fully_loaded = false :: boolean(), - source_penciller :: pid()}). + source_penciller :: pid(), + + ongoing_work = [] :: list()}). %%%============================================================================ @@ -354,12 +314,12 @@ pcl_registersnapshot(Pid, Snapshot) -> pcl_releasesnapshot(Pid, Snapshot) -> gen_server:cast(Pid, {release_snapshot, Snapshot}). -pcl_updatesnapshotcache(Pid, Tree, SQN) -> - gen_server:cast(Pid, {update_snapshotcache, Tree, SQN}). - pcl_loadsnapshot(Pid, Increment) -> gen_server:call(Pid, {load_snapshot, Increment}, infinity). +pcl_updatelevelzero(Pid, L0Cache, L0Size, L0SQN) -> + gen_server:cast(Pid, {load_levelzero, L0Cache, L0Size, L0SQN}). + pcl_close(Pid) -> gen_server:call(Pid, close, 60000). @@ -373,101 +333,123 @@ init([PCLopts]) -> PCLopts#penciller_options.start_snapshot} of {undefined, true} -> SrcPenciller = PCLopts#penciller_options.source_penciller, - {ok, - LedgerSQN, - Manifest, - MemTableCopy} = pcl_registersnapshot(SrcPenciller, self()), - - {ok, #state{memtable_copy=MemTableCopy, - is_snapshot=true, - source_penciller=SrcPenciller, - manifest=Manifest, - ledger_sqn=LedgerSQN}}; + io:format("Registering ledger snapshot~n"), + {ok, State} = pcl_registersnapshot(SrcPenciller, self()), + io:format("Lesger snapshot registered~n"), + {ok, State#state{is_snapshot=true, source_penciller=SrcPenciller}}; %% Need to do something about timeout {_RootPath, false} -> start_from_file(PCLopts) end. -handle_call({push_mem, DumpList}, From, State=#state{is_snapshot=Snap}) - when Snap == false -> - % The process for pushing to memory is as follows - % - Check that the inbound list does not contain any Keys with a lower - % sequence number than any existing keys (assess_sqn/1) - % - Check that any file that had been sent to be written to L0 previously - % is now completed. If it is wipe out the in-memory view as this is now - % safely persisted. This will block waiting for this to complete if it - % hasn't (checkready_pushmem/1). - % - Quick check to see if there is a need to write a L0 file - % (quickcheck_pushmem/3). If there clearly isn't, then we can reply, and - % then add to memory in the background before updating the loop state - % - Push the update into memory (do_pushtomem/3) - % - If we haven't got through quickcheck now need to check if there is a - % definite need to write a new L0 file (roll_memory/3). If all clear this - % will write the file in the background and allow a response to the user. - % If not the change has still been made but the the L0 file will not have - % been prompted - so the reply does not indicate failure but returns the - % atom 'pause' to signal a loose desire for back-pressure to be applied. - StartWatch = os:timestamp(), - case assess_sqn(DumpList) of - {MinSQN, MaxSQN} when MaxSQN >= MinSQN, - MinSQN >= State#state.ledger_sqn -> - case checkready_pushtomem(State) of - {ok, TableSize0, State1} -> - push_and_roll(DumpList, - TableSize0, - State#state.memtable_maxsize, - MaxSQN, - StartWatch, - From, - State1); - timeout -> - io:format("Timeout of ~w microseconds awaiting " ++ - "L0 SFT write~n", - [timer:now_diff(os:timestamp(), StartWatch)]), - {reply, returned, State} - end; - {MinSQN, MaxSQN} -> - io:format("Mismatch of sequence number expectations with push " - ++ "having sequence numbers between ~w and ~w " - ++ "but current sequence number is ~w~n", - [MinSQN, MaxSQN, State#state.ledger_sqn]), - {reply, refused, State}; - empty -> - io:format("Empty request pushed to Penciller~n"), - {reply, ok, State} - end; -handle_call({fetch, Key}, _From, State=#state{is_snapshot=Snap}) +handle_call({push_mem, PushedTree}, From, State=#state{is_snapshot=Snap}) when Snap == false -> + % The push_mem process is as follows: + % + % 1 - Receive a gb_tree containing the latest Key/Value pairs (note that + % we mean value from the perspective of the Ledger, not the full value + % stored in the Inker) + % + % 2 - Check to see if the levelminus1 cache is active, if so the update + % cannot yet be accepted and must be returned (so the Bookie will maintain + % the cache and try again later). + % + % 3 - Check to see if there is a levelzero file pending. If so check if + % the levelzero file is complete. If it is complete, the levelzero tree + % can be flushed, the in-memory manifest updated, and the new tree can + % be accepted as the new levelzero cache. If not, the update must be + % returned. + % + % 3 - Make the new tree the levelminus1 cache, and mark this as active + % + % 4 - The Penciller can now reply to the Bookie to show that the push has + % been accepted + % + % 5 - A background worker clerk can now be triggered to produce a new + % levelzero cache (tree) containing the level minus 1 tree. When this + % completes it will cast back the updated tree, and on receipt of this + % the Penciller may: + % a) Clear down the levelminus1 cache + % b) Determine if the cache is full and it is necessary to build a new + % persisted levelzero file + + SW = os:timestamp(), + + S = case {State#state.levelzero_pending, + State#state.levelminus1_active} of + {_, true} -> + log_pushmem_reply(From, {returned, "L-1 Active"}, SW), + State; + {true, _} -> + L0Pid = State#state.levelzero_constructor, + case checkready(L0Pid) of + timeout -> + log_pushmem_reply(From, + {returned, + "L-0 persist pending"}, + SW), + State; + {ok, SrcFN, StartKey, EndKey} -> + log_pushmem_reply(From, ok, SW), + ManEntry = #manifest_entry{start_key=StartKey, + end_key=EndKey, + owner=L0Pid, + filename=SrcFN}, + % Prompt clerk to ask about work - do this for + % every L0 roll + ok = leveled_pclerk:mergeclerk_prompt(State#state.merge_clerk), + UpdMan = lists:keystore(0, + 1, + State#state.manifest, + {0, [ManEntry]}), + {MinSQN, MaxSQN, Size, _L} = assess_sqn(PushedTree), + if + MinSQN > State#state.ledger_sqn -> + State#state{manifest=UpdMan, + levelzero_cache=PushedTree, + levelzero_cachesize=Size, + levelzero_pending=false, + ledger_sqn=MaxSQN} + end + end; + {false, false} -> + log_pushmem_reply(From, ok, SW), + ok = leveled_pclerk:rollclerk_levelzero(State#state.roll_clerk, + State#state.levelzero_cache, + PushedTree, + State#state.ledger_sqn, + self()), + State#state{levelminus1_active=true, + levelminus1_cache=PushedTree} + end, + io:format("Handling of push completed in ~w microseconds with " + ++ "L0 cache size now ~w~n", + [timer:now_diff(os:timestamp(), SW), + S#state.levelzero_cachesize]), + {noreply, S}; +handle_call({fetch, Key}, _From, State) -> {reply, fetch(Key, State#state.manifest, - State#state.memtable), + State#state.levelminus1_active, + State#state.levelminus1_cache, + State#state.levelzero_cache), State}; -handle_call({fetch, Key}, - _From, - State=#state{snapshot_fully_loaded=Ready}) - when Ready == true -> +handle_call({check_sqn, Key, SQN}, _From, State) -> {reply, - fetch_snap(Key, - State#state.manifest, - State#state.levelzero_snapshot), - State}; -handle_call({check_sqn, Key, SQN}, - _From, - State=#state{snapshot_fully_loaded=Ready}) - when Ready == true -> - {reply, - compare_to_sqn(fetch_snap(Key, - State#state.manifest, - State#state.levelzero_snapshot), + compare_to_sqn(fetch(Key, + State#state.manifest, + State#state.levelminus1_active, + State#state.levelminus1_cache, + State#state.levelzero_cache), SQN), State}; handle_call({fetch_keys, StartKey, EndKey, AccFun, InitAcc}, _From, State=#state{snapshot_fully_loaded=Ready}) when Ready == true -> - L0iter = gb_trees:iterator_from(StartKey, State#state.levelzero_snapshot), + L0iter = gb_trees:iterator_from(StartKey, State#state.levelzero_cache), SFTiter = initiate_rangequery_frommanifest(StartKey, EndKey, State#state.manifest), @@ -480,47 +462,41 @@ handle_call(get_startup_sqn, _From, State) -> {reply, State#state.ledger_sqn, State}; handle_call({register_snapshot, Snapshot}, _From, State) -> Rs = [{Snapshot, State#state.manifest_sqn}|State#state.registered_snapshots], - {reply, - {ok, - State#state.ledger_sqn, - State#state.manifest, - State#state.memtable_copy}, - State#state{registered_snapshots = Rs}}; -handle_call({load_snapshot, Increment}, _From, State) -> - MemTableCopy = State#state.memtable_copy, - {Tree0, TreeSQN0} = roll_new_tree(MemTableCopy#l0snapshot.tree, - MemTableCopy#l0snapshot.increments, - MemTableCopy#l0snapshot.ledger_sqn), - if - TreeSQN0 > MemTableCopy#l0snapshot.ledger_sqn -> - pcl_updatesnapshotcache(State#state.source_penciller, - Tree0, - TreeSQN0); - true -> - io:format("No update required to snapshot cache~n"), - ok - end, - {Tree1, TreeSQN1} = roll_new_tree(Tree0, [Increment], TreeSQN0), - io:format("Snapshot loaded with increments to start at SQN=~w~n", - [TreeSQN1]), - {reply, ok, State#state{levelzero_snapshot=Tree1, - ledger_sqn=TreeSQN1, + {reply, {ok, State}, State#state{registered_snapshots = Rs}}; +handle_call({load_snapshot, BookieIncrTree}, _From, State) -> + {L0T0, _, L0SQN0} = case State#state.levelminus1_active of + true -> + roll_new_tree(State#state.levelzero_cache, + State#state.levelminus1_cache, + State#state.ledger_sqn); + false -> + {State#state.levelzero_cache, + 0, + State#state.ledger_sqn} + end, + {L0T1, _, L0SQN1} = roll_new_tree(L0T0, + BookieIncrTree, + L0SQN0), + io:format("Ledger snapshot loaded with increments to start at SQN=~w~n", + [L0SQN1]), + {reply, ok, State#state{levelzero_cache=L0T1, + ledger_sqn=L0SQN1, + levelminus1_active=false, snapshot_fully_loaded=true}}; handle_call(close, _From, State) -> {stop, normal, ok, State}. -handle_cast({update_snapshotcache, Tree, SQN}, State) -> - MemTableC = cache_tree_in_memcopy(State#state.memtable_copy, Tree, SQN), - {noreply, State#state{memtable_copy=MemTableC}}; + handle_cast({manifest_change, WI}, State) -> {ok, UpdState} = commit_manifest_change(WI, State), - ok = leveled_pclerk:clerk_manifestchange(State#state.clerk, - confirm, - false), + ok = leveled_pclerk:mergeclerk_manifestchange(State#state.merge_clerk, + confirm, + false), {noreply, UpdState}; handle_cast({release_snapshot, Snapshot}, State) -> Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots), - io:format("Penciller snapshot ~w released~n", [Snapshot]), + io:format("Ledger snapshot ~w released~n", [Snapshot]), + io:format("Remaining ledger snapshots are ~w~n", [Rs]), {noreply, State#state{registered_snapshots=Rs}}; handle_cast({confirm_delete, FileName}, State=#state{is_snapshot=Snap}) when Snap == false -> @@ -537,6 +513,37 @@ handle_cast({confirm_delete, FileName}, State=#state{is_snapshot=Snap}) {noreply, State#state{unreferenced_files=UF1}}; _ -> {noreply, State} + end; +handle_cast({load_levelzero, L0Cache, L0Size, L0SQN}, State) -> + if + L0SQN >= State#state.ledger_sqn -> + CacheTooBig = L0Size > State#state.levelzero_maxcachesize, + Level0Free = length(get_item(0, State#state.manifest, [])) == 0, + case {CacheTooBig, Level0Free} of + {true, true} -> + L0Constructor = roll_memory(State, L0Cache), + {noreply, + State#state{levelminus1_active=false, + levelminus1_cache=gb_trees:empty(), + levelzero_cache=L0Cache, + levelzero_cachesize=L0Size, + levelzero_pending=true, + levelzero_constructor=L0Constructor, + ledger_sqn=L0SQN}}; + _ -> + {noreply, + State#state{levelminus1_active=false, + levelminus1_cache=gb_trees:empty(), + levelzero_cache=L0Cache, + levelzero_cachesize=L0Size, + ledger_sqn=L0SQN}} + end; + L0Size == 0 -> + {noreply, + State#state{levelminus1_active=false, + levelminus1_cache=gb_trees:empty(), + levelzero_cache=L0Cache, + levelzero_cachesize=L0Size}} end. handle_info({_Ref, {ok, SrcFN, _StartKey, _EndKey}}, State) -> @@ -566,51 +573,34 @@ terminate(Reason, State) -> %% the penciller looking for a manifest commit %% io:format("Penciller closing for reason - ~w~n", [Reason]), - MC = leveled_pclerk:clerk_manifestchange(State#state.clerk, - return, - true), + MC = leveled_pclerk:mergeclerk_manifestchange(State#state.merge_clerk, + return, + true), UpdState = case MC of {ok, WI} -> {ok, NewState} = commit_manifest_change(WI, State), - Clerk = State#state.clerk, - ok = leveled_pclerk:clerk_manifestchange(Clerk, - confirm, - true), + Clerk = State#state.merge_clerk, + ok = leveled_pclerk:mergeclerk_manifestchange(Clerk, + confirm, + true), NewState; no_change -> State end, - Dump = roll_into_list(State#state.memtable_copy), case {UpdState#state.levelzero_pending, - get_item(0, UpdState#state.manifest, []), - length(Dump)} of - {?L0PEND_RESET, [], L} when L > 0 -> - MSN = UpdState#state.manifest_sqn, - FileName = UpdState#state.root_path - ++ "/" ++ ?FILES_FP ++ "/" - ++ integer_to_list(MSN) ++ "_0_0", - NewSFT = leveled_sft:sft_new(FileName ++ ".pnd", - Dump, - [], - 0), - {ok, L0Pid, {{[], []}, _SK, _HK}} = NewSFT, - io:format("Dump of memory on close to filename ~s~n", - [FileName]), - leveled_sft:sft_close(L0Pid), - file:rename(FileName ++ ".pnd", FileName ++ ".sft"); - {?L0PEND_RESET, [], L} when L == 0 -> - io:format("No keys to dump from memory when closing~n"); - {{true, L0Pid, _TS}, _, _} -> - leveled_sft:sft_close(L0Pid), - io:format("No opportunity to persist memory before closing" - ++ " with ~w keys discarded~n", - [length(Dump)]); + get_item(0, State#state.manifest, [])} of + {true, []} -> + ok = leveled_sft:sft_close(State#state.levelzero_constructor); + {false, []} -> + KL = roll_into_list(State#state.levelzero_cache), + L0Pid = roll_memory(UpdState, KL, true), + ok = leveled_sft:sft_close(L0Pid); _ -> - io:format("No opportunity to persist memory before closing" - ++ " with ~w keys discarded~n", - [length(Dump)]) + ok end, + leveled_pclerk:rollclerk_close(State#state.roll_clerk), + % Tidy shutdown of individual files ok = close_files(0, UpdState#state.manifest), lists:foreach(fun({_FN, Pid, _SN}) -> @@ -636,14 +626,13 @@ start_from_file(PCLopts) -> M -> M end, - % There is no need to export this ets table (hence private) or iterate - % over it (hence set not ordered_set) - TID = ets:new(?MEMTABLE, [set, private]), - {ok, Clerk} = leveled_pclerk:clerk_new(self()), - InitState = #state{memtable=TID, - clerk=Clerk, + + {ok, MergeClerk} = leveled_pclerk:clerk_new(self(), merge), + {ok, RollClerk} = leveled_pclerk:clerk_new(self(), roll), + InitState = #state{merge_clerk=MergeClerk, + roll_clerk=RollClerk, root_path=RootPath, - memtable_maxsize=MaxTableSize}, + levelzero_maxcachesize=MaxTableSize}, %% Open manifest ManifestPath = InitState#state.root_path ++ "/" ++ ?MANIFEST_FP ++ "/", @@ -672,84 +661,60 @@ start_from_file(PCLopts) -> ValidManSQNs), io:format("Store to be started based on " ++ "manifest sequence number of ~w~n", [TopManSQN]), - case TopManSQN of - 0 -> - io:format("Seqence number of 0 indicates no valid manifest~n"), - {ok, InitState}; - _ -> - {ok, Bin} = file:read_file(filepath(InitState#state.root_path, - TopManSQN, - current_manifest)), - Manifest = binary_to_term(Bin), - {UpdManifest, MaxSQN} = open_all_filesinmanifest(Manifest), - io:format("Maximum sequence number of ~w " - ++ "found in nonzero levels~n", - [MaxSQN]), + ManUpdate = case TopManSQN of + 0 -> + io:format("Seqence number of 0 indicates no valid " ++ + "manifest~n"), + {[], 0}; + _ -> + CurrManFile = filepath(InitState#state.root_path, + TopManSQN, + current_manifest), + {ok, Bin} = file:read_file(CurrManFile), + Manifest = binary_to_term(Bin), + open_all_filesinmanifest(Manifest) + end, + + {UpdManifest, MaxSQN} = ManUpdate, + io:format("Maximum sequence number of ~w found in nonzero levels~n", + [MaxSQN]), - %% Find any L0 files - L0FN = filepath(RootPath, - TopManSQN, - new_merge_files) ++ "_0_0.sft", - case filelib:is_file(L0FN) of - true -> - io:format("L0 file found ~s~n", [L0FN]), - {ok, - L0Pid, - {L0StartKey, L0EndKey}} = leveled_sft:sft_open(L0FN), - L0SQN = leveled_sft:sft_getmaxsequencenumber(L0Pid), - ManifestEntry = #manifest_entry{start_key=L0StartKey, - end_key=L0EndKey, - owner=L0Pid, - filename=L0FN}, - UpdManifest2 = lists:keystore(0, - 1, - UpdManifest, - {0, [ManifestEntry]}), - io:format("L0 file had maximum sequence number of ~w~n", - [L0SQN]), - {ok, - InitState#state{manifest=UpdManifest2, - manifest_sqn=TopManSQN, - ledger_sqn=max(MaxSQN, L0SQN)}}; - false -> - io:format("No L0 file found~n"), - {ok, - InitState#state{manifest=UpdManifest, - manifest_sqn=TopManSQN, - ledger_sqn=MaxSQN}} - end + %% Find any L0 files + L0FN = filepath(RootPath, TopManSQN, new_merge_files) ++ "_0_0.sft", + case filelib:is_file(L0FN) of + true -> + io:format("L0 file found ~s~n", [L0FN]), + {ok, + L0Pid, + {L0StartKey, L0EndKey}} = leveled_sft:sft_open(L0FN), + L0SQN = leveled_sft:sft_getmaxsequencenumber(L0Pid), + ManifestEntry = #manifest_entry{start_key=L0StartKey, + end_key=L0EndKey, + owner=L0Pid, + filename=L0FN}, + UpdManifest2 = lists:keystore(0, + 1, + UpdManifest, + {0, [ManifestEntry]}), + io:format("L0 file had maximum sequence number of ~w~n", + [L0SQN]), + {ok, + InitState#state{manifest=UpdManifest2, + manifest_sqn=TopManSQN, + ledger_sqn=max(MaxSQN, L0SQN)}}; + false -> + io:format("No L0 file found~n"), + {ok, + InitState#state{manifest=UpdManifest, + manifest_sqn=TopManSQN, + ledger_sqn=MaxSQN}} end. -checkready_pushtomem(State) -> - case State#state.levelzero_pending of - {true, Pid, _TS} -> - case checkready(Pid) of - timeout -> - timeout; - {ok, SrcFN, StartKey, EndKey} -> - true = ets:delete_all_objects(State#state.memtable), - ManifestEntry = #manifest_entry{start_key=StartKey, - end_key=EndKey, - owner=Pid, - filename=SrcFN}, - % Prompt clerk to ask about work - do this for every - % L0 roll - ok = leveled_pclerk:clerk_prompt(State#state.clerk), - UpdManifest = lists:keystore(0, - 1, - State#state.manifest, - {0, [ManifestEntry]}), - {ok, - 0, - State#state{manifest=UpdManifest, - levelzero_pending=?L0PEND_RESET, - memtable_copy=#l0snapshot{}}} - end; - ?L0PEND_RESET -> - {ok, State#state.table_size, State} - end. - +log_pushmem_reply(From, Reply, SW) -> + io:format("Respone to push_mem of ~w took ~w microseconds~n", + [Reply, timer:now_diff(os:timestamp(), SW)]), + gen_server:reply(From, Reply). checkready(Pid) -> try @@ -759,114 +724,71 @@ checkready(Pid) -> timeout end. +roll_memory(State, L0Tree) -> + roll_memory(State, L0Tree, false). -push_and_roll(DumpList, TableSize, MaxTableSize, MaxSQN, StartWatch, From, State) -> - case quickcheck_pushtomem(DumpList, TableSize, MaxTableSize) of - {twist, TableSize1} -> - gen_server:reply(From, ok), - io:format("Reply made on push in ~w microseconds~n", - [timer:now_diff(os:timestamp(), StartWatch)]), - L0Snap = do_pushtomem(DumpList, - State#state.memtable, - State#state.memtable_copy, - MaxSQN), - io:format("Push completed in ~w microseconds~n", - [timer:now_diff(os:timestamp(), StartWatch)]), - {noreply, - State#state{memtable_copy=L0Snap, - table_size=TableSize1, - ledger_sqn=MaxSQN}}; - {maybe_roll, TableSize1} -> - L0Snap = do_pushtomem(DumpList, - State#state.memtable, - State#state.memtable_copy, - MaxSQN), +roll_memory(State, L0Tree, Wait) -> + MSN = State#state.manifest_sqn, + FileName = State#state.root_path + ++ "/" ++ ?FILES_FP ++ "/" + ++ integer_to_list(MSN) ++ "_0_0", + Opts = #sft_options{wait=Wait}, + {ok, Constructor, _} = leveled_sft:sft_new(FileName, L0Tree, [], 0, Opts), + Constructor. - case roll_memory(State, MaxTableSize, L0Snap) of - {ok, L0Pend, ManSN, TableSize2} -> - io:format("Push completed in ~w microseconds~n", - [timer:now_diff(os:timestamp(), StartWatch)]), - {reply, - ok, - State#state{levelzero_pending=L0Pend, - table_size=TableSize2, - manifest_sqn=ManSN, - memtable_copy=L0Snap, - ledger_sqn=MaxSQN}}; - {pause, Reason, Details} -> - io:format("Excess work due to - " ++ Reason, - Details), - {reply, - pause, - State#state{memtable_copy=L0Snap, - table_size=TableSize1, - ledger_sqn=MaxSQN}} - end +% Merge the Level Minus 1 tree to the Level 0 tree, incrementing the +% SQN, and ensuring all entries do increment the SQN + +roll_new_tree(L0Cache, LMinus1Cache, LedgerSQN) -> + {MinSQN, MaxSQN, Size, LMinus1List} = assess_sqn(LMinus1Cache), + if + MinSQN >= LedgerSQN -> + UpdTree = lists:foldl(fun({Kx, Vx}, TreeAcc) -> + gb_trees:enter(Kx, Vx, TreeAcc) + end, + L0Cache, + LMinus1List), + {UpdTree, gb_trees:size(UpdTree), MaxSQN}; + Size == 0 -> + {L0Cache, gb_trees:size(L0Cache), LedgerSQN} end. -quickcheck_pushtomem(DumpList, TableSize, MaxSize) -> - case TableSize + length(DumpList) of - ApproxTableSize when ApproxTableSize > MaxSize -> - {maybe_roll, ApproxTableSize}; - ApproxTableSize -> - io:format("Table size is approximately ~w~n", [ApproxTableSize]), - {twist, ApproxTableSize} - end. +%% This takes the three parts of a memtable copy - the increments, the tree +%% and the SQN at which the tree was formed, and outputs a sorted list +roll_into_list(Tree) -> + gb_trees:to_list(Tree). -do_pushtomem(DumpList, MemTable, Snapshot, MaxSQN) -> - SW = os:timestamp(), - UpdSnapshot = add_increment_to_memcopy(Snapshot, MaxSQN, DumpList), - % Note that the DumpList must have been taken from a source which - % naturally de-duplicates the keys. It is not possible just to cache - % changes in a list (in the Bookie for example), as the insert method does - % not apply the list in order, and so it is not clear which of a duplicate - % key will be applied - ets:insert(MemTable, DumpList), - io:format("Push into memory timed at ~w microseconds~n", - [timer:now_diff(os:timestamp(), SW)]), - UpdSnapshot. +assess_sqn(Tree) -> + L = roll_into_list(Tree), + FoldFun = fun(KV, {AccMinSQN, AccMaxSQN, AccSize}) -> + SQN = leveled_codec:strip_to_seqonly(KV), + {min(SQN, AccMinSQN), max(SQN, AccMaxSQN), AccSize + 1} + end, + {MinSQN, MaxSQN, Size} = lists:foldl(FoldFun, {infinity, 0, 0}, L), + {MinSQN, MaxSQN, Size, L}. -roll_memory(State, MaxSize, MemTableCopy) -> - case ets:info(State#state.memtable, size) of - Size when Size > MaxSize -> - L0 = get_item(0, State#state.manifest, []), - case L0 of - [] -> - MSN = State#state.manifest_sqn, - FileName = State#state.root_path - ++ "/" ++ ?FILES_FP ++ "/" - ++ integer_to_list(MSN) ++ "_0_0", - Opts = #sft_options{wait=false}, - {ok, L0Pid} = leveled_sft:sft_new(FileName, - MemTableCopy, - [], - 0, - Opts), - {ok, {true, L0Pid, os:timestamp()}, MSN, Size}; - _ -> - {pause, - "L0 file write blocked by L0 file in manifest~n", - []} + +fetch(Key, Manifest, LM1Active, LM1Tree, L0Tree) -> + case LM1Active of + true -> + case gb_trees:lookup(Key, LM1Tree) of + none -> + case gb_trees:lookup(Key, L0Tree) of + none -> + fetch(Key, Manifest, 0, fun leveled_sft:sft_get/2); + {value, Value} -> + {Key, Value} + end; + {value, Value} -> + {Key, Value} end; - Size -> - {ok, ?L0PEND_RESET, State#state.manifest_sqn, Size} - end. - - -fetch_snap(Key, Manifest, Tree) -> - case gb_trees:lookup(Key, Tree) of - {value, Value} -> - {Key, Value}; - none -> - fetch(Key, Manifest, 0, fun leveled_sft:sft_get/2) - end. - -fetch(Key, Manifest, TID) -> - case ets:lookup(TID, Key) of - [Object] -> - Object; - [] -> - fetch(Key, Manifest, 0, fun leveled_sft:sft_get/2) + false -> + case gb_trees:lookup(Key, L0Tree) of + none -> + fetch(Key, Manifest, 0, fun leveled_sft:sft_get/2); + {value, Value} -> + {Key, Value} + end end. fetch(_Key, _Manifest, ?MAX_LEVELS + 1, _FetchFun) -> @@ -934,13 +856,13 @@ return_work(State, From) -> true -> false end, - case element(1, State#state.levelzero_pending) of + case State#state.levelzero_pending of true -> % Once the L0 file is completed there will be more work % - so don't be busy doing other work now io:format("Allocation of work blocked as L0 pending~n"), {State, none}; - _ -> + false -> %% No work currently outstanding %% Can allocate work NextSQN = State#state.manifest_sqn + 1, @@ -965,57 +887,6 @@ return_work(State, From) -> end. -%% This takes the three parts of a memtable copy - the increments, the tree -%% and the SQN at which the tree was formed, and outputs a new tree -roll_new_tree(Tree, [], HighSQN) -> - {Tree, HighSQN}; -roll_new_tree(Tree, [{SQN, KVList}|TailIncs], HighSQN) when SQN >= HighSQN -> - R = lists:foldl(fun({Kx, Vx}, {TreeAcc, MaxSQN}) -> - UpdTree = gb_trees:enter(Kx, Vx, TreeAcc), - SQNx = leveled_codec:strip_to_seqonly({Kx, Vx}), - {UpdTree, max(SQNx, MaxSQN)} - end, - {Tree, HighSQN}, - KVList), - {UpdTree, UpdSQN} = R, - roll_new_tree(UpdTree, TailIncs, UpdSQN); -roll_new_tree(Tree, [_H|TailIncs], HighSQN) -> - roll_new_tree(Tree, TailIncs, HighSQN). - -%% This takes the three parts of a memtable copy - the increments, the tree -%% and the SQN at which the tree was formed, and outputs a sorted list -roll_into_list(MemTableCopy) -> - {Tree, _SQN} = roll_new_tree(MemTableCopy#l0snapshot.tree, - MemTableCopy#l0snapshot.increments, - MemTableCopy#l0snapshot.ledger_sqn), - gb_trees:to_list(Tree). - -%% Update the memtable copy if the tree created advances the SQN -cache_tree_in_memcopy(MemCopy, Tree, SQN) -> - case MemCopy#l0snapshot.ledger_sqn of - CurrentSQN when SQN > CurrentSQN -> - % Discard any merged increments - io:format("Updating cache with new tree at SQN=~w~n", [SQN]), - Incs = lists:foldl(fun({PushSQN, PushL}, Acc) -> - if - SQN >= PushSQN -> - Acc; - true -> - Acc ++ [{PushSQN, PushL}] - end end, - [], - MemCopy#l0snapshot.increments), - #l0snapshot{ledger_sqn = SQN, - increments = Incs, - tree = Tree}; - _CurrentSQN -> - MemCopy - end. - -add_increment_to_memcopy(MemCopy, SQN, KVList) -> - Incs = MemCopy#l0snapshot.increments ++ [{SQN, KVList}], - MemCopy#l0snapshot{increments=Incs}. - close_files(?MAX_LEVELS - 1, _Manifest) -> ok; close_files(Level, Manifest) -> @@ -1371,7 +1242,6 @@ commit_manifest_change(ReturnedWorkItem, State) -> NewManifest, {0, [L0ManEntry]}) end, - print_manifest(RevisedManifest), {ok, State#state{ongoing_work=[], manifest_sqn=NewMSN, manifest=RevisedManifest, @@ -1437,16 +1307,6 @@ confirm_delete(Filename, UnreferencedFiles, RegisteredSnapshots) -> end. -assess_sqn([]) -> - empty; -assess_sqn(DumpList) -> - assess_sqn(DumpList, infinity, 0). - -assess_sqn([], MinSQN, MaxSQN) -> - {MinSQN, MaxSQN}; -assess_sqn([HeadKey|Tail], MinSQN, MaxSQN) -> - SQN = leveled_codec:strip_to_seqonly(HeadKey), - assess_sqn(Tail, min(MinSQN, SQN), max(MaxSQN, SQN)). %%%============================================================================ %%% Test @@ -1473,12 +1333,6 @@ clean_subdir(DirPath) -> ok end. -assess_sqn_test() -> - L1 = [{{}, {5, active, {}}}, {{}, {6, active, {}}}], - ?assertMatch({5, 6}, assess_sqn(L1)), - L2 = [{{}, {5, active, {}}}], - ?assertMatch({5, 5}, assess_sqn(L2)), - ?assertMatch(empty, assess_sqn([])). compaction_work_assessment_test() -> L0 = [{{o, "B1", "K1", null}, {o, "B3", "K3", null}, dummy_pid}], @@ -1522,15 +1376,15 @@ confirm_delete_test() -> maybe_pause_push(PCL, KL) -> - R = pcl_pushmem(PCL, KL), - if - R == pause -> - io:format("Pausing push~n"), - timer:sleep(500), - ok; - R == returned -> + T0 = gb_trees:empty(), + T1 = lists:foldl(fun({K, V}, Acc) -> gb_trees:enter(K, V, Acc) end, + T0, + KL), + case pcl_pushmem(PCL, T1) of + {returned, _Reason} -> + timer:sleep(50), maybe_pause_push(PCL, KL); - true -> + ok -> ok end. @@ -1542,28 +1396,35 @@ simple_server_test() -> Key1 = {{o,"Bucket0001", "Key0001", null}, {1, {active, infinity}, null}}, KL1 = leveled_sft:generate_randomkeys({1000, 2}), Key2 = {{o,"Bucket0002", "Key0002", null}, {1002, {active, infinity}, null}}, - KL2 = leveled_sft:generate_randomkeys({1000, 1002}), - Key3 = {{o,"Bucket0003", "Key0003", null}, {2002, {active, infinity}, null}}, - KL3 = leveled_sft:generate_randomkeys({1000, 2002}), - Key4 = {{o,"Bucket0004", "Key0004", null}, {3002, {active, infinity}, null}}, - KL4 = leveled_sft:generate_randomkeys({1000, 3002}), - ok = pcl_pushmem(PCL, [Key1]), + KL2 = leveled_sft:generate_randomkeys({1000, 1003}), + Key3 = {{o,"Bucket0003", "Key0003", null}, {2003, {active, infinity}, null}}, + KL3 = leveled_sft:generate_randomkeys({1000, 2004}), + Key4 = {{o,"Bucket0004", "Key0004", null}, {3004, {active, infinity}, null}}, + KL4 = leveled_sft:generate_randomkeys({1000, 3005}), + ok = maybe_pause_push(PCL, [Key1]), ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), - ok = pcl_pushmem(PCL, KL1), + ok = maybe_pause_push(PCL, KL1), ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), ok = maybe_pause_push(PCL, [Key2]), ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), + ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), + ok = maybe_pause_push(PCL, KL2), + ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), ok = maybe_pause_push(PCL, [Key3]), ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), ?assertMatch(Key3, pcl_fetch(PCL, {o,"Bucket0003", "Key0003", null})), ok = pcl_close(PCL), + {ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath, max_inmemory_tablesize=1000}), - ?assertMatch(2002, pcl_getstartupsequencenumber(PCLr)), + ?assertMatch(1001, pcl_getstartupsequencenumber(PCLr)), + ok = maybe_pause_push(PCLr, [Key2] ++ KL2 ++ [Key3]), + io:format("Back to starting position with lost data recovered~n"), + ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002", null})), ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003", null})), @@ -1577,7 +1438,7 @@ simple_server_test() -> SnapOpts = #penciller_options{start_snapshot = true, source_penciller = PCLr}, {ok, PclSnap} = pcl_start(SnapOpts), - ok = pcl_loadsnapshot(PclSnap, []), + ok = pcl_loadsnapshot(PclSnap, gb_trees:empty()), ?assertMatch(Key1, pcl_fetch(PclSnap, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key2, pcl_fetch(PclSnap, {o,"Bucket0002", "Key0002", null})), ?assertMatch(Key3, pcl_fetch(PclSnap, {o,"Bucket0003", "Key0003", null})), @@ -1599,18 +1460,18 @@ simple_server_test() -> "Bucket0003", "Key0003", null}, - 2002)), + 2003)), ?assertMatch(true, pcl_checksequencenumber(PclSnap, {o, "Bucket0004", "Key0004", null}, - 3002)), + 3004)), % Add some more keys and confirm that check sequence number still % sees the old version in the previous snapshot, but will see the new version % in a new snapshot - Key1A = {{o,"Bucket0001", "Key0001", null}, {4002, {active, infinity}, null}}, - KL1A = leveled_sft:generate_randomkeys({4002, 2}), + Key1A = {{o,"Bucket0001", "Key0001", null}, {4005, {active, infinity}, null}}, + KL1A = leveled_sft:generate_randomkeys({2000, 4006}), ok = maybe_pause_push(PCLr, [Key1A]), ok = maybe_pause_push(PCLr, KL1A), ?assertMatch(true, pcl_checksequencenumber(PclSnap, @@ -1620,8 +1481,9 @@ simple_server_test() -> null}, 1)), ok = pcl_close(PclSnap), + {ok, PclSnap2} = pcl_start(SnapOpts), - ok = pcl_loadsnapshot(PclSnap2, []), + ok = pcl_loadsnapshot(PclSnap2, gb_trees:empty()), ?assertMatch(false, pcl_checksequencenumber(PclSnap2, {o, "Bucket0001", @@ -1633,7 +1495,7 @@ simple_server_test() -> "Bucket0001", "Key0001", null}, - 4002)), + 4005)), ?assertMatch(true, pcl_checksequencenumber(PclSnap2, {o, "Bucket0002", @@ -1644,55 +1506,6 @@ simple_server_test() -> ok = pcl_close(PCLr), clean_testdir(RootPath). -memcopy_updatecache1_test() -> - KVL1 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), - {X, null, "Val" ++ integer_to_list(X) ++ "A"}} - end, - lists:seq(1, 1000)), - KVL2 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), - {X, null, "Val" ++ integer_to_list(X) ++ "B"}} - end, - lists:seq(1001, 2000)), - KVL3 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), - {X, null, "Val" ++ integer_to_list(X) ++ "C"}} - end, - lists:seq(2001, 3000)), - MemCopy0 = #l0snapshot{}, - MemCopy1 = add_increment_to_memcopy(MemCopy0, 1000, KVL1), - MemCopy2 = add_increment_to_memcopy(MemCopy1, 2000, KVL2), - MemCopy3 = add_increment_to_memcopy(MemCopy2, 3000, KVL3), - ?assertMatch(0, MemCopy3#l0snapshot.ledger_sqn), - {Tree1, HighSQN1} = roll_new_tree(gb_trees:empty(), MemCopy3#l0snapshot.increments, 0), - MemCopy4 = cache_tree_in_memcopy(MemCopy3, Tree1, HighSQN1), - ?assertMatch(0, length(MemCopy4#l0snapshot.increments)), - Size2 = gb_trees:size(MemCopy4#l0snapshot.tree), - ?assertMatch(3000, Size2), - ?assertMatch(3000, MemCopy4#l0snapshot.ledger_sqn). - -memcopy_updatecache2_test() -> - KVL1 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), - {X, null, "Val" ++ integer_to_list(X) ++ "A"}} - end, - lists:seq(1, 1000)), - KVL2 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), - {X, null, "Val" ++ integer_to_list(X) ++ "B"}} - end, - lists:seq(1001, 2000)), - KVL3 = lists:map(fun(X) -> {"Key" ++ integer_to_list(X), - {X, null, "Val" ++ integer_to_list(X) ++ "C"}} - end, - lists:seq(1, 1000)), - MemCopy0 = #l0snapshot{}, - MemCopy1 = add_increment_to_memcopy(MemCopy0, 1000, KVL1), - MemCopy2 = add_increment_to_memcopy(MemCopy1, 2000, KVL2), - MemCopy3 = add_increment_to_memcopy(MemCopy2, 3000, KVL3), - ?assertMatch(0, MemCopy3#l0snapshot.ledger_sqn), - {Tree1, HighSQN1} = roll_new_tree(gb_trees:empty(), MemCopy3#l0snapshot.increments, 0), - MemCopy4 = cache_tree_in_memcopy(MemCopy3, Tree1, HighSQN1), - ?assertMatch(1, length(MemCopy4#l0snapshot.increments)), - Size2 = gb_trees:size(MemCopy4#l0snapshot.tree), - ?assertMatch(2000, Size2), - ?assertMatch(2000, MemCopy4#l0snapshot.ledger_sqn). rangequery_manifest_test() -> {E1, diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 4649f5d..9cebef2 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -234,7 +234,7 @@ sft_new(Filename, KL1, KL2, LevelInfo, Options) -> false -> gen_server:cast(Pid, {sft_new, Filename, KL1, KL2, LevelR}), - {ok, Pid} + {ok, Pid, noreply} end. sft_open(Filename) -> @@ -532,25 +532,29 @@ complete_file(Handle, FileMD, KL1, KL2, LevelR, Rename) -> false -> open_file(FileMD); {true, OldName, NewName} -> - io:format("Renaming file from ~s to ~s~n", [OldName, NewName]), - case filelib:is_file(NewName) of - true -> - io:format("Filename ~s already exists~n", - [NewName]), - AltName = filename:join(filename:dirname(NewName), - filename:basename(NewName)) - ++ ?DISCARD_EXT, - io:format("Rename rogue filename ~s to ~s~n", - [NewName, AltName]), - ok = file:rename(NewName, AltName); - false -> - ok - end, - ok = file:rename(OldName, NewName), + ok = rename_file(OldName, NewName), open_file(FileMD#state{filename=NewName}) end, {ReadHandle, UpdFileMD, KeyRemainders}. +rename_file(OldName, NewName) -> + io:format("Renaming file from ~s to ~s~n", [OldName, NewName]), + case filelib:is_file(NewName) of + true -> + io:format("Filename ~s already exists~n", + [NewName]), + AltName = filename:join(filename:dirname(NewName), + filename:basename(NewName)) + ++ ?DISCARD_EXT, + io:format("Rename rogue filename ~s to ~s~n", + [NewName, AltName]), + ok = file:rename(NewName, AltName); + false -> + ok + end, + file:rename(OldName, NewName). + + %% Fetch a Key and Value from a file, returns %% {value, KV} or not_present %% The key must be pre-checked to ensure it is in the valid range for the file From c6ca973517e73a6a712acaaf8e7a3d8b5a63d6f7 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 27 Oct 2016 21:40:43 +0100 Subject: [PATCH 099/167] Penciller shutdown when empty Stop the penciller from writing an empty file, when shutting down and the L0 Cache is empty. Also parameter fiddle to see impact of the Penciller changes. --- src/leveled_bookie.erl | 2 +- src/leveled_penciller.erl | 11 +++++++---- test/end_to_end/basic_SUITE.erl | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index f89e399..af8b8ed 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -157,7 +157,7 @@ -define(SHUTDOWN_WAITS, 60). -define(SHUTDOWN_PAUSE, 10000). -define(SNAPSHOT_TIMEOUT, 300000). --define(JITTER_PROBABILITY, 0.1). +-define(JITTER_PROBABILITY, 0.01). -record(state, {inker :: pid(), penciller :: pid(), diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 2509ad2..698b9b5 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -588,15 +588,18 @@ terminate(Reason, State) -> State end, case {UpdState#state.levelzero_pending, - get_item(0, State#state.manifest, [])} of - {true, []} -> + get_item(0, State#state.manifest, []), + gb_trees:size(State#state.levelzero_cache)} of + {true, [], _} -> ok = leveled_sft:sft_close(State#state.levelzero_constructor); - {false, []} -> + {false, [], 0} -> + io:format("Level 0 cache empty at close of Penciller~n"); + {false, [], _N} -> KL = roll_into_list(State#state.levelzero_cache), L0Pid = roll_memory(UpdState, KL, true), ok = leveled_sft:sft_close(L0Pid); _ -> - ok + io:format("No level zero action on close of Penciller~n") end, leveled_pclerk:rollclerk_close(State#state.roll_clerk), diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 9096bda..5514f10 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -16,7 +16,7 @@ all() -> [ many_put_fetch_head, journal_compaction, fetchput_snapshot, - load_and_count , + load_and_count, load_and_count_withdelete, space_clear_ondelete ]. From 0e4632ee3181aaa3c57a2316fe9552a1410956d0 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 27 Oct 2016 22:23:19 +0100 Subject: [PATCH 100/167] Test correction In one test run the numbe rof files fluctuated but ended at zero. The ending at zero is the importnat thing. --- test/end_to_end/basic_SUITE.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 5514f10..d8193ec 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -493,7 +493,7 @@ space_clear_ondelete(_Config) -> io:format("Bookie has ~w ledger files " ++ "after second close~n", [length(FNsD_L)]), true = PointB_Journals < length(FNsA_J), - true = length(FNsB_L) =< length(FNsA_L), - true = length(FNsC_L) =< length(FNsB_L), - true = length(FNsD_L) =< length(FNsB_L), + true = length(FNsD_L) < length(FNsA_L), + true = length(FNsD_L) < length(FNsB_L), + true = length(FNsD_L) < length(FNsC_L), true = length(FNsD_L) == 0. \ No newline at end of file From cdb01cd24fbb1ebf3c123f8517c40ba4ee1e072f Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 29 Oct 2016 00:52:49 +0100 Subject: [PATCH 101/167] Quality Review Looked through test coverage and dialyzer output and attempted to fill test gaps and strip out untestable code (to let it crash). --- include/leveled.hrl | 8 +- src/leveled_bookie.erl | 12 +- src/leveled_cdb.erl | 384 +++++++++++------------------- src/leveled_inker.erl | 13 +- src/leveled_penciller.erl | 61 ++++- src/leveled_sft.erl | 76 ++---- test/end_to_end/restart_SUITE.erl | 4 +- 7 files changed, 227 insertions(+), 331 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 93e13e3..209a9b7 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -34,7 +34,7 @@ -record(level, {level :: integer(), is_basement = false :: boolean(), - timestamp :: integer()}). + timestamp :: erlang:timestamp()}). -record(manifest_entry, {start_key :: tuple(), @@ -53,7 +53,8 @@ cdb_options :: #cdb_options{}, start_snapshot = false :: boolean(), source_inker :: pid(), - reload_strategy = [] :: list()}). + reload_strategy = [] :: list(), + max_run_length}). -record(penciller_options, {root_path :: string(), @@ -66,7 +67,8 @@ cache_size :: integer(), max_journalsize :: integer(), snapshot_bookie :: pid(), - reload_strategy = [] :: list()}). + reload_strategy = [] :: list(), + max_run_length :: integer()}). -record(iclerk_options, {inker :: pid(), diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index af8b8ed..206e859 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -162,7 +162,6 @@ -record(state, {inker :: pid(), penciller :: pid(), cache_size :: integer(), - back_pressure :: boolean(), ledger_cache :: gb_trees:tree(), is_snapshot :: boolean()}). @@ -293,14 +292,10 @@ handle_call({put, Bucket, Key, Object, IndexSpecs, Tag}, From, State) -> IndexSpecs), Cache0 = addto_ledgercache(Changes, State#state.ledger_cache), gen_server:reply(From, ok), - case maybepush_ledgercache(State#state.cache_size, + {ok, NewCache} = maybepush_ledgercache(State#state.cache_size, Cache0, - State#state.penciller) of - {ok, NewCache} -> - {noreply, State#state{ledger_cache=NewCache, back_pressure=false}}; - {pause, NewCache} -> - {noreply, State#state{ledger_cache=NewCache, back_pressure=true}} - end; + State#state.penciller), + {noreply, State#state{ledger_cache=NewCache}}; handle_call({get, Bucket, Key, Tag}, _From, State) -> LedgerKey = leveled_codec:to_ledgerkey(Bucket, Key, Tag), case fetch_head(LedgerKey, @@ -532,6 +527,7 @@ set_options(Opts) -> LedgerFP = Opts#bookie_options.root_path ++ "/" ++ ?LEDGER_FP, {#inker_options{root_path = JournalFP, reload_strategy = ReloadStrategy, + max_run_length = Opts#bookie_options.max_run_length, cdb_options = #cdb_options{max_size=MaxJournalSize, binary_mode=true}}, #penciller_options{root_path = LedgerFP}}. diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 63e48d3..9730a53 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -57,6 +57,7 @@ starting/3, writer/3, writer/2, + rolling/2, rolling/3, reader/3, reader/2, @@ -89,7 +90,6 @@ -define(DWORD_SIZE, 8). -define(WORD_SIZE, 4). --define(CRC_CHECK, true). -define(MAX_FILE_SIZE, 3221225472). -define(BINARY_MODE, false). -define(BASE_POSITION, 2048). @@ -106,7 +106,8 @@ max_size :: integer(), binary_mode = false :: boolean(), delete_point = 0 :: integer(), - inker :: pid()}). + inker :: pid(), + deferred_delete = false :: boolean()}). %%%============================================================================ @@ -119,21 +120,13 @@ cdb_open_writer(Filename) -> cdb_open_writer(Filename, Opts) -> {ok, Pid} = gen_fsm:start(?MODULE, [Opts], []), - case gen_fsm:sync_send_event(Pid, {open_writer, Filename}, infinity) of - ok -> - {ok, Pid}; - Error -> - Error - end. + ok = gen_fsm:sync_send_event(Pid, {open_writer, Filename}, infinity), + {ok, Pid}. cdb_open_reader(Filename) -> {ok, Pid} = gen_fsm:start(?MODULE, [#cdb_options{}], []), - case gen_fsm:sync_send_event(Pid, {open_reader, Filename}, infinity) of - ok -> - {ok, Pid}; - Error -> - Error - end. + ok = gen_fsm:sync_send_event(Pid, {open_reader, Filename}, infinity), + {ok, Pid}. cdb_get(Pid, Key) -> gen_fsm:sync_send_event(Pid, {get_kv, Key}, infinity). @@ -280,6 +273,8 @@ writer({put_kv, Key, Value}, _From, State) -> last_key=Key, hashtree=HashTree}} end; +writer({mput_kv, []}, _From, State) -> + {reply, ok, writer, State}; writer({mput_kv, KVList}, _From, State) -> Result = mput(State#state.handle, KVList, @@ -321,8 +316,6 @@ rolling({key_check, Key}, _From, State) -> get_mem(Key, State#state.handle, State#state.hashtree, loose_presence), rolling, State}; -rolling(cdb_filename, _From, State) -> - {reply, State#state.filename, rolling, State}; rolling({return_hashtable, IndexList, HashTreeBin}, _From, State) -> Handle = State#state.handle, {ok, BasePos} = file:position(Handle, State#state.last_position), @@ -333,13 +326,27 @@ rolling({return_hashtable, IndexList, HashTreeBin}, _From, State) -> ok = rename_for_read(State#state.filename, NewName), io:format("Opening file for reading with filename ~s~n", [NewName]), {NewHandle, Index, LastKey} = open_for_readonly(NewName), - {reply, ok, reader, State#state{handle=NewHandle, - last_key=LastKey, - filename=NewName, - hash_index=Index}}; + case State#state.deferred_delete of + true -> + {reply, ok, delete_pending, State#state{handle=NewHandle, + last_key=LastKey, + filename=NewName, + hash_index=Index}}; + false -> + {reply, ok, reader, State#state{handle=NewHandle, + last_key=LastKey, + filename=NewName, + hash_index=Index}} + end; rolling(cdb_kill, _From, State) -> {stop, killed, ok, State}. + +rolling({delete_pending, ManSQN, Inker}, State) -> + {next_state, + rolling, + State#state{delete_point=ManSQN, inker=Inker, deferred_delete=true}}. + reader({get_kv, Key}, _From, State) -> {reply, get_withcache(State#state.handle, Key, State#state.hash_index), @@ -347,10 +354,10 @@ reader({get_kv, Key}, _From, State) -> State}; reader({key_check, Key}, _From, State) -> {reply, - get(State#state.handle, - Key, - loose_presence, - State#state.hash_index), + get_withcache(State#state.handle, + Key, + State#state.hash_index, + loose_presence), reader, State}; reader({get_positions, SampleSize}, _From, State) -> @@ -419,10 +426,10 @@ delete_pending({get_kv, Key}, _From, State) -> ?DELETE_TIMEOUT}; delete_pending({key_check, Key}, _From, State) -> {reply, - get(State#state.handle, - Key, - loose_presence, - State#state.hash_index), + get_withcache(State#state.handle, + Key, + State#state.hash_index, + loose_presence), delete_pending, State, ?DELETE_TIMEOUT}. @@ -499,10 +506,10 @@ handle_sync_event(cdb_close, _From, _StateName, State) -> {stop, normal, ok, State#state{handle=undefined}}. handle_event(_Msg, StateName, State) -> - {next_State, StateName, State}. + {next_state, StateName, State}. handle_info(_Msg, StateName, State) -> - {next_State, StateName, State}. + {next_state, StateName, State}. terminate(Reason, StateName, State) -> io:format("Closing of filename ~s for Reason ~w~n", @@ -548,10 +555,8 @@ create(FileName,KeyValueList) -> %% Given a file name, this function returns a list %% of {key,value} tuples from the CDB. %% -dump(FileName) -> - dump(FileName, ?CRC_CHECK). -dump(FileName, CRCCheck) -> +dump(FileName) -> {ok, Handle} = file:open(FileName, [binary, raw, read]), Fn = fun(Index, Acc) -> {ok, _} = file:position(Handle, ?DWORD_SIZE * Index), @@ -564,17 +569,17 @@ dump(FileName, CRCCheck) -> Fn1 = fun(_I,Acc) -> {KL,VL} = read_next_2_integers(Handle), Key = read_next_term(Handle, KL), - case read_next_term(Handle, VL, crc, CRCCheck) of + case read_next_term(Handle, VL, crc) of {false, _} -> - {ok, CurrLoc} = file:position(Handle, cur), - Return = {crc_wonky, get(Handle, Key)}; - {_, Value} -> - {ok, CurrLoc} = file:position(Handle, cur), - Return = - case get(Handle, Key) of - {Key,Value} -> {Key ,Value}; - X -> {wonky, X} - end + {ok, CurrLoc} = file:position(Handle, cur), + Return = {crc_wonky, get(Handle, Key)}; + {_, Value} -> + {ok, CurrLoc} = file:position(Handle, cur), + Return = + case get(Handle, Key) of + {Key,Value} -> {Key ,Value}; + X -> {wonky, X} + end end, {ok, _} = file:position(Handle, CurrLoc), [Return | Acc] @@ -633,8 +638,6 @@ put(Handle, Key, Value, {LastPosition, HashTree}, BinaryMode, MaxSize) -> put_hashtree(Key, LastPosition, HashTree)} end. -mput(Handle, [], {LastPosition, HashTree0}, _BinaryMode, _MaxSize) -> - {Handle, LastPosition, HashTree0}; mput(Handle, KVList, {LastPosition, HashTree0}, BinaryMode, MaxSize) -> {KPList, Bin, LastKey} = multi_key_value_to_record(KVList, BinaryMode, @@ -663,19 +666,21 @@ put(FileName, Key, Value, {LastPosition, HashTree}) -> %% get(FileName,Key) -> {key,value} %% Given a filename and a key, returns a key and value tuple. %% + + get_withcache(Handle, Key, Cache) -> - get(Handle, Key, ?CRC_CHECK, Cache). + get(Handle, Key, Cache, true). + +get_withcache(Handle, Key, Cache, QuickCheck) -> + get(Handle, Key, Cache, QuickCheck). get(FileNameOrHandle, Key) -> - get(FileNameOrHandle, Key, ?CRC_CHECK). + get(FileNameOrHandle, Key, no_cache, true). -get(FileNameOrHandle, Key, CRCCheck) -> - get(FileNameOrHandle, Key, CRCCheck, no_cache). - -get(FileName, Key, CRCCheck, Cache) when is_list(FileName) -> +get(FileName, Key, Cache, QuickCheck) when is_list(FileName) -> {ok, Handle} = file:open(FileName,[binary, raw, read]), - get(Handle, Key, CRCCheck, Cache); -get(Handle, Key, CRCCheck, Cache) when is_tuple(Handle) -> + get(Handle, Key, Cache, QuickCheck); +get(Handle, Key, Cache, QuickCheck) when is_tuple(Handle) -> Hash = hash(Key), Index = hash_to_index(Hash), {HashTable, Count} = get_index(Handle, Index, Cache), @@ -696,7 +701,9 @@ get(Handle, Key, CRCCheck, Cache) when is_tuple(Handle) -> {L1, L2} = lists:split(Slot, LocList), search_hash_table(Handle, lists:append(L2, L1), - Hash, Key, CRCCheck) + Hash, + Key, + QuickCheck) end. get_index(Handle, Index, no_cache) -> @@ -711,20 +718,20 @@ get_index(_Handle, Index, Cache) -> %% This requires a key dictionary to be passed in (mapping keys to positions) %% Will return {Key, Value} or missing get_mem(Key, FNOrHandle, HashTree) -> - get_mem(Key, FNOrHandle, HashTree, ?CRC_CHECK). + get_mem(Key, FNOrHandle, HashTree, true). -get_mem(Key, Filename, HashTree, CRCCheck) when is_list(Filename) -> +get_mem(Key, Filename, HashTree, QuickCheck) when is_list(Filename) -> {ok, Handle} = file:open(Filename, [binary, raw, read]), - get_mem(Key, Handle, HashTree, CRCCheck); -get_mem(Key, Handle, HashTree, CRCCheck) -> + get_mem(Key, Handle, HashTree, QuickCheck); +get_mem(Key, Handle, HashTree, QuickCheck) -> ListToCheck = get_hashtree(Key, HashTree), - case {CRCCheck, ListToCheck} of + case {QuickCheck, ListToCheck} of {loose_presence, []} -> missing; {loose_presence, _L} -> probably; _ -> - extract_kvpair(Handle, ListToCheck, Key, CRCCheck) + extract_kvpair(Handle, ListToCheck, Key) end. %% Get the next key at a position in the file (or the first key if no position @@ -753,73 +760,6 @@ get_nextkey(Handle, {Position, FirstHashPosition}) -> nomorekeys end. - -%% Fold over all of the objects in the file, applying FoldFun to each object -%% where FoldFun(K, V, Acc0) -> Acc , or FoldFun(K, Acc0) -> Acc if KeyOnly is -%% set to true - -fold(FileName, FoldFun, Acc0) when is_list(FileName) -> - {ok, Handle} = file:open(FileName, [binary, raw, read]), - fold(Handle, FoldFun, Acc0); -fold(Handle, FoldFun, Acc0) -> - {ok, _} = file:position(Handle, bof), - {FirstHashPosition, _} = read_next_2_integers(Handle), - fold(Handle, FoldFun, Acc0, {256 * ?DWORD_SIZE, FirstHashPosition}, false). - -fold(Handle, FoldFun, Acc0, {Position, FirstHashPosition}, KeyOnly) -> - {ok, Position} = file:position(Handle, Position), - case Position of - FirstHashPosition -> - Acc0; - _ -> - case read_next_2_integers(Handle) of - {KeyLength, ValueLength} -> - NextKey = read_next_term(Handle, KeyLength), - NextPosition = Position - + KeyLength + ValueLength + - ?DWORD_SIZE, - case KeyOnly of - true -> - fold(Handle, - FoldFun, - FoldFun(NextKey, Acc0), - {NextPosition, FirstHashPosition}, - KeyOnly); - false -> - case read_next_term(Handle, - ValueLength, - crc, - ?CRC_CHECK) of - {false, _} -> - io:format("Skipping value for Key ~w as CRC - check failed~n", [NextKey]), - fold(Handle, - FoldFun, - Acc0, - {NextPosition, FirstHashPosition}, - KeyOnly); - {_, Value} -> - fold(Handle, - FoldFun, - FoldFun(NextKey, Value, Acc0), - {NextPosition, FirstHashPosition}, - KeyOnly) - end - end; - eof -> - Acc0 - end - end. - - -fold_keys(FileName, FoldFun, Acc0) when is_list(FileName) -> - {ok, Handle} = file:open(FileName, [binary, raw, read]), - fold_keys(Handle, FoldFun, Acc0); -fold_keys(Handle, FoldFun, Acc0) -> - {ok, _} = file:position(Handle, bof), - {FirstHashPosition, _} = read_next_2_integers(Handle), - fold(Handle, FoldFun, Acc0, {256 * ?DWORD_SIZE, FirstHashPosition}, true). - hashtable_calc(HashTree, StartPos) -> Seq = lists:seq(0, 255), SWC = os:timestamp(), @@ -956,22 +896,22 @@ put_hashtree(Key, Position, HashTree) -> end. %% Function to extract a Key-Value pair given a file handle and a position -%% Will confirm that the key matches and do a CRC check when requested -extract_kvpair(_, [], _, _) -> +%% Will confirm that the key matches and do a CRC check +extract_kvpair(_, [], _) -> missing; -extract_kvpair(Handle, [Position|Rest], Key, Check) -> +extract_kvpair(Handle, [Position|Rest], Key) -> {ok, _} = file:position(Handle, Position), {KeyLength, ValueLength} = read_next_2_integers(Handle), case read_next_term(Handle, KeyLength) of Key -> % If same key as passed in, then found! - case read_next_term(Handle, ValueLength, crc, Check) of + case read_next_term(Handle, ValueLength, crc) of {false, _} -> crc_wonky; {_, Value} -> {Key,Value} end; _ -> - extract_kvpair(Handle, Rest, Key, Check) + extract_kvpair(Handle, Rest, Key) end. extract_key(Handle, Position) -> @@ -988,7 +928,7 @@ extract_key_value_check(Handle, Position) -> {ok, _} = file:position(Handle, Position), {KeyLength, ValueLength} = read_next_2_integers(Handle), K = read_next_term(Handle, KeyLength), - {Check, V} = read_next_term(Handle, ValueLength, crc, true), + {Check, V} = read_next_term(Handle, ValueLength, crc), {K, V, Check}. %% Scan through the file until there is a failure to crc check an input, and @@ -1040,7 +980,7 @@ scan_over_file(Handle, Position, FilterFun, Output, LastKey) -> Output, fun extract_valueandsize/1) of {stop, UpdOutput} -> - {NewPosition, UpdOutput}; + {Position, UpdOutput}; {loop, UpdOutput} -> case NewPosition of eof -> @@ -1064,17 +1004,15 @@ check_last_key(LastKey) -> end. %% Read the Key/Value at this point, returning {ok, Key, Value} -%% catch expected exceptiosn associated with file corruption (or end) and +%% catch expected exceptions associated with file corruption (or end) and %% return eof saferead_keyvalue(Handle) -> case read_next_2_integers(Handle) of - {error, einval} -> - false; eof -> false; {KeyL, ValueL} -> case safe_read_next_term(Handle, KeyL) of - {error, einval} -> + {error, _} -> false; eof -> false; @@ -1082,8 +1020,6 @@ saferead_keyvalue(Handle) -> false; Key -> case file:read(Handle, ValueL) of - {error, einval} -> - false; eof -> false; {ok, Value} -> @@ -1141,7 +1077,7 @@ to_dict(FileName) -> dict:from_list(KeyValueList). read_next_term(Handle, Length) -> - case file:read(Handle, Length) of + case file:read(Handle, Length) of {ok, Bin} -> binary_to_term(Bin); ReadError -> @@ -1150,20 +1086,13 @@ read_next_term(Handle, Length) -> %% Read next string where the string has a CRC prepended - stripping the crc %% and checking if requested -read_next_term(Handle, Length, crc, Check) -> - case Check of - true -> - {ok, <>} = file:read(Handle, Length), - case calc_crc(Bin) of - CRC -> - {true, binary_to_term(Bin)}; - _ -> - {false, binary_to_term(Bin)} - end; - false -> - {ok, _} = file:position(Handle, {cur, 4}), - {ok, Bin} = file:read(Handle, Length - 4), - {unchecked, binary_to_term(Bin)} +read_next_term(Handle, Length, crc) -> + {ok, <>} = file:read(Handle, Length), + case calc_crc(Bin) of + CRC -> + {true, binary_to_term(Bin)}; + _ -> + {false, binary_to_term(Bin)} end. %% Extract value and size from binary containing CRC @@ -1202,18 +1131,18 @@ read_integerpairs(<>, Pairs) -> %% false - don't check the CRC before returning key & value %% loose_presence - confirm that the hash of the key is present -search_hash_table(_Handle, [], _Hash, _Key, _CRCCheck) -> +search_hash_table(_Handle, [], _Hash, _Key, _QuickCheck) -> missing; -search_hash_table(Handle, [Entry|RestOfEntries], Hash, Key, CRCCheck) -> +search_hash_table(Handle, [Entry|RestOfEntries], Hash, Key, QuickCheck) -> {ok, _} = file:position(Handle, Entry), {StoredHash, DataLoc} = read_next_2_integers(Handle), case StoredHash of Hash -> - KV = case CRCCheck of + KV = case QuickCheck of loose_presence -> probably; _ -> - extract_kvpair(Handle, [DataLoc], Key, CRCCheck) + extract_kvpair(Handle, [DataLoc], Key) end, case KV of missing -> @@ -1221,7 +1150,7 @@ search_hash_table(Handle, [Entry|RestOfEntries], Hash, Key, CRCCheck) -> RestOfEntries, Hash, Key, - CRCCheck); + QuickCheck); _ -> KV end; @@ -1229,7 +1158,7 @@ search_hash_table(Handle, [Entry|RestOfEntries], Hash, Key, CRCCheck) -> % Hash is 0 so key must be missing as 0 found before Hash matched missing; _ -> - search_hash_table(Handle, RestOfEntries, Hash, Key, CRCCheck) + search_hash_table(Handle, RestOfEntries, Hash, Key, QuickCheck) end. % Write Key and Value tuples into the CDB. Each tuple consists of a @@ -1635,8 +1564,8 @@ search_hash_table_findinslot_test() -> io:format("Slot 2 has Hash ~w Position ~w~n", [ReadH4, ReadP4]), ?assertMatch(0, ReadH4), ?assertMatch({"key1", "value1"}, get(Handle, Key1)), - ?assertMatch(probably, get(Handle, Key1, loose_presence)), - ?assertMatch(missing, get(Handle, "Key99", loose_presence)), + ?assertMatch(probably, get(Handle, Key1, no_cache, loose_presence)), + ?assertMatch(missing, get(Handle, "Key99", no_cache, loose_presence)), {ok, _} = file:position(Handle, FirstHashPosition), FlipH3 = endian_flip(ReadH3), FlipP3 = endian_flip(ReadP3), @@ -1699,91 +1628,6 @@ emptyvalue_fromdict_test() -> ?assertMatch(KVP, D_Result), ok = file:delete("../test/from_dict_test_ev.cdb"). -fold_test() -> - K1 = {"Key1", 1}, - V1 = 2, - K2 = {"Key1", 2}, - V2 = 4, - K3 = {"Key1", 3}, - V3 = 8, - K4 = {"Key1", 4}, - V4 = 16, - K5 = {"Key1", 5}, - V5 = 32, - D = dict:from_list([{K1, V1}, {K2, V2}, {K3, V3}, {K4, V4}, {K5, V5}]), - ok = from_dict("../test/fold_test.cdb", D), - FromSN = 2, - FoldFun = fun(K, V, Acc) -> - {_Key, Seq} = K, - if Seq > FromSN -> - Acc + V; - true -> - Acc - end - end, - ?assertMatch(56, fold("../test/fold_test.cdb", FoldFun, 0)), - ok = file:delete("../test/fold_test.cdb"). - -fold_keys_test() -> - K1 = {"Key1", 1}, - V1 = 2, - K2 = {"Key2", 2}, - V2 = 4, - K3 = {"Key3", 3}, - V3 = 8, - K4 = {"Key4", 4}, - V4 = 16, - K5 = {"Key5", 5}, - V5 = 32, - D = dict:from_list([{K1, V1}, {K2, V2}, {K3, V3}, {K4, V4}, {K5, V5}]), - ok = from_dict("../test/fold_keys_test.cdb", D), - FromSN = 2, - FoldFun = fun(K, Acc) -> - {Key, Seq} = K, - if Seq > FromSN -> - lists:append(Acc, [Key]); - true -> - Acc - end - end, - Result = fold_keys("../test/fold_keys_test.cdb", FoldFun, []), - ?assertMatch(["Key3", "Key4", "Key5"], lists:sort(Result)), - ok = file:delete("../test/fold_keys_test.cdb"). - -fold2_test() -> - K1 = {"Key1", 1}, - V1 = 2, - K2 = {"Key1", 2}, - V2 = 4, - K3 = {"Key1", 3}, - V3 = 8, - K4 = {"Key1", 4}, - V4 = 16, - K5 = {"Key1", 5}, - V5 = 32, - K6 = {"Key2", 1}, - V6 = 64, - D = dict:from_list([{K1, V1}, {K2, V2}, {K3, V3}, - {K4, V4}, {K5, V5}, {K6, V6}]), - ok = from_dict("../test/fold2_test.cdb", D), - FoldFun = fun(K, V, Acc) -> - {Key, Seq} = K, - case dict:find(Key, Acc) of - error -> - dict:store(Key, {Seq, V}, Acc); - {ok, {LSN, _V}} when Seq > LSN -> - dict:store(Key, {Seq, V}, Acc); - _ -> - Acc - end - end, - RD = dict:new(), - RD1 = dict:store("Key1", {5, 32}, RD), - RD2 = dict:store("Key2", {1, 64}, RD1), - Result = fold("../test/fold2_test.cdb", FoldFun, dict:new()), - ?assertMatch(RD2, Result), - ok = file:delete("../test/fold2_test.cdb"). - find_lastkey_test() -> {ok, P1} = cdb_open_writer("../test/lastkey.pnd"), ok = cdb_put(P1, "Key1", "Value1"), @@ -1907,5 +1751,49 @@ mput_test() -> ok = cdb_close(P2), ok = file:delete(F2). +state_test() -> + {ok, P1} = cdb_open_writer("../test/state_test.pnd"), + KVList = generate_sequentialkeys(1000, []), + ok = cdb_mput(P1, KVList), + ?assertMatch(probably, cdb_keycheck(P1, "Key1")), + ?assertMatch({"Key1", "Value1"}, cdb_get(P1, "Key1")), + ok = cdb_roll(P1), + ?assertMatch(probably, cdb_keycheck(P1, "Key1")), + ?assertMatch({"Key1", "Value1"}, cdb_get(P1, "Key1")), + ok = cdb_deletepending(P1), + ?assertMatch(probably, cdb_keycheck(P1, "Key1")), + ?assertMatch({"Key1", "Value1"}, cdb_get(P1, "Key1")), + timer:sleep(500), + ?assertMatch(probably, cdb_keycheck(P1, "Key1")), + ?assertMatch({"Key1", "Value1"}, cdb_get(P1, "Key1")), + ok = cdb_close(P1). + +corruptfile_test() -> + file:delete("../test/corrupt_test.pnd"), + {ok, P1} = cdb_open_writer("../test/corrupt_test.pnd"), + KVList = generate_sequentialkeys(100, []), + ok = cdb_mput(P1, []), % Not relevant to this test, but needs testing + lists:foreach(fun({K, V}) -> cdb_put(P1, K, V) end, KVList), + ?assertMatch(probably, cdb_keycheck(P1, "Key1")), + ?assertMatch({"Key1", "Value1"}, cdb_get(P1, "Key1")), + ?assertMatch({"Key100", "Value100"}, cdb_get(P1, "Key100")), + ok = cdb_close(P1), + lists:foreach(fun(Offset) -> corrupt_testfile_at_offset(Offset) end, + lists:seq(1, 40)), + ok = file:delete("../test/corrupt_test.pnd"). + +corrupt_testfile_at_offset(Offset) -> + {ok, F1} = file:open("../test/corrupt_test.pnd", ?WRITE_OPS), + {ok, EofPos} = file:position(F1, eof), + file:position(F1, EofPos - Offset), + ok = file:truncate(F1), + ok = file:close(F1), + {ok, P2} = cdb_open_writer("../test/corrupt_test.pnd"), + ?assertMatch(probably, cdb_keycheck(P2, "Key1")), + ?assertMatch({"Key1", "Value1"}, cdb_get(P2, "Key1")), + ?assertMatch(missing, cdb_get(P2, "Key100")), + ok = cdb_put(P2, "Key100", "Value100"), + ?assertMatch({"Key100", "Value100"}, cdb_get(P2, "Key100")), + ok = cdb_close(P2). -endif. diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index b929c91..1c8ed17 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -371,19 +371,16 @@ start_from_file(InkOpts) -> filelib:ensure_dir(CompactFP), ManifestFP = filepath(RootPath, manifest_dir), - {ok, ManifestFilenames} = case filelib:is_dir(ManifestFP) of - true -> - file:list_dir(ManifestFP); - false -> - filelib:ensure_dir(ManifestFP), - {ok, []} - end, + ok = filelib:ensure_dir(ManifestFP), + {ok, ManifestFilenames} = file:list_dir(ManifestFP), IClerkCDBOpts = CDBopts#cdb_options{file_path = CompactFP}, ReloadStrategy = InkOpts#inker_options.reload_strategy, + MRL = InkOpts#inker_options.max_run_length, IClerkOpts = #iclerk_options{inker = self(), cdb_options=IClerkCDBOpts, - reload_strategy = ReloadStrategy}, + reload_strategy = ReloadStrategy, + max_run_length = MRL}, {ok, Clerk} = leveled_iclerk:clerk_new(IClerkOpts), {Manifest, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 698b9b5..8522a30 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -930,16 +930,7 @@ print_manifest(Manifest) -> lists:foreach(fun(L) -> io:format("Manifest at Level ~w~n", [L]), Level = get_item(L, Manifest, []), - lists:foreach(fun(M) -> - R = is_record(M, manifest_entry), - case R of - true -> - print_manifest_entry(M); - false -> - {_, M1} = M, - print_manifest_entry(M1) - end end, - Level) + lists:foreach(fun print_manifest_entry/1, Level) end, lists:seq(0, ?MAX_LEVELS - 1)), ok. @@ -1557,7 +1548,10 @@ print_manifest_test() -> M2 = #manifest_entry{start_key={i, self(), {null, "Fld1"}, "K8"}, end_key={i, <<200:32/integer>>, {"Idx1", "Fld9"}, "K93"}, filename="Z1"}, - ?assertMatch(ok, print_manifest([{1, [M1, M2]}])). + M3 = #manifest_entry{start_key={?STD_TAG, self(), {null, "Fld1"}, "K8"}, + end_key={?RIAK_TAG, <<200:32/integer>>, {"Idx1", "Fld9"}, "K93"}, + filename="Z1"}, + print_manifest([{1, [M1, M2, M3]}]). simple_findnextkey_test() -> QueryArray = [ @@ -1689,4 +1683,49 @@ foldwithimm_simple_test() -> {{o, "Bucket1", "Key5"}, 2}, {{o, "Bucket1", "Key6"}, 7}], AccB). +create_file_test() -> + Filename = "../test/new_file.sft", + ok = file:write_file(Filename, term_to_binary("hello")), + {KL1, KL2} = {lists:sort(leveled_sft:generate_randomkeys(10000)), []}, + {ok, SP, noreply} = leveled_sft:sft_new(Filename, + KL1, + KL2, + 0, + #sft_options{wait=false}), + lists:foreach(fun(X) -> + case checkready(SP) of + timeout -> + timer:sleep(X); + _ -> + ok + end end, + [50, 50, 50, 50, 50]), + {ok, SrcFN, StartKey, EndKey} = checkready(SP), + io:format("StartKey ~w EndKey ~w~n", [StartKey, EndKey]), + ?assertMatch({o, _, _, _}, StartKey), + ?assertMatch({o, _, _, _}, EndKey), + ?assertMatch("../test/new_file.sft", SrcFN), + ok = leveled_sft:sft_clear(SP), + {ok, Bin} = file:read_file("../test/new_file.sft.discarded"), + ?assertMatch("hello", binary_to_term(Bin)). + +coverage_test() -> + RootPath = "../test/ledger", + clean_testdir(RootPath), + {ok, PCL} = pcl_start(#penciller_options{root_path=RootPath, + max_inmemory_tablesize=1000}), + Key1 = {{o,"Bucket0001", "Key0001", null}, {1, {active, infinity}, null}}, + KL1 = leveled_sft:generate_randomkeys({1000, 2}), + ok = maybe_pause_push(PCL, [Key1]), + ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), + ok = maybe_pause_push(PCL, KL1), + ok = pcl_close(PCL), + ManifestFP = filepath(RootPath, manifest), + file:write_file(ManifestFP ++ "/yeszero_123.man", term_to_binary("hello")), + {ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath, + max_inmemory_tablesize=1000}), + ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})), + ok = pcl_close(PCLr), + clean_testdir(RootPath). + -endif. \ No newline at end of file diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 9cebef2..c9cfd7f 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -241,9 +241,7 @@ sft_open(Filename) -> {ok, Pid} = gen_server:start(?MODULE, [], []), case gen_server:call(Pid, {sft_open, Filename}, infinity) of {ok, {SK, EK}} -> - {ok, Pid, {SK, EK}}; - Error -> - Error + {ok, Pid, {SK, EK}} end. sft_setfordelete(Pid, Penciller) -> @@ -467,7 +465,7 @@ create_file(FileName) when is_list(FileName) -> {error, Reason} -> io:format("Error opening filename ~s with reason ~w", [FileName, Reason]), - error + {error, Reason} end. @@ -586,17 +584,16 @@ fetch_keyvalue(Handle, FileMD, Key) -> %% Fetches a range of keys returning a list of {Key, SeqN} tuples fetch_range_keysonly(Handle, FileMD, StartKey, EndKey) -> - fetch_range(Handle, FileMD, StartKey, EndKey, [], - fun acc_list_keysonly/2). + fetch_range(Handle, FileMD, StartKey, EndKey, fun acc_list_keysonly/2). fetch_range_keysonly(Handle, FileMD, StartKey, EndKey, ScanWidth) -> - fetch_range(Handle, FileMD, StartKey, EndKey, [], - fun acc_list_keysonly/2, ScanWidth). + fetch_range(Handle, FileMD, StartKey, EndKey, fun acc_list_keysonly/2, + ScanWidth). %% Fetches a range of keys returning the full tuple, including value fetch_range_kv(Handle, FileMD, StartKey, EndKey, ScanWidth) -> - fetch_range(Handle, FileMD, StartKey, EndKey, [], - fun acc_list_kv/2, ScanWidth). + fetch_range(Handle, FileMD, StartKey, EndKey, fun acc_list_kv/2, + ScanWidth). acc_list_keysonly(null, empty) -> []; @@ -630,24 +627,20 @@ acc_list_kv(R, RList) -> %% than keys and values, or other entirely different accumulators can be %% used - e.g. counters, hash-lists to build bloom filters etc -fetch_range(Handle, FileMD, StartKey, EndKey, FunList, AccFun) -> - fetch_range(Handle, FileMD, StartKey, EndKey, FunList, - AccFun, ?ITERATOR_SCANWIDTH). +fetch_range(Handle, FileMD, StartKey, EndKey, AccFun) -> + fetch_range(Handle, FileMD, StartKey, EndKey, AccFun, ?ITERATOR_SCANWIDTH). -fetch_range(Handle, FileMD, StartKey, EndKey, FunList, AccFun, ScanWidth) -> - fetch_range(Handle, FileMD, StartKey, EndKey, FunList, - AccFun, ScanWidth, empty). +fetch_range(Handle, FileMD, StartKey, EndKey, AccFun, ScanWidth) -> + fetch_range(Handle, FileMD, StartKey, EndKey, AccFun, ScanWidth, empty). -fetch_range(_Handle, _FileMD, StartKey, _EndKey, _FunList, - _AccFun, 0, Acc) -> +fetch_range(_Handle, _FileMD, StartKey, _EndKey, _AccFun, 0, Acc) -> {partial, Acc, StartKey}; -fetch_range(Handle, FileMD, StartKey, EndKey, FunList, - AccFun, ScanWidth, Acc) -> +fetch_range(Handle, FileMD, StartKey, EndKey, AccFun, ScanWidth, Acc) -> %% get_nearestkey gets the last key in the index <= StartKey, or the next %% key along if {next, StartKey} is passed case get_nearestkey(FileMD#state.slot_index, StartKey) of {NearestKey, _Filter, {LengthList, PointerB}} -> - fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, FunList, + fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, AccFun, ScanWidth, LengthList, 0, @@ -657,7 +650,7 @@ fetch_range(Handle, FileMD, StartKey, EndKey, FunList, {complete, AccFun(null, Acc)} end. -fetch_range(Handle, FileMD, _StartKey, NearestKey, EndKey, FunList, +fetch_range(Handle, FileMD, _StartKey, NearestKey, EndKey, AccFun, ScanWidth, LengthList, BlockNumber, @@ -665,21 +658,21 @@ fetch_range(Handle, FileMD, _StartKey, NearestKey, EndKey, FunList, Acc) when length(LengthList) == BlockNumber -> %% Reached the end of the slot. Move the start key on one to scan a new slot - fetch_range(Handle, FileMD, {next, NearestKey}, EndKey, FunList, + fetch_range(Handle, FileMD, {next, NearestKey}, EndKey, AccFun, ScanWidth - 1, Acc); -fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, FunList, +fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, AccFun, ScanWidth, LengthList, BlockNumber, Pointer, Acc) -> Block = fetch_block(Handle, LengthList, BlockNumber, Pointer), - Results = scan_block(Block, StartKey, EndKey, FunList, AccFun, Acc), + Results = scan_block(Block, StartKey, EndKey, AccFun, Acc), case Results of {partial, Acc1, StartKey} -> %% Move on to the next block - fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, FunList, + fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, AccFun, ScanWidth, LengthList, BlockNumber + 1, @@ -689,38 +682,20 @@ fetch_range(Handle, FileMD, StartKey, NearestKey, EndKey, FunList, {complete, Acc1} end. -scan_block([], StartKey, _EndKey, _FunList, _AccFun, Acc) -> +scan_block([], StartKey, _EndKey, _AccFun, Acc) -> {partial, Acc, StartKey}; -scan_block([HeadKV|T], StartKey, EndKey, FunList, AccFun, Acc) -> +scan_block([HeadKV|T], StartKey, EndKey, AccFun, Acc) -> K = leveled_codec:strip_to_keyonly(HeadKV), case {StartKey > K, leveled_codec:endkey_passed(EndKey, K)} of {true, _} when StartKey /= all -> - scan_block(T, StartKey, EndKey, FunList, AccFun, Acc); + scan_block(T, StartKey, EndKey, AccFun, Acc); {_, true} when EndKey /= all -> {complete, Acc}; _ -> - case applyfuns(FunList, HeadKV) of - true -> - %% Add result to the accumulator - scan_block(T, StartKey, EndKey, FunList, - AccFun, AccFun(HeadKV, Acc)); - false -> - scan_block(T, StartKey, EndKey, FunList, - AccFun, Acc) - end + scan_block(T, StartKey, EndKey, AccFun, AccFun(HeadKV, Acc)) end. -applyfuns([], _KV) -> - true; -applyfuns([HeadFun|OtherFuns], KV) -> - case HeadFun(KV) of - true -> - applyfuns(OtherFuns, KV); - false -> - false - end. - fetch_keyvalue_fromblock([], _Key, _LengthList, _Handle, _StartOfSlot) -> not_present; fetch_keyvalue_fromblock([BlockNmb|T], Key, LengthList, Handle, StartOfSlot) -> @@ -745,10 +720,7 @@ get_nearestkey(KVList, all) -> [] -> not_found; [H|_Tail] -> - H; - _ -> - io:format("KVList issue ~w~n", [KVList]), - error + H end; get_nearestkey(KVList, Key) -> case Key of diff --git a/test/end_to_end/restart_SUITE.erl b/test/end_to_end/restart_SUITE.erl index 0017035..030cfd6 100644 --- a/test/end_to_end/restart_SUITE.erl +++ b/test/end_to_end/restart_SUITE.erl @@ -15,13 +15,15 @@ retain_strategy(_Config) -> cache_size=1000, max_journalsize=5000000, reload_strategy=[{?RIAK_TAG, retain}]}, + BookOptsAlt = BookOpts#bookie_options{max_run_length=6, + max_journalsize=500000}, {ok, Spcl3, LastV3} = rotating_object_check(BookOpts, "Bucket3", 800), ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3}]), {ok, Spcl4, LastV4} = rotating_object_check(BookOpts, "Bucket4", 1600), ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3}, {"Bucket4", Spcl4, LastV4}]), {ok, Spcl5, LastV5} = rotating_object_check(BookOpts, "Bucket5", 3200), - ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3}, + ok = restart_from_blankledger(BookOptsAlt, [{"Bucket3", Spcl3, LastV3}, {"Bucket5", Spcl5, LastV5}]), {ok, Spcl6, LastV6} = rotating_object_check(BookOpts, "Bucket6", 6400), ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3}, From 807af81b68da4e9ac987ceffc6f920be13c49ce6 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 29 Oct 2016 01:06:00 +0100 Subject: [PATCH 102/167] Pneciller Memory Test The current penciller memory setup is inefficient. Is there an alternative which is still relatively simple and but more efficient? --- src/leveled_pmem.erl | 262 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 src/leveled_pmem.erl diff --git a/src/leveled_pmem.erl b/src/leveled_pmem.erl new file mode 100644 index 0000000..262f5ac --- /dev/null +++ b/src/leveled_pmem.erl @@ -0,0 +1,262 @@ +-module(leveled_pmem). + +-behaviour(gen_server). + +-include("include/leveled.hrl"). + +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + roll_singletree/4, + roll_arraytree/4, + roll_arraylist/4, + terminate/2, + code_change/3]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(ARRAY_WIDTH, 32). +-define(SLOT_WIDTH, 16386). + +-record(state, {}). + +%%%============================================================================ +%%% API +%%%============================================================================ + + +roll_singletree(LevelZero, LevelMinus1, LedgerSQN, PCL) -> + {ok, Pid} = gen_server:start(?MODULE, [], []), + gen_server:call(Pid, {single_tree, LevelZero, LevelMinus1, LedgerSQN, PCL}). + +roll_arraytree(LevelZero, LevelMinus1, LedgerSQN, PCL) -> + {ok, Pid} = gen_server:start(?MODULE, [], []), + gen_server:call(Pid, {array_tree, LevelZero, LevelMinus1, LedgerSQN, PCL}). + +roll_arraylist(LevelZero, LevelMinus1, LedgerSQN, PCL) -> + {ok, Pid} = gen_server:start(?MODULE, [], []), + gen_server:call(Pid, {array_list, LevelZero, LevelMinus1, LedgerSQN, PCL}). + +roll_arrayfilt(LevelZero, LevelMinus1, LedgerSQN, PCL) -> + {ok, Pid} = gen_server:start(?MODULE, [], []), + gen_server:call(Pid, {array_filter, LevelZero, LevelMinus1, LedgerSQN, PCL}). + + +%%%============================================================================ +%%% gen_server callbacks +%%%============================================================================ + +init([]) -> + {ok, #state{}}. + +handle_call({single_tree, LevelZero, LevelMinus1, LedgerSQN, _PCL}, + _From, State) -> + SW = os:timestamp(), + {NewL0, Size, MaxSQN} = leveled_penciller:roll_new_tree(LevelZero, + LevelMinus1, + LedgerSQN), + T = timer:now_diff(os:timestamp(), SW), + io:format("Rolled tree to size ~w in ~w microseconds using single_tree~n", + [Size, T]), + {stop, normal, {NewL0, Size, MaxSQN, T}, State}; +handle_call({array_tree, LevelZero, LevelMinus1, LedgerSQN, _PCL}, + _From, State) -> + SW = os:timestamp(), + {MinSQN, MaxSQN, _Size, SplitTrees} = assess_sqn(LevelMinus1, to_array), + R = lists:foldl(fun(X, {Arr, ArrSize}) -> + LM1 = array:get(X, SplitTrees), + T0 = array:get(X, LevelZero), + T1 = lists:foldl(fun({K, V}, TrAcc) -> + gb_trees:enter(K, V, TrAcc) + end, + T0, + LM1), + {array:set(X, T1, Arr), ArrSize + gb_trees:size(T1)} + end, + {array:new(?ARRAY_WIDTH, {default, gb_trees:empty()}), 0}, + lists:seq(0, ?ARRAY_WIDTH - 1)), + {NextL0, NewSize} = R, + T = timer:now_diff(os:timestamp(), SW), + io:format("Rolled tree to size ~w in ~w microseconds using array_tree~n", + [NewSize, T]), + if + MinSQN >= LedgerSQN -> + {stop, normal, {NextL0, NewSize, MaxSQN, T}, State} + end; +handle_call({array_list, LevelZero, LevelMinus1, LedgerSQN, _PCL}, + _From, State) -> + SW = os:timestamp(), + {MinSQN, MaxSQN, _Size, SplitTrees} = assess_sqn(LevelMinus1, to_array), + R = lists:foldl(fun(X, {Arr, ArrSize}) -> + LM1 = array:get(X, SplitTrees), + T0 = array:get(X, LevelZero), + T1 = lists:foldl(fun({K, V}, TrAcc) -> + [{K, V}|TrAcc] + end, + T0, + LM1), + {array:set(X, T1, Arr), ArrSize + length(T1)} + end, + {array:new(?ARRAY_WIDTH, {default, []}), 0}, + lists:seq(0, ?ARRAY_WIDTH - 1)), + {NextL0, NewSize} = R, + T = timer:now_diff(os:timestamp(), SW), + io:format("Rolled tree to size ~w in ~w microseconds using array_list~n", + [NewSize, T]), + if + MinSQN >= LedgerSQN -> + {stop, normal, {NextL0, NewSize, MaxSQN, T}, State} + end; +handle_call({array_filter, LevelZero, LevelMinus1, LedgerSQN, _PCL}, + _From, State) -> + SW = os:timestamp(), + {MinSQN, MaxSQN, LM1Size, HashList} = assess_sqn(LevelMinus1, to_hashes), + {L0Lookup, L0TreeList, L0Size} = LevelZero, + UpdL0TreeList = [{LedgerSQN, LevelMinus1}|L0TreeList], + UpdL0Lookup = lists:foldl(fun(X, LookupArray) -> + L = array:get(X, LookupArray), + array:set(X, [LedgerSQN|L], LookupArray) + end, + L0Lookup, + HashList), + NewSize = LM1Size + L0Size, + T = timer:now_diff(os:timestamp(), SW), + io:format("Rolled tree to size ~w in ~w microseconds using array_filter~n", + [NewSize, T]), + if + MinSQN >= LedgerSQN -> + {stop, + normal, + {{UpdL0Lookup, UpdL0TreeList, NewSize}, NewSize, MaxSQN, T}, + State} + end. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Msg, State) -> + {stop, normal, ok, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + + +%%%============================================================================ +%%% Internal functions +%%%============================================================================ + + +hash_to_index(Key) -> + erlang:phash2(Key) band (?ARRAY_WIDTH - 1). + +hash_to_slot(Key) -> + erlang:phash2(Key) band (?SLOT_WIDTH - 1). + +roll_into_list(Tree) -> + gb_trees:to_list(Tree). + +assess_sqn(Tree, to_array) -> + L = roll_into_list(Tree), + TmpA = array:new(?ARRAY_WIDTH, {default, []}), + FoldFun = fun({K, V}, {AccMinSQN, AccMaxSQN, AccSize, Array}) -> + SQN = leveled_codec:strip_to_seqonly({K, V}), + Index = hash_to_index(K), + List0 = array:get(Index, Array), + List1 = lists:append(List0, [{K, V}]), + {min(SQN, AccMinSQN), + max(SQN, AccMaxSQN), + AccSize + 1, + array:set(Index, List1, Array)} + end, + lists:foldl(FoldFun, {infinity, 0, 0, TmpA}, L); +assess_sqn(Tree, to_hashes) -> + L = roll_into_list(Tree), + FoldFun = fun({K, V}, {AccMinSQN, AccMaxSQN, AccSize, HashList}) -> + SQN = leveled_codec:strip_to_seqonly({K, V}), + Hash = hash_to_slot(K), + {min(SQN, AccMinSQN), + max(SQN, AccMaxSQN), + AccSize + 1, + [Hash|HashList]} + end, + lists:foldl(FoldFun, {infinity, 0, 0, []}, L). + +%%%============================================================================ +%%% Test +%%%============================================================================ + +-ifdef(TEST). + +generate_randomkeys(Seqn, Count, BucketRangeLow, BucketRangeHigh) -> + generate_randomkeys(Seqn, + Count, + gb_trees:empty(), + BucketRangeLow, + BucketRangeHigh). + +generate_randomkeys(_Seqn, 0, Acc, _BucketLow, _BucketHigh) -> + Acc; +generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) -> + BNumber = string:right(integer_to_list(BucketLow + random:uniform(BRange)), + 4, $0), + KNumber = string:right(integer_to_list(random:uniform(1000)), 4, $0), + {K, V} = {{o, "Bucket" ++ BNumber, "Key" ++ KNumber, null}, + {Seqn, {active, infinity}, null}}, + generate_randomkeys(Seqn + 1, + Count - 1, + gb_trees:enter(K, V, Acc), + BucketLow, + BRange). + + +speed_test() -> + R = lists:foldl(fun(_X, {LedgerSQN, + {L0st, TTst}, + {L0at, TTat}, + {L0al, TTal}, + {L0af, TTaf}}) -> + LM1 = generate_randomkeys(LedgerSQN + 1, 2000, 1, 500), + {NextL0st, S, MaxSQN, Tst} = roll_singletree(L0st, + LM1, + LedgerSQN, + self()), + {NextL0at, S, MaxSQN, Tat} = roll_arraytree(L0at, + LM1, + LedgerSQN, + self()), + {NextL0al, _S, MaxSQN, Tal} = roll_arraylist(L0al, + LM1, + LedgerSQN, + self()), + {NextL0af, _S, MaxSQN, Taf} = roll_arrayfilt(L0af, + LM1, + LedgerSQN, + self()), + {MaxSQN, + {NextL0st, TTst + Tst}, + {NextL0at, TTat + Tat}, + {NextL0al, TTal + Tal}, + {NextL0af, TTaf + Taf}} + end, + {0, + {gb_trees:empty(), 0}, + {array:new(?ARRAY_WIDTH, [{default, gb_trees:empty()}, fixed]), 0}, + {array:new(?ARRAY_WIDTH, [{default, []}, fixed]), 0}, + {{array:new(?SLOT_WIDTH, [{default, []}, fixed]), [], 0}, 0} + }, + lists:seq(1, 16)), + {_, {_, TimeST}, {_, TimeAT}, {_, TimeLT}, {_, TimeAF}} = R, + io:format("Total time for single_tree ~w microseconds ~n", [TimeST]), + io:format("Total time for array_tree ~w microseconds ~n", [TimeAT]), + io:format("Total time for array_list ~w microseconds ~n", [TimeLT]), + io:format("Total time for array_filter ~w microseconds ~n", [TimeAF]), + ?assertMatch(true, false). + + + + +-endif. \ No newline at end of file From c7a56068c51b938607f056005f7a30dcd8a3e99d Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 29 Oct 2016 13:27:21 +0100 Subject: [PATCH 103/167] Refactor of L0 memory Not yet integrated, but there is now unit-tested module for the new way of managing L0 memory cache in the Penciller. This mechansim is considerably more efficient than previous efforts and should allow for further simplification of the code. --- src/leveled_pmem.erl | 398 ++++++++++++++++++++++--------------------- 1 file changed, 201 insertions(+), 197 deletions(-) diff --git a/src/leveled_pmem.erl b/src/leveled_pmem.erl index 262f5ac..9cf498b 100644 --- a/src/leveled_pmem.erl +++ b/src/leveled_pmem.erl @@ -1,189 +1,166 @@ --module(leveled_pmem). +%% -------- PENCILLER MEMORY --------- +%% +%% Module that provides functions for maintaining the L0 memory of the +%% Penciller. +%% +%% It is desirable that the L0Mem can efficiently handle the push of new trees +%% whilst maintaining the capability to quickly snapshot the memory for clones +%% of the Penciller. +%% +%% ETS tables are not used due to complications with managing their mutability. +%% +%% An attempt was made to merge all trees into a single tree on push (in a +%% spawned process), but this proved to have an expensive impact as the tree +%% got larger. +%% +%% This approach is to keep a list of trees which have been received in the +%% order which they were received. There is then a fixed-size array of hashes +%% used to either point lookups at the right tree in the list, or inform the +%% requestor it is not present avoiding any lookups. +%% +%% Tests show this takes one third of the time at push (when compared to +%% merging to a single tree), and is an order of magnitude more efficient as +%% the tree reaches peak size. It is also an order of magnitude more +%% efficient to use the hash index when compared to looking through all the +%% trees. +%% +%% Total time for single_tree 217000 microseconds +%% Total time for array_tree 209000 microseconds +%% Total time for array_list 142000 microseconds +%% Total time for array_filter 69000 microseconds +%% List of 2000 checked without array - success count of 90 in 36000 microseconds +%% List of 2000 checked with array - success count of 90 in 1000 microseconds --behaviour(gen_server). + +-module(leveled_pmem). -include("include/leveled.hrl"). --export([init/1, - handle_call/3, - handle_cast/2, - handle_info/2, - roll_singletree/4, - roll_arraytree/4, - roll_arraylist/4, - terminate/2, - code_change/3]). +-export([ + add_to_index/5, + to_list/1, + new_index/0, + check_levelzero/3, + merge_trees/3 + ]). -include_lib("eunit/include/eunit.hrl"). --define(ARRAY_WIDTH, 32). --define(SLOT_WIDTH, 16386). +-define(SLOT_WIDTH, {4096, 12}). --record(state, {}). %%%============================================================================ %%% API %%%============================================================================ - -roll_singletree(LevelZero, LevelMinus1, LedgerSQN, PCL) -> - {ok, Pid} = gen_server:start(?MODULE, [], []), - gen_server:call(Pid, {single_tree, LevelZero, LevelMinus1, LedgerSQN, PCL}). - -roll_arraytree(LevelZero, LevelMinus1, LedgerSQN, PCL) -> - {ok, Pid} = gen_server:start(?MODULE, [], []), - gen_server:call(Pid, {array_tree, LevelZero, LevelMinus1, LedgerSQN, PCL}). - -roll_arraylist(LevelZero, LevelMinus1, LedgerSQN, PCL) -> - {ok, Pid} = gen_server:start(?MODULE, [], []), - gen_server:call(Pid, {array_list, LevelZero, LevelMinus1, LedgerSQN, PCL}). - -roll_arrayfilt(LevelZero, LevelMinus1, LedgerSQN, PCL) -> - {ok, Pid} = gen_server:start(?MODULE, [], []), - gen_server:call(Pid, {array_filter, LevelZero, LevelMinus1, LedgerSQN, PCL}). - - -%%%============================================================================ -%%% gen_server callbacks -%%%============================================================================ - -init([]) -> - {ok, #state{}}. - -handle_call({single_tree, LevelZero, LevelMinus1, LedgerSQN, _PCL}, - _From, State) -> +add_to_index(L0Index, L0Size, LevelMinus1, LedgerSQN, TreeList) -> SW = os:timestamp(), - {NewL0, Size, MaxSQN} = leveled_penciller:roll_new_tree(LevelZero, - LevelMinus1, - LedgerSQN), - T = timer:now_diff(os:timestamp(), SW), - io:format("Rolled tree to size ~w in ~w microseconds using single_tree~n", - [Size, T]), - {stop, normal, {NewL0, Size, MaxSQN, T}, State}; -handle_call({array_tree, LevelZero, LevelMinus1, LedgerSQN, _PCL}, - _From, State) -> - SW = os:timestamp(), - {MinSQN, MaxSQN, _Size, SplitTrees} = assess_sqn(LevelMinus1, to_array), - R = lists:foldl(fun(X, {Arr, ArrSize}) -> - LM1 = array:get(X, SplitTrees), - T0 = array:get(X, LevelZero), - T1 = lists:foldl(fun({K, V}, TrAcc) -> - gb_trees:enter(K, V, TrAcc) - end, - T0, - LM1), - {array:set(X, T1, Arr), ArrSize + gb_trees:size(T1)} - end, - {array:new(?ARRAY_WIDTH, {default, gb_trees:empty()}), 0}, - lists:seq(0, ?ARRAY_WIDTH - 1)), - {NextL0, NewSize} = R, - T = timer:now_diff(os:timestamp(), SW), - io:format("Rolled tree to size ~w in ~w microseconds using array_tree~n", - [NewSize, T]), + SlotInTreeList = length(TreeList) + 1, + FoldFun = fun({K, V}, {AccMinSQN, AccMaxSQN, HashIndex}) -> + SQN = leveled_codec:strip_to_seqonly({K, V}), + {Hash, Slot} = hash_to_slot(K), + L = array:get(Slot, HashIndex), + {min(SQN, AccMinSQN), + max(SQN, AccMaxSQN), + array:set(Slot, [{Hash, SlotInTreeList}|L], HashIndex)} + end, + LM1List = gb_trees:to_list(LevelMinus1), + {MinSQN, MaxSQN, UpdL0Index} = lists:foldl(FoldFun, + {infinity, 0, L0Index}, + LM1List), + NewL0Size = length(LM1List) + L0Size, + io:format("Rolled tree to size ~w in ~w microseconds~n", + [NewL0Size, timer:now_diff(os:timestamp(), SW)]), if - MinSQN >= LedgerSQN -> - {stop, normal, {NextL0, NewSize, MaxSQN, T}, State} - end; -handle_call({array_list, LevelZero, LevelMinus1, LedgerSQN, _PCL}, - _From, State) -> - SW = os:timestamp(), - {MinSQN, MaxSQN, _Size, SplitTrees} = assess_sqn(LevelMinus1, to_array), - R = lists:foldl(fun(X, {Arr, ArrSize}) -> - LM1 = array:get(X, SplitTrees), - T0 = array:get(X, LevelZero), - T1 = lists:foldl(fun({K, V}, TrAcc) -> - [{K, V}|TrAcc] - end, - T0, - LM1), - {array:set(X, T1, Arr), ArrSize + length(T1)} - end, - {array:new(?ARRAY_WIDTH, {default, []}), 0}, - lists:seq(0, ?ARRAY_WIDTH - 1)), - {NextL0, NewSize} = R, - T = timer:now_diff(os:timestamp(), SW), - io:format("Rolled tree to size ~w in ~w microseconds using array_list~n", - [NewSize, T]), - if - MinSQN >= LedgerSQN -> - {stop, normal, {NextL0, NewSize, MaxSQN, T}, State} - end; -handle_call({array_filter, LevelZero, LevelMinus1, LedgerSQN, _PCL}, - _From, State) -> - SW = os:timestamp(), - {MinSQN, MaxSQN, LM1Size, HashList} = assess_sqn(LevelMinus1, to_hashes), - {L0Lookup, L0TreeList, L0Size} = LevelZero, - UpdL0TreeList = [{LedgerSQN, LevelMinus1}|L0TreeList], - UpdL0Lookup = lists:foldl(fun(X, LookupArray) -> - L = array:get(X, LookupArray), - array:set(X, [LedgerSQN|L], LookupArray) - end, - L0Lookup, - HashList), - NewSize = LM1Size + L0Size, - T = timer:now_diff(os:timestamp(), SW), - io:format("Rolled tree to size ~w in ~w microseconds using array_filter~n", - [NewSize, T]), - if - MinSQN >= LedgerSQN -> - {stop, - normal, - {{UpdL0Lookup, UpdL0TreeList, NewSize}, NewSize, MaxSQN, T}, - State} + MinSQN > LedgerSQN -> + {MaxSQN, + NewL0Size, + UpdL0Index, + lists:append(TreeList, [LevelMinus1])} end. + -handle_cast(_Msg, State) -> - {noreply, State}. +to_list(TreeList) -> + SW = os:timestamp(), + OutList = lists:foldr(fun(Tree, CompleteList) -> + L = gb_trees:to_list(Tree), + lists:umerge(CompleteList, L) + end, + [], + TreeList), + io:format("Rolled tree to list of size ~w in ~w microseconds~n", + [length(OutList), timer:now_diff(os:timestamp(), SW)]), + OutList. -handle_info(_Msg, State) -> - {stop, normal, ok, State}. -terminate(_Reason, _State) -> - ok. +new_index() -> + array:new(element(1, ?SLOT_WIDTH), [{default, []}, fixed]). -code_change(_OldVsn, State, _Extra) -> - {ok, State}. +check_levelzero(Key, L0Index, TreeList) -> + {Hash, Slot} = hash_to_slot(Key), + CheckList = array:get(Slot, L0Index), + SlotList = lists:foldl(fun({H0, S0}, SL) -> + case H0 of + Hash -> + [S0|SL]; + _ -> + SL + end + end, + [], + CheckList), + lists:foldl(fun(SlotToCheck, {Found, KV}) -> + case Found of + true -> + {Found, KV}; + false -> + CheckTree = lists:nth(SlotToCheck, TreeList), + case gb_trees:lookup(Key, CheckTree) of + none -> + {Found, KV}; + {value, Value} -> + {true, {Key, Value}} + end + end + end, + {false, not_found}, + lists:reverse(lists:usort(SlotList))). + + +merge_trees(StartKey, EndKey, TreeList) -> + lists:foldl(fun(Tree, TreeAcc) -> + merge_nexttree(Tree, TreeAcc, StartKey, EndKey) end, + gb_trees:empty(), + TreeList). %%%============================================================================ -%%% Internal functions +%%% Internal Functions %%%============================================================================ -hash_to_index(Key) -> - erlang:phash2(Key) band (?ARRAY_WIDTH - 1). - hash_to_slot(Key) -> - erlang:phash2(Key) band (?SLOT_WIDTH - 1). + H = erlang:phash2(Key), + {H bsr element(2, ?SLOT_WIDTH), H band (element(1, ?SLOT_WIDTH) - 1)}. -roll_into_list(Tree) -> - gb_trees:to_list(Tree). +merge_nexttree(Tree, TreeAcc, StartKey, EndKey) -> + Iter = gb_trees:iterator_from(StartKey, Tree), + merge_nexttree(Iter, TreeAcc, EndKey). -assess_sqn(Tree, to_array) -> - L = roll_into_list(Tree), - TmpA = array:new(?ARRAY_WIDTH, {default, []}), - FoldFun = fun({K, V}, {AccMinSQN, AccMaxSQN, AccSize, Array}) -> - SQN = leveled_codec:strip_to_seqonly({K, V}), - Index = hash_to_index(K), - List0 = array:get(Index, Array), - List1 = lists:append(List0, [{K, V}]), - {min(SQN, AccMinSQN), - max(SQN, AccMaxSQN), - AccSize + 1, - array:set(Index, List1, Array)} - end, - lists:foldl(FoldFun, {infinity, 0, 0, TmpA}, L); -assess_sqn(Tree, to_hashes) -> - L = roll_into_list(Tree), - FoldFun = fun({K, V}, {AccMinSQN, AccMaxSQN, AccSize, HashList}) -> - SQN = leveled_codec:strip_to_seqonly({K, V}), - Hash = hash_to_slot(K), - {min(SQN, AccMinSQN), - max(SQN, AccMaxSQN), - AccSize + 1, - [Hash|HashList]} - end, - lists:foldl(FoldFun, {infinity, 0, 0, []}, L). +merge_nexttree(Iter, TreeAcc, EndKey) -> + case gb_trees:next(Iter) of + none -> + TreeAcc; + {Key, Value, NewIter} -> + case leveled_codec:endkey_passed(EndKey, Key) of + true -> + TreeAcc; + false -> + merge_nexttree(NewIter, + gb_trees:enter(Key, Value, TreeAcc), + EndKey) + end + end. %%%============================================================================ %%% Test @@ -213,49 +190,76 @@ generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) -> BRange). -speed_test() -> - R = lists:foldl(fun(_X, {LedgerSQN, - {L0st, TTst}, - {L0at, TTat}, - {L0al, TTal}, - {L0af, TTaf}}) -> +compare_method_test() -> + R = lists:foldl(fun(_X, {LedgerSQN, L0Size, L0Index, L0TreeList}) -> LM1 = generate_randomkeys(LedgerSQN + 1, 2000, 1, 500), - {NextL0st, S, MaxSQN, Tst} = roll_singletree(L0st, - LM1, - LedgerSQN, - self()), - {NextL0at, S, MaxSQN, Tat} = roll_arraytree(L0at, - LM1, - LedgerSQN, - self()), - {NextL0al, _S, MaxSQN, Tal} = roll_arraylist(L0al, - LM1, - LedgerSQN, - self()), - {NextL0af, _S, MaxSQN, Taf} = roll_arrayfilt(L0af, - LM1, - LedgerSQN, - self()), - {MaxSQN, - {NextL0st, TTst + Tst}, - {NextL0at, TTat + Tat}, - {NextL0al, TTal + Tal}, - {NextL0af, TTaf + Taf}} + add_to_index(L0Index, L0Size, LM1, LedgerSQN, L0TreeList) end, - {0, - {gb_trees:empty(), 0}, - {array:new(?ARRAY_WIDTH, [{default, gb_trees:empty()}, fixed]), 0}, - {array:new(?ARRAY_WIDTH, [{default, []}, fixed]), 0}, - {{array:new(?SLOT_WIDTH, [{default, []}, fixed]), [], 0}, 0} - }, + {0, 0, new_index(), []}, lists:seq(1, 16)), - {_, {_, TimeST}, {_, TimeAT}, {_, TimeLT}, {_, TimeAF}} = R, - io:format("Total time for single_tree ~w microseconds ~n", [TimeST]), - io:format("Total time for array_tree ~w microseconds ~n", [TimeAT]), - io:format("Total time for array_list ~w microseconds ~n", [TimeLT]), - io:format("Total time for array_filter ~w microseconds ~n", [TimeAF]), - ?assertMatch(true, false). - + + {SQN, _Size, Index, TreeList} = R, + ?assertMatch(32000, SQN), + + TestList = gb_trees:to_list(generate_randomkeys(1, 2000, 1, 800)), + + S0 = lists:foldl(fun({Key, _V}, Acc) -> + R0 = lists:foldr(fun(Tree, {Found, KV}) -> + case Found of + true -> + {true, KV}; + false -> + L0 = gb_trees:lookup(Key, Tree), + case L0 of + none -> + {false, not_found}; + {value, Value} -> + {true, {Key, Value}} + end + end + end, + {false, not_found}, + TreeList), + [R0|Acc] + end, + [], + TestList), + + S1 = lists:foldl(fun({Key, _V}, Acc) -> + R0 = check_levelzero(Key, Index, TreeList), + [R0|Acc] + end, + [], + TestList), + + ?assertMatch(S0, S1), + + StartKey = {o, "Bucket0100", null, null}, + EndKey = {o, "Bucket0200", null, null}, + SWa = os:timestamp(), + DumpList = to_list(TreeList), + Q0 = lists:foldl(fun({K, V}, Acc) -> + P = leveled_codec:endkey_passed(EndKey, K), + case {K, P} of + {K, false} when K >= StartKey -> + gb_trees:enter(K, V, Acc); + _ -> + Acc + end + end, + gb_trees:empty(), + DumpList), + Sz0 = gb_trees:size(Q0), + io:format("Crude method took ~w microseconds resulting in tree of " ++ + "size ~w~n", + [timer:now_diff(os:timestamp(), SWa), Sz0]), + SWb = os:timestamp(), + Q1 = merge_trees(StartKey, EndKey, TreeList), + Sz1 = gb_trees:size(Q1), + io:format("Merge method took ~w microseconds resulting in tree of " ++ + "size ~w~n", + [timer:now_diff(os:timestamp(), SWb), Sz1]), + ?assertMatch(Sz0, Sz1). From 95609702bda09966cfc31f0918db5c053dae0b82 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sun, 30 Oct 2016 18:25:30 +0000 Subject: [PATCH 104/167] Penciller Memory Refactor Plugged the ne wpencille rmemory into the Penciller, and took advantage of the increased speed to simplify the callbacks involved. The outcome is much simpler code --- src/leveled_bookie.erl | 2 +- src/leveled_inker.erl | 2 +- src/leveled_pclerk.erl | 73 +++----- src/leveled_penciller.erl | 371 ++++++++++++++++---------------------- src/leveled_pmem.erl | 14 +- src/leveled_sft.erl | 30 ++- 6 files changed, 214 insertions(+), 278 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 206e859..50a886d 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -645,7 +645,7 @@ maybepush_ledgercache(MaxCacheSize, Cache, Penciller) -> case leveled_penciller:pcl_pushmem(Penciller, Cache) of ok -> {ok, gb_trees:empty()}; - {returned, _Reason} -> + returned -> {ok, Cache} end; true -> diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 1c8ed17..f3b3075 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -645,7 +645,7 @@ push_to_penciller(Penciller, KeyTree) -> % the list by order before becoming a de-duplicated list for loading R = leveled_penciller:pcl_pushmem(Penciller, KeyTree), case R of - {returned, _Reason} -> + returned -> timer:sleep(?LOADING_PAUSE), push_to_penciller(Penciller, KeyTree); ok -> diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 792acb4..635d661 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -59,11 +59,9 @@ handle_cast/2, handle_info/2, terminate/2, - clerk_new/2, - mergeclerk_prompt/1, - mergeclerk_manifestchange/3, - rollclerk_levelzero/5, - rollclerk_close/1, + clerk_new/1, + clerk_prompt/1, + clerk_manifestchange/3, code_change/3]). -include_lib("eunit/include/eunit.hrl"). @@ -73,32 +71,25 @@ -record(state, {owner :: pid(), change_pending=false :: boolean(), - work_item :: #penciller_work{}|null, - merge_clerk = false :: boolean(), - roll_clerk = false ::boolean()}). + work_item :: #penciller_work{}|null}). %%%============================================================================ %%% API %%%============================================================================ -clerk_new(Owner, Type) -> +clerk_new(Owner) -> {ok, Pid} = gen_server:start(?MODULE, [], []), - ok = gen_server:call(Pid, {register, Owner, Type}, infinity), + ok = gen_server:call(Pid, {register, Owner}, infinity), io:format("Penciller's clerk ~w started with owner ~w~n", [Pid, Owner]), {ok, Pid}. -mergeclerk_manifestchange(Pid, Action, Closing) -> +clerk_manifestchange(Pid, Action, Closing) -> gen_server:call(Pid, {manifest_change, Action, Closing}, infinity). -mergeclerk_prompt(Pid) -> +clerk_prompt(Pid) -> gen_server:cast(Pid, prompt). -rollclerk_levelzero(Pid, LevelZero, LevelMinus1, LedgerSQN, PCL) -> - gen_server:cast(Pid, - {roll_levelzero, LevelZero, LevelMinus1, LedgerSQN, PCL}). -rollclerk_close(Pid) -> - gen_server:call(Pid, close, infinity). %%%============================================================================ %%% gen_server callbacks @@ -107,18 +98,11 @@ rollclerk_close(Pid) -> init([]) -> {ok, #state{}}. -handle_call({register, Owner, Type}, _From, State) -> - case Type of - merge -> - {reply, - ok, - State#state{owner=Owner, merge_clerk = true}, - ?MIN_TIMEOUT}; - roll -> - {reply, - ok, - State#state{owner=Owner, roll_clerk = true}} - end; +handle_call({register, Owner}, _From, State) -> + {reply, + ok, + State#state{owner=Owner}, + ?MIN_TIMEOUT}; handle_call({manifest_change, return, true}, _From, State) -> io:format("Request for manifest change from clerk on closing~n"), case State#state.change_pending of @@ -150,31 +134,20 @@ handle_call(close, _From, State) -> {stop, normal, ok, State}. handle_cast(prompt, State) -> - {noreply, State, ?MIN_TIMEOUT}; -handle_cast({roll_levelzero, LevelZero, LevelMinus1, LedgerSQN, PCL}, State) -> - SW = os:timestamp(), - {NewL0, Size, MaxSQN} = leveled_penciller:roll_new_tree(LevelZero, - LevelMinus1, - LedgerSQN), - ok = leveled_penciller:pcl_updatelevelzero(PCL, NewL0, Size, MaxSQN), - io:format("Rolled tree to size ~w in ~w microseconds~n", - [Size, timer:now_diff(os:timestamp(), SW)]), - {noreply, State}. + {noreply, State, ?MIN_TIMEOUT}. handle_info(timeout, State=#state{change_pending=Pnd}) when Pnd == false -> - if - State#state.merge_clerk -> - case requestandhandle_work(State) of - {false, Timeout} -> - {noreply, State, Timeout}; - {true, WI} -> - % No timeout now as will wait for call to return manifest - % change - {noreply, - State#state{change_pending=true, work_item=WI}} - end + case requestandhandle_work(State) of + {false, Timeout} -> + {noreply, State, Timeout}; + {true, WI} -> + % No timeout now as will wait for call to return manifest + % change + {noreply, + State#state{change_pending=true, work_item=WI}} end. + terminate(Reason, _State) -> io:format("Penciller's Clerk ~w shutdown now complete for reason ~w~n", [self(), Reason]). diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 8522a30..17554d9 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -185,21 +185,29 @@ %% path. However, an ETS table is mutable, so it does complicate the %% snapshotting of the Ledger. %% -%% Originally the solution had used an ETS tbale for insertion speed as the L0 +%% Originally the solution had used an ETS table for insertion speed as the L0 %% cache. Insertion speed was an order or magnitude faster than gb_trees. To %% resolving issues of trying to have fast start-up snapshots though led to %% keeping a seperate set of trees alongside the ETS table to be used by %% snapshots. %% -%% The current strategy is to perform the expensive operation (merging the +%% The next strategy was to perform the expensive operation (merging the %% Ledger cache into the Level0 cache), within a dedicated Penciller's clerk, %% known as the roll_clerk. This may take 30-40ms, but during this period %% the Penciller will keep a Level -1 cache of the unmerged elements which %% it will wipe once the roll_clerk returns with an updated L0 cache. %% -%% This means that the in-memory cache is now using immutable objects, and -%% so can be shared with snapshots. - +%% This was still a bit complicated, and did a lot of processing to +%% making updates to the large L0 cache - which will have created a lot of GC +%% effort required. The processing was inefficient +%% +%% The current paproach is to simply append each new tree pushed to a list, and +%% use an array of hashes to index for the presence of objects in the list. +%% When attempting to iterate, the caches are all merged for the range relevant +%% to the given iterator only. The main downside to the approahc is that the +%% Penciller cna no longer accurately measure the size of the L0 cache (as it +%% cannot determine how many replacements there are in the Cache - so it may +%% prematurely write a smaller than necessary L0 file. -module(leveled_penciller). @@ -224,11 +232,8 @@ pcl_close/1, pcl_registersnapshot/2, pcl_releasesnapshot/2, - pcl_updatelevelzero/4, pcl_loadsnapshot/2, pcl_getstartupsequencenumber/1, - roll_new_tree/3, - roll_into_list/1, clean_testdir/1]). -include_lib("eunit/include/eunit.hrl"). @@ -249,23 +254,21 @@ -record(state, {manifest = [] :: list(), manifest_sqn = 0 :: integer(), - ledger_sqn = 0 :: integer(), + ledger_sqn = 0 :: integer(), % The highest SQN added to L0 + persisted_sqn = 0 :: integer(), % The highest SQN persisted registered_snapshots = [] :: list(), unreferenced_files = [] :: list(), root_path = "../test" :: string(), - merge_clerk :: pid(), - roll_clerk :: pid(), + clerk :: pid(), levelzero_pending = false :: boolean(), levelzero_constructor :: pid(), - levelzero_cache = gb_trees:empty() :: gb_trees:tree(), - levelzero_cachesize = 0 :: integer(), + levelzero_cache = [] :: list(), % a list of gb_trees + levelzero_index :: erlang:array(), + levelzero_size = 0 :: integer(), levelzero_maxcachesize :: integer(), - levelminus1_active = false :: boolean(), - levelminus1_cache = gb_trees:empty() :: gb_trees:tree(), - is_snapshot = false :: boolean(), snapshot_fully_loaded = false :: boolean(), source_penciller :: pid(), @@ -317,8 +320,6 @@ pcl_releasesnapshot(Pid, Snapshot) -> pcl_loadsnapshot(Pid, Increment) -> gen_server:call(Pid, {load_snapshot, Increment}, infinity). -pcl_updatelevelzero(Pid, L0Cache, L0Size, L0SQN) -> - gen_server:cast(Pid, {load_levelzero, L0Cache, L0Size, L0SQN}). pcl_close(Pid) -> gen_server:call(Pid, close, 60000). @@ -335,7 +336,7 @@ init([PCLopts]) -> SrcPenciller = PCLopts#penciller_options.source_penciller, io:format("Registering ledger snapshot~n"), {ok, State} = pcl_registersnapshot(SrcPenciller, self()), - io:format("Lesger snapshot registered~n"), + io:format("Ledger snapshot registered~n"), {ok, State#state{is_snapshot=true, source_penciller=SrcPenciller}}; %% Need to do something about timeout {_RootPath, false} -> @@ -351,37 +352,26 @@ handle_call({push_mem, PushedTree}, From, State=#state{is_snapshot=Snap}) % we mean value from the perspective of the Ledger, not the full value % stored in the Inker) % - % 2 - Check to see if the levelminus1 cache is active, if so the update - % cannot yet be accepted and must be returned (so the Bookie will maintain - % the cache and try again later). - % - % 3 - Check to see if there is a levelzero file pending. If so check if + % 2 - Check to see if there is a levelzero file pending. If so check if % the levelzero file is complete. If it is complete, the levelzero tree % can be flushed, the in-memory manifest updated, and the new tree can % be accepted as the new levelzero cache. If not, the update must be % returned. % - % 3 - Make the new tree the levelminus1 cache, and mark this as active - % - % 4 - The Penciller can now reply to the Bookie to show that the push has + % 3 - The Penciller can now reply to the Bookie to show that the push has % been accepted % - % 5 - A background worker clerk can now be triggered to produce a new - % levelzero cache (tree) containing the level minus 1 tree. When this - % completes it will cast back the updated tree, and on receipt of this - % the Penciller may: - % a) Clear down the levelminus1 cache - % b) Determine if the cache is full and it is necessary to build a new - % persisted levelzero file + % 4 - Update the cache: + % a) Append the cache to the list + % b) Add hashes for all the elements to the index + % + % Check the approximate size of the cache. If it is over the maximum size, + % trigger a backgroun L0 file write and update state of levelzero_pending. SW = os:timestamp(), - S = case {State#state.levelzero_pending, - State#state.levelminus1_active} of - {_, true} -> - log_pushmem_reply(From, {returned, "L-1 Active"}, SW), - State; - {true, _} -> + S = case State#state.levelzero_pending of + true -> L0Pid = State#state.levelzero_constructor, case checkready(L0Pid) of timeout -> @@ -391,65 +381,71 @@ handle_call({push_mem, PushedTree}, From, State=#state{is_snapshot=Snap}) SW), State; {ok, SrcFN, StartKey, EndKey} -> - log_pushmem_reply(From, ok, SW), + log_pushmem_reply(From, + {ok, + "L-0 persist completed"}, + SW), ManEntry = #manifest_entry{start_key=StartKey, end_key=EndKey, owner=L0Pid, filename=SrcFN}, - % Prompt clerk to ask about work - do this for - % every L0 roll - ok = leveled_pclerk:mergeclerk_prompt(State#state.merge_clerk), UpdMan = lists:keystore(0, 1, State#state.manifest, {0, [ManEntry]}), - {MinSQN, MaxSQN, Size, _L} = assess_sqn(PushedTree), - if - MinSQN > State#state.ledger_sqn -> - State#state{manifest=UpdMan, - levelzero_cache=PushedTree, - levelzero_cachesize=Size, + LedgerSQN = State#state.ledger_sqn, + UpdState = State#state{manifest=UpdMan, levelzero_pending=false, - ledger_sqn=MaxSQN} - end + persisted_sqn=LedgerSQN}, + % Prompt clerk to ask about work - do this for + % every L0 roll + ok = leveled_pclerk:clerk_prompt(State#state.clerk), + NewL0Index = leveled_pmem:new_index(), + update_levelzero(NewL0Index, + 0, + PushedTree, + LedgerSQN, + [], + UpdState) end; - {false, false} -> - log_pushmem_reply(From, ok, SW), - ok = leveled_pclerk:rollclerk_levelzero(State#state.roll_clerk, - State#state.levelzero_cache, - PushedTree, - State#state.ledger_sqn, - self()), - State#state{levelminus1_active=true, - levelminus1_cache=PushedTree} + false -> + log_pushmem_reply(From, {ok, "L0 memory updated"}, SW), + update_levelzero(State#state.levelzero_index, + State#state.levelzero_size, + PushedTree, + State#state.ledger_sqn, + State#state.levelzero_cache, + State) end, io:format("Handling of push completed in ~w microseconds with " ++ "L0 cache size now ~w~n", [timer:now_diff(os:timestamp(), SW), - S#state.levelzero_cachesize]), + S#state.levelzero_size]), {noreply, S}; handle_call({fetch, Key}, _From, State) -> {reply, - fetch(Key, - State#state.manifest, - State#state.levelminus1_active, - State#state.levelminus1_cache, - State#state.levelzero_cache), + fetch_mem(Key, + State#state.manifest, + State#state.levelzero_index, + State#state.levelzero_cache), State}; handle_call({check_sqn, Key, SQN}, _From, State) -> {reply, - compare_to_sqn(fetch(Key, - State#state.manifest, - State#state.levelminus1_active, - State#state.levelminus1_cache, - State#state.levelzero_cache), + compare_to_sqn(fetch_mem(Key, + State#state.manifest, + State#state.levelzero_index, + State#state.levelzero_cache), SQN), State}; handle_call({fetch_keys, StartKey, EndKey, AccFun, InitAcc}, _From, State=#state{snapshot_fully_loaded=Ready}) when Ready == true -> - L0iter = gb_trees:iterator_from(StartKey, State#state.levelzero_cache), + L0AsTree = leveled_pmem:merge_trees(StartKey, + EndKey, + State#state.levelzero_cache, + gb_trees:empty()), + L0iter = gb_trees:iterator(L0AsTree), SFTiter = initiate_rangequery_frommanifest(StartKey, EndKey, State#state.manifest), @@ -459,39 +455,31 @@ handle_call(work_for_clerk, From, State) -> {UpdState, Work} = return_work(State, From), {reply, Work, UpdState}; handle_call(get_startup_sqn, _From, State) -> - {reply, State#state.ledger_sqn, State}; + {reply, State#state.persisted_sqn, State}; handle_call({register_snapshot, Snapshot}, _From, State) -> Rs = [{Snapshot, State#state.manifest_sqn}|State#state.registered_snapshots], {reply, {ok, State}, State#state{registered_snapshots = Rs}}; handle_call({load_snapshot, BookieIncrTree}, _From, State) -> - {L0T0, _, L0SQN0} = case State#state.levelminus1_active of - true -> - roll_new_tree(State#state.levelzero_cache, - State#state.levelminus1_cache, - State#state.ledger_sqn); - false -> - {State#state.levelzero_cache, - 0, - State#state.ledger_sqn} - end, - {L0T1, _, L0SQN1} = roll_new_tree(L0T0, + L0D = leveled_pmem:add_to_index(State#state.levelzero_index, + State#state.levelzero_size, BookieIncrTree, - L0SQN0), - io:format("Ledger snapshot loaded with increments to start at SQN=~w~n", - [L0SQN1]), - {reply, ok, State#state{levelzero_cache=L0T1, - ledger_sqn=L0SQN1, - levelminus1_active=false, - snapshot_fully_loaded=true}}; + State#state.ledger_sqn, + State#state.levelzero_cache), + {LedgerSQN, L0Size, L0Index, L0Cache} = L0D, + {reply, ok, State#state{levelzero_cache=L0Cache, + levelzero_index=L0Index, + levelzero_size=L0Size, + ledger_sqn=LedgerSQN, + snapshot_fully_loaded=true}}; handle_call(close, _From, State) -> {stop, normal, ok, State}. handle_cast({manifest_change, WI}, State) -> {ok, UpdState} = commit_manifest_change(WI, State), - ok = leveled_pclerk:mergeclerk_manifestchange(State#state.merge_clerk, - confirm, - false), + ok = leveled_pclerk:clerk_manifestchange(State#state.clerk, + confirm, + false), {noreply, UpdState}; handle_cast({release_snapshot, Snapshot}, State) -> Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots), @@ -513,39 +501,9 @@ handle_cast({confirm_delete, FileName}, State=#state{is_snapshot=Snap}) {noreply, State#state{unreferenced_files=UF1}}; _ -> {noreply, State} - end; -handle_cast({load_levelzero, L0Cache, L0Size, L0SQN}, State) -> - if - L0SQN >= State#state.ledger_sqn -> - CacheTooBig = L0Size > State#state.levelzero_maxcachesize, - Level0Free = length(get_item(0, State#state.manifest, [])) == 0, - case {CacheTooBig, Level0Free} of - {true, true} -> - L0Constructor = roll_memory(State, L0Cache), - {noreply, - State#state{levelminus1_active=false, - levelminus1_cache=gb_trees:empty(), - levelzero_cache=L0Cache, - levelzero_cachesize=L0Size, - levelzero_pending=true, - levelzero_constructor=L0Constructor, - ledger_sqn=L0SQN}}; - _ -> - {noreply, - State#state{levelminus1_active=false, - levelminus1_cache=gb_trees:empty(), - levelzero_cache=L0Cache, - levelzero_cachesize=L0Size, - ledger_sqn=L0SQN}} - end; - L0Size == 0 -> - {noreply, - State#state{levelminus1_active=false, - levelminus1_cache=gb_trees:empty(), - levelzero_cache=L0Cache, - levelzero_cachesize=L0Size}} end. + handle_info({_Ref, {ok, SrcFN, _StartKey, _EndKey}}, State) -> io:format("Orphaned reply after timeout on L0 file write ~s~n", [SrcFN]), {noreply, State}. @@ -573,37 +531,34 @@ terminate(Reason, State) -> %% the penciller looking for a manifest commit %% io:format("Penciller closing for reason - ~w~n", [Reason]), - MC = leveled_pclerk:mergeclerk_manifestchange(State#state.merge_clerk, - return, - true), + MC = leveled_pclerk:clerk_manifestchange(State#state.clerk, + return, + true), UpdState = case MC of {ok, WI} -> {ok, NewState} = commit_manifest_change(WI, State), - Clerk = State#state.merge_clerk, - ok = leveled_pclerk:mergeclerk_manifestchange(Clerk, - confirm, - true), + Clerk = State#state.clerk, + ok = leveled_pclerk:clerk_manifestchange(Clerk, + confirm, + true), NewState; no_change -> State end, case {UpdState#state.levelzero_pending, get_item(0, State#state.manifest, []), - gb_trees:size(State#state.levelzero_cache)} of + State#state.levelzero_size} of {true, [], _} -> ok = leveled_sft:sft_close(State#state.levelzero_constructor); {false, [], 0} -> io:format("Level 0 cache empty at close of Penciller~n"); {false, [], _N} -> - KL = roll_into_list(State#state.levelzero_cache), - L0Pid = roll_memory(UpdState, KL, true), + L0Pid = roll_memory(UpdState, State#state.levelzero_cache, true), ok = leveled_sft:sft_close(L0Pid); _ -> io:format("No level zero action on close of Penciller~n") end, - leveled_pclerk:rollclerk_close(State#state.roll_clerk), - % Tidy shutdown of individual files ok = close_files(0, UpdState#state.manifest), lists:foreach(fun({_FN, Pid, _SN}) -> @@ -630,11 +585,10 @@ start_from_file(PCLopts) -> M end, - {ok, MergeClerk} = leveled_pclerk:clerk_new(self(), merge), - {ok, RollClerk} = leveled_pclerk:clerk_new(self(), roll), - InitState = #state{merge_clerk=MergeClerk, - roll_clerk=RollClerk, + {ok, MergeClerk} = leveled_pclerk:clerk_new(self()), + InitState = #state{clerk=MergeClerk, root_path=RootPath, + levelzero_index = leveled_pmem:new_index(), levelzero_maxcachesize=MaxTableSize}, %% Open manifest @@ -701,23 +655,61 @@ start_from_file(PCLopts) -> {0, [ManifestEntry]}), io:format("L0 file had maximum sequence number of ~w~n", [L0SQN]), + LedgerSQN = max(MaxSQN, L0SQN), {ok, InitState#state{manifest=UpdManifest2, manifest_sqn=TopManSQN, - ledger_sqn=max(MaxSQN, L0SQN)}}; + ledger_sqn=LedgerSQN, + persisted_sqn=LedgerSQN}}; false -> io:format("No L0 file found~n"), {ok, InitState#state{manifest=UpdManifest, manifest_sqn=TopManSQN, - ledger_sqn=MaxSQN}} + ledger_sqn=MaxSQN, + persisted_sqn=MaxSQN}} end. log_pushmem_reply(From, Reply, SW) -> - io:format("Respone to push_mem of ~w took ~w microseconds~n", - [Reply, timer:now_diff(os:timestamp(), SW)]), - gen_server:reply(From, Reply). + io:format("Respone to push_mem of ~w ~s took ~w microseconds~n", + [element(1, Reply), + element(2, Reply), + timer:now_diff(os:timestamp(), SW)]), + gen_server:reply(From, element(1, Reply)). + + +update_levelzero(L0Index, L0Size, PushedTree, LedgerSQN, L0Cache, State) -> + Update = leveled_pmem:add_to_index(L0Index, + L0Size, + PushedTree, + LedgerSQN, + L0Cache), + {MaxSQN, NewL0Size, UpdL0Index, UpdL0Cache} = Update, + if + MaxSQN >= LedgerSQN -> + UpdState = State#state{levelzero_cache=UpdL0Cache, + levelzero_index=UpdL0Index, + levelzero_size=NewL0Size, + ledger_sqn=MaxSQN}, + CacheTooBig = NewL0Size > State#state.levelzero_maxcachesize, + Level0Free = length(get_item(0, State#state.manifest, [])) == 0, + case {CacheTooBig, Level0Free} of + {true, true} -> + L0Constructor = roll_memory(State, UpdL0Cache), + UpdState#state{levelzero_pending=true, + levelzero_constructor=L0Constructor}; + _ -> + UpdState + end; + NewL0Size == L0Size -> + State#state{levelzero_cache=L0Cache, + levelzero_index=L0Index, + levelzero_size=L0Size, + ledger_sqn=LedgerSQN} + end. + + checkready(Pid) -> try @@ -727,71 +719,28 @@ checkready(Pid) -> timeout end. -roll_memory(State, L0Tree) -> - roll_memory(State, L0Tree, false). +roll_memory(State, L0Cache) -> + roll_memory(State, L0Cache, false). -roll_memory(State, L0Tree, Wait) -> +roll_memory(State, L0Cache, Wait) -> MSN = State#state.manifest_sqn, FileName = State#state.root_path ++ "/" ++ ?FILES_FP ++ "/" ++ integer_to_list(MSN) ++ "_0_0", Opts = #sft_options{wait=Wait}, - {ok, Constructor, _} = leveled_sft:sft_new(FileName, L0Tree, [], 0, Opts), + {ok, Constructor, _} = leveled_sft:sft_newfroml0cache(FileName, + L0Cache, + Opts), Constructor. -% Merge the Level Minus 1 tree to the Level 0 tree, incrementing the -% SQN, and ensuring all entries do increment the SQN -roll_new_tree(L0Cache, LMinus1Cache, LedgerSQN) -> - {MinSQN, MaxSQN, Size, LMinus1List} = assess_sqn(LMinus1Cache), - if - MinSQN >= LedgerSQN -> - UpdTree = lists:foldl(fun({Kx, Vx}, TreeAcc) -> - gb_trees:enter(Kx, Vx, TreeAcc) - end, - L0Cache, - LMinus1List), - {UpdTree, gb_trees:size(UpdTree), MaxSQN}; - Size == 0 -> - {L0Cache, gb_trees:size(L0Cache), LedgerSQN} - end. - -%% This takes the three parts of a memtable copy - the increments, the tree -%% and the SQN at which the tree was formed, and outputs a sorted list -roll_into_list(Tree) -> - gb_trees:to_list(Tree). - -assess_sqn(Tree) -> - L = roll_into_list(Tree), - FoldFun = fun(KV, {AccMinSQN, AccMaxSQN, AccSize}) -> - SQN = leveled_codec:strip_to_seqonly(KV), - {min(SQN, AccMinSQN), max(SQN, AccMaxSQN), AccSize + 1} - end, - {MinSQN, MaxSQN, Size} = lists:foldl(FoldFun, {infinity, 0, 0}, L), - {MinSQN, MaxSQN, Size, L}. - - -fetch(Key, Manifest, LM1Active, LM1Tree, L0Tree) -> - case LM1Active of - true -> - case gb_trees:lookup(Key, LM1Tree) of - none -> - case gb_trees:lookup(Key, L0Tree) of - none -> - fetch(Key, Manifest, 0, fun leveled_sft:sft_get/2); - {value, Value} -> - {Key, Value} - end; - {value, Value} -> - {Key, Value} - end; - false -> - case gb_trees:lookup(Key, L0Tree) of - none -> - fetch(Key, Manifest, 0, fun leveled_sft:sft_get/2); - {value, Value} -> - {Key, Value} - end +fetch_mem(Key, Manifest, L0Index, L0Cache) -> + L0Check = leveled_pmem:check_levelzero(Key, L0Index, L0Cache), + case L0Check of + {false, not_found} -> + fetch(Key, Manifest, 0, fun leveled_sft:sft_get/2); + {true, KV} -> + KV end. fetch(_Key, _Manifest, ?MAX_LEVELS + 1, _FetchFun) -> @@ -1375,7 +1324,7 @@ maybe_pause_push(PCL, KL) -> T0, KL), case pcl_pushmem(PCL, T1) of - {returned, _Reason} -> + returned -> timer:sleep(50), maybe_pause_push(PCL, KL); ok -> @@ -1402,7 +1351,6 @@ simple_server_test() -> ok = maybe_pause_push(PCL, [Key2]), ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), - ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), ok = maybe_pause_push(PCL, KL2), ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), @@ -1686,12 +1634,13 @@ foldwithimm_simple_test() -> create_file_test() -> Filename = "../test/new_file.sft", ok = file:write_file(Filename, term_to_binary("hello")), - {KL1, KL2} = {lists:sort(leveled_sft:generate_randomkeys(10000)), []}, - {ok, SP, noreply} = leveled_sft:sft_new(Filename, - KL1, - KL2, - 0, - #sft_options{wait=false}), + KVL = lists:usort(leveled_sft:generate_randomkeys(10000)), + Tree = gb_trees:from_orddict(KVL), + {ok, + SP, + noreply} = leveled_sft:sft_newfroml0cache(Filename, + [Tree], + #sft_options{wait=false}), lists:foreach(fun(X) -> case checkready(SP) of timeout -> diff --git a/src/leveled_pmem.erl b/src/leveled_pmem.erl index 9cf498b..ae7bf09 100644 --- a/src/leveled_pmem.erl +++ b/src/leveled_pmem.erl @@ -41,7 +41,7 @@ to_list/1, new_index/0, check_levelzero/3, - merge_trees/3 + merge_trees/4 ]). -include_lib("eunit/include/eunit.hrl"). @@ -69,7 +69,7 @@ add_to_index(L0Index, L0Size, LevelMinus1, LedgerSQN, TreeList) -> {infinity, 0, L0Index}, LM1List), NewL0Size = length(LM1List) + L0Size, - io:format("Rolled tree to size ~w in ~w microseconds~n", + io:format("Rolled L0 cache to size ~w in ~w microseconds~n", [NewL0Size, timer:now_diff(os:timestamp(), SW)]), if MinSQN > LedgerSQN -> @@ -84,11 +84,11 @@ to_list(TreeList) -> SW = os:timestamp(), OutList = lists:foldr(fun(Tree, CompleteList) -> L = gb_trees:to_list(Tree), - lists:umerge(CompleteList, L) + lists:ukeymerge(1, CompleteList, L) end, [], TreeList), - io:format("Rolled tree to list of size ~w in ~w microseconds~n", + io:format("L0 cache converted to list of size ~w in ~w microseconds~n", [length(OutList), timer:now_diff(os:timestamp(), SW)]), OutList. @@ -128,11 +128,11 @@ check_levelzero(Key, L0Index, TreeList) -> lists:reverse(lists:usort(SlotList))). -merge_trees(StartKey, EndKey, TreeList) -> +merge_trees(StartKey, EndKey, TreeList, LevelMinus1) -> lists:foldl(fun(Tree, TreeAcc) -> merge_nexttree(Tree, TreeAcc, StartKey, EndKey) end, gb_trees:empty(), - TreeList). + lists:append(TreeList, [LevelMinus1])). %%%============================================================================ %%% Internal Functions @@ -254,7 +254,7 @@ compare_method_test() -> "size ~w~n", [timer:now_diff(os:timestamp(), SWa), Sz0]), SWb = os:timestamp(), - Q1 = merge_trees(StartKey, EndKey, TreeList), + Q1 = merge_trees(StartKey, EndKey, TreeList, gb_trees:empty()), Sz1 = gb_trees:size(Q1), io:format("Merge method took ~w microseconds resulting in tree of " ++ "size ~w~n", diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index c9cfd7f..64b903c 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -152,7 +152,7 @@ terminate/2, code_change/3, sft_new/4, - sft_new/5, + sft_newfroml0cache/3, sft_open/1, sft_get/2, sft_getkvrange/4, @@ -211,10 +211,8 @@ %%% API %%%============================================================================ -sft_new(Filename, KL1, KL2, LevelInfo) -> - sft_new(Filename, KL1, KL2, LevelInfo, #sft_options{}). -sft_new(Filename, KL1, KL2, LevelInfo, Options) -> +sft_new(Filename, KL1, KL2, LevelInfo) -> LevelR = case is_integer(LevelInfo) of true -> #level{level=LevelInfo}; @@ -225,15 +223,30 @@ sft_new(Filename, KL1, KL2, LevelInfo, Options) -> end end, {ok, Pid} = gen_server:start(?MODULE, [], []), + Reply = gen_server:call(Pid, + {sft_new, Filename, KL1, KL2, LevelR}, + infinity), + {ok, Pid, Reply}. + +sft_newfroml0cache(Filename, L0Cache, Options) -> + {ok, Pid} = gen_server:start(?MODULE, [], []), case Options#sft_options.wait of true -> + KL1 = leveled_pmem:to_list(L0Cache), Reply = gen_server:call(Pid, - {sft_new, Filename, KL1, KL2, LevelR}, + {sft_new, + Filename, + KL1, + [], + #level{level=0}}, infinity), {ok, Pid, Reply}; false -> gen_server:cast(Pid, - {sft_new, Filename, KL1, KL2, LevelR}), + {sft_newfromcache, + Filename, + L0Cache, + #level{level=0}}), {ok, Pid, noreply} end. @@ -342,9 +355,10 @@ handle_call({set_for_delete, Penciller}, _From, State) -> handle_call(get_maxsqn, _From, State) -> statecheck_onreply(State#state.highest_sqn, State). -handle_cast({sft_new, Filename, Inp1, [], _LevelR=#level{level=L}}, _State) - when L == 0-> +handle_cast({sft_newfromcache, Filename, L0Cache, _LevelR=#level{level=L}}, + _State) when L == 0-> SW = os:timestamp(), + Inp1 = leveled_pmem:to_list(L0Cache), {ok, State} = create_levelzero(Inp1, Filename), io:format("File creation of L0 file ~s took ~w microseconds~n", [Filename, timer:now_diff(os:timestamp(), SW)]), From 89b574806298b1213373e1a8dae946cb23563e91 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sun, 30 Oct 2016 19:49:01 +0000 Subject: [PATCH 105/167] Remove unnecessary clause --- src/leveled_sft.erl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 64b903c..0004d0a 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -418,13 +418,7 @@ statecheck_onreply(Reply, State) -> %%%============================================================================ -create_levelzero(Inp1, Filename) -> - ListForFile = case is_list(Inp1) of - true -> - Inp1; - false -> - leveled_penciller:roll_into_list(Inp1) - end, +create_levelzero(ListForFile, Filename) -> {TmpFilename, PrmFilename} = generate_filenames(Filename), case create_file(TmpFilename) of {error, Reason} -> From 0e6ee486f86cec0a779822dc8f615603203ab309 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sun, 30 Oct 2016 20:14:11 +0000 Subject: [PATCH 106/167] Make tets less pointless Journla compaction test wouldn't actually cause compaction --- test/end_to_end/basic_SUITE.erl | 39 +++++++++++++++++++++---------- test/end_to_end/restart_SUITE.erl | 2 +- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index d8193ec..2436109 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -12,13 +12,13 @@ ]). all() -> [ - simple_put_fetch_head_delete, - many_put_fetch_head, - journal_compaction, - fetchput_snapshot, - load_and_count, - load_and_count_withdelete, - space_clear_ondelete + % simple_put_fetch_head_delete, + % many_put_fetch_head, + journal_compaction %, + % fetchput_snapshot, + % load_and_count, + % load_and_count_withdelete, + % space_clear_ondelete ]. @@ -106,8 +106,10 @@ many_put_fetch_head(_Config) -> journal_compaction(_Config) -> RootPath = testutil:reset_filestructure(), StartOpts1 = #bookie_options{root_path=RootPath, - max_journalsize=4000000}, + max_journalsize=10000000, + max_run_length=1}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), + ok = leveled_bookie:book_compactjournal(Bookie1, 30000), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), testutil:check_forobject(Bookie1, TestObject), @@ -115,7 +117,7 @@ journal_compaction(_Config) -> lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjList1), - ChkList1 = lists:sublist(lists:sort(ObjList1), 1000), + ChkList1 = lists:sublist(lists:sort(ObjList1), 10000), testutil:check_forlist(Bookie1, ChkList1), testutil:check_forobject(Bookie1, TestObject), {B2, K2, V2, Spec2, MD} = {"Bucket1", @@ -130,17 +132,30 @@ journal_compaction(_Config) -> testutil:check_forlist(Bookie1, ChkList1), testutil:check_forobject(Bookie1, TestObject), testutil:check_forobject(Bookie1, TestObject2), - timer:sleep(5000), % Allow for compaction to complete - io:format("Has journal completed?~n"), testutil:check_forlist(Bookie1, ChkList1), testutil:check_forobject(Bookie1, TestObject), testutil:check_forobject(Bookie1, TestObject2), %% Now replace all the objects - ObjList2 = testutil:generate_objects(5000, 2), + ObjList2 = testutil:generate_objects(50000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjList2), ok = leveled_bookie:book_compactjournal(Bookie1, 30000), + + F = fun leveled_bookie:book_islastcompactionpending/1, + lists:foldl(fun(X, Pending) -> + case Pending of + false -> + false; + true -> + io:format("Loop ~w waiting for journal " + ++ "compaction to complete~n", [X]), + timer:sleep(20000), + F(Bookie1) + end end, + true, + lists:seq(1, 15)), + ChkList3 = lists:sublist(lists:sort(ObjList2), 500), testutil:check_forlist(Bookie1, ChkList3), ok = leveled_bookie:book_close(Bookie1), diff --git a/test/end_to_end/restart_SUITE.erl b/test/end_to_end/restart_SUITE.erl index 030cfd6..9df6fcd 100644 --- a/test/end_to_end/restart_SUITE.erl +++ b/test/end_to_end/restart_SUITE.erl @@ -16,7 +16,7 @@ retain_strategy(_Config) -> max_journalsize=5000000, reload_strategy=[{?RIAK_TAG, retain}]}, BookOptsAlt = BookOpts#bookie_options{max_run_length=6, - max_journalsize=500000}, + max_journalsize=100000}, {ok, Spcl3, LastV3} = rotating_object_check(BookOpts, "Bucket3", 800), ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3}]), {ok, Spcl4, LastV4} = rotating_object_check(BookOpts, "Bucket4", 1600), From 311179964af8d72d24f727121d82fac940a47b03 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sun, 30 Oct 2016 22:06:44 +0000 Subject: [PATCH 107/167] Quality review Minor test fix-up and quality changes --- src/leveled_bookie.erl | 12 ++---------- src/leveled_iclerk.erl | 4 +--- src/leveled_inker.erl | 27 +++++++-------------------- test/end_to_end/basic_SUITE.erl | 15 ++++++++------- test/end_to_end/restart_SUITE.erl | 2 +- 5 files changed, 19 insertions(+), 41 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 50a886d..ee63357 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -403,15 +403,7 @@ handle_info(_Info, State) -> terminate(Reason, State) -> io:format("Bookie closing for reason ~w~n", [Reason]), WaitList = lists:duplicate(?SHUTDOWN_WAITS, ?SHUTDOWN_PAUSE), - ok = case shutdown_wait(WaitList, State#state.inker) of - false -> - io:format("Forcing close of inker following wait of " - ++ "~w milliseconds~n", - [lists:sum(WaitList)]), - leveled_inker:ink_forceclose(State#state.inker); - true -> - ok - end, + ok = shutdown_wait(WaitList, State#state.inker), ok = leveled_penciller:pcl_close(State#state.penciller). code_change(_OldVsn, State, _Extra) -> @@ -503,7 +495,7 @@ shutdown_wait([], _Inker) -> shutdown_wait([TopPause|Rest], Inker) -> case leveled_inker:ink_close(Inker) of ok -> - true; + ok; pause -> io:format("Inker shutdown stil waiting for process to complete" ++ " with further wait of ~w~n", [lists:sum(Rest)]), diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 3a9ecc4..a381ba4 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -164,7 +164,7 @@ handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, Candidates = scan_all_files(Manifest, FilterFun, FilterServer, MaxSQN), BestRun0 = assess_candidates(Candidates, MaxRunLength), case score_run(BestRun0, MaxRunLength) of - Score when Score > 0 -> + Score when Score > 0.0 -> BestRun1 = sort_run(BestRun0), print_compaction_run(BestRun1, MaxRunLength), {ManifestSlice, @@ -372,8 +372,6 @@ sort_run(RunOfFiles) -> lists:sort(CompareFun, RunOfFiles). -compact_files([], _CDBopts, _FilterFun, _FilterServer, _MaxSQN, _RStrategy) -> - {[], 0}; compact_files(BestRun, CDBopts, FilterFun, FilterServer, MaxSQN, RStrategy) -> BatchesOfPositions = get_all_positions(BestRun, []), compact_files(BatchesOfPositions, diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index f3b3075..23b1f49 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -107,7 +107,6 @@ ink_updatemanifest/3, ink_print_manifest/1, ink_close/1, - ink_forceclose/1, build_dummy_journal/0, simple_manifest_reader/2, clean_testdir/1, @@ -165,10 +164,7 @@ ink_confirmdelete(Pid, ManSQN) -> gen_server:call(Pid, {confirm_delete, ManSQN}, 1000). ink_close(Pid) -> - gen_server:call(Pid, {close, false}, infinity). - -ink_forceclose(Pid) -> - gen_server:call(Pid, {close, true}, infinity). + gen_server:call(Pid, close, infinity). ink_loadpcl(Pid, MinSQN, FilterFun, Penciller) -> gen_server:call(Pid, {load_pcl, MinSQN, FilterFun, Penciller}, infinity). @@ -324,11 +320,11 @@ handle_call(compaction_complete, _From, State) -> {reply, ok, State#state{compaction_pending=false}}; handle_call(compaction_pending, _From, State) -> {reply, State#state.compaction_pending, State}; -handle_call({close, Force}, _From, State) -> - case {State#state.compaction_pending, Force} of - {true, false} -> +handle_call(close, _From, State) -> + case State#state.compaction_pending of + true -> {reply, pause, State}; - _ -> + false -> {stop, normal, ok, State} end. @@ -562,8 +558,6 @@ remove_from_manifest(Manifest, Entry) -> io:format("File ~s to be removed from manifest~n", [FN]), lists:keydelete(SQN, 1, Manifest). -find_in_manifest(_SQN, []) -> - error; find_in_manifest(SQN, [{LowSQN, _FN, Pid}|_Tail]) when SQN >= LowSQN -> Pid; find_in_manifest(SQN, [_Head|Tail]) -> @@ -720,18 +714,11 @@ simple_manifest_writer(Manifest, ManSQN, RootPath) -> end. manifest_printer(Manifest) -> - lists:foreach(fun(X) -> - {SQN, FN} = case X of - {A, B, _PID} -> - {A, B}; - {A, B} -> - {A, B} - end, - io:format("At SQN=~w journal has filename ~s~n", + lists:foreach(fun({SQN, FN, _PID}) -> + io:format("At SQN=~w journal has filename ~s~n", [SQN, FN]) end, Manifest). - initiate_penciller_snapshot(Bookie) -> {ok, {LedgerSnap, LedgerCache}, diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 2436109..16fcd0f 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -12,13 +12,13 @@ ]). all() -> [ - % simple_put_fetch_head_delete, - % many_put_fetch_head, - journal_compaction %, - % fetchput_snapshot, - % load_and_count, - % load_and_count_withdelete, - % space_clear_ondelete + simple_put_fetch_head_delete, + many_put_fetch_head, + journal_compaction, + fetchput_snapshot, + load_and_count, + load_and_count_withdelete, + space_clear_ondelete ]. @@ -60,6 +60,7 @@ simple_put_fetch_head_delete(_Config) -> ok = leveled_bookie:book_delete(Bookie3, "Bucket1", "Key2", [{remove, "Index1", "Term1"}]), not_found = leveled_bookie:book_get(Bookie3, "Bucket1", "Key2"), + not_found = leveled_bookie:book_head(Bookie3, "Bucket1", "Key2"), ok = leveled_bookie:book_close(Bookie3), {ok, Bookie4} = leveled_bookie:book_start(StartOpts2), not_found = leveled_bookie:book_get(Bookie4, "Bucket1", "Key2"), diff --git a/test/end_to_end/restart_SUITE.erl b/test/end_to_end/restart_SUITE.erl index 9df6fcd..65efbf7 100644 --- a/test/end_to_end/restart_SUITE.erl +++ b/test/end_to_end/restart_SUITE.erl @@ -15,7 +15,7 @@ retain_strategy(_Config) -> cache_size=1000, max_journalsize=5000000, reload_strategy=[{?RIAK_TAG, retain}]}, - BookOptsAlt = BookOpts#bookie_options{max_run_length=6, + BookOptsAlt = BookOpts#bookie_options{max_run_length=8, max_journalsize=100000}, {ok, Spcl3, LastV3} = rotating_object_check(BookOpts, "Bucket3", 800), ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3}]), From 4cffecf2ca0c214cea1a7856306ebe20be41b39b Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 31 Oct 2016 01:33:33 +0000 Subject: [PATCH 108/167] Handle gen_server:cast slowness There was some unpredictable performance in tests, that was related to the amount of time it took the sft gen_server to accept a cast whihc passed the levelzero_cache. The response time looked to be broadly proportional to the size of the cache - so it appeared to be an issue with passing the large object to the process queue. To avoid this, the penciller now instructs the SFT gen_server to callback to the server for each tree in the cache in turn as it is building the list from the cache. Each of these requests should be reltaively short, and the processing in-between should space out the requests so the Pencille ris not blocked from answering queries when pompting a L0 write. --- include/leveled.hrl | 1 + src/leveled_bookie.erl | 5 +-- src/leveled_penciller.erl | 57 ++++++++++++++++++++++++++------- src/leveled_pmem.erl | 19 ++++++----- src/leveled_sft.erl | 17 +++++----- test/end_to_end/basic_SUITE.erl | 6 ++-- 6 files changed, 73 insertions(+), 32 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 209a9b7..9b32415 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -66,6 +66,7 @@ {root_path :: string(), cache_size :: integer(), max_journalsize :: integer(), + max_pencillercachesize :: integer(), snapshot_bookie :: pid(), reload_strategy = [] :: list(), max_run_length :: integer()}). diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index ee63357..45a1884 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -514,7 +514,7 @@ set_options(Opts) -> AltStrategy = Opts#bookie_options.reload_strategy, ReloadStrategy = leveled_codec:inker_reload_strategy(AltStrategy), - + PCLL0CacheSize = Opts#bookie_options.max_pencillercachesize, JournalFP = Opts#bookie_options.root_path ++ "/" ++ ?JOURNAL_FP, LedgerFP = Opts#bookie_options.root_path ++ "/" ++ ?LEDGER_FP, {#inker_options{root_path = JournalFP, @@ -522,7 +522,8 @@ set_options(Opts) -> max_run_length = Opts#bookie_options.max_run_length, cdb_options = #cdb_options{max_size=MaxJournalSize, binary_mode=true}}, - #penciller_options{root_path = LedgerFP}}. + #penciller_options{root_path = LedgerFP, + max_inmemory_tablesize = PCLL0CacheSize}}. startup(InkerOpts, PencillerOpts) -> {ok, Inker} = leveled_inker:ink_start(InkerOpts), diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 17554d9..18459f9 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -223,6 +223,7 @@ code_change/3, pcl_start/1, pcl_pushmem/2, + pcl_fetchlevelzero/2, pcl_fetch/2, pcl_fetchkeys/5, pcl_checksequencenumber/3, @@ -287,6 +288,9 @@ pcl_start(PCLopts) -> pcl_pushmem(Pid, DumpList) -> %% Bookie to dump memory onto penciller gen_server:call(Pid, {push_mem, DumpList}, infinity). + +pcl_fetchlevelzero(Pid, Slot) -> + gen_server:call(Pid, {fetch_levelzero, Slot}, infinity). pcl_fetch(Pid, Key) -> gen_server:call(Pid, {fetch, Key}, infinity). @@ -471,6 +475,8 @@ handle_call({load_snapshot, BookieIncrTree}, _From, State) -> levelzero_size=L0Size, ledger_sqn=LedgerSQN, snapshot_fully_loaded=true}}; +handle_call({fetch_levelzero, Slot}, _From, State) -> + {reply, lists:nth(Slot, State#state.levelzero_cache), State}; handle_call(close, _From, State) -> {stop, normal, ok, State}. @@ -553,7 +559,7 @@ terminate(Reason, State) -> {false, [], 0} -> io:format("Level 0 cache empty at close of Penciller~n"); {false, [], _N} -> - L0Pid = roll_memory(UpdState, State#state.levelzero_cache, true), + L0Pid = roll_memory(UpdState, true), ok = leveled_sft:sft_close(L0Pid); _ -> io:format("No level zero action on close of Penciller~n") @@ -696,7 +702,7 @@ update_levelzero(L0Index, L0Size, PushedTree, LedgerSQN, L0Cache, State) -> Level0Free = length(get_item(0, State#state.manifest, [])) == 0, case {CacheTooBig, Level0Free} of {true, true} -> - L0Constructor = roll_memory(State, UpdL0Cache), + L0Constructor = roll_memory(UpdState, false), UpdState#state{levelzero_pending=true, levelzero_constructor=L0Constructor}; _ -> @@ -719,19 +725,46 @@ checkready(Pid) -> timeout end. -roll_memory(State, L0Cache) -> - roll_memory(State, L0Cache, false). +%% Casting a large object (the levelzero cache) to the gen_server did not lead +%% to an immediate return as expected. With 32K keys in the TreeList it could +%% take around 35-40ms. +%% +%% To avoid blocking this gen_server, the SFT file cna request each item of the +%% cache one at a time. +%% +%% The Wait is set to false to use a cast when calling this in normal operation +%% where as the Wait of true is used at shutdown -roll_memory(State, L0Cache, Wait) -> +roll_memory(State, false) -> + FileName = levelzero_filename(State), + io:format("Rolling level zero to file ~s~n", [FileName]), + Opts = #sft_options{wait=false}, + PCL = self(), + FetchFun = fun(Slot) -> pcl_fetchlevelzero(PCL, Slot) end, + % FetchFun = fun(Slot) -> lists:nth(Slot, State#state.levelzero_cache) end, + R = leveled_sft:sft_newfroml0cache(FileName, + length(State#state.levelzero_cache), + FetchFun, + Opts), + {ok, Constructor, _} = R, + Constructor; +roll_memory(State, true) -> + FileName = levelzero_filename(State), + Opts = #sft_options{wait=true}, + FetchFun = fun(Slot) -> lists:nth(Slot, State#state.levelzero_cache) end, + R = leveled_sft:sft_newfroml0cache(FileName, + length(State#state.levelzero_cache), + FetchFun, + Opts), + {ok, Constructor, _} = R, + Constructor. + +levelzero_filename(State) -> MSN = State#state.manifest_sqn, FileName = State#state.root_path ++ "/" ++ ?FILES_FP ++ "/" ++ integer_to_list(MSN) ++ "_0_0", - Opts = #sft_options{wait=Wait}, - {ok, Constructor, _} = leveled_sft:sft_newfroml0cache(FileName, - L0Cache, - Opts), - Constructor. + FileName. fetch_mem(Key, Manifest, L0Index, L0Cache) -> @@ -1636,10 +1669,12 @@ create_file_test() -> ok = file:write_file(Filename, term_to_binary("hello")), KVL = lists:usort(leveled_sft:generate_randomkeys(10000)), Tree = gb_trees:from_orddict(KVL), + FetchFun = fun(Slot) -> lists:nth(Slot, [Tree]) end, {ok, SP, noreply} = leveled_sft:sft_newfroml0cache(Filename, - [Tree], + 1, + FetchFun, #sft_options{wait=false}), lists:foreach(fun(X) -> case checkready(SP) of diff --git a/src/leveled_pmem.erl b/src/leveled_pmem.erl index ae7bf09..e7b218a 100644 --- a/src/leveled_pmem.erl +++ b/src/leveled_pmem.erl @@ -38,7 +38,7 @@ -export([ add_to_index/5, - to_list/1, + to_list/2, new_index/0, check_levelzero/3, merge_trees/4 @@ -80,17 +80,19 @@ add_to_index(L0Index, L0Size, LevelMinus1, LedgerSQN, TreeList) -> end. -to_list(TreeList) -> +to_list(Slots, FetchFun) -> SW = os:timestamp(), - OutList = lists:foldr(fun(Tree, CompleteList) -> + SlotList = lists:reverse(lists:seq(1, Slots)), + FullList = lists:foldl(fun(Slot, Acc) -> + Tree = FetchFun(Slot), L = gb_trees:to_list(Tree), - lists:ukeymerge(1, CompleteList, L) + lists:ukeymerge(1, Acc, L) end, [], - TreeList), + SlotList), io:format("L0 cache converted to list of size ~w in ~w microseconds~n", - [length(OutList), timer:now_diff(os:timestamp(), SW)]), - OutList. + [length(FullList), timer:now_diff(os:timestamp(), SW)]), + FullList. new_index() -> @@ -237,7 +239,8 @@ compare_method_test() -> StartKey = {o, "Bucket0100", null, null}, EndKey = {o, "Bucket0200", null, null}, SWa = os:timestamp(), - DumpList = to_list(TreeList), + FetchFun = fun(Slot) -> lists:nth(Slot, TreeList) end, + DumpList = to_list(length(TreeList), FetchFun), Q0 = lists:foldl(fun({K, V}, Acc) -> P = leveled_codec:endkey_passed(EndKey, K), case {K, P} of diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 0004d0a..8f4a870 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -152,7 +152,7 @@ terminate/2, code_change/3, sft_new/4, - sft_newfroml0cache/3, + sft_newfroml0cache/4, sft_open/1, sft_get/2, sft_getkvrange/4, @@ -228,11 +228,11 @@ sft_new(Filename, KL1, KL2, LevelInfo) -> infinity), {ok, Pid, Reply}. -sft_newfroml0cache(Filename, L0Cache, Options) -> +sft_newfroml0cache(Filename, Slots, FetchFun, Options) -> {ok, Pid} = gen_server:start(?MODULE, [], []), case Options#sft_options.wait of true -> - KL1 = leveled_pmem:to_list(L0Cache), + KL1 = leveled_pmem:to_list(Slots, FetchFun), Reply = gen_server:call(Pid, {sft_new, Filename, @@ -243,10 +243,10 @@ sft_newfroml0cache(Filename, L0Cache, Options) -> {ok, Pid, Reply}; false -> gen_server:cast(Pid, - {sft_newfromcache, + {sft_newfroml0cache, Filename, - L0Cache, - #level{level=0}}), + Slots, + FetchFun}), {ok, Pid, noreply} end. @@ -355,10 +355,9 @@ handle_call({set_for_delete, Penciller}, _From, State) -> handle_call(get_maxsqn, _From, State) -> statecheck_onreply(State#state.highest_sqn, State). -handle_cast({sft_newfromcache, Filename, L0Cache, _LevelR=#level{level=L}}, - _State) when L == 0-> +handle_cast({sft_newfroml0cache, Filename, Slots, FetchFun}, _State) -> SW = os:timestamp(), - Inp1 = leveled_pmem:to_list(L0Cache), + Inp1 = leveled_pmem:to_list(Slots, FetchFun), {ok, State} = create_levelzero(Inp1, Filename), io:format("File creation of L0 file ~s took ~w microseconds~n", [Filename, timer:now_diff(os:timestamp(), SW)]), diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 16fcd0f..442390c 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -69,14 +69,16 @@ simple_put_fetch_head_delete(_Config) -> many_put_fetch_head(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath}, + StartOpts1 = #bookie_options{root_path=RootPath, + max_pencillercachesize=16000}, {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), testutil:check_forobject(Bookie1, TestObject), ok = leveled_bookie:book_close(Bookie1), StartOpts2 = #bookie_options{root_path=RootPath, - max_journalsize=1000000000}, + max_journalsize=1000000000, + max_pencillercachesize=32000}, {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), testutil:check_forobject(Bookie2, TestObject), GenList = [2, 20002, 40002, 60002, 80002, From 3b05874b8afab2d507da1d557cc6e4d47f00e80b Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 31 Oct 2016 12:12:06 +0000 Subject: [PATCH 109/167] Add initial timestamp support Covered only by basic unit test at present. --- include/leveled.hrl | 2 +- src/leveled_bookie.erl | 188 ++++++++++++++++++++++++-------- src/leveled_codec.erl | 25 +++-- src/leveled_pclerk.erl | 2 +- test/end_to_end/basic_SUITE.erl | 2 +- 5 files changed, 163 insertions(+), 56 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 9b32415..3a85aa7 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -34,7 +34,7 @@ -record(level, {level :: integer(), is_basement = false :: boolean(), - timestamp :: erlang:timestamp()}). + timestamp :: integer()}). -record(manifest_entry, {start_key :: tuple(), diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 45a1884..2ace726 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -139,6 +139,8 @@ book_riakget/3, book_riakhead/3, book_put/5, + book_put/6, + book_tempput/7, book_delete/4, book_get/3, book_head/3, @@ -183,9 +185,17 @@ book_riakput(Pid, RiakObject, IndexSpecs) -> {Bucket, Key} = leveled_codec:riakto_keydetails(RiakObject), book_put(Pid, Bucket, Key, RiakObject, IndexSpecs, ?RIAK_TAG). +book_tempput(Pid, Bucket, Key, Object, IndexSpecs, Tag, TTL) when is_integer(TTL) -> + book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag, TTL). + book_put(Pid, Bucket, Key, Object, IndexSpecs) -> book_put(Pid, Bucket, Key, Object, IndexSpecs, ?STD_TAG). +book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag) -> + book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag, infinity). + + + %% TODO: %% It is not enough simply to change the value to delete, as the journal %% needs to know the key is a tombstone at compaction time, and currently at @@ -213,8 +223,10 @@ book_riakhead(Pid, Bucket, Key) -> book_head(Pid, Bucket, Key) -> book_head(Pid, Bucket, Key, ?STD_TAG). -book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag) -> - gen_server:call(Pid, {put, Bucket, Key, Object, IndexSpecs, Tag}, infinity). +book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag, TTL) -> + gen_server:call(Pid, + {put, Bucket, Key, Object, IndexSpecs, Tag, TTL}, + infinity). book_get(Pid, Bucket, Key, Tag) -> gen_server:call(Pid, {get, Bucket, Key, Tag}, infinity). @@ -278,18 +290,18 @@ init([Opts]) -> end. -handle_call({put, Bucket, Key, Object, IndexSpecs, Tag}, From, State) -> +handle_call({put, Bucket, Key, Object, IndexSpecs, Tag, TTL}, From, State) -> LedgerKey = leveled_codec:to_ledgerkey(Bucket, Key, Tag), {ok, SQN, ObjSize} = leveled_inker:ink_put(State#state.inker, LedgerKey, Object, - IndexSpecs), + {IndexSpecs, TTL}), Changes = preparefor_ledgercache(no_type_assigned, LedgerKey, SQN, Object, ObjSize, - IndexSpecs), + {IndexSpecs, TTL}), Cache0 = addto_ledgercache(Changes, State#state.ledger_cache), gen_server:reply(From, ok), {ok, NewCache} = maybepush_ledgercache(State#state.cache_size, @@ -308,12 +320,14 @@ handle_call({get, Bucket, Key, Tag}, _From, State) -> case Status of tomb -> {reply, not_found, State}; - {active, _} -> - case fetch_value(LedgerKey, Seqn, State#state.inker) of - not_present -> - {reply, not_found, State}; - Object -> - {reply, {ok, Object}, State} + {active, TS} -> + Active = TS >= leveled_codec:integer_now(), + case {Active, + fetch_value(LedgerKey, Seqn, State#state.inker)} of + {true, Object} -> + {reply, {ok, Object}, State}; + _ -> + {reply, not_found, State} end end end; @@ -329,9 +343,14 @@ handle_call({head, Bucket, Key, Tag}, _From, State) -> case Status of tomb -> {reply, not_found, State}; - {active, _} -> - OMD = leveled_codec:build_metadata_object(LedgerKey, MD), - {reply, {ok, OMD}, State} + {active, TS} -> + case TS >= leveled_codec:integer_now() of + true -> + OMD = leveled_codec:build_metadata_object(LedgerKey, MD), + {reply, {ok, OMD}, State}; + false -> + {reply, not_found, State} + end end end; handle_call({snapshot, _Requestor, SnapType, _Timeout}, _From, State) -> @@ -359,6 +378,13 @@ handle_call({snapshot, _Requestor, SnapType, _Timeout}, _From, State) -> end; handle_call({return_folder, FolderType}, _From, State) -> case FolderType of + {bucket_stats, Bucket} -> + {reply, + bucket_stats(State#state.penciller, + State#state.ledger_cache, + Bucket, + ?STD_TAG), + State}; {riakbucket_stats, Bucket} -> {reply, bucket_stats(State#state.penciller, @@ -425,10 +451,11 @@ bucket_stats(Penciller, LedgerCache, Bucket, Tag) -> LedgerCache), StartKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), EndKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), + AccFun = accumulate_size(), Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, StartKey, EndKey, - fun accumulate_size/3, + AccFun, {0, 0}), ok = leveled_penciller:pcl_close(LedgerSnapshot), Acc @@ -479,10 +506,11 @@ allkey_query(Penciller, LedgerCache, Tag) -> LedgerCache), SK = leveled_codec:to_ledgerkey(null, null, Tag), EK = leveled_codec:to_ledgerkey(null, null, Tag), + AccFun = accumulate_keys(), Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, SK, EK, - fun accumulate_keys/3, + AccFun, []), ok = leveled_penciller:pcl_close(LedgerSnapshot), lists:reverse(Acc) @@ -558,22 +586,31 @@ fetch_value(Key, SQN, Inker) -> not_present end. -accumulate_size(Key, Value, {Size, Count}) -> - case leveled_codec:is_active(Key, Value) of - true -> - {Size + leveled_codec:get_size(Key, Value), Count + 1}; - false -> - {Size, Count} - end. -accumulate_keys(Key, Value, KeyList) -> - case leveled_codec:is_active(Key, Value) of - true -> - [leveled_codec:from_ledgerkey(Key)|KeyList]; - false -> - KeyList - end. +accumulate_size() -> + Now = leveled_codec:integer_now(), + AccFun = fun(Key, Value, {Size, Count}) -> + case leveled_codec:is_active(Key, Value, Now) of + true -> + {Size + leveled_codec:get_size(Key, Value), + Count + 1}; + false -> + {Size, Count} + end + end, + AccFun. +accumulate_keys() -> + Now = leveled_codec:integer_now(), + AccFun = fun(Key, Value, KeyList) -> + case leveled_codec:is_active(Key, Value, Now) of + true -> + [leveled_codec:from_ledgerkey(Key)|KeyList]; + false -> + KeyList + end + end, + AccFun. add_keys(ObjKey, _IdxValue, Acc) -> Acc ++ [ObjKey]. @@ -582,10 +619,11 @@ add_terms(ObjKey, IdxValue, Acc) -> Acc ++ [{IdxValue, ObjKey}]. accumulate_index(TermRe, AddFun) -> + Now = leveled_codec:integer_now(), case TermRe of undefined -> fun(Key, Value, Acc) -> - case leveled_codec:is_active(Key, Value) of + case leveled_codec:is_active(Key, Value, Now) of true -> {_Bucket, ObjKey, @@ -596,7 +634,7 @@ accumulate_index(TermRe, AddFun) -> end end; TermRe -> fun(Key, Value, Acc) -> - case leveled_codec:is_active(Key, Value) of + case leveled_codec:is_active(Key, Value, Now) of true -> {_Bucket, ObjKey, @@ -613,16 +651,21 @@ accumulate_index(TermRe, AddFun) -> end. -preparefor_ledgercache(?INKT_KEYD, LedgerKey, SQN, _Obj, _Size, IndexSpecs) -> +preparefor_ledgercache(?INKT_KEYD, + LedgerKey, SQN, _Obj, _Size, {IndexSpecs, TTL}) -> {Bucket, Key} = leveled_codec:from_ledgerkey(LedgerKey), - leveled_codec:convert_indexspecs(IndexSpecs, Bucket, Key, SQN); -preparefor_ledgercache(_Type, LedgerKey, SQN, Obj, Size, IndexSpecs) -> + leveled_codec:convert_indexspecs(IndexSpecs, Bucket, Key, SQN, TTL); +preparefor_ledgercache(_Type, LedgerKey, SQN, Obj, Size, {IndexSpecs, TTL}) -> {Bucket, Key, PrimaryChange} = leveled_codec:generate_ledgerkv(LedgerKey, SQN, Obj, - Size), - ConvSpecs = leveled_codec:convert_indexspecs(IndexSpecs, Bucket, Key, SQN), - [PrimaryChange] ++ ConvSpecs. + Size, + TTL), + [PrimaryChange] ++ leveled_codec:convert_indexspecs(IndexSpecs, + Bucket, + Key, + SQN, + TTL). addto_ledgercache(Changes, Cache) -> @@ -700,14 +743,26 @@ reset_filestructure() -> RootPath. - - generate_multiple_objects(Count, KeyNumber) -> generate_multiple_objects(Count, KeyNumber, []). - + generate_multiple_objects(0, _KeyNumber, ObjL) -> ObjL; generate_multiple_objects(Count, KeyNumber, ObjL) -> + Key = "Key" ++ integer_to_list(KeyNumber), + Value = crypto:rand_bytes(256), + IndexSpec = [{add, "idx1_bin", "f" ++ integer_to_list(KeyNumber rem 10)}], + generate_multiple_objects(Count - 1, + KeyNumber + 1, + ObjL ++ [{Key, Value, IndexSpec}]). + + +generate_multiple_robjects(Count, KeyNumber) -> + generate_multiple_robjects(Count, KeyNumber, []). + +generate_multiple_robjects(0, _KeyNumber, ObjL) -> + ObjL; +generate_multiple_robjects(Count, KeyNumber, ObjL) -> Obj = {"Bucket", "Key" ++ integer_to_list(KeyNumber), crypto:rand_bytes(1024), @@ -717,7 +772,7 @@ generate_multiple_objects(Count, KeyNumber, ObjL) -> {B1, K1, V1, Spec1, MD} = Obj, Content = #r_content{metadata=MD, value=V1}, Obj1 = #r_object{bucket=B1, key=K1, contents=[Content], vclock=[{'a',1}]}, - generate_multiple_objects(Count - 1, KeyNumber + 1, ObjL ++ [{Obj1, Spec1}]). + generate_multiple_robjects(Count - 1, KeyNumber + 1, ObjL ++ [{Obj1, Spec1}]). single_key_test() -> @@ -758,7 +813,7 @@ multi_key_test() -> C2 = #r_content{metadata=MD2, value=V2}, Obj2 = #r_object{bucket=B2, key=K2, contents=[C2], vclock=[{'a',1}]}, ok = book_riakput(Bookie1, Obj1, Spec1), - ObjL1 = generate_multiple_objects(100, 3), + ObjL1 = generate_multiple_robjects(100, 3), SW1 = os:timestamp(), lists:foreach(fun({O, S}) -> ok = book_riakput(Bookie1, O, S) end, ObjL1), io:format("PUT of 100 objects completed in ~w microseconds~n", @@ -768,7 +823,7 @@ multi_key_test() -> ?assertMatch(F1A, Obj1), {ok, F2A} = book_riakget(Bookie1, B2, K2), ?assertMatch(F2A, Obj2), - ObjL2 = generate_multiple_objects(100, 103), + ObjL2 = generate_multiple_robjects(100, 103), SW2 = os:timestamp(), lists:foreach(fun({O, S}) -> ok = book_riakput(Bookie1, O, S) end, ObjL2), io:format("PUT of 100 objects completed in ~w microseconds~n", @@ -784,7 +839,7 @@ multi_key_test() -> ?assertMatch(F1C, Obj1), {ok, F2C} = book_riakget(Bookie2, B2, K2), ?assertMatch(F2C, Obj2), - ObjL3 = generate_multiple_objects(100, 203), + ObjL3 = generate_multiple_robjects(100, 203), SW3 = os:timestamp(), lists:foreach(fun({O, S}) -> ok = book_riakput(Bookie2, O, S) end, ObjL3), io:format("PUT of 100 objects completed in ~w microseconds~n", @@ -796,4 +851,47 @@ multi_key_test() -> ok = book_close(Bookie2), reset_filestructure(). +ttl_test() -> + RootPath = reset_filestructure(), + {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath}), + ObjL1 = generate_multiple_objects(100, 1), + % Put in all the objects with a TTL in the future + Future = leveled_codec:integer_now() + 300, + lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, + "Bucket", K, V, S, + ?STD_TAG, + Future) end, + ObjL1), + lists:foreach(fun({K, V, _S}) -> + {ok, V} = book_get(Bookie1, "Bucket", K, ?STD_TAG) + end, + ObjL1), + lists:foreach(fun({K, _V, _S}) -> + {ok, _} = book_head(Bookie1, "Bucket", K, ?STD_TAG) + end, + ObjL1), + + ObjL2 = generate_multiple_objects(100, 101), + Past = leveled_codec:integer_now() - 300, + lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, + "Bucket", K, V, S, + ?STD_TAG, + Past) end, + ObjL2), + lists:foreach(fun({K, _V, _S}) -> + not_found = book_get(Bookie1, "Bucket", K, ?STD_TAG) + end, + ObjL2), + lists:foreach(fun({K, _V, _S}) -> + not_found = book_head(Bookie1, "Bucket", K, ?STD_TAG) + end, + ObjL2), + + {async, BucketFolder} = book_returnfolder(Bookie1, + {bucket_stats, "Bucket"}), + {_Size, Count} = BucketFolder(), + ?assertMatch(100, Count), + ok = book_close(Bookie1), + reset_filestructure(). + -endif. \ No newline at end of file diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index afb6792..d212dcc 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -39,7 +39,7 @@ strip_to_statusonly/1, strip_to_keyseqstatusonly/1, striphead_to_details/1, - is_active/2, + is_active/3, endkey_passed/2, key_dominates/2, maybe_reap_expiredkey/2, @@ -58,9 +58,10 @@ generate_ledgerkv/4, generate_ledgerkv/5, get_size/2, - convert_indexspecs/4, + convert_indexspecs/5, riakto_keydetails/1, - generate_uuid/0]). + generate_uuid/0, + integer_now/0]). %% Credit to @@ -117,11 +118,15 @@ maybe_reap(tomb, {true, _CurrTS}) -> maybe_reap(_, _) -> false. -is_active(Key, Value) -> +is_active(Key, Value, Now) -> case strip_to_statusonly({Key, Value}) of {active, infinity} -> true; tomb -> + false; + {active, TS} when Now >= TS -> + true; + {active, _TS} -> false end. @@ -247,12 +252,11 @@ endkey_passed({EK1, EK2, EK3, null}, {CK1, CK2, CK3, _}) -> endkey_passed(EndKey, CheckingKey) -> EndKey < CheckingKey. -convert_indexspecs(IndexSpecs, Bucket, Key, SQN) -> +convert_indexspecs(IndexSpecs, Bucket, Key, SQN, TTL) -> lists:map(fun({IndexOp, IdxField, IdxValue}) -> Status = case IndexOp of add -> - %% TODO: timestamp support - {active, infinity}; + {active, TTL}; remove -> %% TODO: timestamps for delayed reaping tomb @@ -279,7 +283,12 @@ generate_ledgerkv(PrimaryKey, SQN, Obj, Size, TS) -> {PrimaryKey, {SQN, Status, extract_metadata(Obj, Size, Tag)}}}. +integer_now() -> + integer_time(os:timestamp()). +integer_time(TS) -> + DT = calendar:now_to_universal_time(TS), + calendar:datetime_to_gregorian_seconds(DT). extract_metadata(Obj, Size, ?RIAK_TAG) -> riak_extract_metadata(Obj, Size); @@ -353,7 +362,7 @@ indexspecs_test() -> IndexSpecs = [{add, "t1_int", 456}, {add, "t1_bin", "adbc123"}, {remove, "t1_bin", "abdc456"}], - Changes = convert_indexspecs(IndexSpecs, "Bucket", "Key2", 1), + Changes = convert_indexspecs(IndexSpecs, "Bucket", "Key2", 1, infinity), ?assertMatch({{i, "Bucket", {"t1_int", 456}, "Key2"}, {1, {active, infinity}, null}}, lists:nth(1, Changes)), ?assertMatch({{i, "Bucket", {"t1_bin", "adbc123"}, "Key2"}, diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 635d661..34edf96 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -319,7 +319,7 @@ do_merge(KL1, KL2, {SrcLevel, IsB}, {Filepath, MSN}, FileCounter, OutList) -> true -> #level{level = SrcLevel + 1, is_basement = true, - timestamp = os:timestamp()}; + timestamp = leveled_codec:integer_now()}; false -> SrcLevel + 1 end, diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 442390c..612a448 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -46,7 +46,7 @@ simple_put_fetch_head_delete(_Config) -> ok = leveled_bookie:book_put(Bookie2, "Bucket1", "Key2", "Value2", [{add, "Index1", "Term1"}]), {ok, "Value2"} = leveled_bookie:book_get(Bookie2, "Bucket1", "Key2"), - {ok, {62888926, 43}} = leveled_bookie:book_head(Bookie2, + {ok, {62888926, 56}} = leveled_bookie:book_head(Bookie2, "Bucket1", "Key2"), testutil:check_formissingobject(Bookie2, "Bucket1", "Key2"), From 9bef57a78d159542403a48611f5253221a41f170 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 31 Oct 2016 14:01:09 +0000 Subject: [PATCH 110/167] Get Positions - when rolling CT test was call get_positions hilst the sile was rolling - don't want the file to be checked in this state, so just return an empty list. --- src/leveled_cdb.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 9730a53..29bbbfd 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -316,6 +316,8 @@ rolling({key_check, Key}, _From, State) -> get_mem(Key, State#state.handle, State#state.hashtree, loose_presence), rolling, State}; +rolling({get_positions, _SampleSize}, _From, State) -> + {reply, [], rolling, State}; rolling({return_hashtable, IndexList, HashTreeBin}, _From, State) -> Handle = State#state.handle, {ok, BasePos} = file:position(Handle, State#state.last_position), From 6b5b51412e7a47b567f3811164a4d59ffff13eb0 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 31 Oct 2016 15:13:11 +0000 Subject: [PATCH 111/167] Improve TTL unit test Add support for different type of index queries --- src/leveled_bookie.erl | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 2ace726..c76e104 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -891,7 +891,47 @@ ttl_test() -> {bucket_stats, "Bucket"}), {_Size, Count} = BucketFolder(), ?assertMatch(100, Count), + {async, + IndexFolder} = book_returnfolder(Bookie1, + {index_query, + "Bucket", + {"idx1_bin", "f8", "f9"}, + {false, undefined}}), + KeyList = IndexFolder(), + ?assertMatch(20, length(KeyList)), + + {ok, Regex} = re:compile("f8"), + {async, + IndexFolderTR} = book_returnfolder(Bookie1, + {index_query, + "Bucket", + {"idx1_bin", "f8", "f9"}, + {true, Regex}}), + TermKeyList = IndexFolderTR(), + ?assertMatch(10, length(TermKeyList)), + ok = book_close(Bookie1), + {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath}), + + {async, + IndexFolderTR2} = book_returnfolder(Bookie2, + {index_query, + "Bucket", + {"idx1_bin", "f7", "f9"}, + {false, Regex}}), + KeyList2 = IndexFolderTR2(), + ?assertMatch(10, length(KeyList2)), + + lists:foreach(fun({K, _V, _S}) -> + not_found = book_get(Bookie2, "Bucket", K, ?STD_TAG) + end, + ObjL2), + lists:foreach(fun({K, _V, _S}) -> + not_found = book_head(Bookie2, "Bucket", K, ?STD_TAG) + end, + ObjL2), + + ok = book_close(Bookie2), reset_filestructure(). -endif. \ No newline at end of file From 2607792d1f5a6ed881d1e43cdc6323fb3fa509a3 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 31 Oct 2016 15:18:21 +0000 Subject: [PATCH 112/167] Adjust setting If cache size is too small then we're more likely to be not ready to evict a L0 file --- src/leveled_bookie.erl | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index c76e104..cd5a074 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -153,7 +153,7 @@ -include_lib("eunit/include/eunit.hrl"). --define(CACHE_SIZE, 1600). +-define(CACHE_SIZE, 2000). -define(JOURNAL_FP, "journal"). -define(LEDGER_FP, "ledger"). -define(SHUTDOWN_WAITS, 60). @@ -194,17 +194,6 @@ book_put(Pid, Bucket, Key, Object, IndexSpecs) -> book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag) -> book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag, infinity). - - -%% TODO: -%% It is not enough simply to change the value to delete, as the journal -%% needs to know the key is a tombstone at compaction time, and currently at -%% compaction time the clerk only knows the Key and not the Value. -%% -%% The tombstone cannot be removed from the Journal on compaction, as the -%% journal entry the tombstone deletes may not have been reaped - and so if the -%% ledger got erased, the value would be resurrected. - book_riakdelete(Pid, Bucket, Key, IndexSpecs) -> book_put(Pid, Bucket, Key, delete, IndexSpecs, ?RIAK_TAG). From bd6c44e9b024da5ce292a3de2832b36a0870a378 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 31 Oct 2016 16:02:32 +0000 Subject: [PATCH 113/167] Correct is_active Firts part of adding support for scanning for Keys and Hashes. as part of this discovered TTL support did the opposite (only fetched things in the past!). --- src/leveled_bookie.erl | 93 ++++++++++++++++++++++++++++++++++++++++++ src/leveled_codec.erl | 15 ++++++- 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index cd5a074..889be9e 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -397,6 +397,14 @@ handle_call({return_folder, FolderType}, _From, State) -> allkey_query(State#state.penciller, State#state.ledger_cache, Tag), + State}; + {hashtree_query, Tag, JournalCheck} -> + {reply, + hashtree_query(State#state.penciller, + State#state.ledger_cache, + State#state.inker, + Tag, + JournalCheck), State} end; handle_call({compact_journal, Timeout}, _From, State) -> @@ -484,6 +492,34 @@ index_query(Penciller, LedgerCache, end, {async, Folder}. + +hashtree_query(Penciller, LedgerCache, _Inker, + Tag, JournalCheck) -> + PCLopts = #penciller_options{start_snapshot=true, + source_penciller=Penciller}, + {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), + JournalSnapshot = case JournalCheck of + false -> + null + end, + Folder = fun() -> + io:format("Length of increment in snapshot is ~w~n", + [gb_trees:size(LedgerCache)]), + ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, + LedgerCache), + StartKey = leveled_codec:to_ledgerkey(null, null, Tag), + EndKey = leveled_codec:to_ledgerkey(null, null, Tag), + AccFun = accumulate_hashes(), + Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, + StartKey, + EndKey, + AccFun, + []), + ok = leveled_penciller:pcl_close(LedgerSnapshot), + Acc + end, + {async, Folder}. + allkey_query(Penciller, LedgerCache, Tag) -> PCLopts = #penciller_options{start_snapshot=true, source_penciller=Penciller}, @@ -589,6 +625,18 @@ accumulate_size() -> end, AccFun. +accumulate_hashes() -> + Now = leveled_codec:integer_now(), + AccFun = fun(Key, Value, KHList) -> + case leveled_codec:is_active(Key, Value, Now) of + true -> + [leveled_codec:get_keyandhash(Key, Value)|KHList]; + false -> + KHList + end + end, + AccFun. + accumulate_keys() -> Now = leveled_codec:integer_now(), AccFun = fun(Key, Value, KeyList) -> @@ -923,4 +971,49 @@ ttl_test() -> ok = book_close(Bookie2), reset_filestructure(). +hashtree_query_test() -> + RootPath = reset_filestructure(), + {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath, + max_journalsize=1000000, + cache_size=500}), + ObjL1 = generate_multiple_objects(1200, 1), + % Put in all the objects with a TTL in the future + Future = leveled_codec:integer_now() + 300, + lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, + "Bucket", K, V, S, + ?STD_TAG, + Future) end, + ObjL1), + ObjL2 = generate_multiple_objects(20, 1201), + % Put in a few objects with a TTL in the past + Past = leveled_codec:integer_now() - 300, + lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, + "Bucket", K, V, S, + ?STD_TAG, + Past) end, + ObjL2), + % Scan the store for the Bucket, Keys and Hashes + {async, HTFolder} = book_returnfolder(Bookie1, + {hashtree_query, + ?STD_TAG, + false}), + KeyHashList = HTFolder(), + lists:foreach(fun({B, _K, H}) -> + ?assertMatch("Bucket", B), + ?assertMatch(true, is_integer(H)) + end, + KeyHashList), + ?assertMatch(1200, length(KeyHashList)), + ok = book_close(Bookie1), + {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath, + max_journalsize=200000, + cache_size=500}), + {async, HTFolder2} = book_returnfolder(Bookie2, + {hashtree_query, + ?STD_TAG, + false}), + ?assertMatch(KeyHashList, HTFolder2()), + ok = book_close(Bookie2), + reset_filestructure(). + -endif. \ No newline at end of file diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index d212dcc..8015f23 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -58,6 +58,7 @@ generate_ledgerkv/4, generate_ledgerkv/5, get_size/2, + get_keyandhash/2, convert_indexspecs/5, riakto_keydetails/1, generate_uuid/0, @@ -124,7 +125,7 @@ is_active(Key, Value, Now) -> true; tomb -> false; - {active, TS} when Now >= TS -> + {active, TS} when TS >= Now -> true; {active, _TS} -> false @@ -307,6 +308,18 @@ get_size(PK, Value) -> Size end. +get_keyandhash(LK, Value) -> + {Tag, Bucket, Key, _} = LK, + {_, _, MD} = Value, + case Tag of + ?RIAK_TAG -> + {_RMD, _VC, Hash, _Size} = MD, + {Bucket, Key, Hash}; + ?STD_TAG -> + {Hash, _Size} = MD, + {Bucket, Key, Hash} + end. + build_metadata_object(PrimaryKey, MD) -> {Tag, Bucket, Key, null} = PrimaryKey, From 7d3a04428b1927f3a46941d958d18440c458fd80 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 31 Oct 2016 17:26:28 +0000 Subject: [PATCH 114/167] Refactor snapshot Better reuse snapshotting fucntions in the Bookie, and use it to support doing Inker clone checks --- src/leveled_bookie.erl | 140 +++++++++++++++++++++-------------------- src/leveled_codec.erl | 3 + src/leveled_inker.erl | 13 ++++ 3 files changed, 88 insertions(+), 68 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 889be9e..294b11a 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -343,69 +343,36 @@ handle_call({head, Bucket, Key, Tag}, _From, State) -> end end; handle_call({snapshot, _Requestor, SnapType, _Timeout}, _From, State) -> - PCLopts = #penciller_options{start_snapshot=true, - source_penciller=State#state.penciller}, - {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), - case SnapType of - store -> - InkerOpts = #inker_options{start_snapshot=true, - source_inker=State#state.inker}, - {ok, JournalSnapshot} = leveled_inker:ink_start(InkerOpts), - {reply, - {ok, - {LedgerSnapshot, - State#state.ledger_cache}, - JournalSnapshot}, - State}; - ledger -> - {reply, - {ok, - {LedgerSnapshot, - State#state.ledger_cache}, - null}, - State} - end; + Reply = snapshot_store(State, SnapType), + {reply, Reply, State}; handle_call({return_folder, FolderType}, _From, State) -> case FolderType of {bucket_stats, Bucket} -> {reply, - bucket_stats(State#state.penciller, - State#state.ledger_cache, - Bucket, - ?STD_TAG), + bucket_stats(State, Bucket, ?STD_TAG), State}; {riakbucket_stats, Bucket} -> {reply, - bucket_stats(State#state.penciller, - State#state.ledger_cache, - Bucket, - ?RIAK_TAG), + bucket_stats(State, Bucket, ?RIAK_TAG), State}; {index_query, Bucket, {IdxField, StartValue, EndValue}, {ReturnTerms, TermRegex}} -> {reply, - index_query(State#state.penciller, - State#state.ledger_cache, + index_query(State, Bucket, {IdxField, StartValue, EndValue}, {ReturnTerms, TermRegex}), State}; {keylist, Tag} -> - {reply, - allkey_query(State#state.penciller, - State#state.ledger_cache, - Tag), - State}; + {reply, + allkey_query(State, Tag), + State}; {hashtree_query, Tag, JournalCheck} -> - {reply, - hashtree_query(State#state.penciller, - State#state.ledger_cache, - State#state.inker, - Tag, - JournalCheck), - State} + {reply, + hashtree_query(State, Tag, JournalCheck), + State} end; handle_call({compact_journal, Timeout}, _From, State) -> ok = leveled_inker:ink_compactjournal(State#state.inker, @@ -437,10 +404,10 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================ -bucket_stats(Penciller, LedgerCache, Bucket, Tag) -> - PCLopts = #penciller_options{start_snapshot=true, - source_penciller=Penciller}, - {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), +bucket_stats(State, Bucket, Tag) -> + {ok, + {LedgerSnapshot, LedgerCache}, + _JournalSnapshot} = snapshot_store(State, ledger), Folder = fun() -> io:format("Length of increment in snapshot is ~w~n", [gb_trees:size(LedgerCache)]), @@ -459,13 +426,13 @@ bucket_stats(Penciller, LedgerCache, Bucket, Tag) -> end, {async, Folder}. -index_query(Penciller, LedgerCache, +index_query(State, Bucket, {IdxField, StartValue, EndValue}, {ReturnTerms, TermRegex}) -> - PCLopts = #penciller_options{start_snapshot=true, - source_penciller=Penciller}, - {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), + {ok, + {LedgerSnapshot, LedgerCache}, + _JournalSnapshot} = snapshot_store(State, ledger), Folder = fun() -> io:format("Length of increment in snapshot is ~w~n", [gb_trees:size(LedgerCache)]), @@ -493,15 +460,16 @@ index_query(Penciller, LedgerCache, {async, Folder}. -hashtree_query(Penciller, LedgerCache, _Inker, - Tag, JournalCheck) -> - PCLopts = #penciller_options{start_snapshot=true, - source_penciller=Penciller}, - {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), - JournalSnapshot = case JournalCheck of +hashtree_query(State, Tag, JournalCheck) -> + SnapType = case JournalCheck of false -> - null + ledger; + check_presence -> + store end, + {ok, + {LedgerSnapshot, LedgerCache}, + JournalSnapshot} = snapshot_store(State, SnapType), Folder = fun() -> io:format("Length of increment in snapshot is ~w~n", [gb_trees:size(LedgerCache)]), @@ -509,7 +477,7 @@ hashtree_query(Penciller, LedgerCache, _Inker, LedgerCache), StartKey = leveled_codec:to_ledgerkey(null, null, Tag), EndKey = leveled_codec:to_ledgerkey(null, null, Tag), - AccFun = accumulate_hashes(), + AccFun = accumulate_hashes(JournalCheck, JournalSnapshot), Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, StartKey, EndKey, @@ -520,10 +488,10 @@ hashtree_query(Penciller, LedgerCache, _Inker, end, {async, Folder}. -allkey_query(Penciller, LedgerCache, Tag) -> - PCLopts = #penciller_options{start_snapshot=true, - source_penciller=Penciller}, - {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), +allkey_query(State, Tag) -> + {ok, + {LedgerSnapshot, LedgerCache}, + _JournalSnapshot} = snapshot_store(State, ledger), Folder = fun() -> io:format("Length of increment in snapshot is ~w~n", [gb_trees:size(LedgerCache)]), @@ -543,6 +511,22 @@ allkey_query(Penciller, LedgerCache, Tag) -> {async, Folder}. +snapshot_store(State, SnapType) -> + PCLopts = #penciller_options{start_snapshot=true, + source_penciller=State#state.penciller}, + {ok, LedgerSnapshot} = leveled_penciller:pcl_start(PCLopts), + case SnapType of + store -> + InkerOpts = #inker_options{start_snapshot=true, + source_inker=State#state.inker}, + {ok, JournalSnapshot} = leveled_inker:ink_start(InkerOpts), + {ok, {LedgerSnapshot, State#state.ledger_cache}, + JournalSnapshot}; + ledger -> + {ok, {LedgerSnapshot, State#state.ledger_cache}, + null} + end. + shutdown_wait([], _Inker) -> false; shutdown_wait([TopPause|Rest], Inker) -> @@ -625,18 +609,38 @@ accumulate_size() -> end, AccFun. -accumulate_hashes() -> +accumulate_hashes(JournalCheck, InkerClone) -> Now = leveled_codec:integer_now(), - AccFun = fun(Key, Value, KHList) -> - case leveled_codec:is_active(Key, Value, Now) of + AccFun = fun(LK, V, KHList) -> + case leveled_codec:is_active(LK, V, Now) of true -> - [leveled_codec:get_keyandhash(Key, Value)|KHList]; + {B, K, H} = leveled_codec:get_keyandhash(LK, V), + case JournalCheck of + false -> + [{B, K, H}|KHList]; + check_presence -> + case check_presence(LK, V, InkerClone) of + true -> + [{B, K, H}|KHList]; + false -> + KHList + end + end; false -> KHList end end, AccFun. +check_presence(Key, Value, InkerClone) -> + {LedgerKey, SQN} = leveled_codec:strip_to_keyseqonly({Key, Value}), + case leveled_inker:ink_keycheck(InkerClone, LedgerKey, SQN) of + probably -> + true; + missing -> + false + end. + accumulate_keys() -> Now = leveled_codec:integer_now(), AccFun = fun(Key, Value, KeyList) -> diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 8015f23..384d12b 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -38,6 +38,7 @@ strip_to_seqonly/1, strip_to_statusonly/1, strip_to_keyseqstatusonly/1, + strip_to_keyseqonly/1, striphead_to_details/1, is_active/3, endkey_passed/2, @@ -89,6 +90,8 @@ strip_to_statusonly({_, {_, St, _}}) -> St. strip_to_seqonly({_, {SeqN, _, _}}) -> SeqN. +strip_to_keyseqonly({LK, {SeqN, _, _}}) -> {LK, SeqN}. + striphead_to_details({SeqN, St, MD}) -> {SeqN, St, MD}. key_dominates(LeftKey, RightKey) -> diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 23b1f49..b91976c 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -97,6 +97,7 @@ ink_put/4, ink_get/3, ink_fetch/3, + ink_keycheck/3, ink_loadpcl/4, ink_registersnapshot/2, ink_confirmdelete/2, @@ -154,6 +155,9 @@ ink_get(Pid, PrimaryKey, SQN) -> ink_fetch(Pid, PrimaryKey, SQN) -> gen_server:call(Pid, {fetch, PrimaryKey, SQN}, infinity). +ink_keycheck(Pid, PrimaryKey, SQN) -> + gen_server:call(Pid, {key_check, PrimaryKey, SQN}, infinity). + ink_registersnapshot(Pid, Requestor) -> gen_server:call(Pid, {register_snapshot, Requestor}, infinity). @@ -250,6 +254,8 @@ handle_call({fetch, Key, SQN}, _From, State) -> end; handle_call({get, Key, SQN}, _From, State) -> {reply, get_object(Key, SQN, State#state.manifest), State}; +handle_call({key_check, Key, SQN}, _From, State) -> + {reply, key_check(Key, SQN, State#state.manifest), State}; handle_call({load_pcl, StartSQN, FilterFun, Penciller}, _From, State) -> Manifest = lists:reverse(State#state.manifest), Reply = load_from_sequence(StartSQN, FilterFun, Penciller, Manifest), @@ -440,6 +446,13 @@ get_object(LedgerKey, SQN, Manifest) -> Obj = leveled_cdb:cdb_get(JournalP, InkerKey), leveled_codec:from_inkerkv(Obj). +key_check(LedgerKey, SQN, Manifest) -> + JournalP = find_in_manifest(SQN, Manifest), + {InkerKey, _V, true} = leveled_codec:to_inkerkv(LedgerKey, + SQN, + to_fetch, + null), + leveled_cdb:cdb_keycheck(JournalP, InkerKey). build_manifest(ManifestFilenames, RootPath, From b18f7fd1c1b0693f10ca8beeff2e34cd9c9bff66 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 31 Oct 2016 18:51:23 +0000 Subject: [PATCH 115/167] Check presence in Journal on hashtree query Basic happy day unit test added to demonstrate checking presence (with a set probability) of the item in the hashtree query within the Journal. --- src/leveled_bookie.erl | 48 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 294b11a..e67e334 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -159,7 +159,8 @@ -define(SHUTDOWN_WAITS, 60). -define(SHUTDOWN_PAUSE, 10000). -define(SNAPSHOT_TIMEOUT, 300000). --define(JITTER_PROBABILITY, 0.01). +-define(JITTER_PROB, 0.01). +-define(CHECKJOURNAL_PROB, 0.2). -record(state, {inker :: pid(), penciller :: pid(), @@ -484,6 +485,12 @@ hashtree_query(State, Tag, JournalCheck) -> AccFun, []), ok = leveled_penciller:pcl_close(LedgerSnapshot), + case JournalCheck of + false -> + ok; + check_presence -> + leveled_inker:ink_close(JournalSnapshot) + end, Acc end, {async, Folder}. @@ -615,16 +622,17 @@ accumulate_hashes(JournalCheck, InkerClone) -> case leveled_codec:is_active(LK, V, Now) of true -> {B, K, H} = leveled_codec:get_keyandhash(LK, V), - case JournalCheck of - false -> - [{B, K, H}|KHList]; - check_presence -> + Check = random:uniform() < ?CHECKJOURNAL_PROB, + case {JournalCheck, Check} of + {check_presence, true} -> case check_presence(LK, V, InkerClone) of true -> [{B, K, H}|KHList]; false -> KHList - end + end; + _ -> + [{B, K, H}|KHList] end; false -> KHList @@ -737,7 +745,7 @@ maybe_withjitter(CacheSize, MaxCacheSize) -> CacheSize > MaxCacheSize -> R = random:uniform(), if - R < ?JITTER_PROBABILITY -> + R < ?JITTER_PROB -> true; true -> false @@ -1020,4 +1028,30 @@ hashtree_query_test() -> ok = book_close(Bookie2), reset_filestructure(). +hashtree_query_withjournalcheck_test() -> + RootPath = reset_filestructure(), + {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath, + max_journalsize=1000000, + cache_size=500}), + ObjL1 = generate_multiple_objects(800, 1), + % Put in all the objects with a TTL in the future + Future = leveled_codec:integer_now() + 300, + lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, + "Bucket", K, V, S, + ?STD_TAG, + Future) end, + ObjL1), + {async, HTFolder1} = book_returnfolder(Bookie1, + {hashtree_query, + ?STD_TAG, + false}), + KeyHashList = HTFolder1(), + {async, HTFolder2} = book_returnfolder(Bookie1, + {hashtree_query, + ?STD_TAG, + check_presence}), + ?assertMatch(KeyHashList, HTFolder2()), + ok = book_close(Bookie1), + reset_filestructure(). + -endif. \ No newline at end of file From 73004328e1bc76b3f4cf504a3dcf20bde46ae02a Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 31 Oct 2016 20:58:19 +0000 Subject: [PATCH 116/167] Recovery Tests Some initial entropy tests showing loss of data from a corrupted CDB file. --- src/leveled_bookie.erl | 2 + src/leveled_cdb.erl | 4 +- src/leveled_inker.erl | 2 +- .../{restart_SUITE.erl => recovery_SUITE.erl} | 75 ++++++++++++++++++- 4 files changed, 77 insertions(+), 6 deletions(-) rename test/end_to_end/{restart_SUITE.erl => recovery_SUITE.erl} (60%) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index e67e334..8ab54cf 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -314,6 +314,8 @@ handle_call({get, Bucket, Key, Tag}, _From, State) -> Active = TS >= leveled_codec:integer_now(), case {Active, fetch_value(LedgerKey, Seqn, State#state.inker)} of + {_, not_present} -> + {reply, not_found, State}; {true, Object} -> {reply, {ok, Object}, State}; _ -> diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 29bbbfd..454c4ec 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -904,7 +904,7 @@ extract_kvpair(_, [], _) -> extract_kvpair(Handle, [Position|Rest], Key) -> {ok, _} = file:position(Handle, Position), {KeyLength, ValueLength} = read_next_2_integers(Handle), - case read_next_term(Handle, KeyLength) of + case safe_read_next_term(Handle, KeyLength) of Key -> % If same key as passed in, then found! case read_next_term(Handle, ValueLength, crc) of {false, _} -> @@ -1094,7 +1094,7 @@ read_next_term(Handle, Length, crc) -> CRC -> {true, binary_to_term(Bin)}; _ -> - {false, binary_to_term(Bin)} + {false, crc_wonky} end. %% Extract value and size from binary containing CRC diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index b91976c..3eb8272 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -249,7 +249,7 @@ handle_call({fetch, Key, SQN}, _From, State) -> {reply, {ok, Value}, State}; Other -> io:format("Unexpected failure to fetch value for" ++ - "Key=~w SQN=~w with reason ~w", [Key, SQN, Other]), + "Key=~w SQN=~w with reason ~w~n", [Key, SQN, Other]), {reply, not_present, State} end; handle_call({get, Key, SQN}, _From, State) -> diff --git a/test/end_to_end/restart_SUITE.erl b/test/end_to_end/recovery_SUITE.erl similarity index 60% rename from test/end_to_end/restart_SUITE.erl rename to test/end_to_end/recovery_SUITE.erl index 65efbf7..54b058b 100644 --- a/test/end_to_end/restart_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -1,12 +1,14 @@ --module(restart_SUITE). +-module(recovery_SUITE). -include_lib("common_test/include/ct.hrl"). -include("include/leveled.hrl"). -export([all/0]). --export([retain_strategy/1 +-export([retain_strategy/1, + aae_bustedjournal/1 ]). all() -> [ - retain_strategy + retain_strategy, + aae_bustedjournal ]. retain_strategy(_Config) -> @@ -34,6 +36,73 @@ retain_strategy(_Config) -> +aae_bustedjournal(_Config) -> + RootPath = testutil:reset_filestructure(), + StartOpts = #bookie_options{root_path=RootPath, + max_journalsize=20000000}, + {ok, Bookie1} = leveled_bookie:book_start(StartOpts), + {TestObject, TestSpec} = testutil:generate_testobject(), + ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), + testutil:check_forobject(Bookie1, TestObject), + GenList = [2], + _CLs = testutil:load_objects(20000, GenList, Bookie1, TestObject, + fun testutil:generate_objects/2), + ok = leveled_bookie:book_close(Bookie1), + {ok, FNsA_J} = file:list_dir(RootPath ++ "/journal/journal_files"), + {ok, Regex} = re:compile(".*\.cdb"), + CDBFiles = lists:foldl(fun(FN, Acc) -> case re:run(FN, Regex) of + nomatch -> + Acc; + _ -> + [FN|Acc] + end + end, + [], + FNsA_J), + [HeadF|_Rest] = CDBFiles, + io:format("Selected Journal for corruption of ~s~n", [HeadF]), + {ok, Handle} = file:open(RootPath ++ "/journal/journal_files/" ++ HeadF, + [binary, raw, read, write]), + lists:foreach(fun(X) -> + Position = X * 1000 + 2048, + ok = file:pwrite(Handle, Position, <<0:8/integer>>) + end, + lists:seq(1, 1000)), + ok = file:close(Handle), + {ok, Bookie2} = leveled_bookie:book_start(StartOpts), + + {async, KeyF} = leveled_bookie:book_returnfolder(Bookie2, + {keylist, ?RIAK_TAG}), + KeyList = KeyF(), + 20001 = length(KeyList), + HeadCount = lists:foldl(fun({B, K}, Acc) -> + case leveled_bookie:book_riakhead(Bookie2, + B, + K) of + {ok, _} -> Acc + 1; + not_found -> Acc + end + end, + 0, + KeyList), + 20001 = HeadCount, + GetCount = lists:foldl(fun({B, K}, Acc) -> + case leveled_bookie:book_riakget(Bookie2, + B, + K) of + {ok, _} -> Acc + 1; + not_found -> Acc + end + end, + 0, + KeyList), + true = GetCount > 19000, + true = GetCount < HeadCount, + + ok = leveled_bookie:book_close(Bookie2), + testutil:reset_filestructure(). + + rotating_object_check(BookOpts, B, NumberOfObjects) -> {ok, Book1} = leveled_bookie:book_start(BookOpts), {KSpcL1, V1} = testutil:put_indexed_objects(Book1, B, NumberOfObjects), From 7d35ef71268ccac523d48f8890106050261e40d7 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Mon, 31 Oct 2016 22:17:29 +0000 Subject: [PATCH 117/167] Lame AAE hashtree query test Corruption of the values wihtin the journal doesn't get detected by the hashtree query --- test/end_to_end/recovery_SUITE.erl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index 54b058b..71b1d3a 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -99,6 +99,22 @@ aae_bustedjournal(_Config) -> true = GetCount > 19000, true = GetCount < HeadCount, + {async, HashTreeF1} = leveled_bookie:book_returnfolder(Bookie2, + {hashtree_query, + ?RIAK_TAG, + false}), + KeyHashList1 = HashTreeF1(), + 20001 = length(KeyHashList1), + {async, HashTreeF2} = leveled_bookie:book_returnfolder(Bookie2, + {hashtree_query, + ?RIAK_TAG, + check_presence}), + KeyHashList2 = HashTreeF2(), + % The file is still there, and the hashtree is not corrupted + KeyHashList2 = KeyHashList1, + % Will need to remove the file or corrupt the hashtree to get presence to + % fail + ok = leveled_bookie:book_close(Bookie2), testutil:reset_filestructure(). From 84a92b5f9557ab04c39457421e7e76e983d905b3 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 1 Nov 2016 00:46:14 +0000 Subject: [PATCH 118/167] Further testing of compaction Check we avoid crashing in challenging compaction scenarios --- src/leveled_cdb.erl | 7 ++- src/leveled_codec.erl | 6 +- src/leveled_iclerk.erl | 13 +--- test/end_to_end/recovery_SUITE.erl | 99 +++++++++++++++++++++++------- test/end_to_end/testutil.erl | 27 +++++++- 5 files changed, 113 insertions(+), 39 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 454c4ec..fc8af19 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -1025,7 +1025,12 @@ saferead_keyvalue(Handle) -> eof -> false; {ok, Value} -> - {Key, Value, KeyL, ValueL} + case crccheck_value(Value) of + true -> + {Key, Value, KeyL, ValueL}; + false -> + false + end end end end. diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index 384d12b..fed4cff 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -56,7 +56,6 @@ check_forinkertype/2, create_value_for_journal/1, build_metadata_object/2, - generate_ledgerkv/4, generate_ledgerkv/5, get_size/2, get_keyandhash/2, @@ -170,6 +169,8 @@ from_inkerkv(Object) -> from_journalkey({SQN, _Type, LedgerKey}) -> {SQN, LedgerKey}. +compact_inkerkvc({_InkerKey, crc_wonky, false}, _Strategy) -> + skip; compact_inkerkvc({{_SQN, ?INKT_TOMB, _LK}, _V, _CrcCheck}, _Strategy) -> skip; compact_inkerkvc({{SQN, ?INKT_KEYD, LK}, V, CrcCheck}, Strategy) -> @@ -271,9 +272,6 @@ convert_indexspecs(IndexSpecs, Bucket, Key, SQN, TTL) -> end, IndexSpecs). -generate_ledgerkv(PrimaryKey, SQN, Obj, Size) -> - generate_ledgerkv(PrimaryKey, SQN, Obj, Size, infinity). - generate_ledgerkv(PrimaryKey, SQN, Obj, Size, TS) -> {Tag, Bucket, Key, _} = PrimaryKey, Status = case Obj of diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index a381ba4..8fc284c 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -149,12 +149,7 @@ handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, State) -> % Need to fetch manifest at start rather than have it be passed in % Don't want to process a queued call waiting on an old manifest - Manifest = case leveled_inker:ink_getmanifest(Inker) of - [] -> - []; - [_Active|Tail] -> - Tail - end, + [_Active|Manifest] = leveled_inker:ink_getmanifest(Inker), MaxRunLength = State#state.max_run_length, {FilterServer, MaxSQN} = InitiateFun(Checker), CDBopts = State#state.cdb_options, @@ -462,11 +457,7 @@ filter_output(KVCs, FilterFun, FilterServer, MaxSQN, ReloadStrategy) -> {false, true, false, retain} -> {Acc ++ [KVC1], PromptDelete}; {false, true, false, _} -> - {Acc, PromptDelete}; - {_, false, _, _} -> - io:format("Corrupted value found for " - ++ "Journal Key ~w~n", [K]), - {Acc, false} + {Acc, PromptDelete} end end end, diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index 71b1d3a..71b6dd9 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -3,12 +3,14 @@ -include("include/leveled.hrl"). -export([all/0]). -export([retain_strategy/1, - aae_bustedjournal/1 + aae_bustedjournal/1, + journal_compaction_bustedjournal/1 ]). all() -> [ - retain_strategy, - aae_bustedjournal + % retain_strategy, + aae_bustedjournal, + journal_compaction_bustedjournal ]. retain_strategy(_Config) -> @@ -48,27 +50,10 @@ aae_bustedjournal(_Config) -> _CLs = testutil:load_objects(20000, GenList, Bookie1, TestObject, fun testutil:generate_objects/2), ok = leveled_bookie:book_close(Bookie1), - {ok, FNsA_J} = file:list_dir(RootPath ++ "/journal/journal_files"), - {ok, Regex} = re:compile(".*\.cdb"), - CDBFiles = lists:foldl(fun(FN, Acc) -> case re:run(FN, Regex) of - nomatch -> - Acc; - _ -> - [FN|Acc] - end - end, - [], - FNsA_J), + CDBFiles = testutil:find_journals(RootPath), [HeadF|_Rest] = CDBFiles, io:format("Selected Journal for corruption of ~s~n", [HeadF]), - {ok, Handle} = file:open(RootPath ++ "/journal/journal_files/" ++ HeadF, - [binary, raw, read, write]), - lists:foreach(fun(X) -> - Position = X * 1000 + 2048, - ok = file:pwrite(Handle, Position, <<0:8/integer>>) - end, - lists:seq(1, 1000)), - ok = file:close(Handle), + testutil:corrupt_journal(RootPath, HeadF, 1000), {ok, Bookie2} = leveled_bookie:book_start(StartOpts), {async, KeyF} = leveled_bookie:book_returnfolder(Bookie2, @@ -119,6 +104,76 @@ aae_bustedjournal(_Config) -> testutil:reset_filestructure(). +journal_compaction_bustedjournal(_Config) -> + % Simply confirms that none of this causes a crash + RootPath = testutil:reset_filestructure(), + StartOpts1 = #bookie_options{root_path=RootPath, + max_journalsize=10000000, + max_run_length=10}, + {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), + {TestObject, TestSpec} = testutil:generate_testobject(), + ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), + testutil:check_forobject(Bookie1, TestObject), + ObjList1 = testutil:generate_objects(50000, 2), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, + ObjList1), + %% Now replace all the objects + ObjList2 = testutil:generate_objects(50000, 2), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, + ObjList2), + ok = leveled_bookie:book_close(Bookie1), + + CDBFiles = testutil:find_journals(RootPath), + lists:foreach(fun(FN) -> testutil:corrupt_journal(RootPath, FN, 100) end, + CDBFiles), + + {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), + + ok = leveled_bookie:book_compactjournal(Bookie2, 30000), + F = fun leveled_bookie:book_islastcompactionpending/1, + lists:foldl(fun(X, Pending) -> + case Pending of + false -> + false; + true -> + io:format("Loop ~w waiting for journal " + ++ "compaction to complete~n", [X]), + timer:sleep(20000), + F(Bookie2) + end end, + true, + lists:seq(1, 15)), + + ObjList3 = testutil:generate_objects(15000, 50002), + ObjList4 = testutil:generate_objects(15000, 50002), + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, + ObjList3), + %% Now replace all the objects + lists:foreach(fun({_RN, Obj, Spc}) -> + leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, + ObjList4), + + ok = leveled_bookie:book_compactjournal(Bookie2, 30000), + lists:foldl(fun(X, Pending) -> + case Pending of + false -> + false; + true -> + io:format("Loop ~w waiting for journal " + ++ "compaction to complete~n", [X]), + timer:sleep(20000), + F(Bookie2) + end end, + true, + lists:seq(1, 15)), + + ok = leveled_bookie:book_close(Bookie2), + testutil:reset_filestructure(10000). + + rotating_object_check(BookOpts, B, NumberOfObjects) -> {ok, Book1} = leveled_bookie:book_start(BookOpts), {KSpcL1, V1} = testutil:put_indexed_objects(Book1, B, NumberOfObjects), diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index f88e900..3a7585c 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -28,7 +28,9 @@ put_altered_indexed_objects/3, put_altered_indexed_objects/4, check_indexed_objects/4, - rotating_object_check/3]). + rotating_object_check/3, + corrupt_journal/3, + find_journals/1]). -define(RETURN_TERMS, {true, undefined}). @@ -380,3 +382,26 @@ rotating_object_check(RootPath, B, NumberOfObjects) -> ok = leveled_bookie:book_close(Book2), ok. +corrupt_journal(RootPath, FileName, Corruptions) -> + {ok, Handle} = file:open(RootPath ++ "/journal/journal_files/" ++ FileName, + [binary, raw, read, write]), + lists:foreach(fun(X) -> + Position = X * 1000 + 2048, + ok = file:pwrite(Handle, Position, <<0:8/integer>>) + end, + lists:seq(1, Corruptions)), + ok = file:close(Handle). + +find_journals(RootPath) -> + {ok, FNsA_J} = file:list_dir(RootPath ++ "/journal/journal_files"), + {ok, Regex} = re:compile(".*\.cdb"), + CDBFiles = lists:foldl(fun(FN, Acc) -> case re:run(FN, Regex) of + nomatch -> + Acc; + _ -> + [FN|Acc] + end + end, + [], + FNsA_J), + CDBFiles. \ No newline at end of file From ce34235f2f25bb172814bc726ba9dc87f6291e1e Mon Sep 17 00:00:00 2001 From: martinsumner Date: Tue, 1 Nov 2016 01:38:48 +0000 Subject: [PATCH 119/167] Revert commented out test Unintentional commenting --- test/end_to_end/recovery_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index 71b6dd9..723dbd7 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -8,7 +8,7 @@ ]). all() -> [ - % retain_strategy, + retain_strategy, aae_bustedjournal, journal_compaction_bustedjournal ]. From e7506c3c1fd75ab94a4df30f22a1a206eb69e9ae Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 2 Nov 2016 12:58:27 +0000 Subject: [PATCH 120/167] Startup work - baffled Changes the stratup otpions to a prolist to make it easier to get environment variables as default. Tried application:start - and completely baffled as to how to get this to work. --- include/leveled.hrl | 9 ---- src/eleveleddb.app.src | 4 +- src/eleveleddb_sup.erl | 2 +- src/leveled_bookie.erl | 87 +++++++++++++++++------------- test/end_to_end/basic_SUITE.erl | 33 ++++++------ test/end_to_end/iterator_SUITE.erl | 4 +- test/end_to_end/recovery_SUITE.erl | 51 ++++++------------ test/end_to_end/testutil.erl | 6 +-- 8 files changed, 90 insertions(+), 106 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 3a85aa7..028eb95 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -62,15 +62,6 @@ start_snapshot = false :: boolean(), source_penciller :: pid()}). --record(bookie_options, - {root_path :: string(), - cache_size :: integer(), - max_journalsize :: integer(), - max_pencillercachesize :: integer(), - snapshot_bookie :: pid(), - reload_strategy = [] :: list(), - max_run_length :: integer()}). - -record(iclerk_options, {inker :: pid(), max_run_length :: integer(), diff --git a/src/eleveleddb.app.src b/src/eleveleddb.app.src index 6d0069e..37b004d 100644 --- a/src/eleveleddb.app.src +++ b/src/eleveleddb.app.src @@ -1,6 +1,6 @@ {application, eleveleddb, [ - {description, ""}, + {description, "Key Value store based on LSM-Tree and designed for larger values"}, {vsn, "1"}, {registered, []}, {applications, [ @@ -8,5 +8,5 @@ stdlib ]}, {mod, { eleveleddb_app, []}}, - {env, []} + {env, [{root_path, "test"}]} ]}. diff --git a/src/eleveleddb_sup.erl b/src/eleveleddb_sup.erl index 37ef58b..391aca9 100644 --- a/src/eleveleddb_sup.erl +++ b/src/eleveleddb_sup.erl @@ -16,7 +16,7 @@ %% =================================================================== start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). + supervisor:start_link({local, leveled_bookie}, ?MODULE, []). %% =================================================================== %% Supervisor callbacks diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 8ab54cf..4ece04b 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -151,6 +151,9 @@ book_islastcompactionpending/1, book_close/1]). +-export([get_opt/2, + get_opt/3]). + -include_lib("eunit/include/eunit.hrl"). -define(CACHE_SIZE, 2000). @@ -175,9 +178,9 @@ %%%============================================================================ book_start(RootPath, LedgerCacheSize, JournalSize) -> - book_start(#bookie_options{root_path=RootPath, - cache_size=LedgerCacheSize, - max_journalsize=JournalSize}). + book_start([{root_path, RootPath}, + {cache_size, LedgerCacheSize}, + {max_journalsize, JournalSize}]). book_start(Opts) -> gen_server:start(?MODULE, [Opts], []). @@ -247,17 +250,12 @@ book_close(Pid) -> %%%============================================================================ init([Opts]) -> - case Opts#bookie_options.snapshot_bookie of + case get_opt(snapshot_bookie, Opts) of undefined -> % Start from file not snapshot {InkerOpts, PencillerOpts} = set_options(Opts), {Inker, Penciller} = startup(InkerOpts, PencillerOpts), - CacheSize = if - Opts#bookie_options.cache_size == undefined -> - ?CACHE_SIZE; - true -> - Opts#bookie_options.cache_size - end, + CacheSize = get_opt(cache_size, Opts, ?CACHE_SIZE), io:format("Bookie starting with Pcl ~w Ink ~w~n", [Penciller, Inker]), {ok, #state{inker=Inker, @@ -551,21 +549,21 @@ shutdown_wait([TopPause|Rest], Inker) -> set_options(Opts) -> - MaxJournalSize = case Opts#bookie_options.max_journalsize of - undefined -> - 30000; - MS -> - MS - end, + MaxJournalSize = get_opt(max_journalsize, Opts, 10000000000), - AltStrategy = Opts#bookie_options.reload_strategy, + AltStrategy = get_opt(reload_strategy, Opts, []), ReloadStrategy = leveled_codec:inker_reload_strategy(AltStrategy), - PCLL0CacheSize = Opts#bookie_options.max_pencillercachesize, - JournalFP = Opts#bookie_options.root_path ++ "/" ++ ?JOURNAL_FP, - LedgerFP = Opts#bookie_options.root_path ++ "/" ++ ?LEDGER_FP, + + PCLL0CacheSize = get_opt(max_pencillercachesize, Opts), + RootPath = get_opt(root_path, Opts), + JournalFP = RootPath ++ "/" ++ ?JOURNAL_FP, + LedgerFP = RootPath ++ "/" ++ ?LEDGER_FP, + ok =filelib:ensure_dir(JournalFP), + ok =filelib:ensure_dir(LedgerFP), + {#inker_options{root_path = JournalFP, reload_strategy = ReloadStrategy, - max_run_length = Opts#bookie_options.max_run_length, + max_run_length = get_opt(max_run_length, Opts), cdb_options = #cdb_options{max_size=MaxJournalSize, binary_mode=true}}, #penciller_options{root_path = LedgerFP, @@ -781,6 +779,23 @@ load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> end. +get_opt(Key, Opts) -> + get_opt(Key, Opts, undefined). + +get_opt(Key, Opts, Default) -> + case proplists:get_value(Key, Opts) of + undefined -> + case application:get_env(?MODULE, Key) of + {ok, Value} -> + Value; + undefined -> + Default + end; + Value -> + Value + end. + + %%%============================================================================ %%% Test %%%============================================================================ @@ -828,7 +843,7 @@ generate_multiple_robjects(Count, KeyNumber, ObjL) -> single_key_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath}), + {ok, Bookie1} = book_start([{root_path, RootPath}]), {B1, K1, V1, Spec1, MD} = {"Bucket1", "Key1", "Value1", @@ -840,7 +855,7 @@ single_key_test() -> {ok, F1} = book_riakget(Bookie1, B1, K1), ?assertMatch(F1, Object), ok = book_close(Bookie1), - {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath}), + {ok, Bookie2} = book_start([{root_path, RootPath}]), {ok, F2} = book_riakget(Bookie2, B1, K1), ?assertMatch(F2, Object), ok = book_close(Bookie2), @@ -848,7 +863,7 @@ single_key_test() -> multi_key_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath}), + {ok, Bookie1} = book_start([{root_path, RootPath}]), {B1, K1, V1, Spec1, MD1} = {"Bucket", "Key1", "Value1", @@ -885,7 +900,7 @@ multi_key_test() -> ?assertMatch(F2B, Obj2), ok = book_close(Bookie1), % Now reopen the file, and confirm that a fetch is still possible - {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath}), + {ok, Bookie2} = book_start([{root_path, RootPath}]), {ok, F1C} = book_riakget(Bookie2, B1, K1), ?assertMatch(F1C, Obj1), {ok, F2C} = book_riakget(Bookie2, B2, K2), @@ -904,7 +919,7 @@ multi_key_test() -> ttl_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath}), + {ok, Bookie1} = book_start([{root_path, RootPath}]), ObjL1 = generate_multiple_objects(100, 1), % Put in all the objects with a TTL in the future Future = leveled_codec:integer_now() + 300, @@ -962,7 +977,7 @@ ttl_test() -> ?assertMatch(10, length(TermKeyList)), ok = book_close(Bookie1), - {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath}), + {ok, Bookie2} = book_start([{root_path, RootPath}]), {async, IndexFolderTR2} = book_returnfolder(Bookie2, @@ -987,9 +1002,9 @@ ttl_test() -> hashtree_query_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath, - max_journalsize=1000000, - cache_size=500}), + {ok, Bookie1} = book_start([{root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500}]), ObjL1 = generate_multiple_objects(1200, 1), % Put in all the objects with a TTL in the future Future = leveled_codec:integer_now() + 300, @@ -1019,9 +1034,9 @@ hashtree_query_test() -> KeyHashList), ?assertMatch(1200, length(KeyHashList)), ok = book_close(Bookie1), - {ok, Bookie2} = book_start(#bookie_options{root_path=RootPath, - max_journalsize=200000, - cache_size=500}), + {ok, Bookie2} = book_start([{root_path, RootPath}, + {max_journalsize, 200000}, + {cache_size, 500}]), {async, HTFolder2} = book_returnfolder(Bookie2, {hashtree_query, ?STD_TAG, @@ -1032,9 +1047,9 @@ hashtree_query_test() -> hashtree_query_withjournalcheck_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start(#bookie_options{root_path=RootPath, - max_journalsize=1000000, - cache_size=500}), + {ok, Bookie1} = book_start([{root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500}]), ObjL1 = generate_multiple_objects(800, 1), % Put in all the objects with a TTL in the future Future = leveled_codec:integer_now() + 300, diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 612a448..bf21318 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -24,15 +24,15 @@ all() -> [ simple_put_fetch_head_delete(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath}, + StartOpts1 = [{root_path, RootPath}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), testutil:check_forobject(Bookie1, TestObject), testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"), ok = leveled_bookie:book_close(Bookie1), - StartOpts2 = #bookie_options{root_path=RootPath, - max_journalsize=3000000}, + StartOpts2 = [{root_path, RootPath}, + {max_journalsize, 3000000}], {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), testutil:check_forobject(Bookie2, TestObject), ObjList1 = testutil:generate_objects(5000, 2), @@ -69,16 +69,15 @@ simple_put_fetch_head_delete(_Config) -> many_put_fetch_head(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, - max_pencillercachesize=16000}, + StartOpts1 = [{root_path, RootPath}, {max_pencillercachesize, 16000}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), testutil:check_forobject(Bookie1, TestObject), ok = leveled_bookie:book_close(Bookie1), - StartOpts2 = #bookie_options{root_path=RootPath, - max_journalsize=1000000000, - max_pencillercachesize=32000}, + StartOpts2 = [{root_path, RootPath}, + {max_journalsize, 1000000000}, + {max_pencillercachesize, 32000}], {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), testutil:check_forobject(Bookie2, TestObject), GenList = [2, 20002, 40002, 60002, 80002, @@ -108,9 +107,9 @@ many_put_fetch_head(_Config) -> journal_compaction(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, - max_journalsize=10000000, - max_run_length=1}, + StartOpts1 = [{root_path, RootPath}, + {max_journalsize, 10000000}, + {max_run_length, 1}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), ok = leveled_bookie:book_compactjournal(Bookie1, 30000), {TestObject, TestSpec} = testutil:generate_testobject(), @@ -172,7 +171,7 @@ journal_compaction(_Config) -> fetchput_snapshot(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=30000000}, + StartOpts1 = [{root_path, RootPath}, {max_journalsize, 30000000}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), @@ -180,7 +179,7 @@ fetchput_snapshot(_Config) -> lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjList1), - SnapOpts1 = #bookie_options{snapshot_bookie=Bookie1}, + SnapOpts1 = [{snapshot_bookie, Bookie1}], {ok, SnapBookie1} = leveled_bookie:book_start(SnapOpts1), ChkList1 = lists:sublist(lists:sort(ObjList1), 100), testutil:check_forlist(Bookie1, ChkList1), @@ -191,7 +190,7 @@ fetchput_snapshot(_Config) -> io:format("Closed initial bookies~n"), {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), - SnapOpts2 = #bookie_options{snapshot_bookie=Bookie2}, + SnapOpts2 = [{snapshot_bookie, Bookie2}], {ok, SnapBookie2} = leveled_bookie:book_start(SnapOpts2), io:format("Bookies restarted~n"), @@ -279,7 +278,7 @@ load_and_count(_Config) -> % Use artificially small files, and the load keys, counting they're all % present RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=50000000}, + StartOpts1 = [{root_path, RootPath}, {max_journalsize, 50000000}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), @@ -362,7 +361,7 @@ load_and_count(_Config) -> load_and_count_withdelete(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=50000000}, + StartOpts1 = [{root_path, RootPath}, {max_journalsize, 50000000}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), @@ -418,7 +417,7 @@ load_and_count_withdelete(_Config) -> space_clear_ondelete(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, max_journalsize=20000000}, + StartOpts1 = [{root_path, RootPath}, {max_journalsize, 20000000}], {ok, Book1} = leveled_bookie:book_start(StartOpts1), G2 = fun testutil:generate_compressibleobjects/2, testutil:load_objects(20000, diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 5d34786..1353aa8 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -18,8 +18,8 @@ all() -> [ simple_load_with2i(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, - max_journalsize=50000000}, + StartOpts1 = [{root_path, RootPath}, + {max_journalsize, 50000000}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index 723dbd7..b4fbfa2 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -15,12 +15,15 @@ all() -> [ retain_strategy(_Config) -> RootPath = testutil:reset_filestructure(), - BookOpts = #bookie_options{root_path=RootPath, - cache_size=1000, - max_journalsize=5000000, - reload_strategy=[{?RIAK_TAG, retain}]}, - BookOptsAlt = BookOpts#bookie_options{max_run_length=8, - max_journalsize=100000}, + BookOpts = [{root_path, RootPath}, + {cache_size, 1000}, + {max_journalsize, 5000000}, + {reload_strategy, [{?RIAK_TAG, retain}]}], + BookOptsAlt = [{root_path, RootPath}, + {cache_size, 1000}, + {max_journalsize, 100000}, + {reload_strategy, [{?RIAK_TAG, retain}]}, + {max_run_length, 8}], {ok, Spcl3, LastV3} = rotating_object_check(BookOpts, "Bucket3", 800), ok = restart_from_blankledger(BookOpts, [{"Bucket3", Spcl3, LastV3}]), {ok, Spcl4, LastV4} = rotating_object_check(BookOpts, "Bucket4", 1600), @@ -40,8 +43,8 @@ retain_strategy(_Config) -> aae_bustedjournal(_Config) -> RootPath = testutil:reset_filestructure(), - StartOpts = #bookie_options{root_path=RootPath, - max_journalsize=20000000}, + StartOpts = [{root_path, RootPath}, + {max_journalsize, 20000000}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), @@ -107,9 +110,9 @@ aae_bustedjournal(_Config) -> journal_compaction_bustedjournal(_Config) -> % Simply confirms that none of this causes a crash RootPath = testutil:reset_filestructure(), - StartOpts1 = #bookie_options{root_path=RootPath, - max_journalsize=10000000, - max_run_length=10}, + StartOpts1 = [{root_path, RootPath}, + {max_journalsize, 10000000}, + {max_run_length, 10}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), @@ -146,30 +149,6 @@ journal_compaction_bustedjournal(_Config) -> true, lists:seq(1, 15)), - ObjList3 = testutil:generate_objects(15000, 50002), - ObjList4 = testutil:generate_objects(15000, 50002), - lists:foreach(fun({_RN, Obj, Spc}) -> - leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, - ObjList3), - %% Now replace all the objects - lists:foreach(fun({_RN, Obj, Spc}) -> - leveled_bookie:book_riakput(Bookie2, Obj, Spc) end, - ObjList4), - - ok = leveled_bookie:book_compactjournal(Bookie2, 30000), - lists:foldl(fun(X, Pending) -> - case Pending of - false -> - false; - true -> - io:format("Loop ~w waiting for journal " - ++ "compaction to complete~n", [X]), - timer:sleep(20000), - F(Bookie2) - end end, - true, - lists:seq(1, 15)), - ok = leveled_bookie:book_close(Bookie2), testutil:reset_filestructure(10000). @@ -237,7 +216,7 @@ rotating_object_check(BookOpts, B, NumberOfObjects) -> restart_from_blankledger(BookOpts, B_SpcL) -> - leveled_penciller:clean_testdir(BookOpts#bookie_options.root_path ++ + leveled_penciller:clean_testdir(proplists:get_value(root_path, BookOpts) ++ "/ledger"), {ok, Book1} = leveled_bookie:book_start(BookOpts), io:format("Checking index following restart~n"), diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index 3a7585c..1c2536f 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -365,9 +365,9 @@ put_altered_indexed_objects(Book, Bucket, KSpecL, RemoveOld2i) -> {RplKSpecL, V}. rotating_object_check(RootPath, B, NumberOfObjects) -> - BookOpts = #bookie_options{root_path=RootPath, - cache_size=1000, - max_journalsize=5000000}, + BookOpts = [{root_path, RootPath}, + {cache_size, 1000}, + {max_journalsize, 5000000}], {ok, Book1} = leveled_bookie:book_start(BookOpts), {KSpcL1, V1} = testutil:put_indexed_objects(Book1, B, NumberOfObjects), ok = testutil:check_indexed_objects(Book1, B, KSpcL1, V1), From a56ed18ba9b2926f65023976420de68b6c7f1bfb Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 2 Nov 2016 13:27:16 +0000 Subject: [PATCH 121/167] Test timing Look to see if test timing related to intermittent failure --- test/end_to_end/basic_SUITE.erl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index bf21318..555ade5 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -12,12 +12,12 @@ ]). all() -> [ - simple_put_fetch_head_delete, - many_put_fetch_head, - journal_compaction, - fetchput_snapshot, - load_and_count, - load_and_count_withdelete, + % simple_put_fetch_head_delete, + % many_put_fetch_head, + % journal_compaction, + % fetchput_snapshot, + % load_and_count, + % load_and_count_withdelete, space_clear_ondelete ]. @@ -504,7 +504,7 @@ space_clear_ondelete(_Config) -> io:format("This should cause a final ledger merge event~n"), io:format("Will require the penciller to resolve the issue of creating" ++ " an empty file as all keys compact on merge~n"), - timer:sleep(5000), + timer:sleep(12000), ok = leveled_bookie:book_close(Book3), {ok, FNsD_L} = file:list_dir(RootPath ++ "/ledger/ledger_files"), io:format("Bookie has ~w ledger files " ++ From 8601e219d5ec2b9d607813ed5284c7ac9c13f619 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 2 Nov 2016 13:34:34 +0000 Subject: [PATCH 122/167] Revert test commenting Commented out some tests - bring back in --- test/end_to_end/basic_SUITE.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 555ade5..21bf066 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -12,12 +12,12 @@ ]). all() -> [ - % simple_put_fetch_head_delete, - % many_put_fetch_head, - % journal_compaction, - % fetchput_snapshot, - % load_and_count, - % load_and_count_withdelete, + simple_put_fetch_head_delete, + many_put_fetch_head, + journal_compaction, + fetchput_snapshot, + load_and_count, + load_and_count_withdelete, space_clear_ondelete ]. From 898f86a08d4d3575c5bde26ae599f8a28c44f259 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 2 Nov 2016 15:38:51 +0000 Subject: [PATCH 123/167] Add Fold Object by KeyList support --- src/leveled_bookie.erl | 80 ++++++++++++++++++++++++++++++ test/end_to_end/recovery_SUITE.erl | 32 ++++++++++-- 2 files changed, 109 insertions(+), 3 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 4ece04b..61c8fb8 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -373,6 +373,10 @@ handle_call({return_folder, FolderType}, _From, State) -> {hashtree_query, Tag, JournalCheck} -> {reply, hashtree_query(State, Tag, JournalCheck), + State}; + {foldobjects_allkeys, Tag, FoldObjectsFun} -> + {reply, + foldobjects_allkeys(State, Tag, FoldObjectsFun), State} end; handle_call({compact_journal, Timeout}, _From, State) -> @@ -495,6 +499,30 @@ hashtree_query(State, Tag, JournalCheck) -> end, {async, Folder}. + +foldobjects_allkeys(State, Tag, FoldObjectsFun) -> + {ok, + {LedgerSnapshot, LedgerCache}, + JournalSnapshot} = snapshot_store(State, store), + Folder = fun() -> + io:format("Length of increment in snapshot is ~w~n", + [gb_trees:size(LedgerCache)]), + ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, + LedgerCache), + StartKey = leveled_codec:to_ledgerkey(null, null, Tag), + EndKey = leveled_codec:to_ledgerkey(null, null, Tag), + AccFun = accumulate_objects(FoldObjectsFun, JournalSnapshot), + Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, + StartKey, + EndKey, + AccFun, + []), + ok = leveled_penciller:pcl_close(LedgerSnapshot), + ok = leveled_inker:ink_close(JournalSnapshot), + Acc + end, + {async, Folder}. + allkey_query(State, Tag) -> {ok, {LedgerSnapshot, LedgerCache}, @@ -640,6 +668,26 @@ accumulate_hashes(JournalCheck, InkerClone) -> end, AccFun. +accumulate_objects(FoldObjectsFun, InkerClone) -> + Now = leveled_codec:integer_now(), + AccFun = fun(LK, V, Acc) -> + case leveled_codec:is_active(LK, V, Now) of + true -> + SQN = leveled_codec:strip_to_seqonly({LK, V}), + {B, K} = leveled_codec:from_ledgerkey(LK), + R = leveled_inker:ink_fetch(InkerClone, LK, SQN), + case R of + {ok, Value} -> + FoldObjectsFun(B, K, Value, Acc); + not_present -> + Acc + end; + false -> + Acc + end + end, + AccFun. + check_presence(Key, Value, InkerClone) -> {LedgerKey, SQN} = leveled_codec:strip_to_keyseqonly({Key, Value}), case leveled_inker:ink_keycheck(InkerClone, LedgerKey, SQN) of @@ -1071,4 +1119,36 @@ hashtree_query_withjournalcheck_test() -> ok = book_close(Bookie1), reset_filestructure(). +foldobjects_vs_hashtree_test() -> + RootPath = reset_filestructure(), + {ok, Bookie1} = book_start([{root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500}]), + ObjL1 = generate_multiple_objects(800, 1), + % Put in all the objects with a TTL in the future + Future = leveled_codec:integer_now() + 300, + lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, + "Bucket", K, V, S, + ?STD_TAG, + Future) end, + ObjL1), + {async, HTFolder1} = book_returnfolder(Bookie1, + {hashtree_query, + ?STD_TAG, + false}), + KeyHashList1 = lists:usort(HTFolder1()), + io:format("First item ~w~n", [lists:nth(1, KeyHashList1)]), + FoldObjectsFun = fun(B, K, V, Acc) -> + [{B, K, erlang:phash2(term_to_binary(V))}|Acc] end, + {async, HTFolder2} = book_returnfolder(Bookie1, + {foldobjects_allkeys, + ?STD_TAG, + FoldObjectsFun}), + KeyHashList2 = HTFolder2(), + ?assertMatch(KeyHashList1, lists:usort(KeyHashList2)), + + ok = book_close(Bookie1), + reset_filestructure(). + + -endif. \ No newline at end of file diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index b4fbfa2..2992e76 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -8,9 +8,9 @@ ]). all() -> [ - retain_strategy, - aae_bustedjournal, - journal_compaction_bustedjournal + % retain_strategy, + aae_bustedjournal %, + % journal_compaction_bustedjournal ]. retain_strategy(_Config) -> @@ -103,9 +103,35 @@ aae_bustedjournal(_Config) -> % Will need to remove the file or corrupt the hashtree to get presence to % fail + FoldObjectsFun = fun(B, K, V, Acc) -> [{B, K, riak_hash(V)}|Acc] end, + SW = os:timestamp(), + {async, HashTreeF3} = leveled_bookie:book_returnfolder(Bookie2, + {foldobjects_allkeys, + ?RIAK_TAG, + FoldObjectsFun}), + KeyHashList3 = HashTreeF3(), + + true = length(KeyHashList3) > 19000, + true = length(KeyHashList3) < HeadCount, + Delta = length(lists:subtract(KeyHashList1, KeyHashList3)), + true = Delta < 1001, + io:format("Fetch of hashtree using fold objects took ~w microseconds" ++ + " and found a Delta of ~w and an objects count of ~w~n", + [timer:now_diff(os:timestamp(), SW), + Delta, + length(KeyHashList3)]), + ok = leveled_bookie:book_close(Bookie2), testutil:reset_filestructure(). +riak_hash(Obj=#r_object{}) -> + Vclock = vclock(Obj), + UpdObj = set_vclock(Obj, lists:sort(Vclock)), + erlang:phash2(term_to_binary(UpdObj)). + +set_vclock(Object=#r_object{}, VClock) -> Object#r_object{vclock=VClock}. +vclock(#r_object{vclock=VClock}) -> VClock. + journal_compaction_bustedjournal(_Config) -> % Simply confirms that none of this causes a crash From 0572f43b8a499d10c9182a61e8c7634340953973 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 2 Nov 2016 15:40:22 +0000 Subject: [PATCH 124/167] Uncomment test --- test/end_to_end/recovery_SUITE.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index 2992e76..ed32340 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -8,9 +8,9 @@ ]). all() -> [ - % retain_strategy, - aae_bustedjournal %, - % journal_compaction_bustedjournal + retain_strategy, + aae_bustedjournal, + journal_compaction_bustedjournal ]. retain_strategy(_Config) -> From 7147ec0470c912844f2a5191e322d54c4a5b79ed Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 2 Nov 2016 18:14:46 +0000 Subject: [PATCH 125/167] Logging - Phase 1 Abstract out logging and introduce a logbase --- src/leveled_bookie.erl | 30 +++---- src/leveled_codec.erl | 23 ++--- src/leveled_log.erl | 171 ++++++++++++++++++++++++++++++++++++++ src/leveled_pclerk.erl | 36 ++++---- src/leveled_penciller.erl | 116 +++++++++----------------- 5 files changed, 247 insertions(+), 129 deletions(-) create mode 100644 src/leveled_log.erl diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 61c8fb8..2204128 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -256,8 +256,7 @@ init([Opts]) -> {InkerOpts, PencillerOpts} = set_options(Opts), {Inker, Penciller} = startup(InkerOpts, PencillerOpts), CacheSize = get_opt(cache_size, Opts, ?CACHE_SIZE), - io:format("Bookie starting with Pcl ~w Ink ~w~n", - [Penciller, Inker]), + leveled_log:log("B0001", [Inker, Penciller]), {ok, #state{inker=Inker, penciller=Penciller, cache_size=CacheSize, @@ -269,8 +268,7 @@ init([Opts]) -> Inker} = book_snapshotstore(Bookie, self(), ?SNAPSHOT_TIMEOUT), ok = leveled_penciller:pcl_loadsnapshot(Penciller, gb_trees:empty()), - io:format("Snapshot starting with Pcl ~w Ink ~w~n", - [Penciller, Inker]), + leveled_log:log("B0002", [Inker, Penciller]), {ok, #state{penciller=Penciller, inker=Inker, ledger_cache=LedgerCache, @@ -396,7 +394,7 @@ handle_info(_Info, State) -> {noreply, State}. terminate(Reason, State) -> - io:format("Bookie closing for reason ~w~n", [Reason]), + leveled_log:log("B0003", [Reason]), WaitList = lists:duplicate(?SHUTDOWN_WAITS, ?SHUTDOWN_PAUSE), ok = shutdown_wait(WaitList, State#state.inker), ok = leveled_penciller:pcl_close(State#state.penciller). @@ -414,8 +412,7 @@ bucket_stats(State, Bucket, Tag) -> {LedgerSnapshot, LedgerCache}, _JournalSnapshot} = snapshot_store(State, ledger), Folder = fun() -> - io:format("Length of increment in snapshot is ~w~n", - [gb_trees:size(LedgerCache)]), + leveled_log:log("B0004", [gb_trees:size(LedgerCache)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, LedgerCache), StartKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), @@ -439,8 +436,7 @@ index_query(State, {LedgerSnapshot, LedgerCache}, _JournalSnapshot} = snapshot_store(State, ledger), Folder = fun() -> - io:format("Length of increment in snapshot is ~w~n", - [gb_trees:size(LedgerCache)]), + leveled_log:log("B0004", [gb_trees:size(LedgerCache)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, LedgerCache), StartKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, @@ -476,8 +472,7 @@ hashtree_query(State, Tag, JournalCheck) -> {LedgerSnapshot, LedgerCache}, JournalSnapshot} = snapshot_store(State, SnapType), Folder = fun() -> - io:format("Length of increment in snapshot is ~w~n", - [gb_trees:size(LedgerCache)]), + leveled_log:log("B0004", [gb_trees:size(LedgerCache)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, LedgerCache), StartKey = leveled_codec:to_ledgerkey(null, null, Tag), @@ -505,8 +500,7 @@ foldobjects_allkeys(State, Tag, FoldObjectsFun) -> {LedgerSnapshot, LedgerCache}, JournalSnapshot} = snapshot_store(State, store), Folder = fun() -> - io:format("Length of increment in snapshot is ~w~n", - [gb_trees:size(LedgerCache)]), + leveled_log:log("B0004", [gb_trees:size(LedgerCache)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, LedgerCache), StartKey = leveled_codec:to_ledgerkey(null, null, Tag), @@ -528,8 +522,7 @@ allkey_query(State, Tag) -> {LedgerSnapshot, LedgerCache}, _JournalSnapshot} = snapshot_store(State, ledger), Folder = fun() -> - io:format("Length of increment in snapshot is ~w~n", - [gb_trees:size(LedgerCache)]), + leveled_log:log("B0004", [gb_trees:size(LedgerCache)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, LedgerCache), SK = leveled_codec:to_ledgerkey(null, null, Tag), @@ -601,7 +594,7 @@ startup(InkerOpts, PencillerOpts) -> {ok, Inker} = leveled_inker:ink_start(InkerOpts), {ok, Penciller} = leveled_penciller:pcl_start(PencillerOpts), LedgerSQN = leveled_penciller:pcl_getstartupsequencenumber(Penciller), - io:format("LedgerSQN=~w at startup~n", [LedgerSQN]), + leveled_log:log("B0005", [LedgerSQN]), ok = leveled_inker:ink_loadpcl(Inker, LedgerSQN + 1, fun load_fun/5, @@ -816,13 +809,12 @@ load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> Obj, VSize, IndexSpecs), {loop, {MinSQN, MaxSQN, addto_ledgercache(Changes, OutputTree)}}; MaxSQN -> - io:format("Reached end of load batch with SQN ~w~n", [SQN]), + leveled_log:log("B0006", [SQN]), Changes = preparefor_ledgercache(Type, PK, SQN, Obj, VSize, IndexSpecs), {stop, {MinSQN, MaxSQN, addto_ledgercache(Changes, OutputTree)}}; SQN when SQN > MaxSQN -> - io:format("Skipping as exceeded MaxSQN ~w with SQN ~w~n", - [MaxSQN, SQN]), + leveled_log:log("B0007", [MaxSQN, SQN]), {stop, Acc0} end. diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index fed4cff..3c0e598 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -229,20 +229,21 @@ print_key(Key) -> {?IDX_TAG, B, {F, _V}, _K} -> {"Index", B, F} end, - {B_STR, FB} = check_for_string(B_TERM), - {C_STR, FC} = check_for_string(C_TERM), - {A_STR, B_STR, C_STR, FB, FC}. + B_STR = turn_to_string(B_TERM), + C_STR = turn_to_string(C_TERM), + {A_STR, B_STR, C_STR}. -check_for_string(Item) -> +turn_to_string(Item) -> if is_binary(Item) == true -> - {binary_to_list(Item), "~s"}; + binary_to_list(Item); is_integer(Item) == true -> - {integer_to_list(Item), "~s"}; + integer_to_list(Item); is_list(Item) == true -> - {Item, "~s"}; + Item; true -> - {Item, "~w"} + [Output] = io_lib:format("~w", [Item]), + Output end. @@ -392,8 +393,8 @@ endkey_passed_test() -> ?assertMatch(true, endkey_passed(TestKey, K2)). stringcheck_test() -> - ?assertMatch({"Bucket", "~s"}, check_for_string("Bucket")), - ?assertMatch({"Bucket", "~s"}, check_for_string(<<"Bucket">>)), - ?assertMatch({bucket, "~w"}, check_for_string(bucket)). + ?assertMatch("Bucket", turn_to_string("Bucket")), + ?assertMatch("Bucket", turn_to_string(<<"Bucket">>)), + ?assertMatch("bucket", turn_to_string(bucket)). -endif. \ No newline at end of file diff --git a/src/leveled_log.erl b/src/leveled_log.erl new file mode 100644 index 0000000..05c2bba --- /dev/null +++ b/src/leveled_log.erl @@ -0,0 +1,171 @@ +%% Module to abstract from choice of logger, and allow use of logReferences +%% for fast lookup + +-module(leveled_log). + +-include("include/leveled.hrl"). + +-include_lib("eunit/include/eunit.hrl"). + +-export([log/2, + log_timer/3]). + +-define(LOG_LEVEL, [info, warn, error, critical]). +-define(LOGBASE, dict:from_list([ + + {"G0001", + {info, "Generic log point"}}, + {"D0001", + {debug, "Generic debug log"}}, + + {"B0001", + {info, "Bookie starting with Ink ~w Pcl ~w"}}, + {"B0002", + {info, "Snapshot starting with Ink ~w Pcl ~w"}}, + {"B0003", + {info, "Bookie closing for reason ~w"}}, + {"B0004", + {info, "Length of increment in snapshot is ~w"}}, + {"B0005", + {info, "LedgerSQN=~w at startup"}}, + {"B0006", + {info, "Reached end of load batch with SQN ~w"}}, + {"B0007", + {info, "Skipping as exceeded MaxSQN ~w with SQN ~w"}}, + + {"P0001", + {info, "Ledger snapshot ~w registered"}}, + {"P0002", + {info, "Handling of push completed with L0 cache size now ~w"}}, + {"P0003", + {info, "Ledger snapshot ~w released"}}, + {"P0004", + {info, "Remaining ledger snapshots are ~w"}}, + {"P0005", + {info, "Delete confirmed as file is removed from " ++ " + unreferenced files ~w"}}, + {"P0006", + {info, "Orphaned reply after timeout on L0 file write ~s"}}, + {"P0007", + {info, "Sent release message for cloned Penciller following close for " + ++ "reason ~w"}}, + {"P0008", + {info, "Penciller closing for reason ~w"}}, + {"P0009", + {info, "Level 0 cache empty at close of Penciller"}}, + {"P0010", + {info, "No level zero action on close of Penciller"}}, + {"P0011", + {info, "Shutdown complete for Penciller"}}, + {"P0012", + {info, "Store to be started based on manifest sequence number of ~w"}}, + {"P0013", + {warn, "Seqence number of 0 indicates no valid manifest"}}, + {"P0014", + {info, "Maximum sequence number of ~w found in nonzero levels"}}, + {"P0015", + {info, "L0 file found ~s"}}, + {"P0016", + {info, "L0 file had maximum sequence number of ~w"}}, + {"P0017", + {info, "No L0 file found"}}, + {"P0018", + {info, "Respone to push_mem of ~w ~s"}}, + {"P0019", + {info, "Rolling level zero to filename ~s"}}, + {"P0020", + {info, "Work at Level ~w to be scheduled for ~w with ~w " + ++ "queue items outstanding"}}, + {"P0021", + {info, "Allocation of work blocked as L0 pending"}}, + {"P0022", + {info, "Manifest at Level ~w"}}, + {"P0023", + {info, "Manifest entry of startkey ~s ~s ~s endkey ~s ~s ~s " + ++ "filename=~s~n"}}, + {"P0024", + {info, "Outstanding compaction work items of ~w at level ~w"}}, + {"P0025", + {info, "Merge to sqn ~w from Level ~w completed"}}, + {"P0026", + {info, "Merge has been commmitted at sequence number ~w"}}, + {"P0027", + {info, "Rename of manifest from ~s ~w to ~s ~w"}}, + {"P0028", + {info, "Adding cleared file ~s to deletion list"}}, + + {"PC001", + {info, "Penciller's clerk ~w started with owner ~w"}}, + {"PC002", + {info, "Request for manifest change from clerk on closing"}}, + {"PC003", + {info, "Confirmation of manifest change on closing"}}, + {"PC004", + {info, "Prompted confirmation of manifest change"}}, + {"PC005", + {info, "Penciller's Clerk ~w shutdown now complete for reason ~w"}}, + {"PC006", + {info, "Work prompted but none needed ~w"}}, + {"PC007", + {info, "Clerk prompting Penciller regarding manifest change"}}, + {"PC008", + {info, "Merge from level ~w to merge into ~w files below"}}, + {"PC009", + {info, "File ~s to simply switch levels to level ~w"}}, + {"PC010", + {info, "Merge to be commenced for FileToMerge=~s with MSN=~w"}}, + {"PC011", + {info, "Merge completed with MSN=~w Level=~w and FileCounter=~w"}}, + {"PC012", + {info, "File to be created as part of MSN=~w Filename=~s"}}, + {"PC013", + {warn, "Merge resulted in empty file ~s"}}, + {"PC014", + {info, "Empty file ~s to be cleared"}}, + {"PC015", + {info, "File created"}} + + ])). + + +log(LogReference, Subs) -> + {ok, {LogLevel, LogText}} = dict:find(LogReference, ?LOGBASE), + case lists:member(LogLevel, ?LOG_LEVEL) of + true -> + io:format(LogText ++ "~n", Subs); + false -> + ok + end. + +log_timer(LogReference, Subs, StartTime) -> + {ok, {LogLevel, LogText}} = dict:find(LogReference, ?LOGBASE), + case lists:member(LogLevel, ?LOG_LEVEL) of + true -> + MicroS = timer:now_diff(os:timestamp(), StartTime), + {Unit, Time} = case MicroS of + MicroS when MicroS < 100 -> + {"microsec", MicroS}; + MicroS -> + {"ms", MicroS div 1000} + end, + io:format(LogText ++ " with time taken ~w " ++ Unit ++ "~n", + Subs ++ [Time]); + false -> + ok + end. + + + +%%%============================================================================ +%%% Test +%%%============================================================================ + + + +-ifdef(TEST). + +log_test() -> + ?assertMatch(ok, log("D0001", [])), + ?assertMatch(ok, log_timer("D0001", [], os:timestamp())). + +-endif. \ No newline at end of file diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 34edf96..b79a243 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -80,7 +80,7 @@ clerk_new(Owner) -> {ok, Pid} = gen_server:start(?MODULE, [], []), ok = gen_server:call(Pid, {register, Owner}, infinity), - io:format("Penciller's clerk ~w started with owner ~w~n", [Pid, Owner]), + leveled_log:log("PC001", [Pid, Owner]), {ok, Pid}. clerk_manifestchange(Pid, Action, Closing) -> @@ -104,7 +104,7 @@ handle_call({register, Owner}, _From, State) -> State#state{owner=Owner}, ?MIN_TIMEOUT}; handle_call({manifest_change, return, true}, _From, State) -> - io:format("Request for manifest change from clerk on closing~n"), + leveled_log:log("PC002", []), case State#state.change_pending of true -> WI = State#state.work_item, @@ -115,13 +115,13 @@ handle_call({manifest_change, return, true}, _From, State) -> handle_call({manifest_change, confirm, Closing}, From, State) -> case Closing of true -> - io:format("Confirmation of manifest change on closing~n"), + leveled_log:log("PC003", []), WI = State#state.work_item, ok = mark_for_delete(WI#penciller_work.unreferenced_files, State#state.owner), {stop, normal, ok, State}; false -> - io:format("Prompted confirmation of manifest change~n"), + leveled_log:log("PC004", []), gen_server:reply(From, ok), WI = State#state.work_item, ok = mark_for_delete(WI#penciller_work.unreferenced_files, @@ -149,8 +149,7 @@ handle_info(timeout, State=#state{change_pending=Pnd}) when Pnd == false -> terminate(Reason, _State) -> - io:format("Penciller's Clerk ~w shutdown now complete for reason ~w~n", - [self(), Reason]). + leveled_log:log("PC005", [self(), Reason]). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -163,13 +162,13 @@ code_change(_OldVsn, State, _Extra) -> requestandhandle_work(State) -> case leveled_penciller:pcl_workforclerk(State#state.owner) of none -> - io:format("Work prompted but none needed ~w~n", [self()]), + leveled_log:log("PC006", [self()]), {false, ?MAX_TIMEOUT}; WI -> {NewManifest, FilesToDelete} = merge(WI), UpdWI = WI#penciller_work{new_manifest=NewManifest, unreferenced_files=FilesToDelete}, - io:format("Clerk prompting Penciller regarding manifest change~n"), + leveled_log:log("PC007", []), ok = leveled_penciller:pcl_promptmanifestchange(State#state.owner, UpdWI), {true, UpdWI} @@ -186,8 +185,7 @@ merge(WI) -> %% Need to work out if this is the top level %% And then tell merge process to create files at the top level %% Which will include the reaping of expired tombstones - io:format("Merge from level ~w to merge into ~w files below~n", - [SrcLevel, length(Candidates)]), + leveled_log:log("PC008", [SrcLevel, length(Candidates)]), MergedFiles = case length(Candidates) of 0 -> @@ -195,7 +193,7 @@ merge(WI) -> %% %% TODO: need to think still about simply renaming when at %% lower level - io:format("File ~s to simply switch levels to level ~w~n", + leveled_log:log("PC009", [SrcF#manifest_entry.filename, SrcLevel + 1]), [SrcF]; _ -> @@ -293,8 +291,7 @@ select_filetomerge(SrcLevel, Manifest) -> %% The level is the level which the new files should be created at. perform_merge({SrcPid, SrcFN}, CandidateList, LevelInfo, {Filepath, MSN}) -> - io:format("Merge to be commenced for FileToMerge=~s with MSN=~w~n", - [SrcFN, MSN]), + leveled_log:log("PC010", [SrcFN, MSN]), PointerList = lists:map(fun(P) -> {next, P#manifest_entry.owner, all} end, CandidateList), @@ -306,14 +303,12 @@ perform_merge({SrcPid, SrcFN}, CandidateList, LevelInfo, {Filepath, MSN}) -> []). do_merge([], [], {SrcLevel, _IsB}, {_Filepath, MSN}, FileCounter, OutList) -> - io:format("Merge completed with MSN=~w Level=~w and FileCounter=~w~n", - [MSN, SrcLevel, FileCounter]), + leveled_log:log("PC011", [MSN, SrcLevel, FileCounter]), OutList; do_merge(KL1, KL2, {SrcLevel, IsB}, {Filepath, MSN}, FileCounter, OutList) -> FileName = lists:flatten(io_lib:format(Filepath ++ "_~w_~w.sft", [SrcLevel + 1, FileCounter])), - io:format("File to be created as part of MSN=~w Filename=~s~n", - [MSN, FileName]), + leveled_log:log("PC012", [MSN, FileName]), TS1 = os:timestamp(), LevelR = case IsB of true -> @@ -329,8 +324,8 @@ do_merge(KL1, KL2, {SrcLevel, IsB}, {Filepath, MSN}, FileCounter, OutList) -> LevelR), case Reply of {{[], []}, null, _} -> - io:format("Merge resulted in empty file ~s~n", [FileName]), - io:format("Empty file ~s to be cleared~n", [FileName]), + leveled_log:log("PC013", [FileName]), + leveled_log:log("PC014", [FileName]), ok = leveled_sft:sft_clear(Pid), OutList; {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} -> @@ -339,8 +334,7 @@ do_merge(KL1, KL2, {SrcLevel, IsB}, {Filepath, MSN}, FileCounter, OutList) -> end_key=HighestKey, owner=Pid, filename=FileName}]), - MTime = timer:now_diff(os:timestamp(), TS1), - io:format("File creation took ~w microseconds ~n", [MTime]), + leveled_log:log_timer("PC015", [], TS1), do_merge(KL1Rem, KL2Rem, {SrcLevel, IsB}, {Filepath, MSN}, FileCounter + 1, ExtMan) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 18459f9..7336cd6 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -338,9 +338,8 @@ init([PCLopts]) -> PCLopts#penciller_options.start_snapshot} of {undefined, true} -> SrcPenciller = PCLopts#penciller_options.source_penciller, - io:format("Registering ledger snapshot~n"), {ok, State} = pcl_registersnapshot(SrcPenciller, self()), - io:format("Ledger snapshot registered~n"), + leveled_log:log("P0001", [self()]), {ok, State#state{is_snapshot=true, source_penciller=SrcPenciller}}; %% Need to do something about timeout {_RootPath, false} -> @@ -421,10 +420,7 @@ handle_call({push_mem, PushedTree}, From, State=#state{is_snapshot=Snap}) State#state.levelzero_cache, State) end, - io:format("Handling of push completed in ~w microseconds with " - ++ "L0 cache size now ~w~n", - [timer:now_diff(os:timestamp(), SW), - S#state.levelzero_size]), + leveled_log:log_timer("P0002", [S#state.levelzero_size], SW), {noreply, S}; handle_call({fetch, Key}, _From, State) -> {reply, @@ -489,8 +485,8 @@ handle_cast({manifest_change, WI}, State) -> {noreply, UpdState}; handle_cast({release_snapshot, Snapshot}, State) -> Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots), - io:format("Ledger snapshot ~w released~n", [Snapshot]), - io:format("Remaining ledger snapshots are ~w~n", [Rs]), + leveled_log:log("P0003", [Snapshot]), + leveled_log:log("P0004", [Rs]), {noreply, State#state{registered_snapshots=Rs}}; handle_cast({confirm_delete, FileName}, State=#state{is_snapshot=Snap}) when Snap == false -> @@ -500,9 +496,7 @@ handle_cast({confirm_delete, FileName}, State=#state{is_snapshot=Snap}) case Reply of {true, Pid} -> UF1 = lists:keydelete(FileName, 1, State#state.unreferenced_files), - io:format("Filename ~s removed from unreferenced files as delete " - ++ "is confirmed - file should now close~n", - [FileName]), + leveled_log:log("P0005", [FileName]), ok = leveled_sft:sft_deleteconfirmed(Pid), {noreply, State#state{unreferenced_files=UF1}}; _ -> @@ -511,13 +505,12 @@ handle_cast({confirm_delete, FileName}, State=#state{is_snapshot=Snap}) handle_info({_Ref, {ok, SrcFN, _StartKey, _EndKey}}, State) -> - io:format("Orphaned reply after timeout on L0 file write ~s~n", [SrcFN]), + leveled_log:log("P0006", [SrcFN]), {noreply, State}. terminate(Reason, State=#state{is_snapshot=Snap}) when Snap == true -> ok = pcl_releasesnapshot(State#state.source_penciller, self()), - io:format("Sent release message for cloned Penciller following close for " - ++ "reason ~w~n", [Reason]), + leveled_log:log("P0007", [Reason]), ok; terminate(Reason, State) -> %% When a Penciller shuts down it isn't safe to try an manage the safe @@ -536,7 +529,7 @@ terminate(Reason, State) -> %% The cast may not succeed as the clerk could be synchronously calling %% the penciller looking for a manifest commit %% - io:format("Penciller closing for reason - ~w~n", [Reason]), + leveled_log:log("P0008", [Reason]), MC = leveled_pclerk:clerk_manifestchange(State#state.clerk, return, true), @@ -557,12 +550,12 @@ terminate(Reason, State) -> {true, [], _} -> ok = leveled_sft:sft_close(State#state.levelzero_constructor); {false, [], 0} -> - io:format("Level 0 cache empty at close of Penciller~n"); + leveled_log:log("P0009", []); {false, [], _N} -> L0Pid = roll_memory(UpdState, true), ok = leveled_sft:sft_close(L0Pid); _ -> - io:format("No level zero action on close of Penciller~n") + leveled_log:log("P0010", []) end, % Tidy shutdown of individual files @@ -570,7 +563,7 @@ terminate(Reason, State) -> lists:foreach(fun({_FN, Pid, _SN}) -> ok = leveled_sft:sft_close(Pid) end, UpdState#state.unreferenced_files), - io:format("Shutdown complete for Penciller~n"), + leveled_log:log("P0011", []), ok. @@ -622,12 +615,10 @@ start_from_file(PCLopts) -> TopManSQN = lists:foldl(fun(X, MaxSQN) -> max(X, MaxSQN) end, 0, ValidManSQNs), - io:format("Store to be started based on " ++ - "manifest sequence number of ~w~n", [TopManSQN]), + leveled_log:log("P0012", [TopManSQN]), ManUpdate = case TopManSQN of 0 -> - io:format("Seqence number of 0 indicates no valid " ++ - "manifest~n"), + leveled_log:log("P0013", []), {[], 0}; _ -> CurrManFile = filepath(InitState#state.root_path, @@ -639,14 +630,13 @@ start_from_file(PCLopts) -> end, {UpdManifest, MaxSQN} = ManUpdate, - io:format("Maximum sequence number of ~w found in nonzero levels~n", - [MaxSQN]), + leveled_log:log("P0014", [MaxSQN]), %% Find any L0 files L0FN = filepath(RootPath, TopManSQN, new_merge_files) ++ "_0_0.sft", case filelib:is_file(L0FN) of true -> - io:format("L0 file found ~s~n", [L0FN]), + leveled_log:log("P0015", [L0FN]), {ok, L0Pid, {L0StartKey, L0EndKey}} = leveled_sft:sft_open(L0FN), @@ -659,8 +649,7 @@ start_from_file(PCLopts) -> 1, UpdManifest, {0, [ManifestEntry]}), - io:format("L0 file had maximum sequence number of ~w~n", - [L0SQN]), + leveled_log:log("P0016", [L0SQN]), LedgerSQN = max(MaxSQN, L0SQN), {ok, InitState#state{manifest=UpdManifest2, @@ -668,7 +657,7 @@ start_from_file(PCLopts) -> ledger_sqn=LedgerSQN, persisted_sqn=LedgerSQN}}; false -> - io:format("No L0 file found~n"), + leveled_log:log("P0017", []), {ok, InitState#state{manifest=UpdManifest, manifest_sqn=TopManSQN, @@ -678,10 +667,7 @@ start_from_file(PCLopts) -> log_pushmem_reply(From, Reply, SW) -> - io:format("Respone to push_mem of ~w ~s took ~w microseconds~n", - [element(1, Reply), - element(2, Reply), - timer:now_diff(os:timestamp(), SW)]), + leveled_log:log_timer("P0018", [element(1,Reply), element(2,Reply)], SW), gen_server:reply(From, element(1, Reply)). @@ -737,7 +723,7 @@ checkready(Pid) -> roll_memory(State, false) -> FileName = levelzero_filename(State), - io:format("Rolling level zero to file ~s~n", [FileName]), + leveled_log:log("P0019", [FileName]), Opts = #sft_options{wait=false}, PCL = self(), FetchFun = fun(Slot) -> pcl_fetchlevelzero(PCL, Slot) end, @@ -832,9 +818,7 @@ return_work(State, From) -> L when L > 0 -> [{SrcLevel, Manifest}|OtherWork] = WorkQ, Backlog = length(OtherWork), - io:format("Work at Level ~w to be scheduled for ~w with ~w " ++ - "queue items outstanding~n", - [SrcLevel, From, Backlog]), + leveled_log:log("P0020", [SrcLevel, From, Backlog]), IsBasement = if SrcLevel + 1 == BasementL -> true; @@ -845,7 +829,7 @@ return_work(State, From) -> true -> % Once the L0 file is completed there will be more work % - so don't be busy doing other work now - io:format("Allocation of work blocked as L0 pending~n"), + leveled_log:log("P0021", []), {State, none}; false -> %% No work currently outstanding @@ -910,7 +894,7 @@ open_all_filesinmanifest({Manifest, TopSQN}, Level) -> print_manifest(Manifest) -> lists:foreach(fun(L) -> - io:format("Manifest at Level ~w~n", [L]), + leveled_log:log("P0022", [L]), Level = get_item(L, Manifest, []), lists:foreach(fun print_manifest_entry/1, Level) end, @@ -918,16 +902,10 @@ print_manifest(Manifest) -> ok. print_manifest_entry(Entry) -> - {S1, S2, S3, - FS2, FS3} = leveled_codec:print_key(Entry#manifest_entry.start_key), - {E1, E2, E3, - FE2, FE3} = leveled_codec:print_key(Entry#manifest_entry.end_key), - io:format("Manifest entry of " ++ - "startkey ~s " ++ FS2 ++ " " ++ FS3 ++ - " endkey ~s " ++ FE2 ++ " " ++ FE3 ++ - " filename=~s~n", - [S1, S2, S3, E1, E2, E3, - Entry#manifest_entry.filename]). + {S1, S2, S3} = leveled_codec:print_key(Entry#manifest_entry.start_key), + {E1, E2, E3} = leveled_codec:print_key(Entry#manifest_entry.end_key), + leveled_log:log("P0023", + [S1, S2, S3, E1, E2, E3, Entry#manifest_entry.filename]). initiate_rangequery_frommanifest(StartKey, EndKey, Manifest) -> CompareFun = fun(M) -> @@ -1051,9 +1029,6 @@ find_nextkey(QueryArray, LCnt, {BestKeyLevel, BestKV}, QueryFunT) -> % the best key % But we also need to remove the dominated key from the % lower level in the query array - io:format("Key at level ~w with SQN ~w is better than " ++ - "key at lower level ~w with SQN ~w~n", - [LCnt, SQN, BKL, BestSQN]), OldBestEntry = lists:keyfind(BKL, 1, QueryArray), {BKL, [{BestKey, BestVal}|BestTail]} = OldBestEntry, find_nextkey(lists:keyreplace(BKL, @@ -1149,8 +1124,7 @@ assess_workqueue(WorkQ, LevelToAssess, Man, BasementLevel) -> maybe_append_work(WorkQ, Level, Manifest, MaxFiles, FileCount) when FileCount > MaxFiles -> - io:format("Outstanding compaction work items of ~w at level ~w~n", - [FileCount - MaxFiles, Level]), + leveled_log:log("P0024", [FileCount - MaxFiles, Level]), lists:append(WorkQ, [{Level, Manifest}]); maybe_append_work(WorkQ, _Level, _Manifest, _MaxFiles, _FileCount) -> @@ -1183,24 +1157,19 @@ commit_manifest_change(ReturnedWorkItem, State) -> RootPath = State#state.root_path, UnreferencedFiles = State#state.unreferenced_files, - case {SentWorkItem#penciller_work.next_sqn, - SentWorkItem#penciller_work.clerk} of - {NewMSN, _From} -> - MTime = timer:now_diff(os:timestamp(), - SentWorkItem#penciller_work.start_time), + if + NewMSN == SentWorkItem#penciller_work.next_sqn -> WISrcLevel = SentWorkItem#penciller_work.src_level, - io:format("Merge to sqn ~w completed in ~w microseconds " ++ - "from Level ~w~n", - [SentWorkItem#penciller_work.next_sqn, - MTime, - WISrcLevel]), + leveled_log:log_timer("P0025", + [SentWorkItem#penciller_work.next_sqn, + WISrcLevel], + SentWorkItem#penciller_work.start_time), ok = rename_manifest_files(RootPath, NewMSN), FilesToDelete = ReturnedWorkItem#penciller_work.unreferenced_files, UnreferencedFilesUpd = update_deletions(FilesToDelete, NewMSN, UnreferencedFiles), - io:format("Merge has been commmitted at sequence number ~w~n", - [NewMSN]), + leveled_log:log("P0026", [NewMSN]), NewManifest = ReturnedWorkItem#penciller_work.new_manifest, CurrL0 = get_item(0, State#state.manifest, []), @@ -1221,23 +1190,15 @@ commit_manifest_change(ReturnedWorkItem, State) -> {ok, State#state{ongoing_work=[], manifest_sqn=NewMSN, manifest=RevisedManifest, - unreferenced_files=UnreferencedFilesUpd}}; - {MaybeWrongMSN, From} -> - io:format("Merge commit at sqn ~w not matched to expected" ++ - " sqn ~w from Clerk ~w~n", - [NewMSN, MaybeWrongMSN, From]), - {error, State} + unreferenced_files=UnreferencedFilesUpd}} end. rename_manifest_files(RootPath, NewMSN) -> OldFN = filepath(RootPath, NewMSN, pending_manifest), NewFN = filepath(RootPath, NewMSN, current_manifest), - io:format("Rename of manifest from ~s ~w to ~s ~w~n", - [OldFN, - filelib:is_file(OldFN), - NewFN, - filelib:is_file(NewFN)]), + leveled_log:log("P0027", [OldFN, filelib:is_file(OldFN), + NewFN, filelib:is_file(NewFN)]), ok = file:rename(OldFN,NewFN). filepath(RootPath, manifest) -> @@ -1257,8 +1218,7 @@ filepath(RootPath, NewMSN, new_merge_files) -> update_deletions([], _NewMSN, UnreferencedFiles) -> UnreferencedFiles; update_deletions([ClearedFile|Tail], MSN, UnreferencedFiles) -> - io:format("Adding cleared file ~s to deletion list ~n", - [ClearedFile#manifest_entry.filename]), + leveled_log:log("P0028", [ClearedFile#manifest_entry.filename]), update_deletions(Tail, MSN, lists:append(UnreferencedFiles, From 94436d8dfd0faa63258c00c932fc4ce14d8c2b63 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Wed, 2 Nov 2016 18:56:36 +0000 Subject: [PATCH 126/167] Set timing rounding correctly --- src/leveled_log.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 05c2bba..da642f4 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -143,7 +143,7 @@ log_timer(LogReference, Subs, StartTime) -> true -> MicroS = timer:now_diff(os:timestamp(), StartTime), {Unit, Time} = case MicroS of - MicroS when MicroS < 100 -> + MicroS when MicroS < 10000 -> {"microsec", MicroS}; MicroS -> {"ms", MicroS div 1000} From e8a78883977447caf4aca681706604b2281c84a4 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 09:19:02 +0000 Subject: [PATCH 127/167] Experiment with new cache size algorithm Remove the jitter probability and make it a smooth function heading towards the max ache size --- src/leveled_bookie.erl | 9 ++++++--- test/end_to_end/iterator_SUITE.erl | 8 ++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 2204128..b388cae 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -162,7 +162,6 @@ -define(SHUTDOWN_WAITS, 60). -define(SHUTDOWN_PAUSE, 10000). -define(SNAPSHOT_TIMEOUT, 300000). --define(JITTER_PROB, 0.01). -define(CHECKJOURNAL_PROB, 0.2). -record(state, {inker :: pid(), @@ -780,13 +779,15 @@ maybepush_ledgercache(MaxCacheSize, Cache, Penciller) -> maybe_withjitter(CacheSize, MaxCacheSize) -> + if CacheSize > 2 * MaxCacheSize -> true; CacheSize > MaxCacheSize -> - R = random:uniform(), + T = 2 * MaxCacheSize - CacheSize, + R = random:uniform(CacheSize), if - R < ?JITTER_PROB -> + R > T -> true; true -> false @@ -795,6 +796,8 @@ maybe_withjitter(CacheSize, MaxCacheSize) -> false end. + + load_fun(KeyInLedger, ValueInLedger, _Position, Acc0, ExtractFun) -> {MinSQN, MaxSQN, OutputTree} = Acc0, {SQN, Type, PK} = KeyInLedger, diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 1353aa8..7e1fcff 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -7,12 +7,12 @@ -export([all/0]). -export([simple_load_with2i/1, - simple_querycount/1, + query_count/1, rotating_objects/1]). all() -> [ simple_load_with2i, - simple_querycount, + query_count, rotating_objects]. @@ -41,9 +41,9 @@ simple_load_with2i(_Config) -> testutil:reset_filestructure(). -simple_querycount(_Config) -> +query_count(_Config) -> RootPath = testutil:reset_filestructure(), - {ok, Book1} = leveled_bookie:book_start(RootPath, 2500, 50000000), + {ok, Book1} = leveled_bookie:book_start(RootPath, 2000, 50000000), {TestObject, TestSpec} = testutil:generate_testobject("Bucket", "Key1", "Value1", From 37e78dcdc9deaba82703975a279d5eeca5538940 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 12:11:50 +0000 Subject: [PATCH 128/167] Expanded AAE tests to include busted hashtable Busted the hashtable in a Journal file, and demonstrated it can be fixed by changing the extension name (no need to recover from backup if only the hashtable is bust) --- src/leveled_inker.erl | 5 +- test/end_to_end/recovery_SUITE.erl | 74 ++++++++++++++++++++++++++++-- test/end_to_end/testutil.erl | 29 ++++++++++-- 3 files changed, 97 insertions(+), 11 deletions(-) diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 3eb8272..6cbb67d 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -546,8 +546,9 @@ open_all_manifest(Man0, RootPath, CDBOpts) -> Pid} = leveled_cdb:cdb_open_reader(CFN), {LowSQN, FN, Pid}; false -> - {ok, - Pid} = leveled_cdb:cdb_open_reader(PFN), + W = leveled_cdb:cdb_open_writer(PFN, CDBOpts), + {ok, Pid} = W, + ok = leveled_cdb:cdb_roll(Pid), {LowSQN, FN, Pid} end; _ -> diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index ed32340..29f424a 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -8,9 +8,9 @@ ]). all() -> [ - retain_strategy, - aae_bustedjournal, - journal_compaction_bustedjournal + % retain_strategy, + aae_bustedjournal %, + % journal_compaction_bustedjournal ]. retain_strategy(_Config) -> @@ -56,7 +56,7 @@ aae_bustedjournal(_Config) -> CDBFiles = testutil:find_journals(RootPath), [HeadF|_Rest] = CDBFiles, io:format("Selected Journal for corruption of ~s~n", [HeadF]), - testutil:corrupt_journal(RootPath, HeadF, 1000), + testutil:corrupt_journal(RootPath, HeadF, 1000, 2048, 1000), {ok, Bookie2} = leveled_bookie:book_start(StartOpts), {async, KeyF} = leveled_bookie:book_returnfolder(Bookie2, @@ -122,8 +122,74 @@ aae_bustedjournal(_Config) -> length(KeyHashList3)]), ok = leveled_bookie:book_close(Bookie2), + {ok, BytesCopied} = testutil:restore_file(RootPath, HeadF), + io:format("File restored is of size ~w~n", [BytesCopied]), + {ok, Bookie3} = leveled_bookie:book_start(StartOpts), + + SW4 = os:timestamp(), + {async, HashTreeF4} = leveled_bookie:book_returnfolder(Bookie3, + {foldobjects_allkeys, + ?RIAK_TAG, + FoldObjectsFun}), + KeyHashList4 = HashTreeF4(), + + true = length(KeyHashList4) == 20001, + io:format("Fetch of hashtree using fold objects took ~w microseconds" ++ + " and found an object count of ~w~n", + [timer:now_diff(os:timestamp(), SW4), length(KeyHashList4)]), + + ok = leveled_bookie:book_close(Bookie3), + testutil:corrupt_journal(RootPath, HeadF, 500, BytesCopied - 8000, 14), + + {ok, Bookie4} = leveled_bookie:book_start(StartOpts), + + SW5 = os:timestamp(), + {async, HashTreeF5} = leveled_bookie:book_returnfolder(Bookie4, + {foldobjects_allkeys, + ?RIAK_TAG, + FoldObjectsFun}), + KeyHashList5 = HashTreeF5(), + + true = length(KeyHashList5) > 19000, + true = length(KeyHashList5) < HeadCount, + Delta5 = length(lists:subtract(KeyHashList1, KeyHashList5)), + true = Delta5 < 1001, + io:format("Fetch of hashtree using fold objects took ~w microseconds" ++ + " and found a Delta of ~w and an objects count of ~w~n", + [timer:now_diff(os:timestamp(), SW5), + Delta5, + length(KeyHashList5)]), + + {async, HashTreeF6} = leveled_bookie:book_returnfolder(Bookie4, + {hashtree_query, + ?RIAK_TAG, + check_presence}), + KeyHashList6 = HashTreeF6(), + true = length(KeyHashList6) > 19000, + true = length(KeyHashList6) < HeadCount, + + ok = leveled_bookie:book_close(Bookie4), + + testutil:restore_topending(RootPath, HeadF), + + {ok, Bookie5} = leveled_bookie:book_start(StartOpts), + + SW6 = os:timestamp(), + {async, HashTreeF7} = leveled_bookie:book_returnfolder(Bookie5, + {foldobjects_allkeys, + ?RIAK_TAG, + FoldObjectsFun}), + KeyHashList7 = HashTreeF7(), + + true = length(KeyHashList7) == 20001, + io:format("Fetch of hashtree using fold objects took ~w microseconds" ++ + " and found an object count of ~w~n", + [timer:now_diff(os:timestamp(), SW6), length(KeyHashList7)]), + + ok = leveled_bookie:book_close(Bookie5), testutil:reset_filestructure(). + riak_hash(Obj=#r_object{}) -> Vclock = vclock(Obj), UpdObj = set_vclock(Obj, lists:sort(Vclock)), diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index 1c2536f..e11d1a8 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -29,7 +29,9 @@ put_altered_indexed_objects/4, check_indexed_objects/4, rotating_object_check/3, - corrupt_journal/3, + corrupt_journal/5, + restore_file/2, + restore_topending/2, find_journals/1]). -define(RETURN_TERMS, {true, undefined}). @@ -382,16 +384,33 @@ rotating_object_check(RootPath, B, NumberOfObjects) -> ok = leveled_bookie:book_close(Book2), ok. -corrupt_journal(RootPath, FileName, Corruptions) -> - {ok, Handle} = file:open(RootPath ++ "/journal/journal_files/" ++ FileName, - [binary, raw, read, write]), +corrupt_journal(RootPath, FileName, Corruptions, BasePosition, GapSize) -> + OriginalPath = RootPath ++ "/journal/journal_files/" ++ FileName, + BackupPath = RootPath ++ "/journal/journal_files/" ++ + filename:basename(FileName, ".cdb") ++ ".bak", + {ok, _BytesCopied} = file:copy(OriginalPath, BackupPath), + {ok, Handle} = file:open(OriginalPath, [binary, raw, read, write]), lists:foreach(fun(X) -> - Position = X * 1000 + 2048, + Position = X * GapSize + BasePosition, ok = file:pwrite(Handle, Position, <<0:8/integer>>) end, lists:seq(1, Corruptions)), ok = file:close(Handle). + +restore_file(RootPath, FileName) -> + OriginalPath = RootPath ++ "/journal/journal_files/" ++ FileName, + BackupPath = RootPath ++ "/journal/journal_files/" ++ + filename:basename(FileName, ".cdb") ++ ".bak", + file:copy(BackupPath, OriginalPath). + +restore_topending(RootPath, FileName) -> + OriginalPath = RootPath ++ "/journal/journal_files/" ++ FileName, + PndPath = RootPath ++ "/journal/journal_files/" ++ + filename:basename(FileName, ".cdb") ++ ".pnd", + ok = file:rename(OriginalPath, PndPath), + false = filelib:is_file(OriginalPath). + find_journals(RootPath) -> {ok, FNsA_J} = file:list_dir(RootPath ++ "/journal/journal_files"), {ok, Regex} = re:compile(".*\.cdb"), From a75207414897d8a931e8a53038c6a558d55cf224 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 12:12:56 +0000 Subject: [PATCH 129/167] Undo commenting --- test/end_to_end/recovery_SUITE.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index 29f424a..9bd4cd6 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -8,9 +8,9 @@ ]). all() -> [ - % retain_strategy, - aae_bustedjournal %, - % journal_compaction_bustedjournal + retain_strategy, + aae_bustedjournal, + journal_compaction_bustedjournal ]. retain_strategy(_Config) -> From 2f28ae86e4dcf936774dc872fb86df33de717488 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 12:31:00 +0000 Subject: [PATCH 130/167] Journal compaction test to cover deleted objects --- test/end_to_end/basic_SUITE.erl | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 21bf066..d94db71 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -115,7 +115,7 @@ journal_compaction(_Config) -> {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), testutil:check_forobject(Bookie1, TestObject), - ObjList1 = testutil:generate_objects(5000, 2), + ObjList1 = testutil:generate_objects(20000, 2), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjList1), @@ -137,8 +137,18 @@ journal_compaction(_Config) -> testutil:check_forlist(Bookie1, ChkList1), testutil:check_forobject(Bookie1, TestObject), testutil:check_forobject(Bookie1, TestObject2), - %% Now replace all the objects - ObjList2 = testutil:generate_objects(50000, 2), + %% Delete some of the objects + ObjListD = testutil:generate_objects(10000, 2), + lists:foreach(fun({_R, O, _S}) -> + ok = leveled_bookie:book_riakdelete(Bookie1, + O#r_object.bucket, + O#r_object.key, + []) + end, + ObjListD), + + %% Now replace all the other objects + ObjList2 = testutil:generate_objects(40000, 10002), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjList2), From ee39b483133d0f62ba105354424c2cb44889ec1d Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 13:34:06 +0000 Subject: [PATCH 131/167] Messed up test fixed --- test/end_to_end/recovery_SUITE.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index 9bd4cd6..a6269d1 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -221,7 +221,9 @@ journal_compaction_bustedjournal(_Config) -> ok = leveled_bookie:book_close(Bookie1), CDBFiles = testutil:find_journals(RootPath), - lists:foreach(fun(FN) -> testutil:corrupt_journal(RootPath, FN, 100) end, + lists:foreach(fun(FN) -> + testutil:corrupt_journal(RootPath, FN, 100, 2048, 1000) + end, CDBFiles), {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), From c6fc8d1768653deb348a02db2772fd4c04ca27e5 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 13:37:49 +0000 Subject: [PATCH 132/167] Fix log P0005 --- src/leveled_log.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index da642f4..d4d91ad 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -42,8 +42,8 @@ {"P0004", {info, "Remaining ledger snapshots are ~w"}}, {"P0005", - {info, "Delete confirmed as file is removed from " ++ " - unreferenced files ~w"}}, + {info, "Delete confirmed as file ~s is removed from " ++ " + unreferenced files"}}, {"P0006", {info, "Orphaned reply after timeout on L0 file write ~s"}}, {"P0007", From 4e46c9735da34f0a7a268a9630567f541506a06c Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 16:05:43 +0000 Subject: [PATCH 133/167] Log improvements Continuation of log review and conversion to using central log function. Fixup of convoluted shutdown process between Bookie, Inker and Inker's Clerk --- src/leveled_bookie.erl | 21 ++---------- src/leveled_iclerk.erl | 60 +++++++++++++++++++--------------- src/leveled_inker.erl | 57 ++++++++++++-------------------- src/leveled_log.erl | 69 +++++++++++++++++++++++++++++++++++++-- src/leveled_penciller.erl | 6 +++- src/leveled_pmem.erl | 6 ++-- 6 files changed, 128 insertions(+), 91 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index b388cae..3c64b69 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -159,8 +159,6 @@ -define(CACHE_SIZE, 2000). -define(JOURNAL_FP, "journal"). -define(LEDGER_FP, "ledger"). --define(SHUTDOWN_WAITS, 60). --define(SHUTDOWN_PAUSE, 10000). -define(SNAPSHOT_TIMEOUT, 300000). -define(CHECKJOURNAL_PROB, 0.2). @@ -394,8 +392,7 @@ handle_info(_Info, State) -> terminate(Reason, State) -> leveled_log:log("B0003", [Reason]), - WaitList = lists:duplicate(?SHUTDOWN_WAITS, ?SHUTDOWN_PAUSE), - ok = shutdown_wait(WaitList, State#state.inker), + ok = leveled_inker:ink_close(State#state.inker), ok = leveled_penciller:pcl_close(State#state.penciller). code_change(_OldVsn, State, _Extra) -> @@ -552,21 +549,7 @@ snapshot_store(State, SnapType) -> ledger -> {ok, {LedgerSnapshot, State#state.ledger_cache}, null} - end. - -shutdown_wait([], _Inker) -> - false; -shutdown_wait([TopPause|Rest], Inker) -> - case leveled_inker:ink_close(Inker) of - ok -> - ok; - pause -> - io:format("Inker shutdown stil waiting for process to complete" ++ - " with further wait of ~w~n", [lists:sum(Rest)]), - ok = timer:sleep(TopPause), - shutdown_wait(Rest, Inker) - end. - + end. set_options(Opts) -> MaxJournalSize = get_opt(max_journalsize, Opts, 10000000000), diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index 8fc284c..ecd05ab 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -175,27 +175,20 @@ handle_cast({compact, Checker, InitiateFun, FilterFun, Inker, _Timeout}, C#candidate.journal} end, BestRun1), - io:format("Clerk updating Inker as compaction complete of " ++ - "~w files~n", [length(FilesToDelete)]), - {ok, ManSQN} = leveled_inker:ink_updatemanifest(Inker, - ManifestSlice, - FilesToDelete), - ok = leveled_inker:ink_compactioncomplete(Inker), - io:format("Clerk has completed compaction process~n"), - case PromptDelete of + leveled_log:log("IC002", [length(FilesToDelete)]), + case is_process_alive(Inker) of true -> - lists:foreach(fun({_SQN, _FN, J2D}) -> - leveled_cdb:cdb_deletepending(J2D, - ManSQN, - Inker) - end, - FilesToDelete), + update_inker(Inker, + ManifestSlice, + FilesToDelete, + PromptDelete), {noreply, State}; false -> - {noreply, State} + leveled_log:log("IC001", []), + {stop, normal, State} end; Score -> - io:format("No compaction run as highest score=~w~n", [Score]), + leveled_log:log("IC003", [Score]), ok = leveled_inker:ink_compactioncomplete(Inker), {noreply, State} end; @@ -245,7 +238,7 @@ check_single_file(CDB, FilterFun, FilterServer, MaxSQN, SampleSize, BatchSize) - _ -> 100 * ActiveSize / (ActiveSize + ReplacedSize) end, - io:format("Score for filename ~s is ~w~n", [FN, Score]), + leveled_log:log("IC004", [FN, Score]), Score. scan_all_files(Manifest, FilterFun, FilterServer, MaxSQN) -> @@ -352,12 +345,10 @@ score_run(Run, MaxRunLength) -> print_compaction_run(BestRun, MaxRunLength) -> - io:format("Compaction to be performed on ~w files with score of ~w~n", - [length(BestRun), score_run(BestRun, MaxRunLength)]), + leveled_log:log("IC005", [length(BestRun), + score_run(BestRun, MaxRunLength)]), lists:foreach(fun(File) -> - io:format("Filename ~s is part of compaction run~n", - [File#candidate.filename]) - + leveled_log:log("IC006", [File#candidate.filename]) end, BestRun). @@ -366,6 +357,24 @@ sort_run(RunOfFiles) -> Cand1#candidate.low_sqn =< Cand2#candidate.low_sqn end, lists:sort(CompareFun, RunOfFiles). +update_inker(Inker, ManifestSlice, FilesToDelete, PromptDelete) -> + {ok, ManSQN} = leveled_inker:ink_updatemanifest(Inker, + ManifestSlice, + FilesToDelete), + ok = leveled_inker:ink_compactioncomplete(Inker), + leveled_log:log("IC007", []), + case PromptDelete of + true -> + lists:foreach(fun({_SQN, _FN, J2D}) -> + leveled_cdb:cdb_deletepending(J2D, + ManSQN, + Inker) + end, + FilesToDelete), + ok; + false -> + ok + end. compact_files(BestRun, CDBopts, FilterFun, FilterServer, MaxSQN, RStrategy) -> BatchesOfPositions = get_all_positions(BestRun, []), @@ -418,8 +427,7 @@ get_all_positions([], PositionBatches) -> get_all_positions([HeadRef|RestOfBest], PositionBatches) -> SrcJournal = HeadRef#candidate.journal, Positions = leveled_cdb:cdb_getpositions(SrcJournal, all), - io:format("Compaction source ~s has yielded ~w positions~n", - [HeadRef#candidate.filename, length(Positions)]), + leveled_log:log("IC008", [HeadRef#candidate.filename, length(Positions)]), Batches = split_positions_into_batches(lists:sort(Positions), SrcJournal, []), @@ -480,9 +488,7 @@ write_values(KVCList, CDBopts, Journal0, ManSlice0) -> FN = leveled_inker:filepath(FP, SQN, compact_journal), - io:format("Generate journal for compaction" - ++ " with filename ~s~n", - [FN]), + leveled_log:log("IC009", [FN]), leveled_cdb:cdb_open_writer(FN, CDBopts); _ -> diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 6cbb67d..8e2fa11 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -248,8 +248,7 @@ handle_call({fetch, Key, SQN}, _From, State) -> {{SQN, Key}, {Value, _IndexSpecs}} -> {reply, {ok, Value}, State}; Other -> - io:format("Unexpected failure to fetch value for" ++ - "Key=~w SQN=~w with reason ~w~n", [Key, SQN, Other]), + leveled_log:log("I0001", [Key, SQN, Other]), {reply, not_present, State} end; handle_call({get, Key, SQN}, _From, State) -> @@ -263,15 +262,14 @@ handle_call({load_pcl, StartSQN, FilterFun, Penciller}, _From, State) -> handle_call({register_snapshot, Requestor}, _From , State) -> Rs = [{Requestor, State#state.manifest_sqn}|State#state.registered_snapshots], - io:format("Journal snapshot ~w registered at SQN ~w~n", - [Requestor, State#state.manifest_sqn]), + leveled_log:log("I0002", [Requestor, State#state.manifest_sqn]), {reply, {State#state.manifest, State#state.active_journaldb}, State#state{registered_snapshots=Rs}}; handle_call({release_snapshot, Snapshot}, _From , State) -> Rs = lists:keydelete(Snapshot, 1, State#state.registered_snapshots), - io:format("Journal snapshot ~w released~n", [Snapshot]), - io:format("Remaining journal snapshots are ~w~n", [Rs]), + leveled_log:log("I0003", [Snapshot]), + leveled_log:log("I0004", [length(Rs)]), {reply, ok, State#state{registered_snapshots=Rs}}; handle_call({confirm_delete, ManSQN}, _From, State) -> Reply = lists:foldl(fun({_R, SnapSQN}, Bool) -> @@ -300,7 +298,7 @@ handle_call({update_manifest, ManifestSnippet), NewManifestSQN = State#state.manifest_sqn + 1, manifest_printer(Man1), - ok = simple_manifest_writer(Man1, NewManifestSQN, State#state.root_path), + simple_manifest_writer(Man1, NewManifestSQN, State#state.root_path), {reply, {ok, NewManifestSQN}, State#state{manifest=Man1, @@ -327,12 +325,7 @@ handle_call(compaction_complete, _From, State) -> handle_call(compaction_pending, _From, State) -> {reply, State#state.compaction_pending, State}; handle_call(close, _From, State) -> - case State#state.compaction_pending of - true -> - {reply, pause, State}; - false -> - {stop, normal, ok, State} - end. + {stop, normal, ok, State}. handle_cast(_Msg, State) -> {noreply, State}. @@ -345,13 +338,13 @@ terminate(Reason, State) -> true -> ok = ink_releasesnapshot(State#state.source_inker, self()); false -> - io:format("Inker closing journal for reason ~w~n", [Reason]), - io:format("Close triggered with journal_sqn=~w and manifest_sqn=~w~n", - [State#state.journal_sqn, State#state.manifest_sqn]), - io:format("Manifest when closing is: ~n"), + leveled_log:log("I0005", [Reason]), + leveled_log:log("I0006", [State#state.journal_sqn, + State#state.manifest_sqn]), leveled_iclerk:clerk_stop(State#state.clerk), lists:foreach(fun({Snap, _SQN}) -> ok = ink_close(Snap) end, State#state.registered_snapshots), + leveled_log:log("I0007", []), manifest_printer(State#state.manifest), ok = close_allmanifest(State#state.manifest) end. @@ -425,9 +418,7 @@ put_object(LedgerKey, Object, KeyChanges, State) -> ok = leveled_cdb:cdb_put(NewJournalP, JournalKey, JournalBin), - io:format("Put to new active journal " ++ - "with manifest write took ~w microseconds~n", - [timer:now_diff(os:timestamp(),SW)]), + leveled_log:log_timer("I0008", [], SW), {rolling, State#state{journal_sqn=NewSQN, manifest=NewManifest, @@ -489,14 +480,14 @@ build_manifest(ManifestFilenames, % the manifest (must also increment the manifest SQN). UpdManifestSQN = if length(OpenManifest) > length(Manifest) -> - io:format("Updated manifest on startup: ~n"), + leveled_log:log("I0009", []), manifest_printer(OpenManifest), simple_manifest_writer(OpenManifest, ManifestSQN + 1, RootPath), ManifestSQN + 1; true -> - io:format("Unchanged manifest on startup: ~n"), + leveled_log:log("I0010", []), manifest_printer(OpenManifest), ManifestSQN end, @@ -512,7 +503,7 @@ close_allmanifest([H|ManifestT]) -> open_all_manifest([], RootPath, CDBOpts) -> - io:format("Manifest is empty, starting from manifest SQN 1~n"), + leveled_log:log("I0011", []), add_to_manifest([], start_new_activejournal(1, RootPath, CDBOpts)); open_all_manifest(Man0, RootPath, CDBOpts) -> Man1 = lists:reverse(lists:sort(Man0)), @@ -521,8 +512,7 @@ open_all_manifest(Man0, RootPath, CDBOpts) -> PendingHeadFN = HeadFN ++ "." ++ ?PENDING_FILEX, Man2 = case filelib:is_file(CompleteHeadFN) of true -> - io:format("Head manifest entry ~s is complete~n", - [HeadFN]), + leveled_log:log("I0012", [HeadFN]), {ok, HeadR} = leveled_cdb:cdb_open_reader(CompleteHeadFN), {LastSQN, _Type, _PK} = leveled_cdb:cdb_lastkey(HeadR), add_to_manifest(add_to_manifest(ManifestTail, @@ -569,7 +559,7 @@ add_to_manifest(Manifest, Entry) -> remove_from_manifest(Manifest, Entry) -> {SQN, FN, _PidR} = Entry, - io:format("File ~s to be removed from manifest~n", [FN]), + leveled_log:log("I0013", [FN]), lists:keydelete(SQN, 1, Manifest). find_in_manifest(SQN, [{LowSQN, _FN, Pid}|_Tail]) when SQN >= LowSQN -> @@ -623,7 +613,7 @@ load_from_sequence(MinSQN, FilterFun, Penciller, [{_LowSQN, FN, Pid}|Rest]) -> load_between_sequence(MinSQN, MaxSQN, FilterFun, Penciller, CDBpid, StartPos, FN, Rest) -> - io:format("Loading from filename ~s from SQN ~w~n", [FN, MinSQN]), + leveled_log:log("I0014", [FN, MinSQN]), InitAcc = {MinSQN, MaxSQN, gb_trees:empty()}, Res = case leveled_cdb:cdb_scan(CDBpid, FilterFun, InitAcc, StartPos) of {eof, {AccMinSQN, _AccMaxSQN, AccKL}} -> @@ -698,8 +688,7 @@ filepath(CompactFilePath, NewSQN, compact_journal) -> simple_manifest_reader(SQN, RootPath) -> ManifestPath = filepath(RootPath, manifest_dir), - io:format("Opening manifest file at ~s with SQN ~w~n", - [ManifestPath, SQN]), + leveled_log:log("I0015", [ManifestPath, SQN]), {ok, MBin} = file:read_file(filename:join(ManifestPath, integer_to_list(SQN) ++ ".man")), @@ -715,13 +704,8 @@ simple_manifest_writer(Manifest, ManSQN, RootPath) -> MBin = term_to_binary(lists:map(fun({SQN, FN, _PID}) -> {SQN, FN} end, Manifest), [compressed]), case filelib:is_file(NewFN) of - true -> - io:format("Error - trying to write manifest for" - ++ " ManifestSQN=~w which already exists~n", [ManSQN]), - error; false -> - io:format("Writing new version of manifest for " - ++ " manifestSQN=~w~n", [ManSQN]), + leveled_log:log("I0016", [ManSQN]), ok = file:write_file(TmpFN, MBin), ok = file:rename(TmpFN, NewFN), ok @@ -729,8 +713,7 @@ simple_manifest_writer(Manifest, ManSQN, RootPath) -> manifest_printer(Manifest) -> lists:foreach(fun({SQN, FN, _PID}) -> - io:format("At SQN=~w journal has filename ~s~n", - [SQN, FN]) end, + leveled_log:log("I0017", [SQN, FN]) end, Manifest). initiate_penciller_snapshot(Bookie) -> diff --git a/src/leveled_log.erl b/src/leveled_log.erl index d4d91ad..d1e63b5 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -123,7 +123,69 @@ {"PC014", {info, "Empty file ~s to be cleared"}}, {"PC015", - {info, "File created"}} + {info, "File created"}}, + + {"I0001", + {info, "Unexpected failure to fetch value for Key=~w SQN=~w " + ++ "with reason ~w"}}, + {"I0002", + {info, "Journal snapshot ~w registered at SQN ~w"}}, + {"I0003", + {info, "Journal snapshot ~w released"}}, + {"I0004", + {info, "Remaining number of journal snapshots is ~w"}}, + {"I0005", + {info, "Inker closing journal for reason ~w"}}, + {"I0006", + {info, "Close triggered with journal_sqn=~w and manifest_sqn=~w"}}, + {"I0007", + {info, "Inker manifest when closing is:"}}, + {"I0008", + {info, "Put to new active journal required roll and manifest write"}}, + {"I0009", + {info, "Updated manifest on startup:"}}, + {"I0010", + {info, "Unchanged manifest on startup:"}}, + {"I0011", + {info, "Manifest is empty, starting from manifest SQN 1"}}, + {"I0012", + {info, "Head manifest entry ~s is complete so new active journal " + ++ "required"}}, + {"I0013", + {info, "File ~s to be removed from manifest"}}, + {"I0014", + {info, "On startup oading from filename ~s from SQN ~w"}}, + {"I0015", + {info, "Opening manifest file at ~s with SQN ~w"}}, + {"I0016", + {info, "Writing new version of manifest for manifestSQN=~w"}}, + {"I0017", + {info, "At SQN=~w journal has filename ~s"}}, + + {"IC001", + {info, "Inker no longer alive so Clerk to abandon work " + ++ "leaving garbage"}}, + {"IC002", + {info, "Clerk updating Inker as compaction complete of ~w files"}}, + {"IC003", + {info, "No compaction run as highest score=~w"}}, + {"IC004", + {info, "Score for filename ~s is ~w"}}, + {"IC005", + {info, "Compaction to be performed on ~w files with score of ~w"}}, + {"IC006", + {info, "Filename ~s is part of compaction run"}}, + {"IC007", + {info, "Clerk has completed compaction process"}}, + {"IC008", + {info, "Compaction source ~s has yielded ~w positions"}}, + {"IC009", + {info, "Generate journal for compaction with filename ~s"}}, + + {"PM001", + {info, "Indexed new cache entry with total L0 cache size now ~w"}}, + {"PM002", + {info, "Completed dump of L0 cache to list of size ~w"}} ])). @@ -132,7 +194,7 @@ log(LogReference, Subs) -> {ok, {LogLevel, LogText}} = dict:find(LogReference, ?LOGBASE), case lists:member(LogLevel, ?LOG_LEVEL) of true -> - io:format(LogText ++ "~n", Subs); + io:format(LogReference ++ " " ++ LogText ++ "~n", Subs); false -> ok end. @@ -148,7 +210,8 @@ log_timer(LogReference, Subs, StartTime) -> MicroS -> {"ms", MicroS div 1000} end, - io:format(LogText ++ " with time taken ~w " ++ Unit ++ "~n", + io:format(LogReference ++ " " ++ LogText ++ " with time taken ~w " + ++ Unit ++ "~n", Subs ++ [Time]); false -> ok diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 7336cd6..baeec07 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -715,7 +715,7 @@ checkready(Pid) -> %% to an immediate return as expected. With 32K keys in the TreeList it could %% take around 35-40ms. %% -%% To avoid blocking this gen_server, the SFT file cna request each item of the +%% To avoid blocking this gen_server, the SFT file can request each item of the %% cache one at a time. %% %% The Wait is set to false to use a cast when calling this in normal operation @@ -1417,6 +1417,10 @@ simple_server_test() -> 1)), ok = pcl_close(PclSnap), + % Ignore a fake pending mnaifest on startup + ok = file:write_file(RootPath ++ "/" ++ ?MANIFEST_FP ++ "nonzero_99.pnd", + term_to_binary("Hello")), + {ok, PclSnap2} = pcl_start(SnapOpts), ok = pcl_loadsnapshot(PclSnap2, gb_trees:empty()), ?assertMatch(false, pcl_checksequencenumber(PclSnap2, diff --git a/src/leveled_pmem.erl b/src/leveled_pmem.erl index e7b218a..fa30ab9 100644 --- a/src/leveled_pmem.erl +++ b/src/leveled_pmem.erl @@ -69,8 +69,7 @@ add_to_index(L0Index, L0Size, LevelMinus1, LedgerSQN, TreeList) -> {infinity, 0, L0Index}, LM1List), NewL0Size = length(LM1List) + L0Size, - io:format("Rolled L0 cache to size ~w in ~w microseconds~n", - [NewL0Size, timer:now_diff(os:timestamp(), SW)]), + leveled_log:log_timer("PM001", [NewL0Size], SW), if MinSQN > LedgerSQN -> {MaxSQN, @@ -90,8 +89,7 @@ to_list(Slots, FetchFun) -> end, [], SlotList), - io:format("L0 cache converted to list of size ~w in ~w microseconds~n", - [length(FullList), timer:now_diff(os:timestamp(), SW)]), + leveled_log:log_timer("PM002", [length(FullList)], SW), FullList. From 37e20ccdfe4d1f7f859ec7106c525fe860086216 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 16:22:51 +0000 Subject: [PATCH 134/167] L0 cache size counter improved This is now accurate save for hash collisions. It may now be a small under-estimate, whereas previously it could be a large over-estimate. --- src/leveled_pmem.erl | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/leveled_pmem.erl b/src/leveled_pmem.erl index fa30ab9..3784715 100644 --- a/src/leveled_pmem.erl +++ b/src/leveled_pmem.erl @@ -30,7 +30,11 @@ %% Total time for array_filter 69000 microseconds %% List of 2000 checked without array - success count of 90 in 36000 microseconds %% List of 2000 checked with array - success count of 90 in 1000 microseconds - +%% +%% The trade-off taken with the approach is that the size of the L0Cache is +%% uncertain. The Size count is incremented if the hash is not already +%% present, so the size may be lower than the actual size due to hash +%% collisions -module(leveled_pmem). @@ -56,19 +60,26 @@ add_to_index(L0Index, L0Size, LevelMinus1, LedgerSQN, TreeList) -> SW = os:timestamp(), SlotInTreeList = length(TreeList) + 1, - FoldFun = fun({K, V}, {AccMinSQN, AccMaxSQN, HashIndex}) -> + FoldFun = fun({K, V}, {AccMinSQN, AccMaxSQN, AccCount, HashIndex}) -> SQN = leveled_codec:strip_to_seqonly({K, V}), {Hash, Slot} = hash_to_slot(K), L = array:get(Slot, HashIndex), + Count0 = case lists:keymember(Hash, 1, L) of + true -> + AccCount; + false -> + AccCount + 1 + end, {min(SQN, AccMinSQN), max(SQN, AccMaxSQN), + Count0, array:set(Slot, [{Hash, SlotInTreeList}|L], HashIndex)} end, LM1List = gb_trees:to_list(LevelMinus1), - {MinSQN, MaxSQN, UpdL0Index} = lists:foldl(FoldFun, - {infinity, 0, L0Index}, - LM1List), - NewL0Size = length(LM1List) + L0Size, + StartingT = {infinity, 0, L0Size, L0Index}, + {MinSQN, MaxSQN, NewL0Size, UpdL0Index} = lists:foldl(FoldFun, + StartingT, + LM1List), leveled_log:log_timer("PM001", [NewL0Size], SW), if MinSQN > LedgerSQN -> @@ -198,8 +209,9 @@ compare_method_test() -> {0, 0, new_index(), []}, lists:seq(1, 16)), - {SQN, _Size, Index, TreeList} = R, + {SQN, Size, Index, TreeList} = R, ?assertMatch(32000, SQN), + ?assertMatch(true, Size =< 32000), TestList = gb_trees:to_list(generate_randomkeys(1, 2000, 1, 800)), From f41c788bffed419fe68da0f93fb4532c8df5a9f2 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 16:46:25 +0000 Subject: [PATCH 135/167] Minor quibbles Move legacy CDB code used only in unit tests into test area. Fix column width in pmem and comment out the unused case statement (in healthy tests) from the penciller test code --- src/leveled_cdb.erl | 102 ++++++++++++++++++++------------------ src/leveled_penciller.erl | 7 ++- src/leveled_pmem.erl | 13 +++-- 3 files changed, 64 insertions(+), 58 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index fc8af19..59b8d77 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -552,41 +552,6 @@ create(FileName,KeyValueList) -> {BasePos, HashTree} = write_key_value_pairs(Handle, KeyValueList), close_file(Handle, HashTree, BasePos). -%% -%% dump(FileName) -> List -%% Given a file name, this function returns a list -%% of {key,value} tuples from the CDB. -%% - -dump(FileName) -> - {ok, Handle} = file:open(FileName, [binary, raw, read]), - Fn = fun(Index, Acc) -> - {ok, _} = file:position(Handle, ?DWORD_SIZE * Index), - {_, Count} = read_next_2_integers(Handle), - Acc + Count - end, - NumberOfPairs = lists:foldl(Fn, 0, lists:seq(0,255)) bsr 1, - io:format("Count of keys in db is ~w~n", [NumberOfPairs]), - {ok, _} = file:position(Handle, {bof, 2048}), - Fn1 = fun(_I,Acc) -> - {KL,VL} = read_next_2_integers(Handle), - Key = read_next_term(Handle, KL), - case read_next_term(Handle, VL, crc) of - {false, _} -> - {ok, CurrLoc} = file:position(Handle, cur), - Return = {crc_wonky, get(Handle, Key)}; - {_, Value} -> - {ok, CurrLoc} = file:position(Handle, cur), - Return = - case get(Handle, Key) of - {Key,Value} -> {Key ,Value}; - X -> {wonky, X} - end - end, - {ok, _} = file:position(Handle, CurrLoc), - [Return | Acc] - end, - lists:foldr(Fn1, [], lists:seq(0, NumberOfPairs-1)). %% Open an active file - one for which it is assumed the hash tables have not %% yet been written @@ -1069,20 +1034,6 @@ calc_crc(Value) -> erlang:crc32(<>) end. -%% -%% to_dict(FileName) -%% Given a filename returns a dict containing -%% the key value pairs from the dict. -%% -%% @spec to_dict(filename()) -> dictionary() -%% where -%% filename() = string(), -%% dictionary() = dict() -%% -to_dict(FileName) -> - KeyValueList = dump(FileName), - dict:from_list(KeyValueList). - read_next_term(Handle, Length) -> case file:read(Handle, Length) of {ok, Bin} -> @@ -1378,6 +1329,59 @@ multi_key_value_to_record(KVList, BinaryMode, LastPosition) -> %%%%%%%%%%%%%%% -ifdef(TEST). +%% +%% dump(FileName) -> List +%% Given a file name, this function returns a list +%% of {key,value} tuples from the CDB. +%% + +dump(FileName) -> + {ok, Handle} = file:open(FileName, [binary, raw, read]), + Fn = fun(Index, Acc) -> + {ok, _} = file:position(Handle, ?DWORD_SIZE * Index), + {_, Count} = read_next_2_integers(Handle), + Acc + Count + end, + NumberOfPairs = lists:foldl(Fn, 0, lists:seq(0,255)) bsr 1, + io:format("Count of keys in db is ~w~n", [NumberOfPairs]), + {ok, _} = file:position(Handle, {bof, 2048}), + Fn1 = fun(_I,Acc) -> + {KL,VL} = read_next_2_integers(Handle), + Key = read_next_term(Handle, KL), + case read_next_term(Handle, VL, crc) of + {false, _} -> + {ok, CurrLoc} = file:position(Handle, cur), + Return = {crc_wonky, get(Handle, Key)}; + {_, Value} -> + {ok, CurrLoc} = file:position(Handle, cur), + Return = + case get(Handle, Key) of + {Key,Value} -> {Key ,Value}; + X -> {wonky, X} + end + end, + {ok, _} = file:position(Handle, CurrLoc), + [Return | Acc] + end, + lists:foldr(Fn1, [], lists:seq(0, NumberOfPairs-1)). + +%% +%% to_dict(FileName) +%% Given a filename returns a dict containing +%% the key value pairs from the dict. +%% +%% @spec to_dict(filename()) -> dictionary() +%% where +%% filename() = string(), +%% dictionary() = dict() +%% +to_dict(FileName) -> + KeyValueList = dump(FileName), + dict:from_list(KeyValueList). + + + + write_key_value_pairs_1_test() -> {ok,Handle} = file:open("../test/test.cdb",[write]), {_, HashTree} = write_key_value_pairs(Handle, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index baeec07..f0ed89b 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -1260,10 +1260,9 @@ clean_subdir(DirPath) -> {ok, Files} = file:list_dir(DirPath), lists:foreach(fun(FN) -> File = filename:join(DirPath, FN), - case file:delete(File) of - ok -> io:format("Success deleting ~s~n", [File]); - _ -> io:format("Error deleting ~s~n", [File]) - end end, + ok = file:delete(File), + io:format("Success deleting ~s~n", [File]) + end, Files); false -> ok diff --git a/src/leveled_pmem.erl b/src/leveled_pmem.erl index 3784715..d12425b 100644 --- a/src/leveled_pmem.erl +++ b/src/leveled_pmem.erl @@ -7,7 +7,8 @@ %% whilst maintaining the capability to quickly snapshot the memory for clones %% of the Penciller. %% -%% ETS tables are not used due to complications with managing their mutability. +%% ETS tables are not used due to complications with managing their mutability, +%% as the database is snapshotted. %% %% An attempt was made to merge all trees into a single tree on push (in a %% spawned process), but this proved to have an expensive impact as the tree @@ -28,8 +29,8 @@ %% Total time for array_tree 209000 microseconds %% Total time for array_list 142000 microseconds %% Total time for array_filter 69000 microseconds -%% List of 2000 checked without array - success count of 90 in 36000 microseconds -%% List of 2000 checked with array - success count of 90 in 1000 microseconds +%% List of 2000 checked without array - success count of 90 in 36000 microsecs +%% List of 2000 checked with array - success count of 90 in 1000 microsecs %% %% The trade-off taken with the approach is that the size of the L0Cache is %% uncertain. The Size count is incremented if the hash is not already @@ -203,8 +204,10 @@ generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) -> compare_method_test() -> R = lists:foldl(fun(_X, {LedgerSQN, L0Size, L0Index, L0TreeList}) -> - LM1 = generate_randomkeys(LedgerSQN + 1, 2000, 1, 500), - add_to_index(L0Index, L0Size, LM1, LedgerSQN, L0TreeList) + LM1 = generate_randomkeys(LedgerSQN + 1, + 2000, 1, 500), + add_to_index(L0Index, L0Size, LM1, LedgerSQN, + L0TreeList) end, {0, 0, new_index(), []}, lists:seq(1, 16)), From 2716d912ea4107fec7afba08959528be9fd74cbc Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 19:02:50 +0000 Subject: [PATCH 136/167] Timeout and close race Race condition presvented in test - but still not handled nicely. Perhaps need to consider making it a FSM and handling close differently when L0 pending - i.e. don't close immediately, but set a timeout to close on if we don't get the last fetch_levelzero --- src/leveled_penciller.erl | 19 ++++++++++++++----- src/leveled_sft.erl | 2 +- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index f0ed89b..0dd23fa 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -290,7 +290,12 @@ pcl_pushmem(Pid, DumpList) -> gen_server:call(Pid, {push_mem, DumpList}, infinity). pcl_fetchlevelzero(Pid, Slot) -> - gen_server:call(Pid, {fetch_levelzero, Slot}, infinity). + %% Timeout to cause crash of L0 file when it can't get the close signal + %% as it is deadlocked making this call. + %% + %% If the timeout gets hit outside of close scenario the Penciller will + %% be stuck in L0 pending + gen_server:call(Pid, {fetch_levelzero, Slot}, 10000). pcl_fetch(Pid, Key) -> gen_server:call(Pid, {fetch, Key}, infinity). @@ -1661,12 +1666,16 @@ coverage_test() -> clean_testdir(RootPath), {ok, PCL} = pcl_start(#penciller_options{root_path=RootPath, max_inmemory_tablesize=1000}), - Key1 = {{o,"Bucket0001", "Key0001", null}, {1, {active, infinity}, null}}, - KL1 = leveled_sft:generate_randomkeys({1000, 2}), - ok = maybe_pause_push(PCL, [Key1]), + Key1 = {{o,"Bucket0001", "Key0001", null}, {1001, {active, infinity}, null}}, + KL1 = leveled_sft:generate_randomkeys({1000, 1}), + + ok = maybe_pause_push(PCL, KL1 ++ [Key1]), + %% Added together, as split apart there will be a race between the close + %% call to the penciller and the second fetch of the cache entry ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), - ok = maybe_pause_push(PCL, KL1), + ok = pcl_close(PCL), + ManifestFP = filepath(RootPath, manifest), file:write_file(ManifestFP ++ "/yeszero_123.man", term_to_binary("hello")), {ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath, diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 8f4a870..d80116d 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -270,7 +270,7 @@ sft_clear(Pid) -> gen_server:call(Pid, clear, infinity). sft_close(Pid) -> - gen_server:call(Pid, close, infinity). + gen_server:call(Pid, close, 1000). sft_deleteconfirmed(Pid) -> gen_server:cast(Pid, close). From 341e245c090dea4335c1322a241c4479d2d02bad Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 19:34:54 +0000 Subject: [PATCH 137/167] Remove unnecessary no match condition --- src/leveled_penciller.erl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 0dd23fa..ac9ee5c 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -611,10 +611,9 @@ start_from_file(PCLopts) -> nomatch -> Acc; {match, [Int]} when is_list(Int) -> - Acc ++ [list_to_integer(Int)]; - _ -> - Acc - end end, + Acc ++ [list_to_integer(Int)] + end + end, [], Filenames), TopManSQN = lists:foldl(fun(X, MaxSQN) -> max(X, MaxSQN) end, @@ -1677,7 +1676,7 @@ coverage_test() -> ok = pcl_close(PCL), ManifestFP = filepath(RootPath, manifest), - file:write_file(ManifestFP ++ "/yeszero_123.man", term_to_binary("hello")), + ok = file:write_file(ManifestFP ++ "/yeszero_123.man", term_to_binary("hello")), {ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath, max_inmemory_tablesize=1000}), ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})), From 49eed557354e8010bfd0b327b22079f159199b11 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 19:39:23 +0000 Subject: [PATCH 138/167] Remove unnecessary clause --- src/leveled_bookie.erl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 3c64b69..dd1556c 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -761,11 +761,8 @@ maybepush_ledgercache(MaxCacheSize, Cache, Penciller) -> end. -maybe_withjitter(CacheSize, MaxCacheSize) -> - +maybe_withjitter(CacheSize, MaxCacheSize) -> if - CacheSize > 2 * MaxCacheSize -> - true; CacheSize > MaxCacheSize -> T = 2 * MaxCacheSize - CacheSize, R = random:uniform(CacheSize), From 72b9b35dacb8202a86c4c645249672a83e173213 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 19:55:19 +0000 Subject: [PATCH 139/167] Dump test utility - does not need to cope with CRC failure Dump is used in tests only --- src/leveled_cdb.erl | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 59b8d77..4cef684 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -1349,9 +1349,6 @@ dump(FileName) -> {KL,VL} = read_next_2_integers(Handle), Key = read_next_term(Handle, KL), case read_next_term(Handle, VL, crc) of - {false, _} -> - {ok, CurrLoc} = file:position(Handle, cur), - Return = {crc_wonky, get(Handle, Key)}; {_, Value} -> {ok, CurrLoc} = file:position(Handle, cur), Return = From ad9c886b65739d3deacababd8bda39c8c96807a5 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 20:00:55 +0000 Subject: [PATCH 140/167] Inker test for pending manifest Should ignore the corrupted pending manifest file --- src/leveled_inker.erl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 8e2fa11..196919f 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -659,9 +659,7 @@ sequencenumbers_fromfilenames(Filenames, Regex, IntName) -> nomatch -> Acc; {match, [Int]} when is_list(Int) -> - Acc ++ [list_to_integer(Int)]; - _ -> - Acc + Acc ++ [list_to_integer(Int)] end end, [], Filenames). @@ -898,6 +896,11 @@ empty_manifest_test() -> timer:sleep(1000), ?assertMatch(1, length(ink_getmanifest(Ink1))), ok = ink_close(Ink1), + + % Add pending manifest to be ignored + FN = filepath(RootPath, manifest_dir) ++ "999.pnd", + ok = file:write_file(FN, term_to_binary("Hello")), + {ok, Ink2} = ink_start(#inker_options{root_path=RootPath, cdb_options=CDBopts}), ?assertMatch(not_present, ink_fetch(Ink2, "Key1", 1)), From d5ac4d412d8c87a08ebbcc488984099d76ab947e Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 20:06:30 +0000 Subject: [PATCH 141/167] Use filename join Potentiall to avoid *nix vs windows differences --- src/leveled_penciller.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index ac9ee5c..32ee3bc 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -1676,7 +1676,7 @@ coverage_test() -> ok = pcl_close(PCL), ManifestFP = filepath(RootPath, manifest), - ok = file:write_file(ManifestFP ++ "/yeszero_123.man", term_to_binary("hello")), + ok = file:write_file(filename:join(ManifestFP, "yeszero_123.man"), term_to_binary("hello")), {ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath, max_inmemory_tablesize=1000}), ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})), From c3a6489b9354a859d33adad324807e94d96c37f3 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 20:09:38 +0000 Subject: [PATCH 142/167] Ensure manifest dir when starting Penciller Otherwise may fail based on test ordering --- src/leveled_penciller.erl | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 32ee3bc..ae18fcd 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -596,13 +596,9 @@ start_from_file(PCLopts) -> levelzero_maxcachesize=MaxTableSize}, %% Open manifest - ManifestPath = InitState#state.root_path ++ "/" ++ ?MANIFEST_FP ++ "/", - {ok, Filenames} = case filelib:is_dir(ManifestPath) of - true -> - file:list_dir(ManifestPath); - false -> - {ok, []} - end, + ManifestPath = filename:join(InitState#state.root_path, ?MANIFEST_FP), + filelib:ensure_dir(ManifestPath), + {ok, Filenames} = file:list_dir(ManifestPath), CurrRegex = "nonzero_(?[0-9]+)\\." ++ ?CURRENT_FILEX, ValidManSQNs = lists:foldl(fun(FN, Acc) -> case re:run(FN, From dd99d624b17d1a4e60962054191368c351ff316e Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 20:46:56 +0000 Subject: [PATCH 143/167] Tangling with filenames filename join does not work as expected --- src/leveled_penciller.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index ae18fcd..4b78d9e 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -596,8 +596,8 @@ start_from_file(PCLopts) -> levelzero_maxcachesize=MaxTableSize}, %% Open manifest - ManifestPath = filename:join(InitState#state.root_path, ?MANIFEST_FP), - filelib:ensure_dir(ManifestPath), + ManifestPath = InitState#state.root_path ++ ?MANIFEST_FP ++ "/", + ok = filelib:ensure_dir(ManifestPath), {ok, Filenames} = file:list_dir(ManifestPath), CurrRegex = "nonzero_(?[0-9]+)\\." ++ ?CURRENT_FILEX, ValidManSQNs = lists:foldl(fun(FN, Acc) -> From 41f00ba6fab56031baeaa26fd6a185fa92bf0dcf Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 20:48:23 +0000 Subject: [PATCH 144/167] Filename nonsense --- src/leveled_penciller.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 4b78d9e..81216e4 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -596,7 +596,7 @@ start_from_file(PCLopts) -> levelzero_maxcachesize=MaxTableSize}, %% Open manifest - ManifestPath = InitState#state.root_path ++ ?MANIFEST_FP ++ "/", + ManifestPath = InitState#state.root_path ++ "/" ++ ?MANIFEST_FP ++ "/", ok = filelib:ensure_dir(ManifestPath), {ok, Filenames} = file:list_dir(ManifestPath), CurrRegex = "nonzero_(?[0-9]+)\\." ++ ?CURRENT_FILEX, From 9ea74836ee03d6116ee9f716aa9d043dbd7f5d5e Mon Sep 17 00:00:00 2001 From: martinsumner Date: Thu, 3 Nov 2016 21:30:55 +0000 Subject: [PATCH 145/167] Alter impossible clause Starts with {infinity, 0} so can never be {0.0} --- src/leveled_sft.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index d80116d..1f434c6 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -1139,7 +1139,7 @@ pointer_append_queryresults(Results, QueryPid) -> %% Update the sequence numbers update_sequencenumbers(Item, LSN, HSN) when is_tuple(Item) -> update_sequencenumbers(leveled_codec:strip_to_seqonly(Item), LSN, HSN); -update_sequencenumbers(SN, 0, 0) -> +update_sequencenumbers(SN, infinity, 0) -> {SN, SN}; update_sequencenumbers(SN, LSN, HSN) when SN < LSN -> {SN, HSN}; From 68b17c71b33d23b20b3cc08c40ffc0f46fb834da Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 4 Nov 2016 11:01:37 +0000 Subject: [PATCH 146/167] Expand fold objects support Fold over bucket and fold over index added --- src/leveled_bookie.erl | 51 ++++++++++++++++++++++++++---- test/end_to_end/iterator_SUITE.erl | 45 +++++++++++++++++++++++--- test/end_to_end/recovery_SUITE.erl | 12 ++----- test/end_to_end/testutil.erl | 14 ++++++-- 4 files changed, 100 insertions(+), 22 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index dd1556c..dec8ceb 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -372,7 +372,23 @@ handle_call({return_folder, FolderType}, _From, State) -> {foldobjects_allkeys, Tag, FoldObjectsFun} -> {reply, foldobjects_allkeys(State, Tag, FoldObjectsFun), + State}; + {foldobjects_bybucket, Tag, Bucket, FoldObjectsFun} -> + {reply, + foldobjects_bybucket(State, Tag, Bucket, FoldObjectsFun), + State}; + {foldobjects_byindex, + Tag, + Bucket, + {Field, FromTerm, ToTerm}, + FoldObjectsFun} -> + {reply, + foldobjects_byindex(State, + Tag, Bucket, + Field, FromTerm, ToTerm, + FoldObjectsFun), State} + end; handle_call({compact_journal, Timeout}, _From, State) -> ok = leveled_inker:ink_compactjournal(State#state.inker, @@ -492,6 +508,23 @@ hashtree_query(State, Tag, JournalCheck) -> foldobjects_allkeys(State, Tag, FoldObjectsFun) -> + StartKey = leveled_codec:to_ledgerkey(null, null, Tag), + EndKey = leveled_codec:to_ledgerkey(null, null, Tag), + foldobjects(State, Tag, StartKey, EndKey, FoldObjectsFun). + +foldobjects_bybucket(State, Tag, Bucket, FoldObjectsFun) -> + StartKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), + EndKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), + foldobjects(State, Tag, StartKey, EndKey, FoldObjectsFun). + +foldobjects_byindex(State, Tag, Bucket, Field, FromTerm, ToTerm, FoldObjectsFun) -> + StartKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, Field, + FromTerm), + EndKey = leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, Field, + ToTerm), + foldobjects(State, Tag, StartKey, EndKey, FoldObjectsFun). + +foldobjects(State, Tag, StartKey, EndKey, FoldObjectsFun) -> {ok, {LedgerSnapshot, LedgerCache}, JournalSnapshot} = snapshot_store(State, store), @@ -499,9 +532,7 @@ foldobjects_allkeys(State, Tag, FoldObjectsFun) -> leveled_log:log("B0004", [gb_trees:size(LedgerCache)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, LedgerCache), - StartKey = leveled_codec:to_ledgerkey(null, null, Tag), - EndKey = leveled_codec:to_ledgerkey(null, null, Tag), - AccFun = accumulate_objects(FoldObjectsFun, JournalSnapshot), + AccFun = accumulate_objects(FoldObjectsFun, JournalSnapshot, Tag), Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, StartKey, EndKey, @@ -513,6 +544,7 @@ foldobjects_allkeys(State, Tag, FoldObjectsFun) -> end, {async, Folder}. + allkey_query(State, Tag) -> {ok, {LedgerSnapshot, LedgerCache}, @@ -643,14 +675,18 @@ accumulate_hashes(JournalCheck, InkerClone) -> end, AccFun. -accumulate_objects(FoldObjectsFun, InkerClone) -> +accumulate_objects(FoldObjectsFun, InkerClone, Tag) -> Now = leveled_codec:integer_now(), AccFun = fun(LK, V, Acc) -> case leveled_codec:is_active(LK, V, Now) of true -> SQN = leveled_codec:strip_to_seqonly({LK, V}), - {B, K} = leveled_codec:from_ledgerkey(LK), - R = leveled_inker:ink_fetch(InkerClone, LK, SQN), + {B, K} = case leveled_codec:from_ledgerkey(LK) of + {B0, K0} -> {B0, K0}; + {B0, K0, _T0} -> {B0, K0} + end, + QK = leveled_codec:to_ledgerkey(B, K, Tag), + R = leveled_inker:ink_fetch(InkerClone, QK, SQN), case R of {ok, Value} -> FoldObjectsFun(B, K, Value, Acc); @@ -663,6 +699,9 @@ accumulate_objects(FoldObjectsFun, InkerClone) -> end, AccFun. + + + check_presence(Key, Value, InkerClone) -> {LedgerKey, SQN} = leveled_codec:strip_to_keyseqonly({Key, Value}), case leveled_inker:ink_keycheck(InkerClone, LedgerKey, SQN) of diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 7e1fcff..73e9ab4 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -6,17 +6,18 @@ -define(KEY_ONLY, {false, undefined}). -export([all/0]). --export([simple_load_with2i/1, +-export([small_load_with2i/1, query_count/1, rotating_objects/1]). all() -> [ - simple_load_with2i, + small_load_with2i, query_count, - rotating_objects]. + rotating_objects + ]. -simple_load_with2i(_Config) -> +small_load_with2i(_Config) -> RootPath = testutil:reset_filestructure(), StartOpts1 = [{root_path, RootPath}, {max_journalsize, 50000000}], @@ -37,6 +38,42 @@ simple_load_with2i(_Config) -> ChkList1 = lists:sublist(lists:sort(ObjL1), 100), testutil:check_forlist(Bookie1, ChkList1), testutil:check_forobject(Bookie1, TestObject), + + %% Delete the objects from the ChkList removing the indexes + lists:foreach(fun({_RN, Obj, Spc}) -> + DSpc = lists:map(fun({add, F, T}) -> {remove, F, T} + end, + Spc), + {B, K} = leveled_codec:riakto_keydetails(Obj), + leveled_bookie:book_riakdelete(Bookie1, B, K, DSpc) + end, + ChkList1), + %% Get the Buckets Keys and Hashes for the whole bucket + FoldObjectsFun = fun(B, K, V, Acc) -> [{B, K, testutil:riak_hash(V)}|Acc] + end, + {async, HTreeF1} = leveled_bookie:book_returnfolder(Bookie1, + {foldobjects_allkeys, + ?RIAK_TAG, + FoldObjectsFun}), + KeyHashList1 = HTreeF1(), + {async, HTreeF2} = leveled_bookie:book_returnfolder(Bookie1, + {foldobjects_bybucket, + ?RIAK_TAG, + "Bucket", + FoldObjectsFun}), + KeyHashList2 = HTreeF2(), + {async, HTreeF3} = leveled_bookie:book_returnfolder(Bookie1, + {foldobjects_byindex, + ?RIAK_TAG, + "Bucket", + {"idx1_bin", + "#", "~"}, + FoldObjectsFun}), + KeyHashList3 = HTreeF3(), + true = 9901 == length(KeyHashList1), % also includes the test object + true = 9900 == length(KeyHashList2), + true = 9900 == length(KeyHashList3), + ok = leveled_bookie:book_close(Bookie1), testutil:reset_filestructure(). diff --git a/test/end_to_end/recovery_SUITE.erl b/test/end_to_end/recovery_SUITE.erl index a6269d1..de58e4c 100644 --- a/test/end_to_end/recovery_SUITE.erl +++ b/test/end_to_end/recovery_SUITE.erl @@ -103,7 +103,8 @@ aae_bustedjournal(_Config) -> % Will need to remove the file or corrupt the hashtree to get presence to % fail - FoldObjectsFun = fun(B, K, V, Acc) -> [{B, K, riak_hash(V)}|Acc] end, + FoldObjectsFun = fun(B, K, V, Acc) -> [{B, K, testutil:riak_hash(V)}|Acc] + end, SW = os:timestamp(), {async, HashTreeF3} = leveled_bookie:book_returnfolder(Bookie2, {foldobjects_allkeys, @@ -190,15 +191,6 @@ aae_bustedjournal(_Config) -> testutil:reset_filestructure(). -riak_hash(Obj=#r_object{}) -> - Vclock = vclock(Obj), - UpdObj = set_vclock(Obj, lists:sort(Vclock)), - erlang:phash2(term_to_binary(UpdObj)). - -set_vclock(Object=#r_object{}, VClock) -> Object#r_object{vclock=VClock}. -vclock(#r_object{vclock=VClock}) -> VClock. - - journal_compaction_bustedjournal(_Config) -> % Simply confirms that none of this causes a crash RootPath = testutil:reset_filestructure(), diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index e11d1a8..9d9f278 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -32,7 +32,8 @@ corrupt_journal/5, restore_file/2, restore_topending/2, - find_journals/1]). + find_journals/1, + riak_hash/1]). -define(RETURN_TERMS, {true, undefined}). @@ -423,4 +424,13 @@ find_journals(RootPath) -> end, [], FNsA_J), - CDBFiles. \ No newline at end of file + CDBFiles. + + +riak_hash(Obj=#r_object{}) -> + Vclock = vclock(Obj), + UpdObj = set_vclock(Obj, lists:sort(Vclock)), + erlang:phash2(term_to_binary(UpdObj)). + +set_vclock(Object=#r_object{}, VClock) -> Object#r_object{vclock=VClock}. +vclock(#r_object{vclock=VClock}) -> VClock. From ba628c2f40fc261ab6faed542d19b8aa5cbbbcb1 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 4 Nov 2016 12:22:15 +0000 Subject: [PATCH 147/167] Test Rolling of CDB to two files trhough compaction This exposed a potential issue with not opening readers in binary_mode - so now defaults to binary mode. Will add test using object filder to confirm values remain readable in rolled journals after shutdown/startup. --- src/leveled_cdb.erl | 42 +++++++++++++++++++++++++++--------------- src/leveled_iclerk.erl | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 4cef684..2d9746f 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -116,7 +116,7 @@ cdb_open_writer(Filename) -> %% No options passed - cdb_open_writer(Filename, #cdb_options{}). + cdb_open_writer(Filename, #cdb_options{binary_mode=true}). cdb_open_writer(Filename, Opts) -> {ok, Pid} = gen_fsm:start(?MODULE, [Opts], []), @@ -124,7 +124,10 @@ cdb_open_writer(Filename, Opts) -> {ok, Pid}. cdb_open_reader(Filename) -> - {ok, Pid} = gen_fsm:start(?MODULE, [#cdb_options{}], []), + cdb_open_reader(Filename, #cdb_options{binary_mode=true}). + +cdb_open_reader(Filename, Opts) -> + {ok, Pid} = gen_fsm:start(?MODULE, [Opts], []), ok = gen_fsm:sync_send_event(Pid, {open_reader, Filename}, infinity), {ok, Pid}. @@ -1637,7 +1640,8 @@ emptyvalue_fromdict_test() -> ok = file:delete("../test/from_dict_test_ev.cdb"). find_lastkey_test() -> - {ok, P1} = cdb_open_writer("../test/lastkey.pnd"), + {ok, P1} = cdb_open_writer("../test/lastkey.pnd", + #cdb_options{binary_mode=false}), ok = cdb_put(P1, "Key1", "Value1"), ok = cdb_put(P1, "Key3", "Value3"), ok = cdb_put(P1, "Key2", "Value2"), @@ -1645,7 +1649,8 @@ find_lastkey_test() -> ?assertMatch("Key1", cdb_firstkey(P1)), probably = cdb_keycheck(P1, "Key2"), ok = cdb_close(P1), - {ok, P2} = cdb_open_writer("../test/lastkey.pnd"), + {ok, P2} = cdb_open_writer("../test/lastkey.pnd", + #cdb_options{binary_mode=false}), ?assertMatch("Key2", cdb_lastkey(P2)), probably = cdb_keycheck(P2, "Key2"), {ok, F2} = cdb_complete(P2), @@ -1658,13 +1663,14 @@ find_lastkey_test() -> ok = file:delete("../test/lastkey.cdb"). get_keys_byposition_simple_test() -> - {ok, P1} = cdb_open_writer("../test/poskey.pnd"), + {ok, P1} = cdb_open_writer("../test/poskey.pnd", + #cdb_options{binary_mode=false}), ok = cdb_put(P1, "Key1", "Value1"), ok = cdb_put(P1, "Key3", "Value3"), ok = cdb_put(P1, "Key2", "Value2"), KeyList = ["Key1", "Key2", "Key3"], {ok, F2} = cdb_complete(P1), - {ok, P2} = cdb_open_reader(F2), + {ok, P2} = cdb_open_reader(F2, #cdb_options{binary_mode=false}), PositionList = cdb_getpositions(P2, all), io:format("Position list of ~w~n", [PositionList]), ?assertMatch(3, length(PositionList)), @@ -1699,7 +1705,8 @@ generate_sequentialkeys(Count, KVList) -> get_keys_byposition_manykeys_test() -> KeyCount = 1024, - {ok, P1} = cdb_open_writer("../test/poskeymany.pnd"), + {ok, P1} = cdb_open_writer("../test/poskeymany.pnd", + #cdb_options{binary_mode=false}), KVList = generate_sequentialkeys(KeyCount, []), lists:foreach(fun({K, V}) -> cdb_put(P1, K, V) end, KVList), SW1 = os:timestamp(), @@ -1707,7 +1714,7 @@ get_keys_byposition_manykeys_test() -> SW2 = os:timestamp(), io:format("CDB completed in ~w microseconds~n", [timer:now_diff(SW2, SW1)]), - {ok, P2} = cdb_open_reader(F2), + {ok, P2} = cdb_open_reader(F2, #cdb_options{binary_mode=false}), SW3 = os:timestamp(), io:format("CDB opened for read in ~w microseconds~n", [timer:now_diff(SW3, SW2)]), @@ -1729,9 +1736,10 @@ get_keys_byposition_manykeys_test() -> nokeys_test() -> - {ok, P1} = cdb_open_writer("../test/nohash_emptyfile.pnd"), + {ok, P1} = cdb_open_writer("../test/nohash_emptyfile.pnd", + #cdb_options{binary_mode=false}), {ok, F2} = cdb_complete(P1), - {ok, P2} = cdb_open_reader(F2), + {ok, P2} = cdb_open_reader(F2, #cdb_options{binary_mode=false}), io:format("FirstKey is ~s~n", [cdb_firstkey(P2)]), io:format("LastKey is ~s~n", [cdb_lastkey(P2)]), ?assertMatch(empty, cdb_firstkey(P2)), @@ -1741,7 +1749,8 @@ nokeys_test() -> mput_test() -> KeyCount = 1024, - {ok, P1} = cdb_open_writer("../test/nohash_keysinfile.pnd"), + {ok, P1} = cdb_open_writer("../test/nohash_keysinfile.pnd", + #cdb_options{binary_mode=false}), KVList = generate_sequentialkeys(KeyCount, []), ok = cdb_mput(P1, KVList), ?assertMatch({"Key1", "Value1"}, cdb_get(P1, "Key1")), @@ -1749,7 +1758,7 @@ mput_test() -> ?assertMatch(missing, cdb_get(P1, "Key1025")), ?assertMatch(missing, cdb_get(P1, "Key1026")), {ok, F2} = cdb_complete(P1), - {ok, P2} = cdb_open_reader(F2), + {ok, P2} = cdb_open_reader(F2, #cdb_options{binary_mode=false}), ?assertMatch("Key1", cdb_firstkey(P2)), ?assertMatch("Key1024", cdb_lastkey(P2)), ?assertMatch({"Key1", "Value1"}, cdb_get(P2, "Key1")), @@ -1760,7 +1769,8 @@ mput_test() -> ok = file:delete(F2). state_test() -> - {ok, P1} = cdb_open_writer("../test/state_test.pnd"), + {ok, P1} = cdb_open_writer("../test/state_test.pnd", + #cdb_options{binary_mode=false}), KVList = generate_sequentialkeys(1000, []), ok = cdb_mput(P1, KVList), ?assertMatch(probably, cdb_keycheck(P1, "Key1")), @@ -1778,7 +1788,8 @@ state_test() -> corruptfile_test() -> file:delete("../test/corrupt_test.pnd"), - {ok, P1} = cdb_open_writer("../test/corrupt_test.pnd"), + {ok, P1} = cdb_open_writer("../test/corrupt_test.pnd", + #cdb_options{binary_mode=false}), KVList = generate_sequentialkeys(100, []), ok = cdb_mput(P1, []), % Not relevant to this test, but needs testing lists:foreach(fun({K, V}) -> cdb_put(P1, K, V) end, KVList), @@ -1796,7 +1807,8 @@ corrupt_testfile_at_offset(Offset) -> file:position(F1, EofPos - Offset), ok = file:truncate(F1), ok = file:close(F1), - {ok, P2} = cdb_open_writer("../test/corrupt_test.pnd"), + {ok, P2} = cdb_open_writer("../test/corrupt_test.pnd", + #cdb_options{binary_mode=false}), ?assertMatch(probably, cdb_keycheck(P2, "Key1")), ?assertMatch({"Key1", "Value1"}, cdb_get(P2, "Key1")), ?assertMatch(missing, cdb_get(P2, "Key100")), diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index ecd05ab..f955ccb 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -780,4 +780,45 @@ compare_candidate_test() -> ?assertMatch([Candidate1, Candidate2, Candidate3, Candidate4], sort_run([Candidate3, Candidate2, Candidate4, Candidate1])). +compact_singlefile_totwosmallfiles_test() -> + RP = "../test/journal", + CP = "../test/journal/journal_file/post_compact/", + ok = filelib:ensure_dir(CP), + FN1 = leveled_inker:filepath(RP, 1, new_journal), + CDBoptsLarge = #cdb_options{binary_mode=true, max_size=30000000}, + {ok, CDB1} = leveled_cdb:cdb_open_writer(FN1, CDBoptsLarge), + lists:foreach(fun(X) -> + LK = test_ledgerkey("Key" ++ integer_to_list(X)), + Value = term_to_binary({crypto:rand_bytes(1024), []}), + ok = leveled_cdb:cdb_put(CDB1, + {X, ?INKT_STND, LK}, + Value) + end, + lists:seq(1, 1000)), + {ok, NewName} = leveled_cdb:cdb_complete(CDB1), + {ok, CDBr} = leveled_cdb:cdb_open_reader(NewName), + CDBoptsSmall = #cdb_options{binary_mode=true, max_size=400000, file_path=CP}, + BestRun1 = [#candidate{low_sqn=1, + filename=leveled_cdb:cdb_filename(CDBr), + journal=CDBr, + compaction_perc=50.0}], + FakeFilterFun = fun(_FS, _LK, SQN) -> SQN rem 2 == 0 end, + + {ManifestSlice, PromptDelete} = compact_files(BestRun1, + CDBoptsSmall, + FakeFilterFun, + null, + 900, + [{?STD_TAG, recovr}]), + ?assertMatch(2, length(ManifestSlice)), + ?assertMatch(true, PromptDelete), + lists:foreach(fun({_SQN, _FN, CDB}) -> + ok = leveled_cdb:cdb_deletepending(CDB), + ok = leveled_cdb:cdb_destroy(CDB) + end, + ManifestSlice), + ok = leveled_cdb:cdb_deletepending(CDBr), + ok = leveled_cdb:cdb_destroy(CDBr). + + -endif. \ No newline at end of file From eeeee07081155ad61fe86b6f9fb2c7f797cf18fe Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 4 Nov 2016 14:23:37 +0000 Subject: [PATCH 148/167] Fold Objects - Check values test Test that summed values in fold objects before and after restart --- src/leveled_bookie.erl | 10 +++++-- test/end_to_end/iterator_SUITE.erl | 42 +++++++++++++++++++++++++----- test/end_to_end/testutil.erl | 4 +++ 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index dec8ceb..82abb86 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -528,16 +528,22 @@ foldobjects(State, Tag, StartKey, EndKey, FoldObjectsFun) -> {ok, {LedgerSnapshot, LedgerCache}, JournalSnapshot} = snapshot_store(State, store), + {FoldFun, InitAcc} = case is_tuple(FoldObjectsFun) of + true -> + FoldObjectsFun; + false -> + {FoldObjectsFun, []} + end, Folder = fun() -> leveled_log:log("B0004", [gb_trees:size(LedgerCache)]), ok = leveled_penciller:pcl_loadsnapshot(LedgerSnapshot, LedgerCache), - AccFun = accumulate_objects(FoldObjectsFun, JournalSnapshot, Tag), + AccFun = accumulate_objects(FoldFun, JournalSnapshot, Tag), Acc = leveled_penciller:pcl_fetchkeys(LedgerSnapshot, StartKey, EndKey, AccFun, - []), + InitAcc), ok = leveled_penciller:pcl_close(LedgerSnapshot), ok = leveled_inker:ink_close(JournalSnapshot), Acc diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 73e9ab4..78d19bd 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -11,27 +11,30 @@ rotating_objects/1]). all() -> [ - small_load_with2i, - query_count, - rotating_objects + small_load_with2i %, + % query_count, + % rotating_objects ]. small_load_with2i(_Config) -> RootPath = testutil:reset_filestructure(), StartOpts1 = [{root_path, RootPath}, - {max_journalsize, 50000000}], + {max_journalsize, 5000000}], + % low journal size to make sure > 1 created {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = leveled_bookie:book_riakput(Bookie1, TestObject, TestSpec), testutil:check_forobject(Bookie1, TestObject), testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"), testutil:check_forobject(Bookie1, TestObject), + ObjectGen = testutil:get_compressiblevalue_andinteger(), + IndexGen = testutil:get_randomindexes_generator(8), ObjL1 = testutil:generate_objects(10000, uuid, [], - testutil:get_compressiblevalue(), - testutil:get_randomindexes_generator(8)), + ObjectGen, + IndexGen), lists:foreach(fun({_RN, Obj, Spc}) -> leveled_bookie:book_riakput(Bookie1, Obj, Spc) end, ObjL1), @@ -74,7 +77,34 @@ small_load_with2i(_Config) -> true = 9900 == length(KeyHashList2), true = 9900 == length(KeyHashList3), + SumIntegerFun = fun(_B, _K, V, Acc) -> + [C] = V#r_object.contents, + {I, _Bin} = C#r_content.value, + Acc + I + end, + {async, Sum1} = leveled_bookie:book_returnfolder(Bookie1, + {foldobjects_bybucket, + ?RIAK_TAG, + "Bucket", + {SumIntegerFun, + 0}}), + Total1 = Sum1(), + true = Total1 > 100000, + ok = leveled_bookie:book_close(Bookie1), + + {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), + + {async, Sum2} = leveled_bookie:book_returnfolder(Bookie2, + {foldobjects_bybucket, + ?RIAK_TAG, + "Bucket", + {SumIntegerFun, + 0}}), + Total2 = Sum2(), + true = Total2 == Total1, + + ok = leveled_bookie:book_close(Bookie2), testutil:reset_filestructure(). diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index 9d9f278..755cf88 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -21,6 +21,7 @@ get_key/1, get_value/1, get_compressiblevalue/0, + get_compressiblevalue_andinteger/0, get_randomindexes_generator/1, name_list/0, load_objects/5, @@ -147,6 +148,9 @@ generate_compressibleobjects(Count, KeyNumber) -> generate_objects(Count, KeyNumber, [], V). +get_compressiblevalue_andinteger() -> + {random:uniform(1000), get_compressiblevalue()}. + get_compressiblevalue() -> S1 = "111111111111111", S2 = "222222222222222", From 171baefc0c169a62aee274e98ba43169920808ed Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 4 Nov 2016 14:31:19 +0000 Subject: [PATCH 149/167] SFT Background Failure Let it crash approach - stop trying to catch and propgate failure of write --- src/leveled_sft.erl | 62 ++++++++++++------------------ test/end_to_end/iterator_SUITE.erl | 6 +-- 2 files changed, 27 insertions(+), 41 deletions(-) diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 1f434c6..d35f285 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -201,7 +201,6 @@ filename = "not set" :: string(), handle :: file:fd(), background_complete = false :: boolean(), - background_failure :: tuple(), oversized_file = false :: boolean(), ready_for_delete = false ::boolean(), penciller :: pid()}). @@ -334,16 +333,14 @@ handle_call(close, _From, State) -> handle_call(clear, _From, State) -> {stop, normal, ok, State#state{ready_for_delete=true}}; handle_call(background_complete, _From, State) -> - case State#state.background_complete of - true -> + if + State#state.background_complete == true -> {reply, {ok, State#state.filename, State#state.smallest_key, State#state.highest_key}, - State}; - false -> - {reply, {error, State#state.background_failure}, State} + State} end; handle_call({set_for_delete, Penciller}, _From, State) -> io:format("File ~s has been set for delete~n", [State#state.filename]), @@ -419,25 +416,20 @@ statecheck_onreply(Reply, State) -> create_levelzero(ListForFile, Filename) -> {TmpFilename, PrmFilename} = generate_filenames(Filename), - case create_file(TmpFilename) of - {error, Reason} -> - {error, - #state{background_complete=false, background_failure=Reason}}; - {Handle, FileMD} -> - InputSize = length(ListForFile), - io:format("Creating file with input of size ~w~n", [InputSize]), - Rename = {true, TmpFilename, PrmFilename}, - {ReadHandle, - UpdFileMD, - {[], []}} = complete_file(Handle, FileMD, - ListForFile, [], - #level{level=0}, Rename), - {ok, - UpdFileMD#state{handle=ReadHandle, - filename=PrmFilename, - background_complete=true, - oversized_file=InputSize>?MAX_KEYS}} - end. + {Handle, FileMD} = create_file(TmpFilename), + InputSize = length(ListForFile), + io:format("Creating file with input of size ~w~n", [InputSize]), + Rename = {true, TmpFilename, PrmFilename}, + {ReadHandle, + UpdFileMD, + {[], []}} = complete_file(Handle, FileMD, + ListForFile, [], + #level{level=0}, Rename), + {ok, + UpdFileMD#state{handle=ReadHandle, + filename=PrmFilename, + background_complete=true, + oversized_file=InputSize>?MAX_KEYS}}. generate_filenames(RootFilename) -> @@ -461,19 +453,13 @@ generate_filenames(RootFilename) -> create_file(FileName) when is_list(FileName) -> io:format("Opening file with filename ~s~n", [FileName]), ok = filelib:ensure_dir(FileName), - case file:open(FileName, [binary, raw, read, write]) of - {ok, Handle} -> - Header = create_header(initial), - {ok, _} = file:position(Handle, bof), - ok = file:write(Handle, Header), - {ok, StartPos} = file:position(Handle, cur), - FileMD = #state{next_position=StartPos, filename=FileName}, - {Handle, FileMD}; - {error, Reason} -> - io:format("Error opening filename ~s with reason ~w", - [FileName, Reason]), - {error, Reason} - end. + {ok, Handle} = file:open(FileName, [binary, raw, read, write]), + Header = create_header(initial), + {ok, _} = file:position(Handle, bof), + ok = file:write(Handle, Header), + {ok, StartPos} = file:position(Handle, cur), + FileMD = #state{next_position=StartPos, filename=FileName}, + {Handle, FileMD}. create_header(initial) -> diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 78d19bd..ac8c436 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -11,9 +11,9 @@ rotating_objects/1]). all() -> [ - small_load_with2i %, - % query_count, - % rotating_objects + small_load_with2i, + query_count, + rotating_objects ]. From 9abc1d643a38b649f8c511ac23d4a82bb5e4cc0f Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 4 Nov 2016 14:53:19 +0000 Subject: [PATCH 150/167] Logging update Add SFT logs to logbase --- src/leveled_log.erl | 30 +++++++++++++++++++++++++++++- src/leveled_sft.erl | 36 ++++++++++++++---------------------- 2 files changed, 43 insertions(+), 23 deletions(-) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index d1e63b5..ca95b92 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -185,7 +185,35 @@ {"PM001", {info, "Indexed new cache entry with total L0 cache size now ~w"}}, {"PM002", - {info, "Completed dump of L0 cache to list of size ~w"}} + {info, "Completed dump of L0 cache to list of size ~w"}}, + + + {"SFT01", + {info, "Opened filename with name ~s"}}, + {"SFT02", + {info, "File ~s has been set for delete"}}, + {"SFT03", + {info, "File creation of L0 file ~s"}}, + {"SFT04", + {info, "File ~s prompting for delete status check"}}, + {"SFT05", + {info, "Exit called for reason ~w on filename ~s"}}, + {"SFT06", + {info, "Exit called and now clearing ~s"}}, + {"SFT07", + {info, "Creating file with input of size ~w"}}, + {"SFT08", + {info, "Renaming file from ~s to ~s"}}, + {"SFT09", + {warn, "Filename ~s already exists"}}, + {"SFT10", + {warn, "Rename rogue filename ~s to ~s"}}, + {"SFT11", + {error, "Segment filter failed due to ~s"}}, + {"SFT12", + {error, "Segment filter failed due to CRC check ~w did not match ~w"}}, + {"SFT13", + {error, "Segment filter failed due to ~s"}} ])). diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index d35f285..07afbf7 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -312,7 +312,7 @@ handle_call({sft_new, Filename, KL1, KL2, LevelR}, _From, _State) -> end; handle_call({sft_open, Filename}, _From, _State) -> {_Handle, FileMD} = open_file(#state{filename=Filename}), - io:format("Opened filename with name ~s~n", [Filename]), + leveled_log:log("SFT01", [Filename]), {reply, {ok, {FileMD#state.smallest_key, FileMD#state.highest_key}}, @@ -343,7 +343,7 @@ handle_call(background_complete, _From, State) -> State} end; handle_call({set_for_delete, Penciller}, _From, State) -> - io:format("File ~s has been set for delete~n", [State#state.filename]), + leveled_log:log("SFT02", [State#state.filename]), {reply, ok, State#state{ready_for_delete=true, @@ -356,8 +356,7 @@ handle_cast({sft_newfroml0cache, Filename, Slots, FetchFun}, _State) -> SW = os:timestamp(), Inp1 = leveled_pmem:to_list(Slots, FetchFun), {ok, State} = create_levelzero(Inp1, Filename), - io:format("File creation of L0 file ~s took ~w microseconds~n", - [Filename, timer:now_diff(os:timestamp(), SW)]), + leveled_log:log_timer("SFT03", [Filename], SW), {noreply, State}; handle_cast(close, State) -> {stop, normal, State}. @@ -365,8 +364,7 @@ handle_cast(close, State) -> handle_info(timeout, State) -> case State#state.ready_for_delete of true -> - io:format("File ~s prompting for delete status check~n", - [State#state.filename]), + leveled_log:log("SFT05", [State#state.filename]), ok = leveled_penciller:pcl_confirmdelete(State#state.penciller, State#state.filename), {noreply, State, ?DELETE_TIMEOUT}; @@ -375,12 +373,10 @@ handle_info(timeout, State) -> end. terminate(Reason, State) -> - io:format("Exit called for reason ~w on filename ~s~n", - [Reason, State#state.filename]), + leveled_log:log("SFT05", [Reason, State#state.filename]), case State#state.ready_for_delete of true -> - io:format("Exit called and now clearing ~s~n", - [State#state.filename]), + leveled_log:log("SFT06", [State#state.filename]), ok = file:close(State#state.handle), ok = case filelib:is_file(State#state.filename) of true -> @@ -418,7 +414,7 @@ create_levelzero(ListForFile, Filename) -> {TmpFilename, PrmFilename} = generate_filenames(Filename), {Handle, FileMD} = create_file(TmpFilename), InputSize = length(ListForFile), - io:format("Creating file with input of size ~w~n", [InputSize]), + leveled_log:log("SFT07", [InputSize]), Rename = {true, TmpFilename, PrmFilename}, {ReadHandle, UpdFileMD, @@ -451,7 +447,7 @@ generate_filenames(RootFilename) -> %% Start a bare file with an initial header and no further details %% Return the {Handle, metadata record} create_file(FileName) when is_list(FileName) -> - io:format("Opening file with filename ~s~n", [FileName]), + leveled_log:log("SFT01", [FileName]), ok = filelib:ensure_dir(FileName), {ok, Handle} = file:open(FileName, [binary, raw, read, write]), Header = create_header(initial), @@ -529,16 +525,14 @@ complete_file(Handle, FileMD, KL1, KL2, LevelR, Rename) -> {ReadHandle, UpdFileMD, KeyRemainders}. rename_file(OldName, NewName) -> - io:format("Renaming file from ~s to ~s~n", [OldName, NewName]), + leveled_log:log("SFT08", [OldName, NewName]), case filelib:is_file(NewName) of true -> - io:format("Filename ~s already exists~n", - [NewName]), + leveled_log:log("SFT09", [NewName]), AltName = filename:join(filename:dirname(NewName), filename:basename(NewName)) ++ ?DISCARD_EXT, - io:format("Rename rogue filename ~s to ~s~n", - [NewName, AltName]), + leveled_log:log("SFT10", [NewName, AltName]), ok = file:rename(NewName, AltName); false -> ok @@ -1257,7 +1251,7 @@ check_for_segments(SegFilter, SegmentList, CRCCheck) -> [0, 0, 0, 0], 0, Count, []) of {error_so_maybe_present, Reason} -> - io:format("Segment filter failed due to ~s~n", [Reason]), + leveled_log:log("SFT11", [Reason]), error_so_maybe_present; {OutputCheck, BlockList} when OutputCheck == CheckSum, BlockList == [] -> @@ -1265,9 +1259,7 @@ check_for_segments(SegFilter, SegmentList, CRCCheck) -> {OutputCheck, BlockList} when OutputCheck == CheckSum -> {maybe_present, BlockList}; {OutputCheck, _} -> - io:format("Segment filter failed due to CRC check~n - ~w did not match ~w~n", - [OutputCheck, CheckSum]), + leveled_log:log("SFT12", [OutputCheck, CheckSum]), error_so_maybe_present end; false -> @@ -1276,7 +1268,7 @@ check_for_segments(SegFilter, SegmentList, CRCCheck) -> lists:max(SegmentList), 0, Count, []) of {error_so_maybe_present, Reason} -> - io:format("Segment filter failed due to ~s~n", [Reason]), + leveled_log:log("SFT13", [Reason]), error_so_maybe_present; BlockList when BlockList == [] -> not_present; From 479dc3ac8053c890d9754c342670d5b194d56026 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 4 Nov 2016 15:56:57 +0000 Subject: [PATCH 151/167] Registering and releasing of Journal snapshots Added a test of journal compaction with a registered snapshot and it showed that the deleting of files did not correctly check the list of registerd snapshots. Corrected. --- src/leveled_inker.erl | 2 +- src/leveled_log.erl | 6 +++--- src/leveled_sft.erl | 2 +- test/end_to_end/basic_SUITE.erl | 24 +++++++++++++++++++++--- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 196919f..f96800a 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -273,7 +273,7 @@ handle_call({release_snapshot, Snapshot}, _From , State) -> {reply, ok, State#state{registered_snapshots=Rs}}; handle_call({confirm_delete, ManSQN}, _From, State) -> Reply = lists:foldl(fun({_R, SnapSQN}, Bool) -> - case SnapSQN < ManSQN of + case SnapSQN >= ManSQN of true -> Bool; false -> diff --git a/src/leveled_log.erl b/src/leveled_log.erl index ca95b92..5c91ef3 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -42,8 +42,8 @@ {"P0004", {info, "Remaining ledger snapshots are ~w"}}, {"P0005", - {info, "Delete confirmed as file ~s is removed from " ++ " - unreferenced files"}}, + {info, "Delete confirmed as file ~s is removed from " ++ + "unreferenced files"}}, {"P0006", {info, "Orphaned reply after timeout on L0 file write ~s"}}, {"P0007", @@ -233,7 +233,7 @@ log_timer(LogReference, Subs, StartTime) -> true -> MicroS = timer:now_diff(os:timestamp(), StartTime), {Unit, Time} = case MicroS of - MicroS when MicroS < 10000 -> + MicroS when MicroS < 1000 -> {"microsec", MicroS}; MicroS -> {"ms", MicroS div 1000} diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 07afbf7..1d83b20 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -364,7 +364,7 @@ handle_cast(close, State) -> handle_info(timeout, State) -> case State#state.ready_for_delete of true -> - leveled_log:log("SFT05", [State#state.filename]), + leveled_log:log("SFT05", [timeout, State#state.filename]), ok = leveled_penciller:pcl_confirmdelete(State#state.penciller, State#state.filename), {noreply, State, ?DELETE_TIMEOUT}; diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index d94db71..23273de 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -450,6 +450,15 @@ space_clear_ondelete(_Config) -> {ok, FNsA_J} = file:list_dir(RootPath ++ "/journal/journal_files"), io:format("Bookie created ~w journal files and ~w ledger files~n", [length(FNsA_J), length(FNsA_L)]), + + % Get an iterator to lock the inker during compaction + FoldObjectsFun = fun(B, K, V, Acc) -> [{B, K, testutil:riak_hash(V)}|Acc] + end, + {async, HTreeF1} = leveled_bookie:book_returnfolder(Book1, + {foldobjects_allkeys, + ?RIAK_TAG, + FoldObjectsFun}), + % Delete the keys SW2 = os:timestamp(), lists:foreach(fun({Bucket, Key}) -> ok = leveled_bookie:book_riakdelete(Book1, @@ -460,6 +469,9 @@ space_clear_ondelete(_Config) -> KL1), io:format("Deletion took ~w microseconds for 80K keys~n", [timer:now_diff(os:timestamp(), SW2)]), + + + ok = leveled_bookie:book_compactjournal(Book1, 30000), F = fun leveled_bookie:book_islastcompactionpending/1, lists:foldl(fun(X, Pending) -> @@ -474,11 +486,17 @@ space_clear_ondelete(_Config) -> end end, true, lists:seq(1, 15)), - io:format("Waiting for journal deletes~n"), - timer:sleep(20000), + io:format("Waiting for journal deletes - blocked~n"), + timer:sleep(20000), + KeyHashList1 = HTreeF1(), + io:format("Key Hash List returned of length ~w~n", [length(KeyHashList1)]), + true = length(KeyHashList1) == 80000, + io:format("Waiting for journal deletes - unblocked~n"), + timer:sleep(20000), {ok, FNsB_L} = file:list_dir(RootPath ++ "/ledger/ledger_files"), {ok, FNsB_J} = file:list_dir(RootPath ++ "/journal/journal_files"), - {ok, FNsB_PC} = file:list_dir(RootPath ++ "/journal/journal_files/post_compact"), + {ok, FNsB_PC} = file:list_dir(RootPath + ++ "/journal/journal_files/post_compact"), PointB_Journals = length(FNsB_J) + length(FNsB_PC), io:format("Bookie has ~w journal files and ~w ledger files " ++ "after deletes~n", From c306d49a85cebe7c37660e653497b637c9b2ee9e Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 4 Nov 2016 16:11:11 +0000 Subject: [PATCH 152/167] CDB Logging Setup CDB logging to use log base --- src/leveled_cdb.erl | 36 +++++++++++++----------------------- src/leveled_log.erl | 29 ++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 2d9746f..783f545 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -233,7 +233,7 @@ init([Opts]) -> #state{max_size=MaxSize, binary_mode=Opts#cdb_options.binary_mode}}. starting({open_writer, Filename}, _From, State) -> - io:format("Opening file for writing with filename ~s~n", [Filename]), + leveled_log:log("CDB01", [Filename]), {LastPosition, HashTree, LastKey} = open_active_file(Filename), {ok, Handle} = file:open(Filename, [sync | ?WRITE_OPS]), {reply, ok, writer, State#state{handle=Handle, @@ -242,7 +242,7 @@ starting({open_writer, Filename}, _From, State) -> filename=Filename, hashtree=HashTree}}; starting({open_reader, Filename}, _From, State) -> - io:format("Opening file for reading with filename ~s~n", [Filename]), + leveled_log:log("CDB02", [Filename]), {Handle, Index, LastKey} = open_for_readonly(Filename), {reply, ok, reader, State#state{handle=Handle, last_key=LastKey, @@ -329,7 +329,7 @@ rolling({return_hashtable, IndexList, HashTreeBin}, _From, State) -> ok = write_top_index_table(Handle, BasePos, IndexList), file:close(Handle), ok = rename_for_read(State#state.filename, NewName), - io:format("Opening file for reading with filename ~s~n", [NewName]), + leveled_log:log("CDB03", [NewName]), {NewHandle, Index, LastKey} = open_for_readonly(NewName), case State#state.deferred_delete of true -> @@ -449,9 +449,8 @@ delete_pending(timeout, State) -> case leveled_inker:ink_confirmdelete(State#state.inker, ManSQN) of true -> - io:format("Deletion confirmed for file ~s " - ++ "at ManifestSQN ~w~n", - [State#state.filename, ManSQN]), + leveled_log:log("CDB04", [State#state.filename, + ManSQN]), {stop, normal, State}; false -> {next_state, @@ -517,8 +516,7 @@ handle_info(_Msg, StateName, State) -> {next_state, StateName, State}. terminate(Reason, StateName, State) -> - io:format("Closing of filename ~s for Reason ~w~n", - [State#state.filename, Reason]), + leveled_log:log("CDB05", [State#state.filename, Reason]), case {State#state.handle, StateName} of {undefined, _} -> ok; @@ -574,9 +572,7 @@ open_active_file(FileName) when is_list(FileName) -> {ok, LastPosition} -> ok = file:close(Handle); {ok, EndPosition} -> - LogDetails = [LastPosition, EndPosition], - io:format("File to be truncated at last position of ~w " - "with end of file at ~w~n", LogDetails), + leveled_log:log("CDB06", [LastPosition, EndPosition]), {ok, _LastPosition} = file:position(Handle, LastPosition), ok = file:truncate(Handle), ok = file:close(Handle) @@ -738,8 +734,7 @@ hashtable_calc(HashTree, StartPos) -> StartPos, [], <<>>), - io:format("HashTree computed in ~w microseconds~n", - [timer:now_diff(os:timestamp(), SWC)]), + leveled_log:log_timer("CDB07", [], SWC), {IndexList, HashTreeBin}. %%%%%%%%%%%%%%%%%%%% @@ -751,10 +746,7 @@ determine_new_filename(Filename) -> rename_for_read(Filename, NewName) -> %% Rename file - io:format("Renaming file from ~s to ~s " ++ - "for which existence is ~w~n", - [Filename, NewName, - filelib:is_file(NewName)]), + leveled_log:log("CDB08", [Filename, NewName, filelib:is_file(NewName)]), file:rename(Filename, NewName). open_for_readonly(Filename) -> @@ -933,8 +925,7 @@ startup_filter(Key, ValueAsBin, Position, {Hashtree, LastKey}, _ExtractFun) -> scan_over_file(Handle, Position, FilterFun, Output, LastKey) -> case saferead_keyvalue(Handle) of false -> - io:format("Failure to read Key/Value at Position ~w" - ++ " in scan~n", [Position]), + leveled_log:log("CDB09", [Position]), {Position, Output}; {Key, ValueAsBin, KeyLength, ValueLength} -> NewPosition = case Key of @@ -1020,11 +1011,11 @@ crccheck_value(Value) when byte_size(Value) >4 -> Hash -> true; _ -> - io:format("CRC check failed due to mismatch ~n"), + leveled_log:log("CDB10", []), false end; crccheck_value(_) -> - io:format("CRC check failed due to size ~n"), + leveled_log:log("CDB11", []), false. %% Run a crc check filling out any values which don't fit on byte boundary @@ -1157,8 +1148,7 @@ perform_write_hash_tables(Handle, HashTreeBin, StartPos) -> ok = file:write(Handle, HashTreeBin), {ok, EndPos} = file:position(Handle, cur), ok = file:advise(Handle, StartPos, EndPos - StartPos, will_need), - io:format("HashTree written in ~w microseconds~n", - [timer:now_diff(os:timestamp(), SWW)]), + leveled_log:log_timer("CDB12", [], SWW), ok. diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 5c91ef3..d8ef86e 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -213,7 +213,34 @@ {"SFT12", {error, "Segment filter failed due to CRC check ~w did not match ~w"}}, {"SFT13", - {error, "Segment filter failed due to ~s"}} + {error, "Segment filter failed due to ~s"}}, + + + {"CDB01", + {info, "Opening file for writing with filename ~s"}}, + {"CDB02", + {info, "Opening file for reading with filename ~s"}}, + {"CDB03", + {info, "Re-opening file for reading with filename ~s"}}, + {"CDB04", + {info, "Deletion confirmed for file ~s at ManifestSQN ~w"}}, + {"CDB05", + {info, "Closing of filename ~s for Reason ~w"}}, + {"CDB06", + {info, "File to be truncated at last position of ~w with end of " + ++ "file at ~w"}}, + {"CDB07", + {info, "Hashtree computed"}}, + {"CDB08", + {info, "Renaming file from ~s to ~s for which existence is ~w"}}, + {"CDB09", + {info, "Failure to read Key/Value at Position ~w in scan"}}, + {"CDB10", + {info, "CRC check failed due to mismatch"}}, + {"CDB11", + {info, "CRC check failed due to size"}}, + {"CDB12", + {inof, "HashTree written"}} ])). From 2b8a37439d999cc772cc66fcc79feaf9fbdc77d1 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 4 Nov 2016 17:28:04 +0000 Subject: [PATCH 153/167] Log refinement - logging process IDs --- src/leveled_log.erl | 20 ++++++++++++-------- src/leveled_pclerk.erl | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index d8ef86e..d0ab7f5 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -47,7 +47,7 @@ {"P0006", {info, "Orphaned reply after timeout on L0 file write ~s"}}, {"P0007", - {info, "Sent release message for cloned Penciller following close for " + {debug, "Sent release message for cloned Penciller following close for " ++ "reason ~w"}}, {"P0008", {info, "Penciller closing for reason ~w"}}, @@ -105,7 +105,7 @@ {"PC005", {info, "Penciller's Clerk ~w shutdown now complete for reason ~w"}}, {"PC006", - {info, "Work prompted but none needed ~w"}}, + {info, "Work prompted but none needed"}}, {"PC007", {info, "Clerk prompting Penciller regarding manifest change"}}, {"PC008", @@ -249,11 +249,13 @@ log(LogReference, Subs) -> {ok, {LogLevel, LogText}} = dict:find(LogReference, ?LOGBASE), case lists:member(LogLevel, ?LOG_LEVEL) of true -> - io:format(LogReference ++ " " ++ LogText ++ "~n", Subs); + io:format(LogReference ++ " ~w " ++ LogText ++ "~n", + [self()|Subs]); false -> ok end. + log_timer(LogReference, Subs, StartTime) -> {ok, {LogLevel, LogText}} = dict:find(LogReference, ?LOGBASE), case lists:member(LogLevel, ?LOG_LEVEL) of @@ -265,15 +267,17 @@ log_timer(LogReference, Subs, StartTime) -> MicroS -> {"ms", MicroS div 1000} end, - io:format(LogReference ++ " " ++ LogText ++ " with time taken ~w " - ++ Unit ++ "~n", - Subs ++ [Time]); + io:format(LogReference ++ " ~w " ++ LogText + ++ " with time taken ~w " ++ Unit ++ "~n", + [self()|Subs] ++ [Time]); false -> ok end. + + %%%============================================================================ %%% Test %%%============================================================================ @@ -283,7 +287,7 @@ log_timer(LogReference, Subs, StartTime) -> -ifdef(TEST). log_test() -> - ?assertMatch(ok, log("D0001", [])), - ?assertMatch(ok, log_timer("D0001", [], os:timestamp())). + log("D0001", []), + log_timer("D0001", [], os:timestamp()). -endif. \ No newline at end of file diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index b79a243..6b6dfb6 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -162,7 +162,7 @@ code_change(_OldVsn, State, _Extra) -> requestandhandle_work(State) -> case leveled_penciller:pcl_workforclerk(State#state.owner) of none -> - leveled_log:log("PC006", [self()]), + leveled_log:log("PC006", []), {false, ?MAX_TIMEOUT}; WI -> {NewManifest, FilesToDelete} = merge(WI), From a251f3eab036668991dcc1c59354b05e7fe3145d Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 4 Nov 2016 18:20:00 +0000 Subject: [PATCH 154/167] Speed up query count test Less individual querys to make count will speed up this taste, without changing the nature of it --- test/end_to_end/iterator_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index ac8c436..c52cee9 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -340,7 +340,7 @@ count_termsonindex(Bucket, IdxField, Book, QType) -> Acc + Items end, 0, - lists:seq(1901, 2218)). + lists:seq(190, 221)). rotating_objects(_Config) -> From 2299e1ce9d2900c090f9219b2172f97ac694443f Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 4 Nov 2016 19:14:03 +0000 Subject: [PATCH 155/167] Prune unreachbale branches --- src/leveled_pclerk.erl | 4 +--- src/leveled_sft.erl | 22 +++++----------------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 6b6dfb6..c83922a 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -129,9 +129,7 @@ handle_call({manifest_change, confirm, Closing}, From, State) -> {noreply, State#state{work_item=null, change_pending=false}, ?MIN_TIMEOUT} - end; -handle_call(close, _From, State) -> - {stop, normal, ok, State}. + end. handle_cast(prompt, State) -> {noreply, State, ?MIN_TIMEOUT}. diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 1d83b20..b0468bb 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -362,14 +362,12 @@ handle_cast(close, State) -> {stop, normal, State}. handle_info(timeout, State) -> - case State#state.ready_for_delete of - true -> + if + State#state.ready_for_delete == true -> leveled_log:log("SFT05", [timeout, State#state.filename]), ok = leveled_penciller:pcl_confirmdelete(State#state.penciller, State#state.filename), - {noreply, State, ?DELETE_TIMEOUT}; - false -> - {noreply, State} + {noreply, State, ?DELETE_TIMEOUT} end. terminate(Reason, State) -> @@ -378,19 +376,9 @@ terminate(Reason, State) -> true -> leveled_log:log("SFT06", [State#state.filename]), ok = file:close(State#state.handle), - ok = case filelib:is_file(State#state.filename) of - true -> - file:delete(State#state.filename); - false -> - ok - end; + ok = file:delete(State#state.filename); _ -> - case State#state.handle of - undefined -> - ok; - Handle -> - file:close(Handle) - end + ok = file:close(Handle) end. code_change(_OldVsn, State, _Extra) -> From 34a90231e050a52fad7fbbd0571f5dcfa73d5746 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Fri, 4 Nov 2016 19:33:11 +0000 Subject: [PATCH 156/167] Prune dead branches --- src/leveled_iclerk.erl | 8 +------- src/leveled_sft.erl | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index f955ccb..5c69362 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -762,13 +762,7 @@ compact_empty_file_test() -> LedgerSrv1 = [{8, {o, "Bucket", "Key1", null}}, {2, {o, "Bucket", "Key2", null}}, {3, {o, "Bucket", "Key3", null}}], - LedgerFun1 = fun(Srv, Key, ObjSQN) -> - case lists:keyfind(ObjSQN, 1, Srv) of - {ObjSQN, Key} -> - true; - _ -> - false - end end, + LedgerFun1 = fun(_Srv, _Key, _ObjSQN) -> false end, Score1 = check_single_file(CDB2, LedgerFun1, LedgerSrv1, 9, 8, 4), ?assertMatch(100.0, Score1). diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index b0468bb..4833015 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -378,7 +378,7 @@ terminate(Reason, State) -> ok = file:close(State#state.handle), ok = file:delete(State#state.filename); _ -> - ok = file:close(Handle) + ok = file:close(State#state.handle) end. code_change(_OldVsn, State, _Extra) -> From 61c62692003dd4a3ed454a375037752c1a98598c Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 5 Nov 2016 11:22:27 +0000 Subject: [PATCH 157/167] Penciller back-pressure - Phase 1 There were issues with how the Penciller behaves under ehavy write pressure - most particularly where there are a large number of keys per update (i.e. 2i heavy objects). Most immediately the attempt to chekc whether the l0 file was ready slowed down the process of producing the L0 file - so back-pressure created more back-pressure. Going forward want to alter this most significantly as also the work queue can build up unsustainably. there needs to be some pausing prompted by the bookie on 'returned', and the use of 'returend when the work queue exceeds a threshold. --- include/leveled.hrl | 3 +- src/leveled_log.erl | 2 ++ src/leveled_penciller.erl | 73 +++++++++++++++------------------------ src/leveled_sft.erl | 16 +++++++-- 4 files changed, 44 insertions(+), 50 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 028eb95..e685a39 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -17,7 +17,8 @@ -record(sft_options, {wait = true :: boolean(), - expire_tombstones = false :: boolean()}). + expire_tombstones = false :: boolean(), + penciller :: pid()}). -record(penciller_work, {next_sqn :: integer(), diff --git a/src/leveled_log.erl b/src/leveled_log.erl index d0ab7f5..581e02c 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -93,6 +93,8 @@ {info, "Rename of manifest from ~s ~w to ~s ~w"}}, {"P0028", {info, "Adding cleared file ~s to deletion list"}}, + {"P0029", + {info, "L0 completion confirmed and will transition to not pending"}}, {"PC001", {info, "Penciller's clerk ~w started with owner ~w"}}, diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 81216e4..5c1ebce 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -229,6 +229,7 @@ pcl_checksequencenumber/3, pcl_workforclerk/1, pcl_promptmanifestchange/2, + pcl_confirml0complete/4, pcl_confirmdelete/2, pcl_close/1, pcl_registersnapshot/2, @@ -314,6 +315,9 @@ pcl_workforclerk(Pid) -> pcl_promptmanifestchange(Pid, WI) -> gen_server:cast(Pid, {manifest_change, WI}). +pcl_confirml0complete(Pid, FN, StartKey, EndKey) -> + gen_server:cast(Pid, {levelzero_complete, FN, StartKey, EndKey}). + pcl_confirmdelete(Pid, FileName) -> gen_server:cast(Pid, {confirm_delete, FileName}). @@ -360,13 +364,10 @@ handle_call({push_mem, PushedTree}, From, State=#state{is_snapshot=Snap}) % we mean value from the perspective of the Ledger, not the full value % stored in the Inker) % - % 2 - Check to see if there is a levelzero file pending. If so check if - % the levelzero file is complete. If it is complete, the levelzero tree - % can be flushed, the in-memory manifest updated, and the new tree can - % be accepted as the new levelzero cache. If not, the update must be - % returned. + % 2 - Check to see if there is a levelzero file pending. If so, the + % update must be returned. If not the update can be accepted % - % 3 - The Penciller can now reply to the Bookie to show that the push has + % 3 - The Penciller can now reply to the Bookie to show if the push has % been accepted % % 4 - Update the cache: @@ -375,47 +376,12 @@ handle_call({push_mem, PushedTree}, From, State=#state{is_snapshot=Snap}) % % Check the approximate size of the cache. If it is over the maximum size, % trigger a backgroun L0 file write and update state of levelzero_pending. - - SW = os:timestamp(), + SW = os:timestamp(), S = case State#state.levelzero_pending of true -> - L0Pid = State#state.levelzero_constructor, - case checkready(L0Pid) of - timeout -> - log_pushmem_reply(From, - {returned, - "L-0 persist pending"}, - SW), - State; - {ok, SrcFN, StartKey, EndKey} -> - log_pushmem_reply(From, - {ok, - "L-0 persist completed"}, - SW), - ManEntry = #manifest_entry{start_key=StartKey, - end_key=EndKey, - owner=L0Pid, - filename=SrcFN}, - UpdMan = lists:keystore(0, - 1, - State#state.manifest, - {0, [ManEntry]}), - LedgerSQN = State#state.ledger_sqn, - UpdState = State#state{manifest=UpdMan, - levelzero_pending=false, - persisted_sqn=LedgerSQN}, - % Prompt clerk to ask about work - do this for - % every L0 roll - ok = leveled_pclerk:clerk_prompt(State#state.clerk), - NewL0Index = leveled_pmem:new_index(), - update_levelzero(NewL0Index, - 0, - PushedTree, - LedgerSQN, - [], - UpdState) - end; + log_pushmem_reply(From, {returned, "L-0 persist pending"}, SW), + State; false -> log_pushmem_reply(From, {ok, "L0 memory updated"}, SW), update_levelzero(State#state.levelzero_index, @@ -506,7 +472,22 @@ handle_cast({confirm_delete, FileName}, State=#state{is_snapshot=Snap}) {noreply, State#state{unreferenced_files=UF1}}; _ -> {noreply, State} - end. + end; +handle_cast({levelzero_complete, FN, StartKey, EndKey}, State) -> + leveled_log:log("P0029", []), + ManEntry = #manifest_entry{start_key=StartKey, + end_key=EndKey, + owner=State#state.levelzero_constructor, + filename=FN}, + UpdMan = lists:keystore(0, 1, State#state.manifest, {0, [ManEntry]}), + % Prompt clerk to ask about work - do this for every L0 roll + ok = leveled_pclerk:clerk_prompt(State#state.clerk), + {noreply, State#state{levelzero_cache=[], + levelzero_pending=false, + levelzero_constructor=undefined, + levelzero_index=leveled_pmem:new_index(), + levelzero_size=0, + manifest=UpdMan}}. handle_info({_Ref, {ok, SrcFN, _StartKey, _EndKey}}, State) -> @@ -724,7 +705,7 @@ checkready(Pid) -> roll_memory(State, false) -> FileName = levelzero_filename(State), leveled_log:log("P0019", [FileName]), - Opts = #sft_options{wait=false}, + Opts = #sft_options{wait=false, penciller=self()}, PCL = self(), FetchFun = fun(Slot) -> pcl_fetchlevelzero(PCL, Slot) end, % FetchFun = fun(Slot) -> lists:nth(Slot, State#state.levelzero_cache) end, diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 4833015..f6cdeb1 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -245,7 +245,8 @@ sft_newfroml0cache(Filename, Slots, FetchFun, Options) -> {sft_newfroml0cache, Filename, Slots, - FetchFun}), + FetchFun, + Options#sft_options.penciller}), {ok, Pid, noreply} end. @@ -352,12 +353,21 @@ handle_call({set_for_delete, Penciller}, _From, State) -> handle_call(get_maxsqn, _From, State) -> statecheck_onreply(State#state.highest_sqn, State). -handle_cast({sft_newfroml0cache, Filename, Slots, FetchFun}, _State) -> +handle_cast({sft_newfroml0cache, Filename, Slots, FetchFun, PCL}, _State) -> SW = os:timestamp(), Inp1 = leveled_pmem:to_list(Slots, FetchFun), {ok, State} = create_levelzero(Inp1, Filename), leveled_log:log_timer("SFT03", [Filename], SW), - {noreply, State}; + case PCL of + undefined -> + {noreply, State}; + _ -> + ok = leveled_penciller:pcl_confirml0complete(PCL, + Filename, + State#state.smallest_key, + State#state.highest_key), + {noreply, State} + end; handle_cast(close, State) -> {stop, normal, State}. From 309ca2d6a1542ce70951deafffe5a72a1159c48d Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 5 Nov 2016 11:24:48 +0000 Subject: [PATCH 158/167] Handle closing without an active handle (again) --- src/leveled_sft.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index f6cdeb1..a4af4de 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -388,7 +388,12 @@ terminate(Reason, State) -> ok = file:close(State#state.handle), ok = file:delete(State#state.filename); _ -> - ok = file:close(State#state.handle) + case State#state.handle of + undefined -> + ok; + Handle -> + ok = file:close(Handle) + end end. code_change(_OldVsn, State, _Extra) -> From 87b5bd0b18db7b25372034df8fcc6177cbc727ab Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 5 Nov 2016 12:03:21 +0000 Subject: [PATCH 159/167] Set Persisted SQN (regression) As part of previous change had stopped setting the persisted SQN in the ledger - which stopped journal compaction from working) --- src/leveled_penciller.erl | 9 +++++---- src/leveled_sft.erl | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 5c1ebce..6f798bc 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -487,7 +487,8 @@ handle_cast({levelzero_complete, FN, StartKey, EndKey}, State) -> levelzero_constructor=undefined, levelzero_index=leveled_pmem:new_index(), levelzero_size=0, - manifest=UpdMan}}. + manifest=UpdMan, + persisted_sqn=State#state.ledger_sqn}}. handle_info({_Ref, {ok, SrcFN, _StartKey, _EndKey}}, State) -> @@ -531,10 +532,10 @@ terminate(Reason, State) -> State end, case {UpdState#state.levelzero_pending, - get_item(0, State#state.manifest, []), - State#state.levelzero_size} of + get_item(0, UpdState#state.manifest, []), + UpdState#state.levelzero_size} of {true, [], _} -> - ok = leveled_sft:sft_close(State#state.levelzero_constructor); + ok = leveled_sft:sft_close(UpdState#state.levelzero_constructor); {false, [], 0} -> leveled_log:log("P0009", []); {false, [], _N} -> diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index a4af4de..46958e0 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -362,10 +362,10 @@ handle_cast({sft_newfroml0cache, Filename, Slots, FetchFun, PCL}, _State) -> undefined -> {noreply, State}; _ -> - ok = leveled_penciller:pcl_confirml0complete(PCL, - Filename, - State#state.smallest_key, - State#state.highest_key), + leveled_penciller:pcl_confirml0complete(PCL, + Filename, + State#state.smallest_key, + State#state.highest_key), {noreply, State} end; handle_cast(close, State) -> From 9e410d65e3c69514f5c1f2dfa8e4d492d0cf55a1 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 5 Nov 2016 12:49:15 +0000 Subject: [PATCH 160/167] Filename issue - not recording full filename in manifest Broekn by change to get response on L0 completion, SFT was informing penciller of the filename passed in (without extension), not the completed one with the extension. --- src/leveled_sft.erl | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/leveled_sft.erl b/src/leveled_sft.erl index 46958e0..30f1e70 100644 --- a/src/leveled_sft.erl +++ b/src/leveled_sft.erl @@ -363,7 +363,7 @@ handle_cast({sft_newfroml0cache, Filename, Slots, FetchFun, PCL}, _State) -> {noreply, State}; _ -> leveled_penciller:pcl_confirml0complete(PCL, - Filename, + State#state.filename, State#state.smallest_key, State#state.highest_key), {noreply, State} @@ -388,12 +388,7 @@ terminate(Reason, State) -> ok = file:close(State#state.handle), ok = file:delete(State#state.filename); _ -> - case State#state.handle of - undefined -> - ok; - Handle -> - ok = file:close(Handle) - end + ok = file:close(State#state.handle) end. code_change(_OldVsn, State, _Extra) -> From 4556573a5cfc5843c1403b8deea9a4a53d5d744c Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 5 Nov 2016 13:42:44 +0000 Subject: [PATCH 161/167] Rationalise logging on push_mem --- src/leveled_log.erl | 2 -- src/leveled_penciller.erl | 54 +++++++++++++++++---------------------- 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 581e02c..714e7f0 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -35,8 +35,6 @@ {"P0001", {info, "Ledger snapshot ~w registered"}}, - {"P0002", - {info, "Handling of push completed with L0 cache size now ~w"}}, {"P0003", {info, "Ledger snapshot ~w released"}}, {"P0004", diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 6f798bc..f63d462 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -376,23 +376,20 @@ handle_call({push_mem, PushedTree}, From, State=#state{is_snapshot=Snap}) % % Check the approximate size of the cache. If it is over the maximum size, % trigger a backgroun L0 file write and update state of levelzero_pending. - - SW = os:timestamp(), - S = case State#state.levelzero_pending of - true -> - log_pushmem_reply(From, {returned, "L-0 persist pending"}, SW), - State; - false -> - log_pushmem_reply(From, {ok, "L0 memory updated"}, SW), - update_levelzero(State#state.levelzero_index, - State#state.levelzero_size, - PushedTree, - State#state.ledger_sqn, - State#state.levelzero_cache, - State) - end, - leveled_log:log_timer("P0002", [S#state.levelzero_size], SW), - {noreply, S}; + case State#state.levelzero_pending of + true -> + leveled_log:log("P0018", [returned, "L-0 persist pending"]), + {reply, returned, State}; + false -> + leveled_log:log("P0018", [ok, "L0 memory updated"]), + gen_server:reply(From, ok), + {noreply, update_levelzero(State#state.levelzero_index, + State#state.levelzero_size, + PushedTree, + State#state.ledger_sqn, + State#state.levelzero_cache, + State)} + end; handle_call({fetch, Key}, _From, State) -> {reply, fetch_mem(Key, @@ -648,10 +645,6 @@ start_from_file(PCLopts) -> end. -log_pushmem_reply(From, Reply, SW) -> - leveled_log:log_timer("P0018", [element(1,Reply), element(2,Reply)], SW), - gen_server:reply(From, element(1, Reply)). - update_levelzero(L0Index, L0Size, PushedTree, LedgerSQN, L0Cache, State) -> Update = leveled_pmem:add_to_index(L0Index, @@ -684,15 +677,6 @@ update_levelzero(L0Index, L0Size, PushedTree, LedgerSQN, L0Cache, State) -> end. - -checkready(Pid) -> - try - leveled_sft:sft_checkready(Pid) - catch - exit:{timeout, _} -> - timeout - end. - %% Casting a large object (the levelzero cache) to the gen_server did not lead %% to an immediate return as expected. With 32K keys in the TreeList it could %% take around 35-40ms. @@ -1661,4 +1645,14 @@ coverage_test() -> ok = pcl_close(PCLr), clean_testdir(RootPath). + +checkready(Pid) -> + try + leveled_sft:sft_checkready(Pid) + catch + exit:{timeout, _} -> + timeout + end. + + -endif. \ No newline at end of file From 7f456fa993a288bd3c7a02f62f2d32f777c37215 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 5 Nov 2016 14:04:45 +0000 Subject: [PATCH 162/167] Add back-pressure on work queue limit Previously under heavy load, as long as L0 was being cleared, the ledger woud keep accapting. Now there is a formla limit on how far behind the work queue (of compactions required at other levels) before the break is applied on new updates coming in). --- src/leveled_penciller.erl | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index f63d462..4681130 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -252,6 +252,7 @@ -define(MEMTABLE, mem). -define(MAX_TABLESIZE, 32000). -define(PROMPT_WAIT_ONL0, 5). +-define(WORKQUEUE_BACKLOG_TOLERANCE, 4). -record(state, {manifest = [] :: list(), @@ -275,7 +276,8 @@ snapshot_fully_loaded = false :: boolean(), source_penciller :: pid(), - ongoing_work = [] :: list()}). + ongoing_work = [] :: list(), + work_backlog = false :: boolean()}). %%%============================================================================ @@ -376,11 +378,14 @@ handle_call({push_mem, PushedTree}, From, State=#state{is_snapshot=Snap}) % % Check the approximate size of the cache. If it is over the maximum size, % trigger a backgroun L0 file write and update state of levelzero_pending. - case State#state.levelzero_pending of - true -> + case {State#state.levelzero_pending, State#state.work_backlog} of + {true, _} -> leveled_log:log("P0018", [returned, "L-0 persist pending"]), {reply, returned, State}; - false -> + {false, true} -> + leveled_log:log("P0018", [returned, "Merge tree work backlog"]), + {reply, returned, State}; + {false, false} -> leveled_log:log("P0018", [ok, "L0 memory updated"]), gen_server:reply(From, ok), {noreply, update_levelzero(State#state.levelzero_index, @@ -791,12 +796,13 @@ return_work(State, From) -> true -> false end, + Backlog = L >= ?WORKQUEUE_BACKLOG_TOLERANCE, case State#state.levelzero_pending of true -> % Once the L0 file is completed there will be more work % - so don't be busy doing other work now leveled_log:log("P0021", []), - {State, none}; + {State#state{work_backlog=Backlog}, none}; false -> %% No work currently outstanding %% Can allocate work @@ -815,10 +821,10 @@ return_work(State, From) -> ledger_filepath = FP, manifest_file = ManFile, target_is_basement = IsBasement}, - {State#state{ongoing_work=[WI]}, WI} + {State#state{ongoing_work=[WI], work_backlog=Backlog}, WI} end; _ -> - {State, none} + {State#state{work_backlog=false}, none} end. From 70dc637c97e33db52b3f991e3350227776e158bb Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 5 Nov 2016 14:31:10 +0000 Subject: [PATCH 163/167] Add slow offer support on write pressure When there is write pressure on the penciller and it returns to the bookie, the bookie will now punish the next PUT (and itself) with a pause. The longer the back-pressure state has been in place, the more frequent the pauses --- src/leveled_bookie.erl | 30 ++++++++++++++++++++++-------- src/leveled_inker.erl | 2 +- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 82abb86..04d4a5c 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -161,12 +161,14 @@ -define(LEDGER_FP, "ledger"). -define(SNAPSHOT_TIMEOUT, 300000). -define(CHECKJOURNAL_PROB, 0.2). +-define(SLOWOFFER_DELAY, 10). -record(state, {inker :: pid(), penciller :: pid(), cache_size :: integer(), ledger_cache :: gb_trees:tree(), - is_snapshot :: boolean()}). + is_snapshot :: boolean(), + slow_offer = false :: boolean()}). @@ -286,11 +288,24 @@ handle_call({put, Bucket, Key, Object, IndexSpecs, Tag, TTL}, From, State) -> ObjSize, {IndexSpecs, TTL}), Cache0 = addto_ledgercache(Changes, State#state.ledger_cache), + % If the previous push to memory was returned then punish this PUT with a + % delay. If the back-pressure in the Penciller continues, these delays + % will beocme more frequent + case State#state.slow_offer of + true -> + timer:sleep(?SLOWOFFER_DELAY); + false -> + ok + end, gen_server:reply(From, ok), - {ok, NewCache} = maybepush_ledgercache(State#state.cache_size, + case maybepush_ledgercache(State#state.cache_size, Cache0, - State#state.penciller), - {noreply, State#state{ledger_cache=NewCache}}; + State#state.penciller) of + {ok, NewCache} -> + {noreply, State#state{ledger_cache=NewCache, slow_offer=false}}; + {returned, NewCache} -> + {noreply, State#state{ledger_cache=NewCache, slow_offer=true}} + end; handle_call({get, Bucket, Key, Tag}, _From, State) -> LedgerKey = leveled_codec:to_ledgerkey(Bucket, Key, Tag), case fetch_head(LedgerKey, @@ -799,7 +814,7 @@ maybepush_ledgercache(MaxCacheSize, Cache, Penciller) -> ok -> {ok, gb_trees:empty()}; returned -> - {ok, Cache} + {returned, Cache} end; true -> {ok, Cache} @@ -809,10 +824,9 @@ maybepush_ledgercache(MaxCacheSize, Cache, Penciller) -> maybe_withjitter(CacheSize, MaxCacheSize) -> if CacheSize > MaxCacheSize -> - T = 2 * MaxCacheSize - CacheSize, - R = random:uniform(CacheSize), + R = random:uniform(7 * MaxCacheSize), if - R > T -> + (CacheSize - MaxCacheSize) > R -> true; true -> false diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index f96800a..5689274 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -122,7 +122,7 @@ -define(JOURNAL_FILEX, "cdb"). -define(MANIFEST_FILEX, "man"). -define(PENDING_FILEX, "pnd"). --define(LOADING_PAUSE, 5000). +-define(LOADING_PAUSE, 1000). -define(LOADING_BATCH, 1000). -record(state, {manifest = [] :: list(), From 376176eba36be45adc1cb7f69de33678efdae540 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 5 Nov 2016 14:35:01 +0000 Subject: [PATCH 164/167] Correct overlap in naming with Backlog --- src/leveled_penciller.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 4681130..2ddfbbd 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -788,8 +788,7 @@ return_work(State, From) -> case length(WorkQ) of L when L > 0 -> [{SrcLevel, Manifest}|OtherWork] = WorkQ, - Backlog = length(OtherWork), - leveled_log:log("P0020", [SrcLevel, From, Backlog]), + leveled_log:log("P0020", [SrcLevel, From, length(OtherWork)]), IsBasement = if SrcLevel + 1 == BasementL -> true; From a73c2331542e52096fcfaa6c22bce7d7e871e513 Mon Sep 17 00:00:00 2001 From: martinsumner Date: Sat, 5 Nov 2016 15:10:21 +0000 Subject: [PATCH 165/167] Correct the recording of excess work --- src/leveled_log.erl | 2 +- src/leveled_penciller.erl | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/leveled_log.erl b/src/leveled_log.erl index 714e7f0..f1779fc 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -73,7 +73,7 @@ {info, "Rolling level zero to filename ~s"}}, {"P0020", {info, "Work at Level ~w to be scheduled for ~w with ~w " - ++ "queue items outstanding"}}, + ++ "queue items outstanding at all levels"}}, {"P0021", {info, "Allocation of work blocked as L0 pending"}}, {"P0022", diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 2ddfbbd..f232e39 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -787,15 +787,16 @@ return_work(State, From) -> {WorkQ, BasementL} = assess_workqueue([], 0, State#state.manifest, 0), case length(WorkQ) of L when L > 0 -> - [{SrcLevel, Manifest}|OtherWork] = WorkQ, - leveled_log:log("P0020", [SrcLevel, From, length(OtherWork)]), + Excess = lists:foldl(fun({_, _, OH}, Acc) -> Acc+OH end, 0, WorkQ), + [{SrcLevel, Manifest, _Overhead}|_OtherWork] = WorkQ, + leveled_log:log("P0020", [SrcLevel, From, Excess]), IsBasement = if SrcLevel + 1 == BasementL -> true; true -> false end, - Backlog = L >= ?WORKQUEUE_BACKLOG_TOLERANCE, + Backlog = Excess >= ?WORKQUEUE_BACKLOG_TOLERANCE, case State#state.levelzero_pending of true -> % Once the L0 file is completed there will be more work @@ -1095,8 +1096,9 @@ assess_workqueue(WorkQ, LevelToAssess, Man, BasementLevel) -> maybe_append_work(WorkQ, Level, Manifest, MaxFiles, FileCount) when FileCount > MaxFiles -> - leveled_log:log("P0024", [FileCount - MaxFiles, Level]), - lists:append(WorkQ, [{Level, Manifest}]); + Overhead = FileCount - MaxFiles, + leveled_log:log("P0024", [Overhead, Level]), + lists:append(WorkQ, [{Level, Manifest, Overhead}]); maybe_append_work(WorkQ, _Level, _Manifest, _MaxFiles, _FileCount) -> WorkQ. @@ -1246,7 +1248,7 @@ compaction_work_assessment_test() -> {{o, "B2", "K3", null}, {o, "B4", "K4", null}, dummy_pid}], Manifest = [{0, L0}, {1, L1}], {WorkQ1, 1} = assess_workqueue([], 0, Manifest, 0), - ?assertMatch(WorkQ1, [{0, Manifest}]), + ?assertMatch([{0, Manifest, 1}], WorkQ1), L1Alt = lists:append(L1, [{{o, "B5", "K0001", null}, {o, "B5", "K9999", null}, dummy_pid}, @@ -1264,7 +1266,7 @@ compaction_work_assessment_test() -> dummy_pid}]), Manifest3 = [{0, []}, {1, L1Alt}], {WorkQ3, 1} = assess_workqueue([], 0, Manifest3, 0), - ?assertMatch(WorkQ3, [{1, Manifest3}]). + ?assertMatch([{1, Manifest3, 1}], WorkQ3). confirm_delete_test() -> Filename = 'test.sft', From a7ed3e4b857c2d8bba1bbe4bbb6112168d5f3151 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Sat, 5 Nov 2016 15:59:31 +0000 Subject: [PATCH 166/167] Trim dead branches Also an experiment with altering the slowoffer_delay. --- src/leveled_bookie.erl | 4 ++-- src/leveled_penciller.erl | 9 +++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 04d4a5c..0958ee9 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -161,7 +161,7 @@ -define(LEDGER_FP, "ledger"). -define(SNAPSHOT_TIMEOUT, 300000). -define(CHECKJOURNAL_PROB, 0.2). --define(SLOWOFFER_DELAY, 10). +-define(SLOWOFFER_DELAY, 5). -record(state, {inker :: pid(), penciller :: pid(), @@ -1185,4 +1185,4 @@ foldobjects_vs_hashtree_test() -> reset_filestructure(). --endif. \ No newline at end of file +-endif. diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index f232e39..287f122 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -16,7 +16,7 @@ %% may lose data but only in sequence from a particular sequence number. On %% startup the Penciller will inform the Bookie of the highest sequence number %% it has, and the Bookie should load any missing data from that point out of -%5 the journal. +%% the journal. %% %% -------- LEDGER --------- %% @@ -493,8 +493,7 @@ handle_cast({levelzero_complete, FN, StartKey, EndKey}, State) -> persisted_sqn=State#state.ledger_sqn}}. -handle_info({_Ref, {ok, SrcFN, _StartKey, _EndKey}}, State) -> - leveled_log:log("P0006", [SrcFN]), +handle_info(_Info, State) -> {noreply, State}. terminate(Reason, State=#state{is_snapshot=Snap}) when Snap == true -> @@ -1201,8 +1200,6 @@ update_deletions([ClearedFile|Tail], MSN, UnreferencedFiles) -> confirm_delete(Filename, UnreferencedFiles, RegisteredSnapshots) -> case lists:keyfind(Filename, 1, UnreferencedFiles) of - false -> - false; {Filename, Pid, MSN} -> LowSQN = lists:foldl(fun({_, SQN}, MinSQN) -> min(SQN, MinSQN) end, infinity, @@ -1662,4 +1659,4 @@ checkready(Pid) -> end. --endif. \ No newline at end of file +-endif. From 2d590c3077a872ff1945602ec4b957b3e70de00c Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Sat, 5 Nov 2016 17:50:28 +0000 Subject: [PATCH 167/167] More aggressive clerk --- src/leveled_pclerk.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index c83922a..b50c384 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -37,7 +37,7 @@ %% prompt erasing of the removed files). %% %% The clerk will not request work on timeout if the committing of a manifest -%5 change is pending confirmation. +%% change is pending confirmation. %% %% -------- TIMEOUTS --------- %% @@ -67,7 +67,7 @@ -include_lib("eunit/include/eunit.hrl"). -define(MAX_TIMEOUT, 2000). --define(MIN_TIMEOUT, 200). +-define(MIN_TIMEOUT, 50). -record(state, {owner :: pid(), change_pending=false :: boolean(), @@ -459,4 +459,4 @@ select_merge_file_test() -> ?assertMatch(FileRef, {{o, "B1", "K1"}, {o, "B3", "K3"}, dummy_pid}), ?assertMatch(NewManifest, [{0, []}, {1, L1}]). --endif. \ No newline at end of file +-endif.