From 11e4a22c3a7e6a510a4b24edac91e8d01b56cfae Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Fri, 25 Aug 2017 15:53:04 -0400 Subject: [PATCH 01/44] Set up M3 reporter project, build, and thrift compilation --- build.gradle | 17 +- checkstyles/suppressions.xml | 8 + checkstyles/uber_checks.xml | 4 + gradle/wrapper/gradle-wrapper.jar | Bin 54208 -> 54783 bytes m3/build.gradle | 34 ++++ .../java/com/uber/m3/tally/m3/M3Reporter.java | 182 ++++++++++++++++++ m3/src/main/thrift/m3.thrift | 72 +++++++ .../com/uber/m3/tally/m3/M3ReporterTest.java | 48 +++++ settings.gradle | 2 +- 9 files changed, 363 insertions(+), 4 deletions(-) create mode 100644 checkstyles/suppressions.xml create mode 100644 m3/build.gradle create mode 100644 m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java create mode 100644 m3/src/main/thrift/m3.thrift create mode 100644 m3/src/test/java/com/uber/m3/tally/m3/M3ReporterTest.java diff --git a/build.gradle b/build.gradle index 1fecca2..8c705ab 100644 --- a/build.gradle +++ b/build.gradle @@ -206,7 +206,8 @@ def fatJavadocJar(name, projects) { type: Javadoc, dependsOn: projects.collect{it + ':classes'} ]) { - source = projects.collect{ project(it).sourceSets.main.allSource } + classpath = files(projects.collect{ project(it).sourceSets.main.compileClasspath }) + source = projects.collect{ project(it).sourceSets.main.allJava } } return tasks.create([ @@ -221,19 +222,29 @@ def fatJavadocJar(name, projects) { } // -------------------------------------- -// StatsD fat jar +// StatsD reporter fat jar // -------------------------------------- def statsd_fat_jar_name = 'tally-statsd-fat' def statsd_fat_jar_projs = [':tally-core', ':tally-statsd'] +// -------------------------------------- +// M3 reporter fat jar +// -------------------------------------- + +def m3_fat_jar_name = 'tally-m3-fat' +def m3_fat_jar_projs = [':tally-core', ':tally-m3'] + // ------------------- // Fat jars to publish // ------------------- artifacts { archives fatJar(statsd_fat_jar_name, statsd_fat_jar_projs), fatSourcesJar(statsd_fat_jar_name, statsd_fat_jar_projs), - fatJavadocJar(statsd_fat_jar_name, statsd_fat_jar_projs) + fatJavadocJar(statsd_fat_jar_name, statsd_fat_jar_projs), + fatJar(m3_fat_jar_name, m3_fat_jar_projs), + fatSourcesJar(m3_fat_jar_name, m3_fat_jar_projs), + fatJavadocJar(m3_fat_jar_name, m3_fat_jar_projs) } nexusStaging {} diff --git a/checkstyles/suppressions.xml b/checkstyles/suppressions.xml new file mode 100644 index 0000000..c1dd743 --- /dev/null +++ b/checkstyles/suppressions.xml @@ -0,0 +1,8 @@ + + + + + + diff --git a/checkstyles/uber_checks.xml b/checkstyles/uber_checks.xml index 7789ab1..9a6a8f8 100644 --- a/checkstyles/uber_checks.xml +++ b/checkstyles/uber_checks.xml @@ -6,6 +6,10 @@ + + + + diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 37406edbf50e7cd96891199d9f0a9dd7c5683f92..b5a2e12bcaaf27ad5b24ee0d107ea6a5ecb61fe7 100644 GIT binary patch delta 6637 zcmZ8m2RxPk_rI=@k-hicD=RW9E8_|m5v~;_!lle|@kLg|lf6R7DqCju-XWVPBRdi$ z{!jPb|F5s_=k3HApNOFGyU2 zcoMbv#wwW7CVz9^bH{=F2lSc)4-M|;$J~F2qwf_if|v2p6!2PX5v`}!Ro(+&Wxfuz zbOmgfIZ@IfM8#9v&uyN6i`0EQN%R~5*M zy-5wKE6il>bQ4tX9PL<{iJbXLooCH2SG&z<(D;zd&~0C8&pgh0n<=HXAEFd$3cHNo zW;jRQu`&NCP~Y|F+EvOLF=oZdVSa&6rZAzAuW z0bcb%a=CRzqHlA(r6>?qdwQAzJ0Y7-13wQ8RNxC;>=VuM5@-}R_JHo&cbCSBk1gzz z&ys{3Isig++mZn`N-8y8wYCH?Eb$EPs}zizmQ!%jy;0fae7&sf*d>k-^+IH{NN7j( zM7k}iE9#_L_(g~xS;X!d%NaDkmO#)wd zJ z{h`yvDMrylrxo8@D^cMxKb=^T#3RCL)qSOPP?h`Lqa>}ZhZ0;?-QgTCr^gH@VUd;c z+)xV(PG(htDWrRRHbI}i=0Q5MLQ4>{R|z#w(^9L%<`DdgGVDcPbT4I6YZbCK{|>jE zsB?8ff7qo7+Xh+-_Pi3ZV_CQ4VjTn(zD3ZpqC2sACj{rKra3j@6EPtf1j51sfp9_q zgLYa#k(m%BX6J0<;gN1eWKHwY=QN|TUW(}%naPb(>s{Ms!t~BgYm$FrgCT)D!F1sw z3^UwYH*T8_bXQaa%uYO89+rnuuPUdF>7`X}TW*HZ66tN`i8_9mXiTlvim8O9R=cER ztjF^Go}W=I8rOJ4yf&b4y7Bw$$Jvh3-m&YGuTl_CRCpu-)jJz&2J!LIAC6OxH7<+y zt<=3~sdM&kYi9K}ZgahVB#H|=dVp(l z5*@hEvD5QqA#s3x?I({lI(oi5L(z&yexk4g!fdt?bgGS zs=$(?R20WpX6)HMWr2U3a)RsuWT@#YBB8(l^@ejofJ8r z!Y-eQrMh;nsJyy%R}Zc;rDSh)vwZD zQwuvb^V~0&iPgSA2B9D7>id|l}tq^k=S2v5Y)>YPc`XXL$#xZdB z^(q3fHgHr?z0994vhW@6t7exT3nOz_2isQ_d1ZHts~3sKzSrwb*RY^kwe2_WuWJTQ zH|@2do?hr=iy+NlTpWwm2-NRQPc9R*Uw*zZ@Nmf6gv;K`>Ra)#&&&p!VdO77>1fU3 zE2(@J(`&?T&k6P=uSXvvMw364w|pU*(c!n$e2JX*xT~m;&ZeFBWG1#%L6}J8Gs4~Q zO+14rcYee06D|IAc8WD8%Z}psgyOWRE1VQ-UMT9y1{;C(N$%uYtr-P-Sl$?7R!~^R z#z*VQ;7x*)DIUv4M;X8|yS~tVVZhUB=dw$(d;0ok-Oh?WgO{_2Iic_aHN>;#l9DS| zqPM(aVn|nV`R2CUdR~<9+&AIkgmH!@08~Om3S0c?4Vyo|G+aDt{vzmy5R0oVd+9N} z_W(6iW+=_KoZI_~EVe`$sBWBRR3O(O@RgKO5~VbJR2_3Dms~4hK8@&V991}dzSVzJ z5oL)3|Lvk7#MvDa8*zYNGyS>HQ>y8z&d4M+BT9MsLrArZ<`xd|7l#S)iU^;jNw_=z ztJm7@jkVn>YcI;39G#}we#Ajj`xNUdX8lmJ{)W^gl_6}~6{9@`-|lS{Go-~|?zzMW z?~uS1v6NfWtrsB!Os3~6{F;*ukD3~*pe{o8PRmhSt4R$uGntRpYe{(@W+F-s|4x1xL^mH@$fvB>y`d`7a);WVNSAjd?$QZ6X0-zbw7Qx3?K*F|N`e!^;mtIrGq(+bI9K_c`8l0j z9=JSz>+O_~5aGcon=YIr<5Ksq{9`It#sAnm{rtfGtM&>w*6$1hI=WPPXmTDjB@01 zh=AM9(ooxW)8yxV7%)sqULy4#p+QciQ8wnu3HyLLUy?m9v^(_& z71vp9_j9}lX#n>`IF}8vxo$5-QPq}HuDZd4v;ae3Wp5nNt)vmMXE;pi9TK%JH_X59-iU-9{BIHp56+B$@>bf z551xKbw~c|JAdi)NM6yz z_$Aq5LUwd{QB7?1Z`H%fc&)wJNmF++(HyvB|LD)|6Trlj&Cz|ivo)*YVHfd$Q$KQc5{nJ6>tiw_Y1%rk22aF zax)@W5>M)LT6150`(B-aViT-&E8~fOcjp+Ui?IHgXhpG}(=z(aN=RU_K*7HByK^Rw zP)mD(nK){$-&>L{Wbd2sRhIZkPYqsJsWx0MCtMM4l2li)`besXD(}@P zuu`1-tUR72pww7#a*C=KPh{iuvPoVP?vD8Qc_o76p#GvWV&L$sZ8FG~v8Z+Y%o;zV zLfBh7x7b^^_)O2~z;(2C4Z*>)U9z}PL2H3nKjvhes#)w?NT@WL&U$>ex+I!)rQT1A zH^$eq=I||@YItvM&5e80-&aK5JCo2V8rzWg2svMawo#N)wi2Myy6_L-kitt{WpGJP z%7mMEhi%{DUpD!l_SHb&6F7>0lAOVh?xa);SV*b%92u!4+zX=Gf6Y~F%N(NjeW49u z9!;?jxdVS6T4WFr{8h`?xR5{ISeQ2Ei?L!4?Qv@Tz=O!w_nqE{eytvKBCJ%0exz{T zco#}RU&-5;npHN3(ta=_;HvNysW4%Cf$Lre-=Jh=ScI#FenCN$Di^XA)~ z5w1Wto^1OvPO5HOyPIwU(VdA0Y?je3Q+8vUB_&hl^h6p5B9}XUj2S6K)^1Fub(FBH z?MH@%*4#nPkUFEDJ*^S7{naP>Xh7yxapNP=s;}lVhV^tcL*osrjFmM{pyQt=L_9N0 zfW!IQgTXFbA}U)0o$(2EH?u{92g49^7J&M%{RRJ`rK~e~?v~VH?iP_cdDrX`smSfs zva0;7Z*_bhxk<&}zbrizLge%z+CGlSuKx6*gf%O%e1E=#Qm#X(Advw_b(wog-7gPp z*?%6@u>Zr*EI>)sMyb0f0JuI6_l)Y6*TW-~;^6is=HHbrP>u$U979)DwSJo!zK@V= z3XnRY8eHm=d-`_q+HJ}%S-BsuG}^}`VH|8Ud?K``vN_S!4M8=FPZ@?4_G}vBO~+sF z5&JW^avQNZpn~3~3nEk>#2D=0J2*r&>@+JF=sZiTt?@D>&eDfa`uR|lCE^;yzwW5t zV+(qq&=m1w=}EAW{+hRn@>P_`HQmIWUy2zq@O-?b1m<_#^9dbdEh~DbvmKdpC$*_6 zespC&?vD&`cDOb17*sE+O5>~3vh(q>|MH<7`n)j|SdBVVpOqRtzJiA!T$s_bG&@iv zg@^dDx6M&D?GPMx(IYt?qv$Tom=^gS4`j#4iHWn3ZMm)I`N(&-$;(UkE^ghmF%`OV zA%fBCWfJnjt_Uk!ecNxB(pZt$#Py2^dtL1k=OJ7tO!y}9?6(;U(>5Xb9i-gjS3f_< zrF1Ob&^$sVv9Oc7vAlI@)sL!B3e>w6^hrd+Hz@WhPxtsI??-+|YIHd~3_ke$Z{uY6 z%x=b)X+X27d~PlhbS{4)N}kKN%8QlXlk0phVN%y++~aNZU`pTgLwLydStD5zEpiE2 zjj@ifihZ@|Wwj~l_QZ!HxKSs*hL59;-tc`lIT7%~K@Apn#eDK1*ixKJN%ac~JVl=1 z`AMvd-z?`RnTD(XP|mgJu}ktrMbusw>0Jo=ys>L87x!tB)*p9c*G+EF^VKq+i@k|! zzh>IHe`y(iq*Qusm=&+`pqR}Ybyk8Md$oB@`25i52~OU@2i~%IV`+!P>)bc(E0Q!3Wv|VXPNH)4pU~FT7f5pL61_gC7pkiTibr-DDGr% zJF9d8JInWlH9ZKv)KIlLPOz-s0w~Xdr?rz9{a_C?c!N~C3#G)hC&kO@U!^I%8$^V? z$TUh(2{C&l@w!#oiG}IWG0O5r>yg#@@5wPe@wqH;XweH^5`RA@_1#_Xy4!nr=sJ2l zw0E(0$9!Hsde%Z5bd^SZNBF~iTD87I_2J<qzV(t( ze6)Asif)n%>Z^u9ky?eswfiFU#fc?arUL0pg2|*nU=A(d_J*Wl!0|P9aK!JvhJnwd#o=P8w+_2lwFYmlV%7S% z7GilTacqPb+|8m>EN`-A2TR#Bu;HRTPII;x%Ey5~@^B##IbhF|6`%kvRH%7$W4-IQ zTfmBJR<2>G_;NUw`u$iQ>x@D`J{88CSI!tVq{x?0>{E(I!&rTUsY4hFAZvx;UG`+u z0jyfDaN&b_zCbwPGwMnWmM!QR_!|dG1Nl3F1L|6({x^aHzPHMwtpQSPVi>}{O$I|` zv?*bT(oQDexQ!n8)X9fokaRIYO$aapmM#{KKQaF^ha(3Qb`?ZOgI518`3u-d3?~pi za*^W?_|K%~@1W){6aq;l`v0*Fu+&9`2A@BM{{}D7|38@Q{GkL|+G)_Tc9YvFNU;1U zEu%r1y-F#>THR0A9*~Act5CeFfQ=q_w1rh*H zx>?cS)lE~6LJ%|qUR>wEyrLlZ62^*kCli!h4E#+8_;Zl~FFP?YA$Y)^!V2bWQxF0H zwf$>x%V5C-z@=_XZ2%Ts%s4mD4!jk=_;&~L9ZXP49U!Zn3s7NX0G@YXblV;|3a^1- zU;>W^-@i0NAj~ijEY0yJDR+GP7vew{m53maOXonEMrcdvtgL`O*4y)E``>7OGc*`H z=|Ge;bnP364K*{aIjLp%*#! zj=8ylpW8i{$e(K$=Rg_uKh{!juNNiUd-^KPbiqJfJJ_yGS9CmL);t~{Cpx+!?!zeRq@ z7Yn8m`EM6c9idzK3R-v!214=N{&=3Cg>r`Z(b^@QZ&ho9+VQ}Ks&Ni@FAxnZbmavs z`mkB19Atvh1*7@;g94ZU{TyO~%0;1h0Y%J!^B^V_)y<_INuc6FurQU*DVC1`!GZw! zJ34?3+c8+2=*k8_@C*o+JqH$zLxW$BV*1dzHhd0vGadxW|4*alh{I+as4)%P*ZJ0< zn~2uP-^+mUUu=X4DxQMod5myjc!R@CQ08=y#|@BC|~Xq0&Al5UVLDS=(-QeXuo1yLF$ z1^kv>{y#t7bLPx*?mYLocV_O)Iq&=}!EZ0Xr+lP@Pe2O-k&uAM{4OiTP;!x+_Txv- zc4DM-=$L-FZn@%t{*jJ-F{DCv0&L$8!}WWD*uMIb0V(OF(~!SUcKJj6IO%h*bULu^V!V3JecJ)hEcK$-Y1c+C-rCgwockf92?8 zU({9hBRV!?R`CO|^?DH(QY#j&M>Tq`OYBX%Sy>`Z4tMcyGetREiK(7rw=&^>mMT78 z8PjeF?nAX$n32~z+>fcRaG0aKca+R_cqnmC)Cg@BI%ckMjc@GuQS&;Km1HNFjeyo= zrRx0)nR35sHe$1;HBVB+l-joSqqQG(B(*cMdb~v6wHV2?$i3>XhH4Zmjcx0NU&o)@ zq1$-+a{Ebl0Hgf$7is;>j*2hTJ|Uxj)Wu-ejDjhc5AGhz{B3V}WbGtgWI88+&p7_e z%{TF?6fi%=_thnX!w4{vrWCX?Y=VN_CxF$oFnj9eca4aD+#Hx6`}3x((km zNuma~1f5gtqGN-qUi|SgmXxm949(g{g!Duo*9zB16l zqo6EfjP9L)@#r0xl&IfeHuVyIu^(0R;|F8daMk!yaWO6IQl+0s8!9?|#I((_+|yz( zNhhZb_3R)!G)X^Kt{QP)DvGa#ozI@_@^(-f$!HBKE#SQaz zbpg#XUl9{+Tq#f^Z~h(~`S`NUmaUbH2zj^TXjAz|7Kg9gHq7rFM#XKQ->n@cp2S@I z;J6a~r}eG4%KVbG+wYsB+y~QBW2~dWOWMXW%?pT@4Maa;e>iI9_z6e~8HadwBg3{s z_b7-ZBcW4-DCGHTLj1%{feRbrnG03V_GWeQH;`_wUQb4k@>ouKZ^KUXI>{l^R>orG zjMne}T)VbA^Ob|=+^RiuXGiNejly0J$QX~!i5i!&4~sJ4}TD*V4;e@C}w zAOTnEPvk!Q{kdNmtZls?#=-tl0K%gsYRj1!mML(zX;?ICCA+8H#bntO5hS48OLx^K z%z7NjYgn^vS}V}eq>~d-+oS^{tg$t1;TE^%v_q-9~ zgNHxj7NyE>|5`?O$5fu&pI>jyS?Kf}PHAZGy$M@6cs%~5LfHGA+FhQCY)?PqOvLK? z{Ugm=^>y_mFNFvDBURw32qLIwzAvgaU)W=haH4n7zTWo{be(3nfwgYQKINL%prn(# zRq3j-XNTv<&rGb$(is(of@|h*SA=BhE*07DI>QG`u0nmD9?B#HhiUL)L$Rd6$m70Wn|hrPYZI*xkfN~w4Vah0_4vV#2zH;z=7Quw5=N^j3YM1j?Ic?<0$_ZKMf zBHcHQh*2V;J?H=1D|;Mku~ z_qMljFjBN^s7EO)5)ow<_w$?ov+*jS%igyV_rl&e45;}8f=196#Ppcdb)~rYxgOk0 z`M%P7v}G^$Zb|p!ZbK?Px|yI6sx&gUGn#^y*AF#*yO^h;)KZxvu;1qp>v&ClQ4Gq% z5pGogT^h+<{QS=G$56he=vVY(82TrnM_=R`L-H4nsWx%T zUdocIa`DgSypWTlh;LqFsUcrJ@da_EHK$ft)K!%7bGs<_3b_a_shD50;w1lY{~;xV zkZPBu|1E~Tp4Eh1TT=jD{{Aw!vP3hOIU!NI_~L}EX*Ikqw2-?lWnR*y1@3rpN?AJa zkIE5uKp*Y(@HIZ(h{oX(>XD4%LHLro`?NNj0-b{^IxEaW1MxKUA#xZ7J_)Ce-!a`05v!m!Xf<37TotBZLv6fV2S9~B{_aixot zp0bBIXHvOTJDyb~GS&2MTCUtxv7j+;g~OEjJWgG*1lwKaUX}r5ifadaQd3>aP*>EY zm$m?z{?hxX>-GezM%pFMv0?tI@*)jKf^644)75zcZt>*7he%5O*2SUk_Tzezw_EN$ z{?5}bm@TZV z&d>gmYGo11n#!o2<#XLmQ|PT(UUNcn=bqx)l66tG!c4kY5SN>Bd?ETk{k^bKi8_a# zmcgKwhvbdf+@xoo|>fc45NVM+S zirK@0v|$#eH&^r-N(|JX#J@6}s>REU<*RnpUf+5#RQR+^vZp z+9a$j`Wd8gufRNMj4S2vSH&8g4AD|`Bt@U;z_Vq#X4C&KG1IwcAd|Sj)t8Sl)ym2-BUH&VH+``2PP?@g& z&{xs#&r^4qj3FjrAM)^C9Z}2&@Gf{&QE{w>eJbVGFjzP$TsS~}=CO++YxS~mIOz9G zuU_!hr@xb|fL|^qQgE2_WoYroqSV%3w$x^Y&`cdBC(`k}1dS!-iJ`GS2~XUspuhW^ zAl|BoorLT+b{+Q})~}-Mv>13r9GP?*?-*+D1upmTYN%JQ{k`U)R{0xk`qAGPn(?vE zt2mF(ZsGb-smOflV20yJdk%w8r!13)F78)y3Jzn+jY5-y1G* zyiQeU2@=k>&KI+DhK?`zeDfZC68GX)L*~hz^@~x#Ps)13nUc_f*3PMGc>;jZ7=^&dM&MAC^hQ$yRr3kMcOPt;1^LBp%u-v=vte(h(-X zrlcepDqM>i9)_>XOXqhzh~Qf!FYOSVwNOOzHnX(J4;sx&x2Ky7vgv4Y2i&(R{nit) zTI~BAnmBCGUYs#(QFBGm%sn=|lUzrHg_I6aQ3o5IzC_Ar46Srn8{vGmc^LY*JykSAG8<&nPPfB%_s|tZ=q#dLA8pED@^!%a z^9u*GQl_t0RTN?`hk=-}v)<%l{Q@l|2QuAvSdQU0%{pE)wcMs23o_@1tjxWAw~k{` z*-ZhK57mr0C=S0g-#2s942^qQ>;3>MB2l2L4jN-L}7i(LvkrfHSme>&y>ryo} zb<+_KKXL}j+ZR}tIFf8Hn5pC(x-~H{qjI$NZDWbB`CFK<9|zJ|Tg%vqi$ZY04WWiy zk?#@w6dQhBy-~U0#Tt~n!J=X1m_(YXwf0358G45&-0_Rbb&udZ1Jv|4uusPyGW5)J zZ6LlMy>qmSc2DTQOVXrVDEzMQ1NMUCNs<0=QK&~bFOn+Zy`&D&3VT3EyJ)m8Re*Nh ze)gE!`nE<{(&{{@wp)1n+6F=&4^dC$9!y0y%)D;rM;}7I%(hNJ)x8q~n&?4^-AI{r zRLxY1W`ELNnPGh8BI&khhW#qZj9NQ8>1&y&ELgWBr>`IX0uEoJE955n7JJ; zx8+Nam0G*CCqeFPg@1BeVvWUwe_cjiu8kDYt^3@2OD?Sk{M(snI)pCH&q>JiFQe1r z%I=MQ^{;+fMXnnZZ;ZdPkVUz%tT#XB?;ls;imE%RJi{1yGX z=`1znU0h#|yJ9Q+`3HQU-w?Aeli4p3nZ=6Zm|;Bap{93|+UbCBTga zzLCVoq1NmIY8<-ql9dF9m^+5fJPJKx&vzegEhtVlBx*=dV8E}XH3NX}7fwNufg@9?uF*Z6tS%n7p+<#{G{Qv>q zqQ=@wKk(tWy*&^x9@8230-&nI2(b53{Clo(LeYcCnX%(1C=wtS#l?PF9;g5Bu>>HH z3@He74KuTUpFI>y_R@s@3p|5KQkfHDrw1O)1cId4<1<=$(fz9LGlfb_uFzAx-na)`}LCi$0Z((fofJqM{*10~+ zcqN+*1mb1Hv~$j#>R4xcP6hz+zg7K$S-mntto=eaH&&qk!V}pT%po}-0)hC?31}K) ziH?25fRQc@z~6^UcOhulh!sO*!L-Zg9IlxGZ~KJUPb>4(juFLvRA)V|i!zJthhpm( zI2wU~Tg`yVkDP$h2r+PHlo9J|M0ImC2jdJU27yG*`DR&S$(w_0IDt0<5OBN=#>N6% zbfy4q3{c?g{zDM3ofDS(Zb%p_Dr{sCm5&iy#x$ODqD-z>;;K6dATdaRC8yaN3>Dyk zK>3)nb>keF&<#sIO?PSpiX}(f5wrKhkjXJ*$#Z0|JD}Ar4LJAXRAB`26igVR)#=>M z5yw5Sa|j$@#^oQ_4*`pN0W$qez!U2${|S6E9fQqboEbL*VIh95SqiZ^V4@-Qx1 zP+kp#6*UNi!c+zKIWm7BhAfJUw1rtIbS&0ca2SrW{rv<1AEp4zV{+K}$<~qV$zbNz zi#aAz=Y$5*u#TI9M8LEM9pEr@w)`>}0kcd%wMF1`ORysPUXDp|n6y7&u8{ND*yLb{ zoB-?-?m7evh9TfPC0OBcwk)Su-vI+n~&XZ@TQvq#s ziZ5Tjuq}t#UnPu_&^d>eCX53M0C8aWSEE9h;@4>`NmoFJ$JIO^o!``=wmf_Wc5y|VxRJ&m~!{@Hq)$ diff --git a/m3/build.gradle b/m3/build.gradle new file mode 100644 index 0000000..e66ee73 --- /dev/null +++ b/m3/build.gradle @@ -0,0 +1,34 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +plugins { + id "org.jruyi.thrift" version "0.4.0" +} + +description = 'tally M3 reporter' + +dependencies { + compile('org.apache.thrift:libthrift:0.9.3') + compile project(':tally-core') +} + +compileThrift { + generator 'java', 'private-members' +} diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java new file mode 100644 index 0000000..b3e5903 --- /dev/null +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -0,0 +1,182 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.tally.m3; + +import com.uber.m3.tally.Buckets; +import com.uber.m3.tally.CachedCounter; +import com.uber.m3.tally.CachedGauge; +import com.uber.m3.tally.CachedHistogram; +import com.uber.m3.tally.CachedStatsReporter; +import com.uber.m3.tally.CachedTimer; +import com.uber.m3.tally.Capabilities; +import com.uber.m3.tally.CapableOf; +import com.uber.m3.util.ImmutableMap; + +import java.util.Map; + +public class M3Reporter implements CachedStatsReporter, AutoCloseable { + private static final int DEFAULT_MAX_QUEUE_SIZE = 4096; + private static final int DEFAULT_MAX_PACKET_SIZE = 1440; + private static final String DEFAULT_HISTOGRAM_BUCKET_ID_NAME = "bucketid"; + private static final String DEFAULT_HISTOGRAM_BUCKET_NAME = "bucket"; + private static final int DEFAULT_HISTOGRAM_BUCKET_TAG_PRECISION = 6; + + private String[] hostPorts; + private String service; + private String env; + private ImmutableMap commonTags; + private boolean includeHost; + //TODO protocol? + private int maxQueueSize; + private int maxPacketSizeBytes; + private String histogramBucketIdName; + private String histogramBucketName; + private int histogramBucketTagPrecision; + + private M3Reporter(Builder builder) { + hostPorts = builder.hostPorts; + service = builder.service; + env = builder.env; + commonTags = builder.commonTags; + includeHost = builder.includeHost; + maxQueueSize = builder.maxQueueSize; + maxPacketSizeBytes = builder.maxPacketSizeBytes; + histogramBucketIdName = builder.histogramBucketIdName; + histogramBucketName = builder.histogramBucketName; + histogramBucketTagPrecision = builder.histogramBucketTagPrecision; + } + + @Override + public CachedCounter allocateCounter(String name, Map tags) { + //TODO + return null; + } + + @Override + public CachedGauge allocateGauge(String name, Map tags) { + //TODO + return null; + } + + @Override + public CachedTimer allocateTimer(String name, Map tags) { + //TODO + return null; + } + + @Override + public CachedHistogram allocateHistogram(String name, Map tags, Buckets buckets) { + //TODO + return null; + } + + @Override + public Capabilities capabilities() { + return CapableOf.REPORTING_TAGGING; + } + + @Override + public void flush() { + //TODO + } + + @Override + public void close() { + //TODO + } + + public static class Builder { + private String[] hostPorts; + private String service; + private String env; + private ImmutableMap commonTags; + private boolean includeHost; + private int maxQueueSize = DEFAULT_MAX_QUEUE_SIZE; + private int maxPacketSizeBytes = DEFAULT_MAX_PACKET_SIZE; + private String histogramBucketIdName = DEFAULT_HISTOGRAM_BUCKET_ID_NAME; + private String histogramBucketName = DEFAULT_HISTOGRAM_BUCKET_NAME; + private int histogramBucketTagPrecision = DEFAULT_HISTOGRAM_BUCKET_TAG_PRECISION; + + public Builder hostPorts(String[] hostPorts) { + this.hostPorts = hostPorts; + + return this; + } + + public Builder service(String service) { + this.service = service; + + return this; + } + + public Builder env(String env) { + this.env = env; + + return this; + } + + public Builder commonTags(ImmutableMap commonTags) { + this.commonTags = commonTags; + + return this; + } + + public Builder includeHost(boolean includeHost) { + this.includeHost = includeHost; + + return this; + } + + public Builder maxQueueSize(int maxQueueSize) { + this.maxQueueSize = maxQueueSize; + + return this; + } + + public Builder maxPacketSizeBytes(int maxPacketSizeBytes) { + this.maxPacketSizeBytes = maxPacketSizeBytes; + + return this; + } + + public Builder histogramBucketIdName(String histogramBucketIdName) { + this.histogramBucketIdName = histogramBucketIdName; + + return this; + } + + public Builder histogramBucketName(String histogramBucketName) { + this.histogramBucketName = histogramBucketName; + + return this; + } + + public Builder histogramBucketTagPrecision(int histogramBucketTagPrecision) { + this.histogramBucketTagPrecision = histogramBucketTagPrecision; + + return this; + } + + public M3Reporter build() { + return new M3Reporter(this); + } + } +} diff --git a/m3/src/main/thrift/m3.thrift b/m3/src/main/thrift/m3.thrift new file mode 100644 index 0000000..294a371 --- /dev/null +++ b/m3/src/main/thrift/m3.thrift @@ -0,0 +1,72 @@ +namespace java com.uber.m3.thrift.generated + +/** +* Different types of count values +*/ +struct CountValue { + 1: optional i64 i64Value +} + +/** +* Different types of gauge values +*/ +struct GaugeValue { + 1: optional i64 i64Value + 2: optional double dValue +} + +/** +* Different types of timer values +*/ +struct TimerValue { + 1: optional i64 i64Value + 2: optional double dValue +} + +/** +* Different types of values that m3 emits. Each metric +* must contain one of these values +*/ +struct MetricValue { + 1: optional CountValue count + 2: optional GaugeValue gauge + 3: optional TimerValue timer +} + +/** +* Tags that can be applied to a metric +*/ +struct MetricTag { + 1: string tagName, + 2: optional string tagValue +} + +/** +* The metric that is being emitted +*/ +struct Metric { + 1: string name, + 2: optional MetricValue metricValue, + 3: optional i64 timestamp, + 4: optional set tags +} + +/** +* Structure that holds a group of metrics which share +* common properties like the cluster and service. +*/ +struct MetricBatch { + 1: list metrics + 2: optional set commonTags +} + +/** +* M3 Metrics Service +*/ +service M3 { + + /** + * Emits a batch of metrics. + */ + oneway void emitMetricBatch(1: MetricBatch batch) +} diff --git a/m3/src/test/java/com/uber/m3/tally/m3/M3ReporterTest.java b/m3/src/test/java/com/uber/m3/tally/m3/M3ReporterTest.java new file mode 100644 index 0000000..260c3d9 --- /dev/null +++ b/m3/src/test/java/com/uber/m3/tally/m3/M3ReporterTest.java @@ -0,0 +1,48 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.tally.m3; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class M3ReporterTest { + @BeforeClass + public static void start() { + + } + + @Before + public void setUp() { + + } + + @AfterClass + public static void stop() { + + } + + @Test + public void capabilities() { + + } +} diff --git a/settings.gradle b/settings.gradle index 2042209..cffc174 100644 --- a/settings.gradle +++ b/settings.gradle @@ -18,7 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -include 'core', 'example', 'statsd' +include 'core', 'example', 'statsd', 'm3' rootProject.name = 'tally' From 5d426d989b29faeab67cde83cff9942edfc334b7 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Fri, 25 Aug 2017 15:55:02 -0400 Subject: [PATCH 02/44] Update DEVELOPER.md about Thrift --- DEVELOPER.md | 1 + 1 file changed, 1 insertion(+) diff --git a/DEVELOPER.md b/DEVELOPER.md index 2127e5f..a41f7aa 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -7,6 +7,7 @@ This document summarizes information relevant to Java tally contributors. ### Prerequisites In order to build this project, you must have: - JDK-7 or later +- Apache Thrift 0.9.x ### Building and testing Gradle is used to build tally. Run this from the top-level directory to build the project: From fa23527959964ce8a8cb862e6f32840602c676d8 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Fri, 25 Aug 2017 17:53:20 -0400 Subject: [PATCH 03/44] Initial commit of UDP transport --- .../java/com/uber/m3/util/TUdpClient.java | 44 ++++++ .../java/com/uber/m3/util/TUdpServer.java | 53 +++++++ .../java/com/uber/m3/util/TUdpTransport.java | 145 ++++++++++++++++++ 3 files changed, 242 insertions(+) create mode 100644 m3/src/main/java/com/uber/m3/util/TUdpClient.java create mode 100644 m3/src/main/java/com/uber/m3/util/TUdpServer.java create mode 100644 m3/src/main/java/com/uber/m3/util/TUdpTransport.java diff --git a/m3/src/main/java/com/uber/m3/util/TUdpClient.java b/m3/src/main/java/com/uber/m3/util/TUdpClient.java new file mode 100644 index 0000000..2eb52c1 --- /dev/null +++ b/m3/src/main/java/com/uber/m3/util/TUdpClient.java @@ -0,0 +1,44 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.util; + +import org.apache.thrift.transport.TTransportException; + +import java.net.InetSocketAddress; +import java.net.SocketException; + +/** + * A client for sending data via Thrift UDP + */ +public class TUdpClient extends TUdpTransport implements AutoCloseable { + public TUdpClient(String host, int port) throws SocketException { + super(host, port); + } + + @Override + public void open() throws TTransportException { + try { + socket.connect(new InetSocketAddress(host, port)); + } catch (SocketException e) { + throw new TTransportException("Error opening transport", e); + } + } +} diff --git a/m3/src/main/java/com/uber/m3/util/TUdpServer.java b/m3/src/main/java/com/uber/m3/util/TUdpServer.java new file mode 100644 index 0000000..4b03b7b --- /dev/null +++ b/m3/src/main/java/com/uber/m3/util/TUdpServer.java @@ -0,0 +1,53 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.util; + +import org.apache.thrift.transport.TTransportException; + +import java.net.InetSocketAddress; +import java.net.SocketException; + +/** + * A server for receiving data via Thrift UDP + */ +public class TUdpServer extends TUdpTransport implements AutoCloseable { + private int timeoutMillis; + + public TUdpServer(String host, int port) throws SocketException { + this(host, port, 0); + } + + public TUdpServer(String host, int port, int timeoutMillis) throws SocketException { + super(host, port); + + this.timeoutMillis = timeoutMillis; + } + + @Override + public void open() throws TTransportException { + try { + socket.bind(new InetSocketAddress(host, port)); + socket.setSoTimeout(timeoutMillis); + } catch (SocketException e) { + throw new TTransportException("Error opening transport", e); + } + } +} diff --git a/m3/src/main/java/com/uber/m3/util/TUdpTransport.java b/m3/src/main/java/com/uber/m3/util/TUdpTransport.java new file mode 100644 index 0000000..a9b9a65 --- /dev/null +++ b/m3/src/main/java/com/uber/m3/util/TUdpTransport.java @@ -0,0 +1,145 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.util; + +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.SocketException; +import java.nio.ByteBuffer; + +/** + * Abstract class that supports Thrift UDP functionality + */ +public abstract class TUdpTransport extends TTransport implements AutoCloseable { + public static final int MAX_PACKET_SIZE = 14400; + public static final int MAX_BUFFER_SIZE = 65536; + + public final Object receiveLock = new Object(); + public final Object sendLock = new Object(); + + protected final DatagramSocket socket = new DatagramSocket(null);; + protected byte[] receiveBuf; + protected int receiveOffSet = -1; + protected int receiveLength = 0; + protected ByteBuffer writeButter; + + protected String host; + protected int port; + + protected TUdpTransport(String host, int port) throws SocketException { + this.host = host; + this.port = port; + } + + @Override + public boolean isOpen() { + return !socket.isClosed(); + } + + @Override + public abstract void open() throws TTransportException; + + @Override + public void close() { + socket.close(); + } + + @Override + public int getBytesRemainingInBuffer() { + return Integer.MAX_VALUE; + } + + @Override + public int read(byte[] bytes, int offset, int len) throws TTransportException { + synchronized (receiveLock) { + if (!isOpen()) { + throw new TTransportException(TTransportException.NOT_OPEN); + } + + if (receiveOffSet == -1) { + receiveBuf = new byte[MAX_BUFFER_SIZE]; + DatagramPacket dg = new DatagramPacket(receiveBuf, MAX_BUFFER_SIZE); + + try { + socket.receive(dg); + } catch (IOException e) { + throw new TTransportException("Error from underlying socket", e); + } + + receiveOffSet = 0; + receiveLength = dg.getLength(); + } + + int curDataSize = receiveLength - receiveOffSet; + + if (curDataSize <= len) { + System.arraycopy(receiveBuf, receiveOffSet, bytes, offset, curDataSize); + receiveOffSet = -1; + return curDataSize; + } else { + System.arraycopy(receiveBuf, receiveOffSet, bytes, offset, len); + receiveOffSet += len; + return len; + } + } + } + + @Override + public void write(byte[] bytes, int offset, int len) throws TTransportException { + synchronized (sendLock) { + if (!isOpen()) { + throw new TTransportException(TTransportException.NOT_OPEN); + } + if (writeButter == null) { + writeButter = ByteBuffer.allocate(MAX_BUFFER_SIZE); + } + if (writeButter.position() + len > MAX_PACKET_SIZE) { + throw new TTransportException("Message size too large: " + len + " > " + MAX_BUFFER_SIZE); + } + writeButter.put(bytes, offset, len); + } + } + + @Override + public void flush() throws TTransportException { + synchronized (sendLock) { + if (writeButter != null) { + byte[] bytes = new byte[MAX_BUFFER_SIZE]; + int len = writeButter.position(); + + writeButter.flip(); + writeButter.get(bytes, 0, len); + + try { + socket.send(new DatagramPacket(bytes, len)); + } catch (IOException e) { + throw new TTransportException("Cannot flush closed transport", e); + } finally { + writeButter = null; + } + } + } + } +} From d9785ee70b63112c7fbc67e8f2541a0c9160ffa1 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Fri, 25 Aug 2017 17:59:44 -0400 Subject: [PATCH 04/44] Use safer synchronized Objects over locks --- .../java/com/uber/m3/tally/ScopeImpl.java | 123 ++++++++---------- .../java/com/uber/m3/tally/TimerImpl.java | 2 +- 2 files changed, 58 insertions(+), 67 deletions(-) diff --git a/core/src/main/java/com/uber/m3/tally/ScopeImpl.java b/core/src/main/java/com/uber/m3/tally/ScopeImpl.java index cafcc36..6e4868e 100644 --- a/core/src/main/java/com/uber/m3/tally/ScopeImpl.java +++ b/core/src/main/java/com/uber/m3/tally/ScopeImpl.java @@ -29,7 +29,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; /** * Default {@link Scope} implementation. @@ -55,10 +54,10 @@ class ScopeImpl implements Scope { private Map timers = new ConcurrentHashMap<>(); private Map histograms = new ConcurrentHashMap<>(); - private Lock counterAllocationLock = new ReentrantLock(); - private Lock gaugeAllocationLock = new ReentrantLock(); - private Lock timerAllocationLock = new ReentrantLock(); - private Lock histogramAllocationLock = new ReentrantLock(); + private final Object counterAllocationLock = new Object(); + private final Object gaugeAllocationLock = new Object(); + private final Object timerAllocationLock = new Object(); + private final Object histogramAllocationLock = new Object(); // Private ScopeImpl constructor. Root scopes should be built using the RootScopeBuilder class ScopeImpl(ScheduledExecutorService scheduler, Registry registry, ScopeBuilder builder) { @@ -82,22 +81,20 @@ public Counter counter(String name) { return counter; } - counterAllocationLock.lock(); + synchronized(counterAllocationLock) { + if (!counters.containsKey(name)) { + CachedCounter cachedCounter = null; - if (!counters.containsKey(name)) { - CachedCounter cachedCounter = null; + if (cachedReporter != null) { + cachedCounter = cachedReporter.allocateCounter(fullyQualifiedName(name), tags); + } - if (cachedReporter != null) { - cachedCounter = cachedReporter.allocateCounter(fullyQualifiedName(name), tags); + counters.put(name, new CounterImpl(cachedCounter)); } - counters.put(name, new CounterImpl(cachedCounter)); + counter = counters.get(name); } - counter = counters.get(name); - - counterAllocationLock.unlock(); - return counter; } @@ -109,22 +106,20 @@ public Gauge gauge(String name) { return gauge; } - gaugeAllocationLock.lock(); + synchronized(gaugeAllocationLock) { + if (!gauges.containsKey(name)) { + CachedGauge cachedGauge = null; - if (!gauges.containsKey(name)) { - CachedGauge cachedGauge = null; + if (cachedReporter != null) { + cachedGauge = cachedReporter.allocateGauge(fullyQualifiedName(name), tags); + } - if (cachedReporter != null) { - cachedGauge = cachedReporter.allocateGauge(fullyQualifiedName(name), tags); + gauges.put(name, new GaugeImpl(cachedGauge)); } - gauges.put(name, new GaugeImpl(cachedGauge)); + gauge = gauges.get(name); } - gauge = gauges.get(name); - - gaugeAllocationLock.unlock(); - return gauge; } @@ -136,24 +131,22 @@ public Timer timer(String name) { return timer; } - timerAllocationLock.lock(); + synchronized(timerAllocationLock) { + if (!timers.containsKey(name)) { + CachedTimer cachedTimer = null; - if (!timers.containsKey(name)) { - CachedTimer cachedTimer = null; + String fullName = fullyQualifiedName(name); - String fullName = fullyQualifiedName(name); + if (cachedReporter != null) { + cachedTimer = cachedReporter.allocateTimer(fullName, tags); + } - if (cachedReporter != null) { - cachedTimer = cachedReporter.allocateTimer(fullName, tags); + timers.put(name, new TimerImpl(fullName, tags, reporter, cachedTimer)); } - timers.put(name, new TimerImpl(fullName, tags, reporter, cachedTimer)); + timer = timers.get(name); } - timer = timers.get(name); - - timerAllocationLock.unlock(); - return timer; } @@ -169,24 +162,22 @@ public Histogram histogram(String name, Buckets buckets) { return histogram; } - histogramAllocationLock.lock(); + synchronized(histogramAllocationLock) { + if (!histograms.containsKey(name)) { + CachedHistogram cachedHistogram = null; - if (!histograms.containsKey(name)) { - CachedHistogram cachedHistogram = null; + String fullName = fullyQualifiedName(name); - String fullName = fullyQualifiedName(name); + if (cachedReporter != null) { + cachedHistogram = cachedReporter.allocateHistogram(fullName, tags, buckets); + } - if (cachedReporter != null) { - cachedHistogram = cachedReporter.allocateHistogram(fullName, tags, buckets); + histograms.put(name, new HistogramImpl(fullName, tags, reporter, buckets, cachedHistogram)); } - histograms.put(name, new HistogramImpl(fullName, tags, reporter, buckets, cachedHistogram)); + histogram = histograms.get(name); } - histogram = histograms.get(name); - - histogramAllocationLock.unlock(); - return histogram; } @@ -398,25 +389,25 @@ private Scope subScopeHelper(String prefix, Map tags) { String key = keyForPrefixedStringMap(prefix, mergedTags); - registry.allocationLock.lock(); - - if (!registry.subscopes.containsKey(key)) { - registry.subscopes.put( - key, - new ScopeBuilder(scheduler, registry) - .reporter(reporter) - .cachedReporter(cachedReporter) - .prefix(prefix) - .separator(separator) - .tags(mergedTags) - .defaultBuckets(defaultBuckets) - .build() - ); - } - - Scope subscope = registry.subscopes.get(key); + Scope subscope; + + synchronized(registry.allocationLock) { + if (!registry.subscopes.containsKey(key)) { + registry.subscopes.put( + key, + new ScopeBuilder(scheduler, registry) + .reporter(reporter) + .cachedReporter(cachedReporter) + .prefix(prefix) + .separator(separator) + .tags(mergedTags) + .defaultBuckets(defaultBuckets) + .build() + ); + } - registry.allocationLock.unlock(); + subscope = registry.subscopes.get(key); + } return subscope; } @@ -445,7 +436,7 @@ public void run() { } static class Registry { - Lock allocationLock = new ReentrantLock(); + final Object allocationLock = new Object(); Map subscopes = new ConcurrentHashMap<>(); } } diff --git a/core/src/main/java/com/uber/m3/tally/TimerImpl.java b/core/src/main/java/com/uber/m3/tally/TimerImpl.java index 6ef1a8e..e900d79 100644 --- a/core/src/main/java/com/uber/m3/tally/TimerImpl.java +++ b/core/src/main/java/com/uber/m3/tally/TimerImpl.java @@ -94,7 +94,7 @@ Duration[] snapshot() { } static class Values { - private ReadWriteLock rwlock = new ReentrantReadWriteLock(); + private final ReadWriteLock rwlock = new ReentrantReadWriteLock(); private List values = new ArrayList<>(); Lock readLock() { From 6b749f468e297ee7e426a03773c1ada5c0c3e064 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Mon, 28 Aug 2017 10:18:08 -0400 Subject: [PATCH 05/44] Fix checkstyle errors --- core/src/main/java/com/uber/m3/tally/ScopeImpl.java | 11 +++++------ m3/src/main/java/com/uber/m3/util/TUdpTransport.java | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/uber/m3/tally/ScopeImpl.java b/core/src/main/java/com/uber/m3/tally/ScopeImpl.java index 6e4868e..db852be 100644 --- a/core/src/main/java/com/uber/m3/tally/ScopeImpl.java +++ b/core/src/main/java/com/uber/m3/tally/ScopeImpl.java @@ -28,7 +28,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.locks.Lock; /** * Default {@link Scope} implementation. @@ -81,7 +80,7 @@ public Counter counter(String name) { return counter; } - synchronized(counterAllocationLock) { + synchronized (counterAllocationLock) { if (!counters.containsKey(name)) { CachedCounter cachedCounter = null; @@ -106,7 +105,7 @@ public Gauge gauge(String name) { return gauge; } - synchronized(gaugeAllocationLock) { + synchronized (gaugeAllocationLock) { if (!gauges.containsKey(name)) { CachedGauge cachedGauge = null; @@ -131,7 +130,7 @@ public Timer timer(String name) { return timer; } - synchronized(timerAllocationLock) { + synchronized (timerAllocationLock) { if (!timers.containsKey(name)) { CachedTimer cachedTimer = null; @@ -162,7 +161,7 @@ public Histogram histogram(String name, Buckets buckets) { return histogram; } - synchronized(histogramAllocationLock) { + synchronized (histogramAllocationLock) { if (!histograms.containsKey(name)) { CachedHistogram cachedHistogram = null; @@ -391,7 +390,7 @@ private Scope subScopeHelper(String prefix, Map tags) { Scope subscope; - synchronized(registry.allocationLock) { + synchronized (registry.allocationLock) { if (!registry.subscopes.containsKey(key)) { registry.subscopes.put( key, diff --git a/m3/src/main/java/com/uber/m3/util/TUdpTransport.java b/m3/src/main/java/com/uber/m3/util/TUdpTransport.java index a9b9a65..395c96a 100644 --- a/m3/src/main/java/com/uber/m3/util/TUdpTransport.java +++ b/m3/src/main/java/com/uber/m3/util/TUdpTransport.java @@ -39,7 +39,7 @@ public abstract class TUdpTransport extends TTransport implements AutoCloseable public final Object receiveLock = new Object(); public final Object sendLock = new Object(); - protected final DatagramSocket socket = new DatagramSocket(null);; + protected final DatagramSocket socket = new DatagramSocket(null); protected byte[] receiveBuf; protected int receiveOffSet = -1; protected int receiveLength = 0; From 668e99a8ead695a76b1701248fb11a4cc415d218 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Tue, 29 Aug 2017 17:40:16 -0400 Subject: [PATCH 06/44] Big chunk of M3 reporter functionality --- .../java/com/uber/m3/tally/MetricType.java | 27 ++++++++++++ .../com/uber/m3/tally/m3/SizedMetric.java | 4 ++ .../com/uber/m3/tally/m3/ThriftProtocol.java | 6 +++ .../java/com/uber/m3/util/TCalcTransport.java | 41 +++++++++++++++++++ 4 files changed, 78 insertions(+) create mode 100644 m3/src/main/java/com/uber/m3/tally/MetricType.java create mode 100644 m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java create mode 100644 m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java create mode 100644 m3/src/main/java/com/uber/m3/util/TCalcTransport.java diff --git a/m3/src/main/java/com/uber/m3/tally/MetricType.java b/m3/src/main/java/com/uber/m3/tally/MetricType.java new file mode 100644 index 0000000..a571b63 --- /dev/null +++ b/m3/src/main/java/com/uber/m3/tally/MetricType.java @@ -0,0 +1,27 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.tally; + +public enum MetricType { + COUNTER, + GAUGE, + TIMER, +} diff --git a/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java b/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java new file mode 100644 index 0000000..c21e8ba --- /dev/null +++ b/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java @@ -0,0 +1,4 @@ +package com.uber.m3.tally.m3; + +public class SizedMetric { +} diff --git a/m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java b/m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java new file mode 100644 index 0000000..22bbcf9 --- /dev/null +++ b/m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java @@ -0,0 +1,6 @@ +package com.uber.m3.tally.m3; + +public enum TProtocol { + COMPACT, + BINARY +} diff --git a/m3/src/main/java/com/uber/m3/util/TCalcTransport.java b/m3/src/main/java/com/uber/m3/util/TCalcTransport.java new file mode 100644 index 0000000..e33a982 --- /dev/null +++ b/m3/src/main/java/com/uber/m3/util/TCalcTransport.java @@ -0,0 +1,41 @@ +package com.uber.m3.thrift.transport; + +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +/** + * Created by sliu on 12/23/15. + * for counting, very much not thread safe + */ +public class TCalcTransport extends TTransport { + + private int count = 0; + @Override + public boolean isOpen() { + return true; + } + + @Override + public void open() throws TTransportException { + } + + @Override + public void close() { + } + + @Override + public int read(byte[] bytes, int i, int i1) throws TTransportException { + return 0; + } + + @Override + public void write(byte[] bytes, int i, int len) throws TTransportException { + this.count += len; + } + + public int getWrittenCount() { + int tmp = this.count; + this.count = 0; + return tmp; + } +} From 53bf961fcd5a12527b814300c2a7b096f40bdd98 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Tue, 29 Aug 2017 17:43:04 -0400 Subject: [PATCH 07/44] Include missing changes from previous commit --- .../java/com/uber/m3/tally/ScopeImpl.java | 21 +- .../java/com/uber/m3/util/ImmutableSet.java | 36 ++ .../java/com/uber/m3/tally/m3/M3Reporter.java | 381 ++++++++++++++++-- .../com/uber/m3/tally/m3/SizedMetric.java | 40 +- .../com/uber/m3/tally/m3/ThriftProtocol.java | 22 +- .../java/com/uber/m3/util/TCalcTransport.java | 85 ++-- .../java/com/uber/m3/util/TUdpClient.java | 4 + .../java/com/uber/m3/util/TUdpTransport.java | 15 + 8 files changed, 523 insertions(+), 81 deletions(-) diff --git a/core/src/main/java/com/uber/m3/tally/ScopeImpl.java b/core/src/main/java/com/uber/m3/tally/ScopeImpl.java index db852be..a6b5a3d 100644 --- a/core/src/main/java/com/uber/m3/tally/ScopeImpl.java +++ b/core/src/main/java/com/uber/m3/tally/ScopeImpl.java @@ -48,15 +48,10 @@ class ScopeImpl implements Scope { // performance upside to the consequence of reporting a newly-made metric in // the middle of looping and reporting through all metrics. Therefore, locks // are generally only used when having to allocate new metrics. - private Map counters = new ConcurrentHashMap<>(); - private Map gauges = new ConcurrentHashMap<>(); - private Map timers = new ConcurrentHashMap<>(); - private Map histograms = new ConcurrentHashMap<>(); - - private final Object counterAllocationLock = new Object(); - private final Object gaugeAllocationLock = new Object(); - private final Object timerAllocationLock = new Object(); - private final Object histogramAllocationLock = new Object(); + private final Map counters = new ConcurrentHashMap<>(); + private final Map gauges = new ConcurrentHashMap<>(); + private final Map timers = new ConcurrentHashMap<>(); + private final Map histograms = new ConcurrentHashMap<>(); // Private ScopeImpl constructor. Root scopes should be built using the RootScopeBuilder class ScopeImpl(ScheduledExecutorService scheduler, Registry registry, ScopeBuilder builder) { @@ -80,7 +75,7 @@ public Counter counter(String name) { return counter; } - synchronized (counterAllocationLock) { + synchronized (counters) { if (!counters.containsKey(name)) { CachedCounter cachedCounter = null; @@ -105,7 +100,7 @@ public Gauge gauge(String name) { return gauge; } - synchronized (gaugeAllocationLock) { + synchronized (gauges) { if (!gauges.containsKey(name)) { CachedGauge cachedGauge = null; @@ -130,7 +125,7 @@ public Timer timer(String name) { return timer; } - synchronized (timerAllocationLock) { + synchronized (timers) { if (!timers.containsKey(name)) { CachedTimer cachedTimer = null; @@ -161,7 +156,7 @@ public Histogram histogram(String name, Buckets buckets) { return histogram; } - synchronized (histogramAllocationLock) { + synchronized (histograms) { if (!histograms.containsKey(name)) { CachedHistogram cachedHistogram = null; diff --git a/core/src/main/java/com/uber/m3/util/ImmutableSet.java b/core/src/main/java/com/uber/m3/util/ImmutableSet.java index 6c8f26d..006fa21 100644 --- a/core/src/main/java/com/uber/m3/util/ImmutableSet.java +++ b/core/src/main/java/com/uber/m3/util/ImmutableSet.java @@ -119,4 +119,40 @@ public boolean equals(Object other) { public int hashCode() { return set.hashCode(); } + + /** + * Helper class to construct {@link ImmutableSet}s. + * @param the type of elements in this set + */ + public static class Builder { + private HashSet set; + + public Builder() { + this(16, 0.75f); + } + + public Builder(int initialCapacity) { + this(initialCapacity, 1); + } + + public Builder(int initialCapacity, float loadFactor) { + set = new HashSet<>(initialCapacity, loadFactor); + } + + public Builder add(E element) { + set.add(element); + + return this; + } + + public Builder addAll(Set otherSet) { + set.addAll(otherSet); + + return this; + } + + public ImmutableSet build() { + return new ImmutableSet<>(set); + } + } } diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index b3e5903..e2c890c 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -28,58 +28,164 @@ import com.uber.m3.tally.CachedTimer; import com.uber.m3.tally.Capabilities; import com.uber.m3.tally.CapableOf; +import com.uber.m3.tally.MetricType; +import com.uber.m3.thrift.generated.CountValue; +import com.uber.m3.thrift.generated.GaugeValue; +import com.uber.m3.thrift.generated.M3; +import com.uber.m3.thrift.generated.Metric; +import com.uber.m3.thrift.generated.MetricBatch; +import com.uber.m3.thrift.generated.MetricTag; +import com.uber.m3.thrift.generated.MetricValue; +import com.uber.m3.thrift.generated.TimerValue; +import com.uber.m3.util.Duration; import com.uber.m3.util.ImmutableMap; - +import com.uber.m3.util.ImmutableSet; +import com.uber.m3.util.TCalcTransport; +import com.uber.m3.util.TUdpClient; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TTransport; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Phaser; public class M3Reporter implements CachedStatsReporter, AutoCloseable { + private static final int DEFAULT_METRIC_SIZE = 100; + private static final int DEFAULT_MAX_QUEUE_SIZE = 4096; private static final int DEFAULT_MAX_PACKET_SIZE = 1440; + + private static final String SERVICE_TAG = "service"; + private static final String ENV_TAG = "env"; + private static final String HOST_TAG = "host"; + private static final String DEFAULT_HISTOGRAM_BUCKET_ID_NAME = "bucketid"; private static final String DEFAULT_HISTOGRAM_BUCKET_NAME = "bucket"; private static final int DEFAULT_HISTOGRAM_BUCKET_TAG_PRECISION = 6; - private String[] hostPorts; - private String service; - private String env; - private ImmutableMap commonTags; - private boolean includeHost; - //TODO protocol? - private int maxQueueSize; - private int maxPacketSizeBytes; - private String histogramBucketIdName; - private String histogramBucketName; - private int histogramBucketTagPrecision; + private static final int EMIT_METRIC_BATCH_OVERHEAD = 19; + private static final int MIN_METRIC_BUCKET_ID_TAG_LENGTH = 4; + + private M3.Client client; + + private final MetricBatch metricBatch = new MetricBatch(); + + private TCalcTransport calc; + private TProtocol calcProtocol; + private final Object calcLock = new Object(); + + private ImmutableSet commonTags; + + private int freeBytes; + + private String bucketIdTagName; + private String bucketTagName; + private String bucketValFmt; + + private BlockingQueue metricQueue; + private Phaser phaser = new Phaser(1); private M3Reporter(Builder builder) { - hostPorts = builder.hostPorts; - service = builder.service; - env = builder.env; - commonTags = builder.commonTags; - includeHost = builder.includeHost; - maxQueueSize = builder.maxQueueSize; - maxPacketSizeBytes = builder.maxPacketSizeBytes; - histogramBucketIdName = builder.histogramBucketIdName; - histogramBucketName = builder.histogramBucketName; - histogramBucketTagPrecision = builder.histogramBucketTagPrecision; + try { + // Builder verifies non-null, non-empty hostPorts + String[] hostPorts = builder.hostPorts; + + TProtocolFactory protocolFactory; + if (builder.protocol == ThriftProtocol.COMPACT) { + protocolFactory = new TCompactProtocol.Factory(); + } else { + protocolFactory = new TBinaryProtocol.Factory(); + } + + TTransport transport = new TUdpClient(hostPorts[0]); + //TODO multi UDP client transport + + client = new M3.Client(protocolFactory.getProtocol(transport)); + + Set tags = metricTagSetFromMap(builder.commonTags); + + if (!builder.commonTags.containsKey(SERVICE_TAG)) { + if (builder.service == null || builder.service.isEmpty()) { + throw new IllegalArgumentException(String.format("%s common tag is required", SERVICE_TAG)); + } + + MetricTag tag = new MetricTag(SERVICE_TAG); + tag.setTagValue(builder.service); + tags.add(tag); + } + if (!builder.commonTags.containsKey(ENV_TAG)) { + if (builder.env == null || builder.env.isEmpty()) { + throw new IllegalArgumentException(String.format("%s common tag is required", ENV_TAG)); + } + + MetricTag tag = new MetricTag(ENV_TAG); + tag.setTagValue(builder.env); + tags.add(tag); + } + if (builder.includeHost) { + if (!builder.commonTags.containsKey(HOST_TAG)) { + String hostname = InetAddress.getLocalHost().getHostName(); + + MetricTag tag = new MetricTag(HOST_TAG); + tag.setTagValue(hostname); + tags.add(tag); + } + } + + commonTags = new ImmutableSet<>(tags); + + metricBatch.setCommonTags(commonTags); + calcProtocol = protocolFactory.getProtocol(new TCalcTransport()); + metricBatch.write(calcProtocol); + calc = (TCalcTransport) calcProtocol.getTransport(); + int numOverheadBytes = EMIT_METRIC_BATCH_OVERHEAD + calc.getCount(); + + freeBytes = builder.maxPacketSizeBytes - numOverheadBytes; + if (freeBytes <= 0) { + throw new IllegalArgumentException("Common tags serialized size exceeds packet size"); + } + + bucketIdTagName = builder.histogramBucketIdName; + bucketTagName = builder.histogramBucketName; + bucketValFmt = String.format("%%.%df", builder.histogramBucketTagPrecision); + + metricQueue = new LinkedBlockingQueue<>(builder.maxQueueSize); + + //TODO add to Phaser + } catch (Exception e) { + throw new IllegalArgumentException("Exception creating M3Reporter", e); + } } @Override public CachedCounter allocateCounter(String name, Map tags) { - //TODO - return null; + Metric counter = newMetric(name, tags, MetricType.COUNTER); + + return new CachedCounterImpl(counter, calculateSize(counter)); } @Override public CachedGauge allocateGauge(String name, Map tags) { - //TODO - return null; + Metric gauge = newMetric(name, tags, MetricType.GAUGE); + + return new CachedGaugeImpl(gauge, calculateSize(gauge)); } @Override public CachedTimer allocateTimer(String name, Map tags) { - //TODO - return null; + Metric timer = newMetric(name, tags, MetricType.TIMER); + + return new CachedTimerImpl(timer, calculateSize(timer)); } @Override @@ -95,12 +201,212 @@ public Capabilities capabilities() { @Override public void flush() { - //TODO + try { + metricQueue.put(new SizedMetric(null, 0)); + } catch (InterruptedException e) { + //TODO log exception? + } + } + + private void flush(List metrics) throws TException { + synchronized (metricBatch) { + try { + metricBatch.setMetrics(metrics); + client.emitMetricBatch(metricBatch); + } finally { + metricBatch.setMetrics(null); + } + } } @Override public void close() { - //TODO + phaser.arriveAndAwaitAdvance(); + } + + private static Metric newMetric(String name, Map tags, MetricType metricType) { + Metric metric = new Metric(name); + metric.setTimestamp(Long.MAX_VALUE); + + MetricValue metricValue = new MetricValue(); + + switch (metricType) { + case COUNTER: + CountValue countValue = new CountValue(); + countValue.setI64Value(Long.MAX_VALUE); + metricValue.setCount(countValue); + break; + case GAUGE: + GaugeValue gaugeValue = new GaugeValue(); + gaugeValue.setDValue(Double.MAX_VALUE); + metricValue.setGauge(gaugeValue); + break; + case TIMER: + TimerValue timerValue = new TimerValue(); + timerValue.setI64Value(Long.MAX_VALUE); + metricValue.setTimer(timerValue); + break; + } + + metric.setMetricValue(metricValue); + + metric.setTags(metricTagSetFromMap(tags)); + + return metric; + } + + private static Set metricTagSetFromMap(Map tags) { + if (tags == null) { + return null; + } + + Set metricTags = new HashSet<>(tags.size(), 1); + + for (Map.Entry tag : tags.entrySet()) { + MetricTag metricTag = new MetricTag(tag.getKey()); + metricTag.setTagValue(tag.getValue()); + + metricTags.add(metricTag); + } + + return metricTags; + } + + private int calculateSize(Metric metric) { + try { + synchronized (calcLock) { + metric.write(calcProtocol); + + return calc.getCount(); + } + } catch (TException e) { + //TODO log exception? + return DEFAULT_METRIC_SIZE; + } + } + + private void reportCopyMetric( + Metric metric, + int size, + MetricType metricType, + long iValue, + double dValue + ) { + Metric metricCopy = new Metric(metric.getName()); + metricCopy.setTags(metric.getTags()); + // Unfortunately, there is no finer time resolution for Java 7 + metricCopy.setTimestamp(System.currentTimeMillis() * Duration.NANOS_PER_MILLI); + metricCopy.setMetricValue(new MetricValue()); + + switch (metricType) { + case COUNTER: + CountValue countValue = new CountValue(); + countValue.setI64Value(iValue); + metricCopy.getMetricValue().setCount(countValue); + break; + case GAUGE: + GaugeValue gaugeValue = new GaugeValue(); + gaugeValue.setDValue(dValue); + metricCopy.getMetricValue().setGauge(gaugeValue); + break; + case TIMER: + TimerValue timerValue = new TimerValue(); + timerValue.setI64Value(iValue); + metricCopy.getMetricValue().setTimer(timerValue); + break; + } + + try { + metricQueue.put(new SizedMetric(metricCopy, size)); + } catch (InterruptedException e) { + //TODO log exception? + } + } + + private void process() { + try { + List metrics = new ArrayList<>(freeBytes / 10); + int bytes = 0; + + while (!metricQueue.isEmpty()) { + SizedMetric sizedMetric = metricQueue.poll(); + Metric metric = sizedMetric.getMetric(); + + if (metric == null) { + // Explicit flush requested + if (metrics.size() > 0) { + metrics.clear(); + bytes = 0; + } + + continue; + } + + int size = sizedMetric.getSize(); + + if (bytes + size > freeBytes) { + flush(metrics); + metrics.clear(); + bytes = 0; + } + + metrics.add(metric); + bytes += size; + } + + if (metrics.size() > 0) { + // Final flush + flush(metrics); + } + } catch (TException e) { + throw new IllegalStateException("Bad client state. Metrics to fail to flush", e); + } finally { + // Always arrive at phaser to prevent deadlock + phaser.arrive(); + } + } + + private abstract class CachedMetric { + Metric metric; + int size; + + CachedMetric(Metric metric, int size) { + this.metric = metric; + this.size = size; + } + } + + public class CachedCounterImpl extends CachedMetric implements CachedCounter { + private CachedCounterImpl(Metric metric, int size) { + super(metric, size); + } + + @Override + public void reportCount(long value) { + reportCopyMetric(metric, size, MetricType.COUNTER, value, 0); + } + } + + public class CachedGaugeImpl extends CachedMetric implements CachedGauge { + private CachedGaugeImpl(Metric metric, int size) { + super(metric, size); + } + + @Override + public void reportGauge(double value) { + reportCopyMetric(metric, size, MetricType.GAUGE, 0, value); + } + } + + public class CachedTimerImpl extends CachedMetric implements CachedTimer { + private CachedTimerImpl(Metric metric, int size) { + super(metric, size); + } + + @Override + public void reportTimer(Duration interval) { + reportCopyMetric(metric, size, MetricType.TIMER, interval.getNanos(), 0); + } } public static class Builder { @@ -109,16 +415,19 @@ public static class Builder { private String env; private ImmutableMap commonTags; private boolean includeHost; + private ThriftProtocol protocol; private int maxQueueSize = DEFAULT_MAX_QUEUE_SIZE; private int maxPacketSizeBytes = DEFAULT_MAX_PACKET_SIZE; private String histogramBucketIdName = DEFAULT_HISTOGRAM_BUCKET_ID_NAME; private String histogramBucketName = DEFAULT_HISTOGRAM_BUCKET_NAME; private int histogramBucketTagPrecision = DEFAULT_HISTOGRAM_BUCKET_TAG_PRECISION; - public Builder hostPorts(String[] hostPorts) { - this.hostPorts = hostPorts; + public Builder(String[] hostPorts) { + if (hostPorts == null || hostPorts.length == 0) { + throw new IllegalArgumentException("Must specify at least one host port"); + } - return this; + this.hostPorts = hostPorts; } public Builder service(String service) { @@ -145,6 +454,12 @@ public Builder includeHost(boolean includeHost) { return this; } + public Builder protocol(ThriftProtocol protocol) { + this.protocol = protocol; + + return this; + } + public Builder maxQueueSize(int maxQueueSize) { this.maxQueueSize = maxQueueSize; diff --git a/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java b/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java index c21e8ba..7e58116 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java @@ -1,4 +1,42 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package com.uber.m3.tally.m3; -public class SizedMetric { +import com.uber.m3.thrift.generated.Metric; + +public class SizedMetric extends Metric { + private Metric metric; + private int size; + + public SizedMetric(Metric metric, int size) { + // Recreate metric to prevent underlying metric from being modified + this.metric = new Metric(metric); + this.size = size; + } + + public Metric getMetric() { + return metric; + } + + public int getSize() { + return size; + } } diff --git a/m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java b/m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java index 22bbcf9..76f0c03 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java @@ -1,6 +1,26 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package com.uber.m3.tally.m3; -public enum TProtocol { +public enum ThriftProtocol { COMPACT, BINARY } diff --git a/m3/src/main/java/com/uber/m3/util/TCalcTransport.java b/m3/src/main/java/com/uber/m3/util/TCalcTransport.java index e33a982..6e03227 100644 --- a/m3/src/main/java/com/uber/m3/util/TCalcTransport.java +++ b/m3/src/main/java/com/uber/m3/util/TCalcTransport.java @@ -1,41 +1,60 @@ -package com.uber.m3.thrift.transport; +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.util; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; /** - * Created by sliu on 12/23/15. - * for counting, very much not thread safe + * Dummy transport used for calculating size only */ public class TCalcTransport extends TTransport { - - private int count = 0; - @Override - public boolean isOpen() { - return true; - } - - @Override - public void open() throws TTransportException { - } - - @Override - public void close() { - } - - @Override - public int read(byte[] bytes, int i, int i1) throws TTransportException { - return 0; - } - - @Override - public void write(byte[] bytes, int i, int len) throws TTransportException { - this.count += len; - } - - public int getWrittenCount() { - int tmp = this.count; - this.count = 0; - return tmp; - } + private int count = 0; + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void open() throws TTransportException { + } + + @Override + public void close() { + } + + @Override + public int read(byte[] bytes, int i, int i1) throws TTransportException { + return 0; + } + + @Override + public void write(byte[] bytes, int i, int len) throws TTransportException { + this.count += len; + } + + public int getCount() { + int tmp = this.count; + this.count = 0; + return tmp; + } } diff --git a/m3/src/main/java/com/uber/m3/util/TUdpClient.java b/m3/src/main/java/com/uber/m3/util/TUdpClient.java index 2eb52c1..d4338d5 100644 --- a/m3/src/main/java/com/uber/m3/util/TUdpClient.java +++ b/m3/src/main/java/com/uber/m3/util/TUdpClient.java @@ -33,6 +33,10 @@ public TUdpClient(String host, int port) throws SocketException { super(host, port); } + public TUdpClient(String hostPort) throws SocketException { + super(hostPort); + } + @Override public void open() throws TTransportException { try { diff --git a/m3/src/main/java/com/uber/m3/util/TUdpTransport.java b/m3/src/main/java/com/uber/m3/util/TUdpTransport.java index 395c96a..5d2965c 100644 --- a/m3/src/main/java/com/uber/m3/util/TUdpTransport.java +++ b/m3/src/main/java/com/uber/m3/util/TUdpTransport.java @@ -53,6 +53,21 @@ protected TUdpTransport(String host, int port) throws SocketException { this.port = port; } + protected TUdpTransport(String hostPort) throws SocketException { + try { + int colonIdx = hostPort.lastIndexOf(':'); + + if (colonIdx == -1) { + throw new IllegalArgumentException(); + } + + this.host = hostPort.substring(0, colonIdx); + this.port = Integer.parseInt(hostPort.substring(colonIdx + 1)); + } catch (Exception e) { + throw new IllegalArgumentException("Invalid host/port argument: " + hostPort, e); + } + } + @Override public boolean isOpen() { return !socket.isClosed(); From b6b543d8282039168969b82f396e6308dc57260b Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Wed, 30 Aug 2017 15:56:08 -0400 Subject: [PATCH 08/44] Add ExecutorService and ObjectPool --- .../java/com/uber/m3/tally/ScopeImpl.java | 1 + .../java/com/uber/m3/tally/m3/M3Reporter.java | 95 ++++++++++++------- .../main/java/com/uber/m3/util/Construct.java | 25 +++++ .../java/com/uber/m3/util/ObjectPool.java | 59 ++++++++++++ 4 files changed, 148 insertions(+), 32 deletions(-) create mode 100644 m3/src/main/java/com/uber/m3/util/Construct.java create mode 100644 m3/src/main/java/com/uber/m3/util/ObjectPool.java diff --git a/core/src/main/java/com/uber/m3/tally/ScopeImpl.java b/core/src/main/java/com/uber/m3/tally/ScopeImpl.java index a6b5a3d..e83b20e 100644 --- a/core/src/main/java/com/uber/m3/tally/ScopeImpl.java +++ b/core/src/main/java/com/uber/m3/tally/ScopeImpl.java @@ -255,6 +255,7 @@ void cachedReport(CachedStatsReporter cachedReporter) { } // Serializes a map to generate a key for a prefix/map combination + // Non-generic EMPTY ImmutableMap will never contain any elements @SuppressWarnings("unchecked") static String keyForPrefixedStringMap(String prefix, ImmutableMap stringMap) { if (prefix == null) { diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index e2c890c..2f4587c 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -56,15 +56,23 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Phaser; +/** + * An M3 reporter + */ public class M3Reporter implements CachedStatsReporter, AutoCloseable { private static final int DEFAULT_METRIC_SIZE = 100; private static final int DEFAULT_MAX_QUEUE_SIZE = 4096; private static final int DEFAULT_MAX_PACKET_SIZE = 1440; + private static final int THREAD_POOL_SIZE = 8; + private static final long THREAD_POOL_KEEPALIVE_SECS = 60; + private static final String SERVICE_TAG = "service"; private static final String ENV_TAG = "env"; private static final String HOST_TAG = "host"; @@ -94,7 +102,9 @@ public class M3Reporter implements CachedStatsReporter, AutoCloseable { private BlockingQueue metricQueue; private Phaser phaser = new Phaser(1); + private ExecutorService executor; + // Use inner Builder class to construct an M3Reporter private M3Reporter(Builder builder) { try { // Builder verifies non-null, non-empty hostPorts @@ -161,12 +171,20 @@ private M3Reporter(Builder builder) { metricQueue = new LinkedBlockingQueue<>(builder.maxQueueSize); - //TODO add to Phaser + executor = builder.executor; + + addAndRunProcessor(); } catch (Exception e) { throw new IllegalArgumentException("Exception creating M3Reporter", e); } } + public void addAndRunProcessor() { + phaser.register(); + + executor.execute(new Processor()); + } + @Override public CachedCounter allocateCounter(String name, Map tags) { Metric counter = newMetric(name, tags, MetricType.COUNTER); @@ -323,46 +341,49 @@ private void reportCopyMetric( } } - private void process() { - try { - List metrics = new ArrayList<>(freeBytes / 10); - int bytes = 0; + private class Processor implements Runnable { + @Override + public void run() { + try { + List metrics = new ArrayList<>(freeBytes / 10); + int bytes = 0; + + while (!metricQueue.isEmpty()) { + SizedMetric sizedMetric = metricQueue.poll(); + Metric metric = sizedMetric.getMetric(); + + if (metric == null) { + // Explicit flush requested + if (metrics.size() > 0) { + metrics.clear(); + bytes = 0; + } + + continue; + } - while (!metricQueue.isEmpty()) { - SizedMetric sizedMetric = metricQueue.poll(); - Metric metric = sizedMetric.getMetric(); + int size = sizedMetric.getSize(); - if (metric == null) { - // Explicit flush requested - if (metrics.size() > 0) { + if (bytes + size > freeBytes) { + flush(metrics); metrics.clear(); bytes = 0; } - continue; + metrics.add(metric); + bytes += size; } - int size = sizedMetric.getSize(); - - if (bytes + size > freeBytes) { + if (metrics.size() > 0) { + // Final flush flush(metrics); - metrics.clear(); - bytes = 0; } - - metrics.add(metric); - bytes += size; - } - - if (metrics.size() > 0) { - // Final flush - flush(metrics); + } catch (TException e) { + throw new IllegalStateException("Bad client state. Metrics to fail to flush", e); + } finally { + // Always arrive at phaser to prevent deadlock + phaser.arrive(); } - } catch (TException e) { - throw new IllegalStateException("Bad client state. Metrics to fail to flush", e); - } finally { - // Always arrive at phaser to prevent deadlock - phaser.arrive(); } } @@ -413,15 +434,19 @@ public static class Builder { private String[] hostPorts; private String service; private String env; - private ImmutableMap commonTags; + // Non-generic EMPTY ImmutableMap will never contain any elements + @SuppressWarnings("unchecked") + private ImmutableMap commonTags = ImmutableMap.EMPTY; private boolean includeHost; - private ThriftProtocol protocol; + private ThriftProtocol protocol = ThriftProtocol.COMPACT; private int maxQueueSize = DEFAULT_MAX_QUEUE_SIZE; private int maxPacketSizeBytes = DEFAULT_MAX_PACKET_SIZE; private String histogramBucketIdName = DEFAULT_HISTOGRAM_BUCKET_ID_NAME; private String histogramBucketName = DEFAULT_HISTOGRAM_BUCKET_NAME; private int histogramBucketTagPrecision = DEFAULT_HISTOGRAM_BUCKET_TAG_PRECISION; + private ExecutorService executor = Executors.newFixedThreadPool(THREAD_POOL_SIZE); + public Builder(String[] hostPorts) { if (hostPorts == null || hostPorts.length == 0) { throw new IllegalArgumentException("Must specify at least one host port"); @@ -490,6 +515,12 @@ public Builder histogramBucketTagPrecision(int histogramBucketTagPrecision) { return this; } + public Builder executor(ExecutorService executor) { + this.executor = executor; + + return this; + } + public M3Reporter build() { return new M3Reporter(this); } diff --git a/m3/src/main/java/com/uber/m3/util/Construct.java b/m3/src/main/java/com/uber/m3/util/Construct.java new file mode 100644 index 0000000..22592ad --- /dev/null +++ b/m3/src/main/java/com/uber/m3/util/Construct.java @@ -0,0 +1,25 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.util; + +public interface Construct { + E instance(); +} diff --git a/m3/src/main/java/com/uber/m3/util/ObjectPool.java b/m3/src/main/java/com/uber/m3/util/ObjectPool.java new file mode 100644 index 0000000..63c5217 --- /dev/null +++ b/m3/src/main/java/com/uber/m3/util/ObjectPool.java @@ -0,0 +1,59 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.util; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +public class ObjectPool { + private BlockingQueue objects; + private Construct construct; + + public ObjectPool(int capacity, Construct construct) { + this(capacity, construct, false); + } + + public ObjectPool(int capacity, Construct construct, boolean fair) { + objects = new ArrayBlockingQueue<>(capacity, fair); + + this.construct = construct; + } + + public void init() { + for (int i = 0; i < objects.size(); i++) { + objects.add(construct.instance()); + } + } + + public E get() throws InterruptedException { + if (objects.isEmpty()) { + // Rather than block + return construct.instance(); + } + + return objects.take(); + } + + public void put(E element) throws InterruptedException { + // Should in practice never block because + objects.put(element); + } +} From 11ba23d679048a689d6d9653e3a97b9e93909052 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Wed, 30 Aug 2017 17:55:13 -0400 Subject: [PATCH 09/44] Add ResourcePool --- .../java/com/uber/m3/tally/m3/M3Reporter.java | 5 +- .../com/uber/m3/tally/m3/ResourcePool.java | 218 ++++++++++++++++++ .../java/com/uber/m3/util/ObjectPool.java | 19 +- .../java/com/uber/m3/util/TCalcTransport.java | 18 +- 4 files changed, 246 insertions(+), 14 deletions(-) create mode 100644 m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 2f4587c..2b64e98 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -71,7 +71,6 @@ public class M3Reporter implements CachedStatsReporter, AutoCloseable { private static final int DEFAULT_MAX_PACKET_SIZE = 1440; private static final int THREAD_POOL_SIZE = 8; - private static final long THREAD_POOL_KEEPALIVE_SECS = 60; private static final String SERVICE_TAG = "service"; private static final String ENV_TAG = "env"; @@ -158,7 +157,7 @@ private M3Reporter(Builder builder) { calcProtocol = protocolFactory.getProtocol(new TCalcTransport()); metricBatch.write(calcProtocol); calc = (TCalcTransport) calcProtocol.getTransport(); - int numOverheadBytes = EMIT_METRIC_BATCH_OVERHEAD + calc.getCount(); + int numOverheadBytes = EMIT_METRIC_BATCH_OVERHEAD + calc.getCountAndReset(); freeBytes = builder.maxPacketSizeBytes - numOverheadBytes; if (freeBytes <= 0) { @@ -295,7 +294,7 @@ private int calculateSize(Metric metric) { synchronized (calcLock) { metric.write(calcProtocol); - return calc.getCount(); + return calc.getCountAndReset(); } } catch (TException e) { //TODO log exception? diff --git a/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java b/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java new file mode 100644 index 0000000..c86757c --- /dev/null +++ b/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java @@ -0,0 +1,218 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.tally.m3; + +import com.uber.m3.thrift.generated.CountValue; +import com.uber.m3.thrift.generated.GaugeValue; +import com.uber.m3.thrift.generated.Metric; +import com.uber.m3.thrift.generated.MetricBatch; +import com.uber.m3.thrift.generated.MetricTag; +import com.uber.m3.thrift.generated.MetricValue; +import com.uber.m3.thrift.generated.TimerValue; +import com.uber.m3.util.Construct; +import com.uber.m3.util.ObjectPool; +import com.uber.m3.util.TCalcTransport; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; + +import java.util.HashSet; + +public class ResourcePool { + private static final int BATCH_POOL_SIZE = 10; + private static final int PROTOCOL_POOL_SIZE = 10; + + private ObjectPool metricBatchPool; + private ObjectPool metricPool; + private ObjectPool tagPool; + private ObjectPool metricValuePool; + private ObjectPool counterPool; + private ObjectPool gaugePool; + private ObjectPool timerPool; + private ObjectPool protocolPool; + + ResourcePool(int maxQueueSize, final TProtocolFactory protocolFactory) { + metricBatchPool = new ObjectPool<>(BATCH_POOL_SIZE, new Construct() { + @Override + public MetricBatch instance() { + return new MetricBatch(); + } + }); + + metricPool = new ObjectPool<>(maxQueueSize, new Construct() { + @Override + public Metric instance() { + return new Metric(); + } + }); + + tagPool = new ObjectPool<>(maxQueueSize, new Construct() { + @Override + public MetricTag instance() { + return new MetricTag(); + } + }); + + metricValuePool = new ObjectPool<>(maxQueueSize, new Construct() { + @Override + public MetricValue instance() { + return new MetricValue(); + } + }); + + counterPool = new ObjectPool<>(maxQueueSize, new Construct() { + @Override + public CountValue instance() { + return new CountValue(); + } + }); + + gaugePool = new ObjectPool<>(maxQueueSize, new Construct() { + @Override + public GaugeValue instance() { + return new GaugeValue(); + } + }); + + timerPool = new ObjectPool<>(maxQueueSize, new Construct() { + @Override + public TimerValue instance() { + return new TimerValue(); + } + }); + + protocolPool = new ObjectPool<>(PROTOCOL_POOL_SIZE, new Construct() { + @Override + public TProtocol instance() { + return protocolFactory.getProtocol(new TCalcTransport()); + } + }); + } + + public MetricBatch getBatch() { + return metricBatchPool.get(); + } + + public Metric getMetric() { + return metricPool.get(); + } + + public HashSet getTagList() { + return new HashSet<>(); + } + + public MetricTag getTag() { + return tagPool.get(); + } + + public MetricValue getValue() { + return metricValuePool.get(); + } + + public CountValue getCounter() { + return counterPool.get(); + } + + public GaugeValue getGauge() { + return gaugePool.get(); + } + + public TimerValue getTimer() { + return timerPool.get(); + } + + public TProtocol getProtocol() { + return protocolPool.get(); + } + + public void releaseProtocol(TProtocol protocol) { + ((TCalcTransport) protocol.getTransport()).resetCount(); + + protocolPool.put(protocol); + } + + public void releaseBatch(MetricBatch metricBatch) { + metricBatch.setCommonTags(null); + + for (Metric metric : metricBatch.getMetrics()) { + releaseMetric(metric); + } + + metricBatch.setMetrics(null); + + metricBatchPool.put(metricBatch); + } + + public void releaseMetric(Metric metric) { + metric.setName(""); + + for (MetricTag tag : metric.getTags()) { + tag.setTagName(""); + tag.setTagValue(null); + tagPool.put(tag); + } + + metric.setTags(null); + + releaseShallowMetric(metric); + } + + public void releaseShallowMetric(Metric metric) { + metric.setTimestamp(0); + + MetricValue metricValue = metric.getMetricValue(); + + if (metricValue == null) { + metricPool.put(metric); + return; + } + + releaseMetricValue(metricValue); + metric.setMetricValue(null); + + metricPool.put(metric); + } + + public void releaseMetricValue(MetricValue metricValue) { + if (metricValue.isSetCount()) { + metricValue.getCount().setI64Value(0); + + counterPool.put(metricValue.getCount()); + + metricValue.unsetCount(); + } else if (metricValue.isSetGauge()) { + metricValue.getGauge().setI64Value(0); + metricValue.getGauge().setDValue(0); + + gaugePool.put(metricValue.getGauge()); + + metricValue.unsetGauge(); + } else if (metricValue.isSetTimer()) { + metricValue.getTimer().setI64Value(0); + metricValue.getTimer().setDValue(0); + + timerPool.put(metricValue.getTimer()); + + metricValue.unsetTimer(); + } + + metricValuePool.put(metricValue); + } +} diff --git a/m3/src/main/java/com/uber/m3/util/ObjectPool.java b/m3/src/main/java/com/uber/m3/util/ObjectPool.java index 63c5217..3e33f00 100644 --- a/m3/src/main/java/com/uber/m3/util/ObjectPool.java +++ b/m3/src/main/java/com/uber/m3/util/ObjectPool.java @@ -35,25 +35,30 @@ public ObjectPool(int capacity, Construct construct, boolean fair) { objects = new ArrayBlockingQueue<>(capacity, fair); this.construct = construct; - } - public void init() { for (int i = 0; i < objects.size(); i++) { objects.add(construct.instance()); } } - public E get() throws InterruptedException { + public E get() { if (objects.isEmpty()) { // Rather than block return construct.instance(); } - return objects.take(); + try { + return objects.take(); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } } - public void put(E element) throws InterruptedException { - // Should in practice never block because - objects.put(element); + public void put(E element) { + try { + objects.put(element); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } } } diff --git a/m3/src/main/java/com/uber/m3/util/TCalcTransport.java b/m3/src/main/java/com/uber/m3/util/TCalcTransport.java index 6e03227..18e1f78 100644 --- a/m3/src/main/java/com/uber/m3/util/TCalcTransport.java +++ b/m3/src/main/java/com/uber/m3/util/TCalcTransport.java @@ -49,12 +49,22 @@ public int read(byte[] bytes, int i, int i1) throws TTransportException { @Override public void write(byte[] bytes, int i, int len) throws TTransportException { - this.count += len; + count += len; + } + + public int getCountAndReset() { + int currentCount = count; + + resetCount(); + + return currentCount; } public int getCount() { - int tmp = this.count; - this.count = 0; - return tmp; + return count; + } + + public void resetCount() { + count = 0; } } From 66c988872797debb8b2980d153c6389a4535fc00 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Thu, 31 Aug 2017 13:58:19 -0400 Subject: [PATCH 10/44] Use ResourcePool instead of allocating new --- .../java/com/uber/m3/tally/m3/M3Reporter.java | 110 ++++++++++-------- .../com/uber/m3/tally/m3/ResourcePool.java | 53 +++++---- 2 files changed, 92 insertions(+), 71 deletions(-) diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 2b64e98..a18d7f6 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -39,7 +39,6 @@ import com.uber.m3.thrift.generated.TimerValue; import com.uber.m3.util.Duration; import com.uber.m3.util.ImmutableMap; -import com.uber.m3.util.ImmutableSet; import com.uber.m3.util.TCalcTransport; import com.uber.m3.util.TUdpClient; import org.apache.thrift.TException; @@ -51,7 +50,6 @@ import java.net.InetAddress; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -70,7 +68,7 @@ public class M3Reporter implements CachedStatsReporter, AutoCloseable { private static final int DEFAULT_MAX_QUEUE_SIZE = 4096; private static final int DEFAULT_MAX_PACKET_SIZE = 1440; - private static final int THREAD_POOL_SIZE = 8; + private static final int THREAD_POOL_SIZE = 10; private static final String SERVICE_TAG = "service"; private static final String ENV_TAG = "env"; @@ -85,16 +83,18 @@ public class M3Reporter implements CachedStatsReporter, AutoCloseable { private M3.Client client; - private final MetricBatch metricBatch = new MetricBatch(); + private MetricBatch metricBatch; + private final Object calcLock = new Object(); private TCalcTransport calc; private TProtocol calcProtocol; - private final Object calcLock = new Object(); - private ImmutableSet commonTags; + private Set commonTags; private int freeBytes; + private ResourcePool resourcePool; + private String bucketIdTagName; private String bucketTagName; private String bucketValFmt; @@ -120,41 +120,36 @@ private M3Reporter(Builder builder) { //TODO multi UDP client transport client = new M3.Client(protocolFactory.getProtocol(transport)); + resourcePool = new ResourcePool(builder.maxQueueSize, protocolFactory); - Set tags = metricTagSetFromMap(builder.commonTags); + commonTags = addMetricTagsToSet(builder.commonTags); + // Set and ensure required tags if (!builder.commonTags.containsKey(SERVICE_TAG)) { if (builder.service == null || builder.service.isEmpty()) { throw new IllegalArgumentException(String.format("%s common tag is required", SERVICE_TAG)); } - MetricTag tag = new MetricTag(SERVICE_TAG); - tag.setTagValue(builder.service); - tags.add(tag); + commonTags.add(createMetricTag(SERVICE_TAG, builder.service)); } if (!builder.commonTags.containsKey(ENV_TAG)) { if (builder.env == null || builder.env.isEmpty()) { throw new IllegalArgumentException(String.format("%s common tag is required", ENV_TAG)); } - MetricTag tag = new MetricTag(ENV_TAG); - tag.setTagValue(builder.env); - tags.add(tag); + commonTags.add(createMetricTag(ENV_TAG, builder.env)); } if (builder.includeHost) { if (!builder.commonTags.containsKey(HOST_TAG)) { - String hostname = InetAddress.getLocalHost().getHostName(); - - MetricTag tag = new MetricTag(HOST_TAG); - tag.setTagValue(hostname); - tags.add(tag); + commonTags.add(createMetricTag(HOST_TAG, InetAddress.getLocalHost().getHostName())); } } - commonTags = new ImmutableSet<>(tags); - + metricBatch = resourcePool.getMetricBatch(); metricBatch.setCommonTags(commonTags); - calcProtocol = protocolFactory.getProtocol(new TCalcTransport()); + metricBatch.setMetrics(new ArrayList()); + + calcProtocol = resourcePool.getProtocol(); metricBatch.write(calcProtocol); calc = (TCalcTransport) calcProtocol.getTransport(); int numOverheadBytes = EMIT_METRIC_BATCH_OVERHEAD + calc.getCountAndReset(); @@ -225,15 +220,24 @@ public void flush() { } } - private void flush(List metrics) throws TException { + private List flush(List metrics) throws TException { synchronized (metricBatch) { + metricBatch.setMetrics(metrics); + try { - metricBatch.setMetrics(metrics); client.emitMetricBatch(metricBatch); - } finally { - metricBatch.setMetrics(null); + } catch (TException e) { + //TODO log exception? } + + metricBatch.setMetrics(null); } + + resourcePool.releaseShallowMetrics(metrics); + + metrics.clear(); + + return metrics; } @Override @@ -241,25 +245,29 @@ public void close() { phaser.arriveAndAwaitAdvance(); } - private static Metric newMetric(String name, Map tags, MetricType metricType) { - Metric metric = new Metric(name); + private Metric newMetric(String name, Map tags, MetricType metricType) { + Metric metric = resourcePool.getMetric(); + metric.setName(name); + + metric.setTags(addMetricTagsToSet(tags)); + metric.setTimestamp(Long.MAX_VALUE); - MetricValue metricValue = new MetricValue(); + MetricValue metricValue = resourcePool.getMetricValue(); switch (metricType) { case COUNTER: - CountValue countValue = new CountValue(); + CountValue countValue = resourcePool.getCountValue(); countValue.setI64Value(Long.MAX_VALUE); metricValue.setCount(countValue); break; case GAUGE: - GaugeValue gaugeValue = new GaugeValue(); + GaugeValue gaugeValue = resourcePool.getGaugeValue(); gaugeValue.setDValue(Double.MAX_VALUE); metricValue.setGauge(gaugeValue); break; case TIMER: - TimerValue timerValue = new TimerValue(); + TimerValue timerValue = resourcePool.getTimerValue(); timerValue.setI64Value(Long.MAX_VALUE); metricValue.setTimer(timerValue); break; @@ -267,26 +275,32 @@ private static Metric newMetric(String name, Map tags, MetricTyp metric.setMetricValue(metricValue); - metric.setTags(metricTagSetFromMap(tags)); - return metric; } - private static Set metricTagSetFromMap(Map tags) { + private Set addMetricTagsToSet(Map tags) { + Set metricTagSet = resourcePool.getTagList(); + if (tags == null) { - return null; + return metricTagSet; } - Set metricTags = new HashSet<>(tags.size(), 1); - for (Map.Entry tag : tags.entrySet()) { - MetricTag metricTag = new MetricTag(tag.getKey()); - metricTag.setTagValue(tag.getValue()); + metricTagSet.add(createMetricTag(tag.getKey(), tag.getValue())); + } + + return metricTagSet; + } + + public MetricTag createMetricTag(String tagName, String tagValue) { + MetricTag metricTag = resourcePool.getMetricTag(); + metricTag.setTagName(tagName); - metricTags.add(metricTag); + if (tagValue != null && !tagValue.isEmpty()) { + metricTag.setTagValue(tagValue); } - return metricTags; + return metricTag; } private int calculateSize(Metric metric) { @@ -309,25 +323,26 @@ private void reportCopyMetric( long iValue, double dValue ) { - Metric metricCopy = new Metric(metric.getName()); + Metric metricCopy = resourcePool.getMetric(); + metricCopy.setName(metric.getName()); metricCopy.setTags(metric.getTags()); // Unfortunately, there is no finer time resolution for Java 7 metricCopy.setTimestamp(System.currentTimeMillis() * Duration.NANOS_PER_MILLI); - metricCopy.setMetricValue(new MetricValue()); + metricCopy.setMetricValue(resourcePool.getMetricValue()); switch (metricType) { case COUNTER: - CountValue countValue = new CountValue(); + CountValue countValue = resourcePool.getCountValue(); countValue.setI64Value(iValue); metricCopy.getMetricValue().setCount(countValue); break; case GAUGE: - GaugeValue gaugeValue = new GaugeValue(); + GaugeValue gaugeValue = resourcePool.getGaugeValue(); gaugeValue.setDValue(dValue); metricCopy.getMetricValue().setGauge(gaugeValue); break; case TIMER: - TimerValue timerValue = new TimerValue(); + TimerValue timerValue = resourcePool.getTimerValue(); timerValue.setI64Value(iValue); metricCopy.getMetricValue().setTimer(timerValue); break; @@ -364,8 +379,7 @@ public void run() { int size = sizedMetric.getSize(); if (bytes + size > freeBytes) { - flush(metrics); - metrics.clear(); + metrics = flush(metrics); bytes = 0; } diff --git a/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java b/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java index c86757c..ea21439 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java @@ -34,6 +34,7 @@ import org.apache.thrift.protocol.TProtocolFactory; import java.util.HashSet; +import java.util.List; public class ResourcePool { private static final int BATCH_POOL_SIZE = 10; @@ -41,11 +42,11 @@ public class ResourcePool { private ObjectPool metricBatchPool; private ObjectPool metricPool; - private ObjectPool tagPool; + private ObjectPool metricTagPool; private ObjectPool metricValuePool; - private ObjectPool counterPool; - private ObjectPool gaugePool; - private ObjectPool timerPool; + private ObjectPool counterValuePool; + private ObjectPool gaugeValuePool; + private ObjectPool timerValuePool; private ObjectPool protocolPool; ResourcePool(int maxQueueSize, final TProtocolFactory protocolFactory) { @@ -63,7 +64,7 @@ public Metric instance() { } }); - tagPool = new ObjectPool<>(maxQueueSize, new Construct() { + metricTagPool = new ObjectPool<>(maxQueueSize, new Construct() { @Override public MetricTag instance() { return new MetricTag(); @@ -77,21 +78,21 @@ public MetricValue instance() { } }); - counterPool = new ObjectPool<>(maxQueueSize, new Construct() { + counterValuePool = new ObjectPool<>(maxQueueSize, new Construct() { @Override public CountValue instance() { return new CountValue(); } }); - gaugePool = new ObjectPool<>(maxQueueSize, new Construct() { + gaugeValuePool = new ObjectPool<>(maxQueueSize, new Construct() { @Override public GaugeValue instance() { return new GaugeValue(); } }); - timerPool = new ObjectPool<>(maxQueueSize, new Construct() { + timerValuePool = new ObjectPool<>(maxQueueSize, new Construct() { @Override public TimerValue instance() { return new TimerValue(); @@ -106,7 +107,7 @@ public TProtocol instance() { }); } - public MetricBatch getBatch() { + public MetricBatch getMetricBatch() { return metricBatchPool.get(); } @@ -118,24 +119,24 @@ public HashSet getTagList() { return new HashSet<>(); } - public MetricTag getTag() { - return tagPool.get(); + public MetricTag getMetricTag() { + return metricTagPool.get(); } - public MetricValue getValue() { + public MetricValue getMetricValue() { return metricValuePool.get(); } - public CountValue getCounter() { - return counterPool.get(); + public CountValue getCountValue() { + return counterValuePool.get(); } - public GaugeValue getGauge() { - return gaugePool.get(); + public GaugeValue getGaugeValue() { + return gaugeValuePool.get(); } - public TimerValue getTimer() { - return timerPool.get(); + public TimerValue getTimerValue() { + return timerValuePool.get(); } public TProtocol getProtocol() { @@ -148,7 +149,7 @@ public void releaseProtocol(TProtocol protocol) { protocolPool.put(protocol); } - public void releaseBatch(MetricBatch metricBatch) { + public void releaseMetricBatch(MetricBatch metricBatch) { metricBatch.setCommonTags(null); for (Metric metric : metricBatch.getMetrics()) { @@ -166,7 +167,7 @@ public void releaseMetric(Metric metric) { for (MetricTag tag : metric.getTags()) { tag.setTagName(""); tag.setTagValue(null); - tagPool.put(tag); + metricTagPool.put(tag); } metric.setTags(null); @@ -190,25 +191,31 @@ public void releaseShallowMetric(Metric metric) { metricPool.put(metric); } + public void releaseShallowMetrics(List metrics) { + for (Metric metric : metrics) { + releaseShallowMetric(metric); + } + } + public void releaseMetricValue(MetricValue metricValue) { if (metricValue.isSetCount()) { metricValue.getCount().setI64Value(0); - counterPool.put(metricValue.getCount()); + counterValuePool.put(metricValue.getCount()); metricValue.unsetCount(); } else if (metricValue.isSetGauge()) { metricValue.getGauge().setI64Value(0); metricValue.getGauge().setDValue(0); - gaugePool.put(metricValue.getGauge()); + gaugeValuePool.put(metricValue.getGauge()); metricValue.unsetGauge(); } else if (metricValue.isSetTimer()) { metricValue.getTimer().setI64Value(0); metricValue.getTimer().setDValue(0); - timerPool.put(metricValue.getTimer()); + timerValuePool.put(metricValue.getTimer()); metricValue.unsetTimer(); } From 88119c2fab1adaae1ede65f1b261aadde8db0864 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Thu, 31 Aug 2017 17:56:52 -0400 Subject: [PATCH 11/44] Fix exceptions; add JavaDocs --- .../java/com/uber/m3/tally/MetricType.java | 3 + .../java/com/uber/m3/tally/m3/M3Reporter.java | 95 ++++++++++++++++--- .../com/uber/m3/tally/m3/ResourcePool.java | 73 ++++++++++++++ .../com/uber/m3/tally/m3/SizedMetric.java | 16 ++++ .../com/uber/m3/tally/m3/ThriftProtocol.java | 3 + .../main/java/com/uber/m3/util/Construct.java | 8 ++ .../java/com/uber/m3/util/ObjectPool.java | 46 +++++++-- 7 files changed, 224 insertions(+), 20 deletions(-) diff --git a/m3/src/main/java/com/uber/m3/tally/MetricType.java b/m3/src/main/java/com/uber/m3/tally/MetricType.java index a571b63..721f3ab 100644 --- a/m3/src/main/java/com/uber/m3/tally/MetricType.java +++ b/m3/src/main/java/com/uber/m3/tally/MetricType.java @@ -20,6 +20,9 @@ package com.uber.m3.tally; +/** + * Enum of the type of metric. + */ public enum MetricType { COUNTER, GAUGE, diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index a18d7f6..eb12e9b 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -60,7 +60,7 @@ import java.util.concurrent.Phaser; /** - * An M3 reporter + * An M3 reporter. */ public class M3Reporter implements CachedStatsReporter, AutoCloseable { private static final int DEFAULT_METRIC_SIZE = 100; @@ -83,7 +83,7 @@ public class M3Reporter implements CachedStatsReporter, AutoCloseable { private M3.Client client; - private MetricBatch metricBatch; + private final MetricBatch metricBatch; private final Object calcLock = new Object(); private TCalcTransport calc; @@ -173,7 +173,7 @@ private M3Reporter(Builder builder) { } } - public void addAndRunProcessor() { + private void addAndRunProcessor() { phaser.register(); executor.execute(new Processor()); @@ -216,18 +216,20 @@ public void flush() { try { metricQueue.put(new SizedMetric(null, 0)); } catch (InterruptedException e) { - //TODO log exception? + throw new RuntimeException(e); } } private List flush(List metrics) throws TException { + TException exception = null; + synchronized (metricBatch) { metricBatch.setMetrics(metrics); try { client.emitMetricBatch(metricBatch); } catch (TException e) { - //TODO log exception? + exception = e; } metricBatch.setMetrics(null); @@ -237,6 +239,10 @@ private List flush(List metrics) throws TException { metrics.clear(); + if (exception != null) { + throw exception; + } + return metrics; } @@ -292,7 +298,7 @@ private Set addMetricTagsToSet(Map tags) { return metricTagSet; } - public MetricTag createMetricTag(String tagName, String tagValue) { + private MetricTag createMetricTag(String tagName, String tagValue) { MetricTag metricTag = resourcePool.getMetricTag(); metricTag.setTagName(tagName); @@ -311,7 +317,6 @@ private int calculateSize(Metric metric) { return calc.getCountAndReset(); } } catch (TException e) { - //TODO log exception? return DEFAULT_METRIC_SIZE; } } @@ -351,7 +356,7 @@ private void reportCopyMetric( try { metricQueue.put(new SizedMetric(metricCopy, size)); } catch (InterruptedException e) { - //TODO log exception? + throw new RuntimeException(e); } } @@ -392,7 +397,7 @@ public void run() { flush(metrics); } } catch (TException e) { - throw new IllegalStateException("Bad client state. Metrics to fail to flush", e); + throw new RuntimeException("Failed to emit metrics", e); } finally { // Always arrive at phaser to prevent deadlock phaser.arrive(); @@ -410,7 +415,7 @@ private abstract class CachedMetric { } } - public class CachedCounterImpl extends CachedMetric implements CachedCounter { + private class CachedCounterImpl extends CachedMetric implements CachedCounter { private CachedCounterImpl(Metric metric, int size) { super(metric, size); } @@ -421,7 +426,7 @@ public void reportCount(long value) { } } - public class CachedGaugeImpl extends CachedMetric implements CachedGauge { + private class CachedGaugeImpl extends CachedMetric implements CachedGauge { private CachedGaugeImpl(Metric metric, int size) { super(metric, size); } @@ -432,7 +437,7 @@ public void reportGauge(double value) { } } - public class CachedTimerImpl extends CachedMetric implements CachedTimer { + private class CachedTimerImpl extends CachedMetric implements CachedTimer { private CachedTimerImpl(Metric metric, int size) { super(metric, size); } @@ -443,6 +448,9 @@ public void reportTimer(Duration interval) { } } + /** + * Builder pattern to construct an {@link M3Reporter}. + */ public static class Builder { private String[] hostPorts; private String service; @@ -460,6 +468,10 @@ public static class Builder { private ExecutorService executor = Executors.newFixedThreadPool(THREAD_POOL_SIZE); + /** + * Constructs a {@link Builder}. Having at least one host/port is required. + * @param hostPorts the array of host/ports + */ public Builder(String[] hostPorts) { if (hostPorts == null || hostPorts.length == 0) { throw new IllegalArgumentException("Must specify at least one host port"); @@ -468,72 +480,131 @@ public Builder(String[] hostPorts) { this.hostPorts = hostPorts; } + /** + * Configures the service of this {@link Builder}. + * @param service the value to set + * @return this {@link Builder} with the new value set + */ public Builder service(String service) { this.service = service; return this; } + /** + * Configures the env of this {@link Builder}. + * @param env the value to set + * @return this {@link Builder} with the new value set + */ public Builder env(String env) { this.env = env; return this; } + /** + * Configures the common tags of this {@link Builder}. + * @param commonTags the value to set + * @return this {@link Builder} with the new value set + */ public Builder commonTags(ImmutableMap commonTags) { this.commonTags = commonTags; return this; } + /** + * Configures whether to include the host tag of this {@link Builder}. + * @param includeHost the value to set + * @return this {@link Builder} with the new value set + */ public Builder includeHost(boolean includeHost) { this.includeHost = includeHost; return this; } + /** + * Configures the protocol of this {@link Builder}. + * @param protocol the value to set + * @return this {@link Builder} with the new value set + */ public Builder protocol(ThriftProtocol protocol) { this.protocol = protocol; return this; } + /** + * Configures the maximum queue size of this {@link Builder}. + * @param maxQueueSize the value to set + * @return this {@link Builder} with the new value set + */ public Builder maxQueueSize(int maxQueueSize) { this.maxQueueSize = maxQueueSize; return this; } + /** + * Configures the maximum packet size in bytes of this {@link Builder}. + * @param maxPacketSizeBytes the value to set + * @return this {@link Builder} with the new value set + */ public Builder maxPacketSizeBytes(int maxPacketSizeBytes) { this.maxPacketSizeBytes = maxPacketSizeBytes; return this; } + /** + * Configures the histogram bucket ID name of this {@link Builder}. + * @param histogramBucketIdName the value to set + * @return this {@link Builder} with the new value set + */ public Builder histogramBucketIdName(String histogramBucketIdName) { this.histogramBucketIdName = histogramBucketIdName; return this; } + /** + * Configures the histogram bucket name of this {@link Builder}. + * @param histogramBucketName the value to set + * @return this {@link Builder} with the new value set + */ public Builder histogramBucketName(String histogramBucketName) { this.histogramBucketName = histogramBucketName; return this; } + /** + * Configures the histogram bucket tag precision of this {@link Builder}. + * @param histogramBucketTagPrecision the value to set + * @return this {@link Builder} with the new value set + */ public Builder histogramBucketTagPrecision(int histogramBucketTagPrecision) { this.histogramBucketTagPrecision = histogramBucketTagPrecision; return this; } + /** + * Configures the executor service of this {@link Builder}. + * @param executor the value to set + * @return this {@link Builder} with the new value set + */ public Builder executor(ExecutorService executor) { this.executor = executor; return this; } + /** + * Builds and returns an {@link M3Reporter} with the configured paramters. + * @return a new {@link M3Reporter} instance with the configured paramters + */ public M3Reporter build() { return new M3Reporter(this); } diff --git a/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java b/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java index ea21439..fa065c5 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java @@ -36,6 +36,9 @@ import java.util.HashSet; import java.util.List; +/** + * A collection of {@link ObjectPool}s used for the {@link M3Reporter}. + */ public class ResourcePool { private static final int BATCH_POOL_SIZE = 10; private static final int PROTOCOL_POOL_SIZE = 10; @@ -49,6 +52,11 @@ public class ResourcePool { private ObjectPool timerValuePool; private ObjectPool protocolPool; + /** + * Creates a {@link ResourcePool}, initializing all its inner pools. + * @param maxQueueSize the maximum metric queue size + * @param protocolFactory a factory to create the {@link TProtocol} we want + */ ResourcePool(int maxQueueSize, final TProtocolFactory protocolFactory) { metricBatchPool = new ObjectPool<>(BATCH_POOL_SIZE, new Construct() { @Override @@ -107,48 +115,94 @@ public TProtocol instance() { }); } + /** + * Returns a {@link MetricBatch} from the pool. + * @return a {@link MetricBatch} from the pool + */ public MetricBatch getMetricBatch() { return metricBatchPool.get(); } + /** + * Returns a {@link Metric} from the pool. + * @return a {@link Metric} from the pool + */ public Metric getMetric() { return metricPool.get(); } + /** + * Returns an empty {@code Set} of {@link MetricTag}s. + * @return an empty {@code Set} of {@link MetricTag}s + */ public HashSet getTagList() { return new HashSet<>(); } + /** + * Returns a {@link MetricTag} from the pool. + * @return a {@link MetricTag} from the pool + */ public MetricTag getMetricTag() { return metricTagPool.get(); } + /** + * Returns a {@link MetricValue} from the pool. + * @return a {@link MetricValue} from the pool + */ public MetricValue getMetricValue() { return metricValuePool.get(); } + /** + * Returns a {@link CountValue} from the pool. + * @return a {@link CountValue} from the pool + */ public CountValue getCountValue() { return counterValuePool.get(); } + /** + * Returns a {@link GaugeValue} from the pool. + * @return a {@link GaugeValue} from the pool + */ public GaugeValue getGaugeValue() { return gaugeValuePool.get(); } + /** + * Returns a {@link TimerValue} from the pool. + * @return a {@link TimerValue} from the pool + */ public TimerValue getTimerValue() { return timerValuePool.get(); } + /** + * Returns a {@link TProtocol} from the pool. + * @return a {@link TProtocol} from the pool + */ public TProtocol getProtocol() { return protocolPool.get(); } + /** + * Puts the given {@link TProtocol} back to the pool, reseting the + * {@link TCalcTransport} internal count. + * @param protocol the {@link TProtocol} to put back to the pool + */ public void releaseProtocol(TProtocol protocol) { ((TCalcTransport) protocol.getTransport()).resetCount(); protocolPool.put(protocol); } + /** + * Puts the given {@link MetricBatch} back to the pool after reseting inner + * values and releasing each of its internal {@link Metric}s as well. + * @param metricBatch the {@link MetricBatch} to put back to the pool + */ public void releaseMetricBatch(MetricBatch metricBatch) { metricBatch.setCommonTags(null); @@ -161,6 +215,11 @@ public void releaseMetricBatch(MetricBatch metricBatch) { metricBatchPool.put(metricBatch); } + /** + * Puts the given {@link Metric} back to the pool after reseting its + * internal attributes. + * @param metric the {@link Metric} to put back to the pool + */ public void releaseMetric(Metric metric) { metric.setName(""); @@ -175,6 +234,11 @@ public void releaseMetric(Metric metric) { releaseShallowMetric(metric); } + /** + * Puts a tag-less {@link Metric} back to the pool as well as + * its {@link MetricValue}. + * @param metric the {@link Metric} to put back to the pool + */ public void releaseShallowMetric(Metric metric) { metric.setTimestamp(0); @@ -191,12 +255,21 @@ public void releaseShallowMetric(Metric metric) { metricPool.put(metric); } + /** + * Puts a list of metrics back to the pool. + * @param metrics the list of metrics to put back to the pool + */ public void releaseShallowMetrics(List metrics) { for (Metric metric : metrics) { releaseShallowMetric(metric); } } + /** + * Puts the given {@link MetricValue} back to the pool after reseting its + * internal attributes and values. + * @param metricValue the {@link MetricValue} to put back to the pool + */ public void releaseMetricValue(MetricValue metricValue) { if (metricValue.isSetCount()) { metricValue.getCount().setI64Value(0); diff --git a/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java b/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java index 7e58116..f70c573 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java @@ -22,20 +22,36 @@ import com.uber.m3.thrift.generated.Metric; +/** + * A metric along with its associated size. + */ public class SizedMetric extends Metric { private Metric metric; private int size; + /** + * Creates a new {@link SizedMetric} with the given metric and size. + * @param metric the metric + * @param size the size of this metric + */ public SizedMetric(Metric metric, int size) { // Recreate metric to prevent underlying metric from being modified this.metric = new Metric(metric); this.size = size; } + /** + * Returns the metric. + * @return the metric + */ public Metric getMetric() { return metric; } + /** + * Returns the size of this metric. + * @return the size of this metric + */ public int getSize() { return size; } diff --git a/m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java b/m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java index 76f0c03..27f6e7c 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java @@ -20,6 +20,9 @@ package com.uber.m3.tally.m3; +/** + * Enum of Thrift protocols. + */ public enum ThriftProtocol { COMPACT, BINARY diff --git a/m3/src/main/java/com/uber/m3/util/Construct.java b/m3/src/main/java/com/uber/m3/util/Construct.java index 22592ad..5be3290 100644 --- a/m3/src/main/java/com/uber/m3/util/Construct.java +++ b/m3/src/main/java/com/uber/m3/util/Construct.java @@ -20,6 +20,14 @@ package com.uber.m3.util; +/** + * A wrapper around a constructor for the type E. + * @param the type this constructor instantiates + */ public interface Construct { + /** + * Returns a new instance of E. + * @return a new instance of E + */ E instance(); } diff --git a/m3/src/main/java/com/uber/m3/util/ObjectPool.java b/m3/src/main/java/com/uber/m3/util/ObjectPool.java index 3e33f00..9db3369 100644 --- a/m3/src/main/java/com/uber/m3/util/ObjectPool.java +++ b/m3/src/main/java/com/uber/m3/util/ObjectPool.java @@ -23,14 +23,32 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; +/** + * An object pool used to reduce the number of objects required to be allocated + * and subsequently garbage-collected. + * @param the type of object this is a pool of + */ public class ObjectPool { private BlockingQueue objects; private Construct construct; + /** + * Initializes an {@link ObjectPool} and preallocates objects in it up to capacity. + * Defaults fair-ness (FIFO for requests) to false. This improves throughput + * with the slight chance of starvation. + * @param capacity capacity of this pool + * @param construct the constructor of E + */ public ObjectPool(int capacity, Construct construct) { this(capacity, construct, false); } + /** + * Initializes an {@link ObjectPool} and preallocates objects in it up to capacity. + * @param capacity capacity of this pool + * @param construct the constructor of E + * @param fair whether the pool is fair (FIFO for requests) + */ public ObjectPool(int capacity, Construct construct, boolean fair) { objects = new ArrayBlockingQueue<>(capacity, fair); @@ -41,24 +59,36 @@ public ObjectPool(int capacity, Construct construct, boolean fair) { } } + /** + * Gets an object from the pool. If the pool is empty, it will create and return + * a new instance instead of blocking until a new + * @return an object from the pool + */ public E get() { if (objects.isEmpty()) { - // Rather than block + // Rather than block, return a newly a newly created instance instead return construct.instance(); } try { return objects.take(); } catch (InterruptedException e) { - throw new IllegalStateException(e); + Thread.currentThread().interrupt(); + + return construct.instance(); } } - public void put(E element) { - try { - objects.put(element); - } catch (InterruptedException e) { - throw new IllegalStateException(e); - } + /** + * Puts an object back into the pool. + * @param element the element to put back + * @return whether the element got put back or not + */ + public boolean put(E element) { + // Use offer instead of put (which blocks until there is space), because a full + // ObjectPool is a fine state to be in. We can get into this state because the + // get() function will create a new instance instead of waiting for an object + // to be in the pool. + return objects.offer(element); } } From 38fb9d93d3e5d6d2c00e65f8ac7d9d12ce99c83e Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Thu, 31 Aug 2017 17:58:40 -0400 Subject: [PATCH 12/44] Move MetricType into correct package --- m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java | 1 - m3/src/main/java/com/uber/m3/tally/{ => m3}/MetricType.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) rename m3/src/main/java/com/uber/m3/tally/{ => m3}/MetricType.java (97%) diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index eb12e9b..87205f3 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -28,7 +28,6 @@ import com.uber.m3.tally.CachedTimer; import com.uber.m3.tally.Capabilities; import com.uber.m3.tally.CapableOf; -import com.uber.m3.tally.MetricType; import com.uber.m3.thrift.generated.CountValue; import com.uber.m3.thrift.generated.GaugeValue; import com.uber.m3.thrift.generated.M3; diff --git a/m3/src/main/java/com/uber/m3/tally/MetricType.java b/m3/src/main/java/com/uber/m3/tally/m3/MetricType.java similarity index 97% rename from m3/src/main/java/com/uber/m3/tally/MetricType.java rename to m3/src/main/java/com/uber/m3/tally/m3/MetricType.java index 721f3ab..19f5814 100644 --- a/m3/src/main/java/com/uber/m3/tally/MetricType.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/MetricType.java @@ -18,7 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -package com.uber.m3.tally; +package com.uber.m3.tally.m3; /** * Enum of the type of metric. From 80c68bd7f983b5df78b6afb74c7117e1d600800a Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Thu, 31 Aug 2017 18:14:58 -0400 Subject: [PATCH 13/44] Revert to previous gradle-wrapper --- gradle/wrapper/gradle-wrapper.jar | Bin 54783 -> 54208 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index b5a2e12bcaaf27ad5b24ee0d107ea6a5ecb61fe7..37406edbf50e7cd96891199d9f0a9dd7c5683f92 100644 GIT binary patch delta 5988 zcmY*d1yq#J*It$mL1`A2QdkxVNu_LQL`qt^8>C|~Xq0&Al5UVLDS=(-QeXuo1yLF$ z1^kv>{y#t7bLPx*?mYLocV_O)Iq&=}!EZ0Xr+lP@Pe2O-k&uAM{4OiTP;!x+_Txv- zc4DM-=$L-FZn@%t{*jJ-F{DCv0&L$8!}WWD*uMIb0V(OF(~!SUcKJj6IO%h*bULu^V!V3JecJ)hEcK$-Y1c+C-rCgwockf92?8 zU({9hBRV!?R`CO|^?DH(QY#j&M>Tq`OYBX%Sy>`Z4tMcyGetREiK(7rw=&^>mMT78 z8PjeF?nAX$n32~z+>fcRaG0aKca+R_cqnmC)Cg@BI%ckMjc@GuQS&;Km1HNFjeyo= zrRx0)nR35sHe$1;HBVB+l-joSqqQG(B(*cMdb~v6wHV2?$i3>XhH4Zmjcx0NU&o)@ zq1$-+a{Ebl0Hgf$7is;>j*2hTJ|Uxj)Wu-ejDjhc5AGhz{B3V}WbGtgWI88+&p7_e z%{TF?6fi%=_thnX!w4{vrWCX?Y=VN_CxF$oFnj9eca4aD+#Hx6`}3x((km zNuma~1f5gtqGN-qUi|SgmXxm949(g{g!Duo*9zB16l zqo6EfjP9L)@#r0xl&IfeHuVyIu^(0R;|F8daMk!yaWO6IQl+0s8!9?|#I((_+|yz( zNhhZb_3R)!G)X^Kt{QP)DvGa#ozI@_@^(-f$!HBKE#SQaz zbpg#XUl9{+Tq#f^Z~h(~`S`NUmaUbH2zj^TXjAz|7Kg9gHq7rFM#XKQ->n@cp2S@I z;J6a~r}eG4%KVbG+wYsB+y~QBW2~dWOWMXW%?pT@4Maa;e>iI9_z6e~8HadwBg3{s z_b7-ZBcW4-DCGHTLj1%{feRbrnG03V_GWeQH;`_wUQb4k@>ouKZ^KUXI>{l^R>orG zjMne}T)VbA^Ob|=+^RiuXGiNejly0J$QX~!i5i!&4~sJ4}TD*V4;e@C}w zAOTnEPvk!Q{kdNmtZls?#=-tl0K%gsYRj1!mML(zX;?ICCA+8H#bntO5hS48OLx^K z%z7NjYgn^vS}V}eq>~d-+oS^{tg$t1;TE^%v_q-9~ zgNHxj7NyE>|5`?O$5fu&pI>jyS?Kf}PHAZGy$M@6cs%~5LfHGA+FhQCY)?PqOvLK? z{Ugm=^>y_mFNFvDBURw32qLIwzAvgaU)W=haH4n7zTWo{be(3nfwgYQKINL%prn(# zRq3j-XNTv<&rGb$(is(of@|h*SA=BhE*07DI>QG`u0nmD9?B#HhiUL)L$Rd6$m70Wn|hrPYZI*xkfN~w4Vah0_4vV#2zH;z=7Quw5=N^j3YM1j?Ic?<0$_ZKMf zBHcHQh*2V;J?H=1D|;Mku~ z_qMljFjBN^s7EO)5)ow<_w$?ov+*jS%igyV_rl&e45;}8f=196#Ppcdb)~rYxgOk0 z`M%P7v}G^$Zb|p!ZbK?Px|yI6sx&gUGn#^y*AF#*yO^h;)KZxvu;1qp>v&ClQ4Gq% z5pGogT^h+<{QS=G$56he=vVY(82TrnM_=R`L-H4nsWx%T zUdocIa`DgSypWTlh;LqFsUcrJ@da_EHK$ft)K!%7bGs<_3b_a_shD50;w1lY{~;xV zkZPBu|1E~Tp4Eh1TT=jD{{Aw!vP3hOIU!NI_~L}EX*Ikqw2-?lWnR*y1@3rpN?AJa zkIE5uKp*Y(@HIZ(h{oX(>XD4%LHLro`?NNj0-b{^IxEaW1MxKUA#xZ7J_)Ce-!a`05v!m!Xf<37TotBZLv6fV2S9~B{_aixot zp0bBIXHvOTJDyb~GS&2MTCUtxv7j+;g~OEjJWgG*1lwKaUX}r5ifadaQd3>aP*>EY zm$m?z{?hxX>-GezM%pFMv0?tI@*)jKf^644)75zcZt>*7he%5O*2SUk_Tzezw_EN$ z{?5}bm@TZV z&d>gmYGo11n#!o2<#XLmQ|PT(UUNcn=bqx)l66tG!c4kY5SN>Bd?ETk{k^bKi8_a# zmcgKwhvbdf+@xoo|>fc45NVM+S zirK@0v|$#eH&^r-N(|JX#J@6}s>REU<*RnpUf+5#RQR+^vZp z+9a$j`Wd8gufRNMj4S2vSH&8g4AD|`Bt@U;z_Vq#X4C&KG1IwcAd|Sj)t8Sl)ym2-BUH&VH+``2PP?@g& z&{xs#&r^4qj3FjrAM)^C9Z}2&@Gf{&QE{w>eJbVGFjzP$TsS~}=CO++YxS~mIOz9G zuU_!hr@xb|fL|^qQgE2_WoYroqSV%3w$x^Y&`cdBC(`k}1dS!-iJ`GS2~XUspuhW^ zAl|BoorLT+b{+Q})~}-Mv>13r9GP?*?-*+D1upmTYN%JQ{k`U)R{0xk`qAGPn(?vE zt2mF(ZsGb-smOflV20yJdk%w8r!13)F78)y3Jzn+jY5-y1G* zyiQeU2@=k>&KI+DhK?`zeDfZC68GX)L*~hz^@~x#Ps)13nUc_f*3PMGc>;jZ7=^&dM&MAC^hQ$yRr3kMcOPt;1^LBp%u-v=vte(h(-X zrlcepDqM>i9)_>XOXqhzh~Qf!FYOSVwNOOzHnX(J4;sx&x2Ky7vgv4Y2i&(R{nit) zTI~BAnmBCGUYs#(QFBGm%sn=|lUzrHg_I6aQ3o5IzC_Ar46Srn8{vGmc^LY*JykSAG8<&nPPfB%_s|tZ=q#dLA8pED@^!%a z^9u*GQl_t0RTN?`hk=-}v)<%l{Q@l|2QuAvSdQU0%{pE)wcMs23o_@1tjxWAw~k{` z*-ZhK57mr0C=S0g-#2s942^qQ>;3>MB2l2L4jN-L}7i(LvkrfHSme>&y>ryo} zb<+_KKXL}j+ZR}tIFf8Hn5pC(x-~H{qjI$NZDWbB`CFK<9|zJ|Tg%vqi$ZY04WWiy zk?#@w6dQhBy-~U0#Tt~n!J=X1m_(YXwf0358G45&-0_Rbb&udZ1Jv|4uusPyGW5)J zZ6LlMy>qmSc2DTQOVXrVDEzMQ1NMUCNs<0=QK&~bFOn+Zy`&D&3VT3EyJ)m8Re*Nh ze)gE!`nE<{(&{{@wp)1n+6F=&4^dC$9!y0y%)D;rM;}7I%(hNJ)x8q~n&?4^-AI{r zRLxY1W`ELNnPGh8BI&khhW#qZj9NQ8>1&y&ELgWBr>`IX0uEoJE955n7JJ; zx8+Nam0G*CCqeFPg@1BeVvWUwe_cjiu8kDYt^3@2OD?Sk{M(snI)pCH&q>JiFQe1r z%I=MQ^{;+fMXnnZZ;ZdPkVUz%tT#XB?;ls;imE%RJi{1yGX z=`1znU0h#|yJ9Q+`3HQU-w?Aeli4p3nZ=6Zm|;Bap{93|+UbCBTga zzLCVoq1NmIY8<-ql9dF9m^+5fJPJKx&vzegEhtVlBx*=dV8E}XH3NX}7fwNufg@9?uF*Z6tS%n7p+<#{G{Qv>q zqQ=@wKk(tWy*&^x9@8230-&nI2(b53{Clo(LeYcCnX%(1C=wtS#l?PF9;g5Bu>>HH z3@He74KuTUpFI>y_R@s@3p|5KQkfHDrw1O)1cId4<1<=$(fz9LGlfb_uFzAx-na)`}LCi$0Z((fofJqM{*10~+ zcqN+*1mb1Hv~$j#>R4xcP6hz+zg7K$S-mntto=eaH&&qk!V}pT%po}-0)hC?31}K) ziH?25fRQc@z~6^UcOhulh!sO*!L-Zg9IlxGZ~KJUPb>4(juFLvRA)V|i!zJthhpm( zI2wU~Tg`yVkDP$h2r+PHlo9J|M0ImC2jdJU27yG*`DR&S$(w_0IDt0<5OBN=#>N6% zbfy4q3{c?g{zDM3ofDS(Zb%p_Dr{sCm5&iy#x$ODqD-z>;;K6dATdaRC8yaN3>Dyk zK>3)nb>keF&<#sIO?PSpiX}(f5wrKhkjXJ*$#Z0|JD}Ar4LJAXRAB`26igVR)#=>M z5yw5Sa|j$@#^oQ_4*`pN0W$qez!U2${|S6E9fQqboEbL*VIh95SqiZ^V4@-Qx1 zP+kp#6*UNi!c+zKIWm7BhAfJUw1rtIbS&0ca2SrW{rv<1AEp4zV{+K}$<~qV$zbNz zi#aAz=Y$5*u#TI9M8LEM9pEr@w)`>}0kcd%wMF1`ORysPUXDp|n6y7&u8{ND*yLb{ zoB-?-?m7evh9TfPC0OBcwk)Su-vI+n~&XZ@TQvq#s ziZ5Tjuq}t#UnPu_&^d>eCX53M0C8aWSEE9h;@4>`NmoFJ$JIO^o!``=wmf_Wc5y|VxRJ&m~!{@Hq)$ delta 6637 zcmZ8m2RxPk_rI=@k-hicD=RW9E8_|m5v~;_!lle|@kLg|lf6R7DqCju-XWVPBRdi$ z{!jPb|F5s_=k3HApNOFGyU2 zcoMbv#wwW7CVz9^bH{=F2lSc)4-M|;$J~F2qwf_if|v2p6!2PX5v`}!Ro(+&Wxfuz zbOmgfIZ@IfM8#9v&uyN6i`0EQN%R~5*M zy-5wKE6il>bQ4tX9PL<{iJbXLooCH2SG&z<(D;zd&~0C8&pgh0n<=HXAEFd$3cHNo zW;jRQu`&NCP~Y|F+EvOLF=oZdVSa&6rZAzAuW z0bcb%a=CRzqHlA(r6>?qdwQAzJ0Y7-13wQ8RNxC;>=VuM5@-}R_JHo&cbCSBk1gzz z&ys{3Isig++mZn`N-8y8wYCH?Eb$EPs}zizmQ!%jy;0fae7&sf*d>k-^+IH{NN7j( zM7k}iE9#_L_(g~xS;X!d%NaDkmO#)wd zJ z{h`yvDMrylrxo8@D^cMxKb=^T#3RCL)qSOPP?h`Lqa>}ZhZ0;?-QgTCr^gH@VUd;c z+)xV(PG(htDWrRRHbI}i=0Q5MLQ4>{R|z#w(^9L%<`DdgGVDcPbT4I6YZbCK{|>jE zsB?8ff7qo7+Xh+-_Pi3ZV_CQ4VjTn(zD3ZpqC2sACj{rKra3j@6EPtf1j51sfp9_q zgLYa#k(m%BX6J0<;gN1eWKHwY=QN|TUW(}%naPb(>s{Ms!t~BgYm$FrgCT)D!F1sw z3^UwYH*T8_bXQaa%uYO89+rnuuPUdF>7`X}TW*HZ66tN`i8_9mXiTlvim8O9R=cER ztjF^Go}W=I8rOJ4yf&b4y7Bw$$Jvh3-m&YGuTl_CRCpu-)jJz&2J!LIAC6OxH7<+y zt<=3~sdM&kYi9K}ZgahVB#H|=dVp(l z5*@hEvD5QqA#s3x?I({lI(oi5L(z&yexk4g!fdt?bgGS zs=$(?R20WpX6)HMWr2U3a)RsuWT@#YBB8(l^@ejofJ8r z!Y-eQrMh;nsJyy%R}Zc;rDSh)vwZD zQwuvb^V~0&iPgSA2B9D7>id|l}tq^k=S2v5Y)>YPc`XXL$#xZdB z^(q3fHgHr?z0994vhW@6t7exT3nOz_2isQ_d1ZHts~3sKzSrwb*RY^kwe2_WuWJTQ zH|@2do?hr=iy+NlTpWwm2-NRQPc9R*Uw*zZ@Nmf6gv;K`>Ra)#&&&p!VdO77>1fU3 zE2(@J(`&?T&k6P=uSXvvMw364w|pU*(c!n$e2JX*xT~m;&ZeFBWG1#%L6}J8Gs4~Q zO+14rcYee06D|IAc8WD8%Z}psgyOWRE1VQ-UMT9y1{;C(N$%uYtr-P-Sl$?7R!~^R z#z*VQ;7x*)DIUv4M;X8|yS~tVVZhUB=dw$(d;0ok-Oh?WgO{_2Iic_aHN>;#l9DS| zqPM(aVn|nV`R2CUdR~<9+&AIkgmH!@08~Om3S0c?4Vyo|G+aDt{vzmy5R0oVd+9N} z_W(6iW+=_KoZI_~EVe`$sBWBRR3O(O@RgKO5~VbJR2_3Dms~4hK8@&V991}dzSVzJ z5oL)3|Lvk7#MvDa8*zYNGyS>HQ>y8z&d4M+BT9MsLrArZ<`xd|7l#S)iU^;jNw_=z ztJm7@jkVn>YcI;39G#}we#Ajj`xNUdX8lmJ{)W^gl_6}~6{9@`-|lS{Go-~|?zzMW z?~uS1v6NfWtrsB!Os3~6{F;*ukD3~*pe{o8PRmhSt4R$uGntRpYe{(@W+F-s|4x1xL^mH@$fvB>y`d`7a);WVNSAjd?$QZ6X0-zbw7Qx3?K*F|N`e!^;mtIrGq(+bI9K_c`8l0j z9=JSz>+O_~5aGcon=YIr<5Ksq{9`It#sAnm{rtfGtM&>w*6$1hI=WPPXmTDjB@01 zh=AM9(ooxW)8yxV7%)sqULy4#p+QciQ8wnu3HyLLUy?m9v^(_& z71vp9_j9}lX#n>`IF}8vxo$5-QPq}HuDZd4v;ae3Wp5nNt)vmMXE;pi9TK%JH_X59-iU-9{BIHp56+B$@>bf z551xKbw~c|JAdi)NM6yz z_$Aq5LUwd{QB7?1Z`H%fc&)wJNmF++(HyvB|LD)|6Trlj&Cz|ivo)*YVHfd$Q$KQc5{nJ6>tiw_Y1%rk22aF zax)@W5>M)LT6150`(B-aViT-&E8~fOcjp+Ui?IHgXhpG}(=z(aN=RU_K*7HByK^Rw zP)mD(nK){$-&>L{Wbd2sRhIZkPYqsJsWx0MCtMM4l2li)`besXD(}@P zuu`1-tUR72pww7#a*C=KPh{iuvPoVP?vD8Qc_o76p#GvWV&L$sZ8FG~v8Z+Y%o;zV zLfBh7x7b^^_)O2~z;(2C4Z*>)U9z}PL2H3nKjvhes#)w?NT@WL&U$>ex+I!)rQT1A zH^$eq=I||@YItvM&5e80-&aK5JCo2V8rzWg2svMawo#N)wi2Myy6_L-kitt{WpGJP z%7mMEhi%{DUpD!l_SHb&6F7>0lAOVh?xa);SV*b%92u!4+zX=Gf6Y~F%N(NjeW49u z9!;?jxdVS6T4WFr{8h`?xR5{ISeQ2Ei?L!4?Qv@Tz=O!w_nqE{eytvKBCJ%0exz{T zco#}RU&-5;npHN3(ta=_;HvNysW4%Cf$Lre-=Jh=ScI#FenCN$Di^XA)~ z5w1Wto^1OvPO5HOyPIwU(VdA0Y?je3Q+8vUB_&hl^h6p5B9}XUj2S6K)^1Fub(FBH z?MH@%*4#nPkUFEDJ*^S7{naP>Xh7yxapNP=s;}lVhV^tcL*osrjFmM{pyQt=L_9N0 zfW!IQgTXFbA}U)0o$(2EH?u{92g49^7J&M%{RRJ`rK~e~?v~VH?iP_cdDrX`smSfs zva0;7Z*_bhxk<&}zbrizLge%z+CGlSuKx6*gf%O%e1E=#Qm#X(Advw_b(wog-7gPp z*?%6@u>Zr*EI>)sMyb0f0JuI6_l)Y6*TW-~;^6is=HHbrP>u$U979)DwSJo!zK@V= z3XnRY8eHm=d-`_q+HJ}%S-BsuG}^}`VH|8Ud?K``vN_S!4M8=FPZ@?4_G}vBO~+sF z5&JW^avQNZpn~3~3nEk>#2D=0J2*r&>@+JF=sZiTt?@D>&eDfa`uR|lCE^;yzwW5t zV+(qq&=m1w=}EAW{+hRn@>P_`HQmIWUy2zq@O-?b1m<_#^9dbdEh~DbvmKdpC$*_6 zespC&?vD&`cDOb17*sE+O5>~3vh(q>|MH<7`n)j|SdBVVpOqRtzJiA!T$s_bG&@iv zg@^dDx6M&D?GPMx(IYt?qv$Tom=^gS4`j#4iHWn3ZMm)I`N(&-$;(UkE^ghmF%`OV zA%fBCWfJnjt_Uk!ecNxB(pZt$#Py2^dtL1k=OJ7tO!y}9?6(;U(>5Xb9i-gjS3f_< zrF1Ob&^$sVv9Oc7vAlI@)sL!B3e>w6^hrd+Hz@WhPxtsI??-+|YIHd~3_ke$Z{uY6 z%x=b)X+X27d~PlhbS{4)N}kKN%8QlXlk0phVN%y++~aNZU`pTgLwLydStD5zEpiE2 zjj@ifihZ@|Wwj~l_QZ!HxKSs*hL59;-tc`lIT7%~K@Apn#eDK1*ixKJN%ac~JVl=1 z`AMvd-z?`RnTD(XP|mgJu}ktrMbusw>0Jo=ys>L87x!tB)*p9c*G+EF^VKq+i@k|! zzh>IHe`y(iq*Qusm=&+`pqR}Ybyk8Md$oB@`25i52~OU@2i~%IV`+!P>)bc(E0Q!3Wv|VXPNH)4pU~FT7f5pL61_gC7pkiTibr-DDGr% zJF9d8JInWlH9ZKv)KIlLPOz-s0w~Xdr?rz9{a_C?c!N~C3#G)hC&kO@U!^I%8$^V? z$TUh(2{C&l@w!#oiG}IWG0O5r>yg#@@5wPe@wqH;XweH^5`RA@_1#_Xy4!nr=sJ2l zw0E(0$9!Hsde%Z5bd^SZNBF~iTD87I_2J<qzV(t( ze6)Asif)n%>Z^u9ky?eswfiFU#fc?arUL0pg2|*nU=A(d_J*Wl!0|P9aK!JvhJnwd#o=P8w+_2lwFYmlV%7S% z7GilTacqPb+|8m>EN`-A2TR#Bu;HRTPII;x%Ey5~@^B##IbhF|6`%kvRH%7$W4-IQ zTfmBJR<2>G_;NUw`u$iQ>x@D`J{88CSI!tVq{x?0>{E(I!&rTUsY4hFAZvx;UG`+u z0jyfDaN&b_zCbwPGwMnWmM!QR_!|dG1Nl3F1L|6({x^aHzPHMwtpQSPVi>}{O$I|` zv?*bT(oQDexQ!n8)X9fokaRIYO$aapmM#{KKQaF^ha(3Qb`?ZOgI518`3u-d3?~pi za*^W?_|K%~@1W){6aq;l`v0*Fu+&9`2A@BM{{}D7|38@Q{GkL|+G)_Tc9YvFNU;1U zEu%r1y-F#>THR0A9*~Act5CeFfQ=q_w1rh*H zx>?cS)lE~6LJ%|qUR>wEyrLlZ62^*kCli!h4E#+8_;Zl~FFP?YA$Y)^!V2bWQxF0H zwf$>x%V5C-z@=_XZ2%Ts%s4mD4!jk=_;&~L9ZXP49U!Zn3s7NX0G@YXblV;|3a^1- zU;>W^-@i0NAj~ijEY0yJDR+GP7vew{m53maOXonEMrcdvtgL`O*4y)E``>7OGc*`H z=|Ge;bnP364K*{aIjLp%*#! zj=8ylpW8i{$e(K$=Rg_uKh{!juNNiUd-^KPbiqJfJJ_yGS9CmL);t~{Cpx+!?!zeRq@ z7Yn8m`EM6c9idzK3R-v!214=N{&=3Cg>r`Z(b^@QZ&ho9+VQ}Ks&Ni@FAxnZbmavs z`mkB19Atvh1*7@;g94ZU{TyO~%0;1h0Y%J!^B^V_)y<_INuc6FurQU*DVC1`!GZw! zJ34?3+c8+2=*k8_@C*o+JqH$zLxW$BV*1dzHhd0vGadxW|4*alh{I+as4)%P*ZJ0< zn~2uP-^+mUUu=X4DxQMod5myjc!R@CQ08=y#|@B Date: Fri, 1 Sep 2017 12:55:02 -0400 Subject: [PATCH 14/44] Use Gradle 3.5 --- build.gradle | 11 ++++------- gradle/wrapper/gradle-wrapper.jar | Bin 54208 -> 54783 bytes gradle/wrapper/gradle-wrapper.properties | 4 ++-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/build.gradle b/build.gradle index 8c705ab..4469650 100644 --- a/build.gradle +++ b/build.gradle @@ -126,13 +126,6 @@ allprojects { } } - // ---------------------- - // Build & Release - // ---------------------- - task wrapper(type: Wrapper) { - gradleVersion = '3.3' - } - release { tagTemplate = '$name-$version' } @@ -171,6 +164,10 @@ subprojects { ext."signing.secretKeyRingFile" = "ci/${ext.secretKeyRingFileName}" +task wrapper(type: Wrapper) { + gradleVersion = '3.5' +} + // ---------------------- // Fat jar task factories // ---------------------- diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 37406edbf50e7cd96891199d9f0a9dd7c5683f92..dcdd34d9ee44425d0ea9ff5cc7505dbed7d701ca 100644 GIT binary patch delta 6637 zcmZ8m2RxPk_rI=@k-hicD=RW9E8_|m5v~;_!lle|@kLg|lf6R7DqCju-XWVPBRdi$ z{!jPb|F5s_=k3HApNOFGyU2 zcoMbv#wwW7CVz9^bH{=F2lSc)4-M|;$J~F2qwf_if|v2p6!2PX5v`}!Ro(+&Wxfuz zbOmgfIZ@IfM8#9v&uyN6i`0EQN%R~5*M zy-5wKE6il>bQ4tX9PL<{iJbXLooCH2SG&z<(D;zd&~0C8&pgh0n<=HXAEFd$3cHNo zW;jRQu`&NCP~Y|F+EvOLF=oZdVSa&6rZAzAuW z0bcb%a=CRzqHlA(r6>?qdwQAzJ0Y7-13wQ8RNxC;>=VuM5@-}R_JHo&cbCSBk1gzz z&ys{3Isig++mZn`N-8y8wYCH?Eb$EPs}zizmQ!%jy;0fae7&sf*d>k-^+IH{NN7j( zM7k}iE9#_L_(g~xS;X!d%NaDkmO#)wd zJ z{h`yvDMrylrxo8@D^cMxKb=^T#3RCL)qSOPP?h`Lqa>}ZhZ0;?-QgTCr^gH@VUd;c z+)xV(PG(htDWrRRHbI}i=0Q5MLQ4>{R|z#w(^9L%<`DdgGVDcPbT4I6YZbCK{|>jE zsB?8ff7qo7+Xh+-_Pi3ZV_CQ4VjTn(zD3ZpqC2sACj{rKra3j@6EPtf1j51sfp9_q zgLYa#k(m%BX6J0<;gN1eWKHwY=QN|TUW(}%naPb(>s{Ms!t~BgYm$FrgCT)D!F1sw z3^UwYH*T8_bXQaa%uYO89+rnuuPUdF>7`X}TW*HZ66tN`i8_9mXiTlvim8O9R=cER ztjF^Go}W=I8rOJ4yf&b4y7Bw$$Jvh3-m&YGuTl_CRCpu-)jJz&2J!LIAC6OxH7<+y zt<=3~sdM&kYi9K}ZgahVB#H|=dVp(l z5*@hEvD5QqA#s3x?I({lI(oi5L(z&yexk4g!fdt?bgGS zs=$(?R20WpX6)HMWr2U3a)RsuWT@#YBB8(l^@ejofJ8r z!Y-eQrMh;nsJyy%R}Zc;rDSh)vwZD zQwuvb^V~0&iPgSA2B9D7>id|l}tq^k=S2v5Y)>YPc`XXL$#xZdB z^(q3fHgHr?z0994vhW@6t7exT3nOz_2isQ_d1ZHts~3sKzSrwb*RY^kwe2_WuWJTQ zH|@2do?hr=iy+NlTpWwm2-NRQPc9R*Uw*zZ@Nmf6gv;K`>Ra)#&&&p!VdO77>1fU3 zE2(@J(`&?T&k6P=uSXvvMw364w|pU*(c!n$e2JX*xT~m;&ZeFBWG1#%L6}J8Gs4~Q zO+14rcYee06D|IAc8WD8%Z}psgyOWRE1VQ-UMT9y1{;C(N$%uYtr-P-Sl$?7R!~^R z#z*VQ;7x*)DIUv4M;X8|yS~tVVZhUB=dw$(d;0ok-Oh?WgO{_2Iic_aHN>;#l9DS| zqPM(aVn|nV`R2CUdR~<9+&AIkgmH!@08~Om3S0c?4Vyo|G+aDt{vzmy5R0oVd+9N} z_W(6iW+=_KoZI_~EVe`$sBWBRR3O(O@RgKO5~VbJR2_3Dms~4hK8@&V991}dzSVzJ z5oL)3|Lvk7#MvDa8*zYNGyS>HQ>y8z&d4M+BT9MsLrArZ<`xd|7l#S)iU^;jNw_=z ztJm7@jkVn>YcI;39G#}we#Ajj`xNUdX8lmJ{)W^gl_6}~6{9@`-|lS{Go-~|?zzMW z?~uS1v6NfWtrsB!Os3~6{F;*ukD3~*pe{o8PRmhSt4R$uGntRpYe{(@W+F-s|4x1xL^mH@$fvB>y`d`7a);WVNSAjd?$QZ6X0-zbw7Qx3?K*F|N`e!^;mtIrGq(+bI9K_c`8l0j z9=JSz>+O_~5aGcon=YIr<5Ksq{9`It#sAnm{rtfGtM&>w*6$1hI=WPPXmTDjB@01 zh=AM9(ooxW)8yxV7%)sqULy4#p+QciQ8wnu3HyLLUy?m9v^(_& z71vp9_j9}lX#n>`IF}8vxo$5-QPq}HuDZd4v;ae3Wp5nNt)vmMXE;pi9TK%JH_X59-iU-9{BIHp56+B$@>bf z551xKbw~c|JAdi)NM6yz z_$Aq5LUwd{QB7?1Z`H%fc&)wJNmF++(HyvB|LD)|6Trlj&Cz|ivo)*YVHfd$Q$KQc5{nJ6>tiw_Y1%rk22aF zax)@W5>M)LT6150`(B-aViT-&E8~fOcjp+Ui?IHgXhpG}(=z(aN=RU_K*7HByK^Rw zP)mD(nK){$-&>L{Wbd2sRhIZkPYqsJsWx0MCtMM4l2li)`besXD(}@P zuu`1-tUR72pww7#a*C=KPh{iuvPoVP?vD8Qc_o76p#GvWV&L$sZ8FG~v8Z+Y%o;zV zLfBh7x7b^^_)O2~z;(2C4Z*>)U9z}PL2H3nKjvhes#)w?NT@WL&U$>ex+I!)rQT1A zH^$eq=I||@YItvM&5e80-&aK5JCo2V8rzWg2svMawo#N)wi2Myy6_L-kitt{WpGJP z%7mMEhi%{DUpD!l_SHb&6F7>0lAOVh?xa);SV*b%92u!4+zX=Gf6Y~F%N(NjeW49u z9!;?jxdVS6T4WFr{8h`?xR5{ISeQ2Ei?L!4?Qv@Tz=O!w_nqE{eytvKBCJ%0exz{T zco#}RU&-5;npHN3(ta=_;HvNysW4%Cf$Lre-=Jh=ScI#FenCN$Di^XA)~ z5w1Wto^1OvPO5HOyPIwU(VdA0Y?je3Q+8vUB_&hl^h6p5B9}XUj2S6K)^1Fub(FBH z?MH@%*4#nPkUFEDJ*^S7{naP>Xh7yxapNP=s;}lVhV^tcL*osrjFmM{pyQt=L_9N0 zfW!IQgTXFbA}U)0o$(2EH?u{92g49^7J&M%{RRJ`rK~e~?v~VH?iP_cdDrX`smSfs zva0;7Z*_bhxk<&}zbrizLge%z+CGlSuKx6*gf%O%e1E=#Qm#X(Advw_b(wog-7gPp z*?%6@u>Zr*EI>)sMyb0f0JuI6_l)Y6*TW-~;^6is=HHbrP>u$U979)DwSJo!zK@V= z3XnRY8eHm=d-`_q+HJ}%S-BsuG}^}`VH|8Ud?K``vN_S!4M8=FPZ@?4_G}vBO~+sF z5&JW^avQNZpn~3~3nEk>#2D=0J2*r&>@+JF=sZiTt?@D>&eDfa`uR|lCE^;yzwW5t zV+(qq&=m1w=}EAW{+hRn@>P_`HQmIWUy2zq@O-?b1m<_#^9dbdEh~DbvmKdpC$*_6 zespC&?vD&`cDOb17*sE+O5>~3vh(q>|MH<7`n)j|SdBVVpOqRtzJiA!T$s_bG&@iv zg@^dDx6M&D?GPMx(IYt?qv$Tom=^gS4`j#4iHWn3ZMm)I`N(&-$;(UkE^ghmF%`OV zA%fBCWfJnjt_Uk!ecNxB(pZt$#Py2^dtL1k=OJ7tO!y}9?6(;U(>5Xb9i-gjS3f_< zrF1Ob&^$sVv9Oc7vAlI@)sL!B3e>w6^hrd+Hz@WhPxtsI??-+|YIHd~3_ke$Z{uY6 z%x=b)X+X27d~PlhbS{4)N}kKN%8QlXlk0phVN%y++~aNZU`pTgLwLydStD5zEpiE2 zjj@ifihZ@|Wwj~l_QZ!HxKSs*hL59;-tc`lIT7%~K@Apn#eDK1*ixKJN%ac~JVl=1 z`AMvd-z?`RnTD(XP|mgJu}ktrMbusw>0Jo=ys>L87x!tB)*p9c*G+EF^VKq+i@k|! zzh>IHe`y(iq*Qusm=&+`pqR}Ybyk8Md$oB@`25i52~OU@2i~%IV`+!P>)bc(E0Q!3Wv|VXPNH)4pU~FT7f5pL61_gC7pkiTibr-DDGr% zJF9d8JInWlH9ZKv)KIlLPOz-s0w~Xdr?rz9{a_C?c!N~C3#G)hC&kO@U!^I%8$^V? z$TUh(2{C&l@w!#oiG}IWG0O5r>yg#@@5wPe@wqH;XweH^5`RA@_1#_Xy4!nr=sJ2l zw0E(0$9!Hsde%Z5bd^SZNBF~iTD87I_2J<qzV(t( ze6)Asif)n%>Z^u9ky?eswfiFU#fc?arUL0pg2|*nU=A(d_J*Wl!0|P9aK!JvhJnwd#o=P8w+_2lwFYmlV%7S% z7GilTacqPb+|8m>EN`-A2TR#Bu;HRTPII;x%Ey5~@^B##IbhF|6`%kvRH%7$W4-IQ zTfmBJR<2>G_;NUw`u$iQ>x@D`J{88CSI!tVq{x?0>{E(I!&rTUsY4hFAZvx;729*_ z09LJ6xbVR|Um%?D8Fi%w%NFzu{EY*pf&87o0d=iX{~N&p-&^I;)&QwCF$`hfCW9d| z+LSOvX(tnK+(r+4>g2;PNV=GyCIpxPOBV~rpP2ud!;ymty9y$tL92h4`~~bJh7$-M zxybPc{AW`0cTn>e3W1~&{r}hoSn8regU=tse}k9k{~t_t{!juf?KEgvyUFboBv^hF z7aZYkjAqbiQFRpW7UQrNy31h{&lL<;L2L7f4{JF@0mz|iH5IkT{VFh!xDF}gp z+Wxh;Ww2lZ;8HiHHUNt*W}F*n2i}Tb{JR7B4kjq24v^K(1*kAG0M9!xx^0gfh1b9^ zFoDN|?_ZiB5M~$%mge}Alsmrt3vr-}N<+Sio{ckkC85#_p zbRf!_8d&Sa26v&235sihea4BlxEwNU#RkfP<2nd8c>P@z_JCWD1jiq@{w%Pr(2E>< z$J|`O&+Q&ebD#`+Aojf!5YUf_gg4)kku|tUCK3ol;T%x#9tdOvSOzX}{4u-6 zP5Mm}JkC0xZHaThC1;?_kp)m3<;LXmc|Q}>+Xc-d^CJJBrN2$+zcHV51;GM;Bjeo1 z1by=e;Pl`Iiu$P0)-|dLe<*>Cq!-M^c{fu$(ZEFqe1Lz56AiW^S02p)-4wj=-y*-{ ziv?4O{I?6Jj?k@q1uZ-V1EF|se>_jnLOH|yXzh~Dx2m;4?Ra2A)i?*d7l;NHy7B@R zeb_8h4l+UMg3)~aK>gH09Bv5f7SeQ!Z6wAkeU_pTV z9UZ`i?HDXhbY%k|cm@Q^o&$@wIg_ zO+;(t?`6RFFE+vi6;DC)JVv-Myuo26D04c<;|54aSTQ{P0Vb$>CYl#BAcWyHg1pjf zkjDl58o=~3b8nZV7htNoz~;<%&O`Gm0QHg!_&kI?tI_*0gV*JPX@ zgJnqqrcVBx@ReqO?UCZ2^21nu3}%9~6-XM?L<^FiwcP^W0TA%b3}HVfxY!1;v~vEh nKnTGC|~Xq0&Al5UVLDS=(-QeXuo1yLF$ z1^kv>{y#t7bLPx*?mYLocV_O)Iq&=}!EZ0Xr+lP@Pe2O-k&uAM{4OiTP;!x+_Txv- zc4DM-=$L-FZn@%t{*jJ-F{DCv0&L$8!}WWD*uMIb0V(OF(~!SUcKJj6IO%h*bULu^V!V3JecJ)hEcK$-Y1c+C-rCgwockf92?8 zU({9hBRV!?R`CO|^?DH(QY#j&M>Tq`OYBX%Sy>`Z4tMcyGetREiK(7rw=&^>mMT78 z8PjeF?nAX$n32~z+>fcRaG0aKca+R_cqnmC)Cg@BI%ckMjc@GuQS&;Km1HNFjeyo= zrRx0)nR35sHe$1;HBVB+l-joSqqQG(B(*cMdb~v6wHV2?$i3>XhH4Zmjcx0NU&o)@ zq1$-+a{Ebl0Hgf$7is;>j*2hTJ|Uxj)Wu-ejDjhc5AGhz{B3V}WbGtgWI88+&p7_e z%{TF?6fi%=_thnX!w4{vrWCX?Y=VN_CxF$oFnj9eca4aD+#Hx6`}3x((km zNuma~1f5gtqGN-qUi|SgmXxm949(g{g!Duo*9zB16l zqo6EfjP9L)@#r0xl&IfeHuVyIu^(0R;|F8daMk!yaWO6IQl+0s8!9?|#I((_+|yz( zNhhZb_3R)!G)X^Kt{QP)DvGa#ozI@_@^(-f$!HBKE#SQaz zbpg#XUl9{+Tq#f^Z~h(~`S`NUmaUbH2zj^TXjAz|7Kg9gHq7rFM#XKQ->n@cp2S@I z;J6a~r}eG4%KVbG+wYsB+y~QBW2~dWOWMXW%?pT@4Maa;e>iI9_z6e~8HadwBg3{s z_b7-ZBcW4-DCGHTLj1%{feRbrnG03V_GWeQH;`_wUQb4k@>ouKZ^KUXI>{l^R>orG zjMne}T)VbA^Ob|=+^RiuXGiNejly0J$QX~!i5i!&4~sJ4}TD*V4;e@C}w zAOTnEPvk!Q{kdNmtZls?#=-tl0K%gsYRj1!mML(zX;?ICCA+8H#bntO5hS48OLx^K z%z7NjYgn^vS}V}eq>~d-+oS^{tg$t1;TE^%v_q-9~ zgNHxj7NyE>|5`?O$5fu&pI>jyS?Kf}PHAZGy$M@6cs%~5LfHGA+FhQCY)?PqOvLK? z{Ugm=^>y_mFNFvDBURw32qLIwzAvgaU)W=haH4n7zTWo{be(3nfwgYQKINL%prn(# zRq3j-XNTv<&rGb$(is(of@|h*SA=BhE*07DI>QG`u0nmD9?B#HhiUL)L$Rd6$m70Wn|hrPYZI*xkfN~w4Vah0_4vV#2zH;z=7Quw5=N^j3YM1j?Ic?<0$_ZKMf zBHcHQh*2V;J?H=1D|;Mku~ z_qMljFjBN^s7EO)5)ow<_w$?ov+*jS%igyV_rl&e45;}8f=196#Ppcdb)~rYxgOk0 z`M%P7v}G^$Zb|p!ZbK?Px|yI6sx&gUGn#^y*AF#*yO^h;)KZxvu;1qp>v&ClQ4Gq% z5pGogT^h+<{QS=G$56he=vVY(82TrnM_=R`L-H4nsWx%T zUdocIa`DgSypWTlh;LqFsUcrJ@da_EHK$ft)K!%7bGs<_3b_a_shD50;w1lY{~;xV zkZPBu|1E~Tp4Eh1TT=jD{{Aw!vP3hOIU!NI_~L}EX*Ikqw2-?lWnR*y1@3rpN?AJa zkIE5uKp*Y(@HIZ(h{oX(>XD4%LHLro`?NNj0-b{^IxEaW1MxKUA#xZ7J_)Ce-!a`05v!m!Xf<37TotBZLv6fV2S9~B{_aixot zp0bBIXHvOTJDyb~GS&2MTCUtxv7j+;g~OEjJWgG*1lwKaUX}r5ifadaQd3>aP*>EY zm$m?z{?hxX>-GezM%pFMv0?tI@*)jKf^644)75zcZt>*7he%5O*2SUk_Tzezw_EN$ z{?5}bm@TZV z&d>gmYGo11n#!o2<#XLmQ|PT(UUNcn=bqx)l66tG!c4kY5SN>Bd?ETk{k^bKi8_a# zmcgKwhvbdf+@xoo|>fc45NVM+S zirK@0v|$#eH&^r-N(|JX#J@6}s>REU<*RnpUf+5#RQR+^vZp z+9a$j`Wd8gufRNMj4S2vSH&8g4AD|`Bt@U;z_Vq#X4C&KG1IwcAd|Sj)t8Sl)ym2-BUH&VH+``2PP?@g& z&{xs#&r^4qj3FjrAM)^C9Z}2&@Gf{&QE{w>eJbVGFjzP$TsS~}=CO++YxS~mIOz9G zuU_!hr@xb|fL|^qQgE2_WoYroqSV%3w$x^Y&`cdBC(`k}1dS!-iJ`GS2~XUspuhW^ zAl|BoorLT+b{+Q})~}-Mv>13r9GP?*?-*+D1upmTYN%JQ{k`U)R{0xk`qAGPn(?vE zt2mF(ZsGb-smOflV20yJdk%w8r!13)F78)y3Jzn+jY5-y1G* zyiQeU2@=k>&KI+DhK?`zeDfZC68GX)L*~hz^@~x#Ps)13nUc_f*3PMGc>;jZ7=^&dM&MAC^hQ$yRr3kMcOPt;1^LBp%u-v=vte(h(-X zrlcepDqM>i9)_>XOXqhzh~Qf!FYOSVwNOOzHnX(J4;sx&x2Ky7vgv4Y2i&(R{nit) zTI~BAnmBCGUYs#(QFBGm%sn=|lUzrHg_I6aQ3o5IzC_Ar46Srn8{vGmc^LY*JykSAG8<&nPPfB%_s|tZ=q#dLA8pED@^!%a z^9u*GQl_t0RTN?`hk=-}v)<%l{Q@l|2QuAvSdQU0%{pE)wcMs23o_@1tjxWAw~k{` z*-ZhK57mr0C=S0g-#2s942^qQ>;3>MB2l2L4jN-L}7i(LvkrfHSme>&y>ryo} zb<+_KKXL}j+ZR}tIFf8Hn5pC(x-~H{qjI$NZDWbB`CFK<9|zJ|Tg%vqi$ZY04WWiy zk?#@w6dQhBy-~U0#Tt~n!J=X1m_(YXwf0358G45&-0_Rbb&udZ1Jv|4uusPyGW5)J zZ6LlMy>qmSc2DTQOVXrVDEzMQ1NMUCNs<0=QK&~bFOn+Zy`&D&3VT3EyJ)m8Re*Nh ze)gE!`nE<{(&{{@wp)1n+6F=&4^dC$9!y0y%)D;rM;}7I%(hNJ)x8q~n&?4^-AI{r zRLxY1W`ELNnPGh8BI&khhW#qZj9NQ8>1&y&ELgWBr>`IX0uEoJE955n7JJ; zx8+Nam0G*CCqeFPg@1BeVvWUwe_cjiu8kDYt^3@2OD?Sk{M(snI)pCH&q>JiFQe1r z%I=MQ^{;+fMXnnZZ;ZdPkVUz%tT#XB?;ls;imE%RJi{1yGX z=`1znU0h#|yJ9Q+`3HQU-w?Aeli4p3nZ=6Zm|;Bap{93|+UbCBTga zzLCVoq1NmIY8<-ql9dF9m^+5fJPJKx&vzegEhtVlBx*=dV8E}XH3NX}7fwNufg@9?uF*Z6tS%n7p+<#{G{Qv>q zqQ=@wKk(tWy*&^x9@8230-&nI2(b53{Clo(LeYcCnX%(1C=wtS#l?PF9;g5Bu>>HH z3@He74KuTUpFI>y_R@s@3p|5KQkfHDrw1O)1cId4<1<=$(fz9LGlfb_uFzAx-na)`}LCi$0Z((fofJqM{*10~+ zcqN+*1mb1Hv~$j#>R4xcP6hz+zg7K$S-mntto=eaH&&qk!V}pT%po}-0)hC?31}K) ziH?25fRQc@z~6^UcOhulh!sO*!L-Zg9IlxGZ~KJUPb>4(juFLvRA)V|i!zJthhpm( zI2wU~Tg`yVkDP$h2r+PHlo9J|M0ImC2jdJU27yG*`DR&S$(w_0IDt0<5OBN=#>N6% zbfy4q3{c?g{zDM3ofDS(Zb%p_Dr{sCm5&iy#x$ODqD-z>;;K6dATdaRC8yaN3>Dyk zK>3)nb>keF&<#sIO?PSpiX}(f5wrKhkjXJ*$#Z0|JD}Ar4LJAXRAB`26igVR)#=>M z5yw5Sa|j$@#^oQ_4*`pN0W$qez!U2${|S6E9fQqboEbL*VIh95SqiZ^V4@-Qx1 zP+kp#6*UNi!c+zKIWm7BhAfJUw1rtIbS&0ca2SrW{rv<1AEp4zV{+K}$<~qV$zbNz zi#aAz=Y$5*u#TI9M8LEM9pEr@w)`>}0kcd%wMF1`ORysPUXDp|n6y7&u8{ND*yLb{ zoB-?-?m7evh9TfPC0OBcwk)Su-vI+n~&XZ@TQvq#s ziZ5Tjuq}t#UnPu_&^d>eCX53M0C8aWSEE9h;@4>`NmoFJ$JIO^o!``=wmf_Wc5y|VxRJ&m~!{@Hq)$ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 08227bc..25af6b2 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Fri Jul 07 11:19:17 EDT 2017 +#Fri Sep 01 12:45:24 EDT 2017 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-3.5-bin.zip From fcca24891e0dac366d34f6db26e291585ba4b80e Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Fri, 8 Sep 2017 14:10:23 -0400 Subject: [PATCH 15/44] Install Thrift on Travis --- ci/before_install.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/before_install.sh b/ci/before_install.sh index e26aa9d..f9410eb 100755 --- a/ci/before_install.sh +++ b/ci/before_install.sh @@ -10,3 +10,10 @@ wget "https://bouncycastle.org/download/${BCPROV_FILENAME}" sudo mv $BCPROV_FILENAME /usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext sudo perl -pi.bak -e 's/^(security\.provider\.)([0-9]+)/$1.($2+1)/ge' /etc/java-7-openjdk/security/java.security echo "security.provider.1=org.bouncycastle.jce.provider.BouncyCastleProvider" | sudo tee -a /etc/java-7-openjdk/security/java.security + +# Install Thrift 0.9.3 +sudo apt-get update -qq +sudo apt-get install automake bison flex g++ git libboost1.55-all-dev libevent-dev libssl-dev libtool make pkg-config +wget http://www.us.apache.org/dist/thrift/0.9.3/thrift-0.9.3.tar.gz +tar xfz thrift-0.9.3.tar.gz +cd thrift-0.9.3 && ./configure && sudo make install From 46bd28e5bba698c659a8c91bdd26c528d9cda27b Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Fri, 8 Sep 2017 14:43:07 -0400 Subject: [PATCH 16/44] Install different packages to install Thrift --- ci/before_install.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/before_install.sh b/ci/before_install.sh index f9410eb..42bc3a8 100755 --- a/ci/before_install.sh +++ b/ci/before_install.sh @@ -13,7 +13,10 @@ echo "security.provider.1=org.bouncycastle.jce.provider.BouncyCastleProvider" | # Install Thrift 0.9.3 sudo apt-get update -qq -sudo apt-get install automake bison flex g++ git libboost1.55-all-dev libevent-dev libssl-dev libtool make pkg-config +sudo apt-get install libboost-dev libboost-test-dev libboost-program-options-dev libevent-dev automake libtool flex bison pkg-config g++ libssl-dev +pushd . +cd wget http://www.us.apache.org/dist/thrift/0.9.3/thrift-0.9.3.tar.gz tar xfz thrift-0.9.3.tar.gz cd thrift-0.9.3 && ./configure && sudo make install +popd From 3b45aa7335b4f5768638f7a74f7c641f6cb51301 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Mon, 11 Sep 2017 15:40:07 -0400 Subject: [PATCH 17/44] More M3 reporter functionality --- .../com/uber/m3/tally/BucketPairImpl.java | 76 +++++++- .../java/com/uber/m3/tally/HistogramImpl.java | 74 +------- .../com/uber/m3/tally/HistogramImplTest.java | 4 +- .../java/com/uber/m3/tally/m3/M3Reporter.java | 176 +++++++++++++++--- 4 files changed, 231 insertions(+), 99 deletions(-) diff --git a/core/src/main/java/com/uber/m3/tally/BucketPairImpl.java b/core/src/main/java/com/uber/m3/tally/BucketPairImpl.java index ab2882f..8be0256 100644 --- a/core/src/main/java/com/uber/m3/tally/BucketPairImpl.java +++ b/core/src/main/java/com/uber/m3/tally/BucketPairImpl.java @@ -22,10 +22,12 @@ import com.uber.m3.util.Duration; +import java.util.Collections; + /** * Default implementation of a {@link BucketPair} */ -class BucketPairImpl implements BucketPair { +public class BucketPairImpl implements BucketPair { private double lowerBoundValue; private double upperBoundValue; private Duration lowerBoundDuration; @@ -43,6 +45,78 @@ class BucketPairImpl implements BucketPair { this.upperBoundDuration = upperBoundDuration; } + public static BucketPair[] bucketPairs(Buckets buckets) { + if (buckets == null || buckets.size() < 1) { + return new BucketPair[]{ + new BucketPairImpl( + -Double.MAX_VALUE, + Double.MAX_VALUE, + Duration.MIN_VALUE, + Duration.MAX_VALUE + ) + }; + } + + if (buckets instanceof DurationBuckets) { + // If using duration buckets separating negative times and + // positive times is very much desirable as depending on the + // reporter will create buckets "-infinity,0" and "0,{first_bucket}" + // instead of just "-infinity,{first_bucket}" which for time + // durations is not desirable nor pragmatic + boolean hasZero = false; + + for (Duration duration : buckets.asDurations()) { + if (duration.equals(Duration.ZERO)) { + hasZero = true; + break; + } + } + + if (!hasZero) { + ((DurationBuckets) buckets).add(Duration.ZERO); + } + } + + Collections.sort(buckets); + + Double[] asValueBuckets = buckets.asValues(); + Duration[] asDurationBuckets = buckets.asDurations(); + BucketPair[] pairs = new BucketPair[buckets.size() + 1]; + + // Add lower bound + pairs[0] = new BucketPairImpl( + -Double.MAX_VALUE, + asValueBuckets[0], + Duration.MIN_VALUE, + asDurationBuckets[0] + ); + + double prevValueBucket = asValueBuckets[0]; + Duration prevDurationBucket = asDurationBuckets[0]; + + for (int i = 1; i < buckets.size(); i++) { + pairs[i] = new BucketPairImpl( + prevValueBucket, + asValueBuckets[i], + prevDurationBucket, + asDurationBuckets[i] + ); + + prevValueBucket = asValueBuckets[i]; + prevDurationBucket = asDurationBuckets[i]; + } + + // Add upper bound + pairs[pairs.length - 1] = new BucketPairImpl( + prevValueBucket, + Double.MAX_VALUE, + prevDurationBucket, + Duration.MAX_VALUE + ); + + return pairs; + } + @Override public double lowerBoundValue() { return lowerBoundValue; diff --git a/core/src/main/java/com/uber/m3/tally/HistogramImpl.java b/core/src/main/java/com/uber/m3/tally/HistogramImpl.java index fc5b8fc..9a59108 100644 --- a/core/src/main/java/com/uber/m3/tally/HistogramImpl.java +++ b/core/src/main/java/com/uber/m3/tally/HistogramImpl.java @@ -55,7 +55,7 @@ class HistogramImpl implements Histogram { type = Type.VALUE; } - BucketPair[] pairs = bucketPairs(buckets); + BucketPair[] pairs = BucketPairImpl.bucketPairs(buckets); int pairsLen = pairs.length; this.name = name; @@ -201,78 +201,6 @@ Map snapshotDurations() { return durations; } - static BucketPair[] bucketPairs(Buckets buckets) { - if (buckets == null || buckets.size() < 1) { - return new BucketPair[]{ - new BucketPairImpl( - -Double.MAX_VALUE, - Double.MAX_VALUE, - Duration.MIN_VALUE, - Duration.MAX_VALUE - ) - }; - } - - if (buckets instanceof DurationBuckets) { - // If using duration buckets separating negative times and - // positive times is very much desirable as depending on the - // reporter will create buckets "-infinity,0" and "0,{first_bucket}" - // instead of just "-infinity,{first_bucket}" which for time - // durations is not desirable nor pragmatic - boolean hasZero = false; - - for (Duration duration : buckets.asDurations()) { - if (duration.equals(Duration.ZERO)) { - hasZero = true; - break; - } - } - - if (!hasZero) { - ((DurationBuckets) buckets).add(Duration.ZERO); - } - } - - Collections.sort(buckets); - - Double[] asValueBuckets = buckets.asValues(); - Duration[] asDurationBuckets = buckets.asDurations(); - BucketPair[] pairs = new BucketPair[buckets.size() + 1]; - - // Add lower bound - pairs[0] = new BucketPairImpl( - -Double.MAX_VALUE, - asValueBuckets[0], - Duration.MIN_VALUE, - asDurationBuckets[0] - ); - - double prevValueBucket = asValueBuckets[0]; - Duration prevDurationBucket = asDurationBuckets[0]; - - for (int i = 1; i < buckets.size(); i++) { - pairs[i] = new BucketPairImpl( - prevValueBucket, - asValueBuckets[i], - prevDurationBucket, - asDurationBuckets[i] - ); - - prevValueBucket = asValueBuckets[i]; - prevDurationBucket = asDurationBuckets[i]; - } - - // Add upper bound - pairs[pairs.length - 1] = new BucketPairImpl( - prevValueBucket, - Double.MAX_VALUE, - prevDurationBucket, - Duration.MAX_VALUE - ); - - return pairs; - } - class HistogramBucket { CounterImpl samples; double valueLowerBound; diff --git a/core/src/test/java/com/uber/m3/tally/HistogramImplTest.java b/core/src/test/java/com/uber/m3/tally/HistogramImplTest.java index 68aa161..c46006d 100644 --- a/core/src/test/java/com/uber/m3/tally/HistogramImplTest.java +++ b/core/src/test/java/com/uber/m3/tally/HistogramImplTest.java @@ -166,7 +166,7 @@ public void bucketPairs() { ) }; - assertArrayEquals(expectedPairs, HistogramImpl.bucketPairs(null)); + assertArrayEquals(expectedPairs, BucketPairImpl.bucketPairs(null)); expectedPairs = new BucketPair[]{ new BucketPairImpl( @@ -197,7 +197,7 @@ public void bucketPairs() { assertArrayEquals( expectedPairs, - HistogramImpl.bucketPairs(ValueBuckets.linear(0, 50, 3)) + BucketPairImpl.bucketPairs(ValueBuckets.linear(0, 50, 3)) ); } } diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 87205f3..f0c0fc3 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -20,10 +20,13 @@ package com.uber.m3.tally.m3; +import com.uber.m3.tally.BucketPair; +import com.uber.m3.tally.BucketPairImpl; import com.uber.m3.tally.Buckets; import com.uber.m3.tally.CachedCounter; import com.uber.m3.tally.CachedGauge; import com.uber.m3.tally.CachedHistogram; +import com.uber.m3.tally.CachedHistogramBucket; import com.uber.m3.tally.CachedStatsReporter; import com.uber.m3.tally.CachedTimer; import com.uber.m3.tally.Capabilities; @@ -49,6 +52,7 @@ import java.net.InetAddress; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -116,7 +120,6 @@ private M3Reporter(Builder builder) { } TTransport transport = new TUdpClient(hostPorts[0]); - //TODO multi UDP client transport client = new M3.Client(protocolFactory.getProtocol(transport)); resourcePool = new ResourcePool(builder.maxQueueSize, protocolFactory); @@ -182,27 +185,71 @@ private void addAndRunProcessor() { public CachedCounter allocateCounter(String name, Map tags) { Metric counter = newMetric(name, tags, MetricType.COUNTER); - return new CachedCounterImpl(counter, calculateSize(counter)); + return new CachedMetric(counter, calculateSize(counter)); } @Override public CachedGauge allocateGauge(String name, Map tags) { Metric gauge = newMetric(name, tags, MetricType.GAUGE); - return new CachedGaugeImpl(gauge, calculateSize(gauge)); + return new CachedMetric(gauge, calculateSize(gauge)); } @Override public CachedTimer allocateTimer(String name, Map tags) { Metric timer = newMetric(name, tags, MetricType.TIMER); - return new CachedTimerImpl(timer, calculateSize(timer)); + return new CachedMetric(timer, calculateSize(timer)); } @Override public CachedHistogram allocateHistogram(String name, Map tags, Buckets buckets) { - //TODO - return null; + List cachedValueBuckets = new ArrayList<>(); + List cachedDurationBuckets = new ArrayList<>(); + + int bucketIdLen = String.valueOf(buckets.size()).length(); + bucketIdLen = Math.max(bucketIdLen, MIN_METRIC_BUCKET_ID_TAG_LENGTH); + + String bucketIdFmt = String.format("%%0%sd", bucketIdLen); + + BucketPair[] bucketPairs = BucketPairImpl.bucketPairs(buckets); + + for (int i = 0; i < bucketPairs.length; i++) { + Map valueTags = new HashMap<>(tags); + Map durationTags = new HashMap<>(tags); + + String idTagValue = String.format(bucketIdFmt, i); + + valueTags.put(bucketIdTagName, idTagValue); + valueTags.put(bucketTagName, String.format("%s-%s", + valueBucketString(bucketPairs[i].lowerBoundValue()), + valueBucketString(bucketPairs[i].upperBoundValue()))); + + cachedValueBuckets.add(new CachedHistogramBucketImpl( + bucketPairs[i].upperBoundValue(), + bucketPairs[i].upperBoundDuration(), + (CachedMetric) allocateCounter(name, valueTags) + )); + + durationTags.put(bucketIdTagName, idTagValue); + durationTags.put(bucketTagName, String.format("%s-%s", + durationBucketString(bucketPairs[i].lowerBoundDuration()), + durationBucketString(bucketPairs[i].upperBoundDuration()))); + + cachedDurationBuckets.add(new CachedHistogramBucketImpl( + bucketPairs[i].upperBoundValue(), + bucketPairs[i].upperBoundDuration(), + (CachedMetric) allocateCounter(name, durationTags) + )); + } + + return new CachedHistogramImpl( + name, + tags, + buckets, + cachedValueBuckets, + cachedDurationBuckets + ); } @Override @@ -228,6 +275,9 @@ private List flush(List metrics) throws TException { try { client.emitMetricBatch(metricBatch); } catch (TException e) { + // Store exception to throw later after clean up. + // Don't want to put everything in a finally block here, + // which will hold the lock unnecessarily long exception = e; } @@ -359,6 +409,30 @@ private void reportCopyMetric( } } + private String valueBucketString(double bucketBound) { + if (bucketBound == Double.MAX_VALUE) { + return "infinity"; + } + + if (bucketBound == -Double.MAX_VALUE) { + return "-infinity"; + } + + return String.format(bucketValFmt, bucketBound); + } + + private String durationBucketString(Duration bucketBound) { + if (Duration.MAX_VALUE.equals(bucketBound)) { + return "infinity"; + } + + if (Duration.MIN_VALUE.equals(bucketBound)) { + return "-infinity"; + } + + return bucketBound.toString(); + } + private class Processor implements Runnable { @Override public void run() { @@ -404,7 +478,8 @@ public void run() { } } - private abstract class CachedMetric { + private class CachedMetric implements + CachedCounter, CachedGauge, CachedTimer, CachedHistogramBucket { Metric metric; int size; @@ -412,38 +487,93 @@ private abstract class CachedMetric { this.metric = metric; this.size = size; } - } - private class CachedCounterImpl extends CachedMetric implements CachedCounter { - private CachedCounterImpl(Metric metric, int size) { - super(metric, size); + @Override + public void reportCount(long value) { + reportCopyMetric(metric, size, MetricType.COUNTER, value, 0); } @Override - public void reportCount(long value) { + public void reportGauge(double value) { + reportCopyMetric(metric, size, MetricType.GAUGE, 0, value); + } + + @Override + public void reportTimer(Duration interval) { + reportCopyMetric(metric, size, MetricType.TIMER, interval.getNanos(), 0); + } + + @Override + public void reportSamples(long value) { reportCopyMetric(metric, size, MetricType.COUNTER, value, 0); } } - private class CachedGaugeImpl extends CachedMetric implements CachedGauge { - private CachedGaugeImpl(Metric metric, int size) { - super(metric, size); + private class CachedHistogramImpl implements CachedHistogram { + String name; + Map tags; + Buckets buckets; + List cachedValueBuckets; + List cachedDurationBuckets; + + CachedHistogramImpl( + String name, + Map tags, + Buckets buckets, + List cachedValueBuckets, + List cachedDurationBuckets + ) { + this.name = name; + this.tags = tags; + this.buckets = buckets; + this.cachedValueBuckets = cachedValueBuckets; + this.cachedDurationBuckets = cachedDurationBuckets; } @Override - public void reportGauge(double value) { - reportCopyMetric(metric, size, MetricType.GAUGE, 0, value); + public CachedHistogramBucket valueBucket(double bucketLowerBound, double bucketUpperBound) { + for (CachedHistogramBucketImpl bucket : cachedValueBuckets) { + if (bucket.getValueUpperBound() >= bucketUpperBound) { + return bucket.getMetric(); + } + } + + return null; + } + + @Override + public CachedHistogramBucket durationBucket(Duration bucketLowerBound, Duration bucketUpperBound) { + for (CachedHistogramBucketImpl bucket : cachedValueBuckets) { + if (bucket.getDurationUpperBound().compareTo(bucketUpperBound) >= 0) { + return bucket.getMetric(); + } + } + + return null; } } - private class CachedTimerImpl extends CachedMetric implements CachedTimer { - private CachedTimerImpl(Metric metric, int size) { - super(metric, size); + private class CachedHistogramBucketImpl { + private double valueUpperBound; + private Duration durationUpperBound; + private CachedHistogramBucket metric; + + CachedHistogramBucketImpl(double valueUpperBound, Duration durationUpperBound, CachedMetric metric) { + this.valueUpperBound = valueUpperBound; + this.durationUpperBound = durationUpperBound; + this.metric = metric; + } + + double getValueUpperBound() { + return valueUpperBound; } - @Override - public void reportTimer(Duration interval) { - reportCopyMetric(metric, size, MetricType.TIMER, interval.getNanos(), 0); + Duration getDurationUpperBound() { + return durationUpperBound; + } + + CachedHistogramBucket getMetric() { + return metric; } } From d0aef44cdfd2b9054a980c329173be08c3034300 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Mon, 11 Sep 2017 17:56:36 -0400 Subject: [PATCH 18/44] Improve comments and Javadocs --- .../com/uber/m3/tally/BucketPairImpl.java | 2 +- .../java/com/uber/m3/tally/m3/M3Reporter.java | 20 ++++++----- .../com/uber/m3/tally/m3/ResourcePool.java | 2 +- .../java/com/uber/m3/util/ObjectPool.java | 10 +++--- .../java/com/uber/m3/util/TCalcTransport.java | 36 ++++++++++++------- .../java/com/uber/m3/util/TUdpClient.java | 14 +++++++- .../java/com/uber/m3/util/TUdpServer.java | 15 +++++++- .../java/com/uber/m3/util/TUdpTransport.java | 2 +- 8 files changed, 70 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/com/uber/m3/tally/BucketPairImpl.java b/core/src/main/java/com/uber/m3/tally/BucketPairImpl.java index 8be0256..17732c8 100644 --- a/core/src/main/java/com/uber/m3/tally/BucketPairImpl.java +++ b/core/src/main/java/com/uber/m3/tally/BucketPairImpl.java @@ -33,7 +33,7 @@ public class BucketPairImpl implements BucketPair { private Duration lowerBoundDuration; private Duration upperBoundDuration; - BucketPairImpl( + public BucketPairImpl( double lowerBoundValue, double upperBoundValue, Duration lowerBoundDuration, diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index f0c0fc3..4589c55 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -92,8 +92,6 @@ public class M3Reporter implements CachedStatsReporter, AutoCloseable { private TCalcTransport calc; private TProtocol calcProtocol; - private Set commonTags; - private int freeBytes; private ResourcePool resourcePool; @@ -102,9 +100,13 @@ public class M3Reporter implements CachedStatsReporter, AutoCloseable { private String bucketTagName; private String bucketValFmt; + // Holds metrics to be flushed private BlockingQueue metricQueue; - private Phaser phaser = new Phaser(1); + // The service used to execute metric flushes private ExecutorService executor; + // Used to keep track of how many processors are in progress in + // order to wait for them to finish before closing + private Phaser phaser = new Phaser(1); // Use inner Builder class to construct an M3Reporter private M3Reporter(Builder builder) { @@ -124,7 +126,7 @@ private M3Reporter(Builder builder) { client = new M3.Client(protocolFactory.getProtocol(transport)); resourcePool = new ResourcePool(builder.maxQueueSize, protocolFactory); - commonTags = addMetricTagsToSet(builder.commonTags); + Set commonTags = addMetricTagsToSet(builder.commonTags); // Set and ensure required tags if (!builder.commonTags.containsKey(SERVICE_TAG)) { @@ -154,7 +156,7 @@ private M3Reporter(Builder builder) { calcProtocol = resourcePool.getProtocol(); metricBatch.write(calcProtocol); calc = (TCalcTransport) calcProtocol.getTransport(); - int numOverheadBytes = EMIT_METRIC_BATCH_OVERHEAD + calc.getCountAndReset(); + int numOverheadBytes = EMIT_METRIC_BATCH_OVERHEAD + calc.getSizeAndReset(); freeBytes = builder.maxPacketSizeBytes - numOverheadBytes; if (freeBytes <= 0) { @@ -260,6 +262,8 @@ public Capabilities capabilities() { @Override public void flush() { try { + // A null metric signifies to processors that a flush was called, + // which causes the metric queue to be cleared metricQueue.put(new SizedMetric(null, 0)); } catch (InterruptedException e) { throw new RuntimeException(e); @@ -297,6 +301,7 @@ private List flush(List metrics) throws TException { @Override public void close() { + // Wait (block) until all processors to return from flushing phaser.arriveAndAwaitAdvance(); } @@ -363,7 +368,7 @@ private int calculateSize(Metric metric) { synchronized (calcLock) { metric.write(calcProtocol); - return calc.getCountAndReset(); + return calc.getSizeAndReset(); } } catch (TException e) { return DEFAULT_METRIC_SIZE; @@ -478,8 +483,7 @@ public void run() { } } - private class CachedMetric implements - CachedCounter, CachedGauge, CachedTimer, CachedHistogramBucket { + private class CachedMetric implements CachedCounter, CachedGauge, CachedTimer, CachedHistogramBucket { Metric metric; int size; diff --git a/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java b/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java index fa065c5..5c58539 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java @@ -193,7 +193,7 @@ public TProtocol getProtocol() { * @param protocol the {@link TProtocol} to put back to the pool */ public void releaseProtocol(TProtocol protocol) { - ((TCalcTransport) protocol.getTransport()).resetCount(); + ((TCalcTransport) protocol.getTransport()).resetSize(); protocolPool.put(protocol); } diff --git a/m3/src/main/java/com/uber/m3/util/ObjectPool.java b/m3/src/main/java/com/uber/m3/util/ObjectPool.java index 9db3369..1faec7e 100644 --- a/m3/src/main/java/com/uber/m3/util/ObjectPool.java +++ b/m3/src/main/java/com/uber/m3/util/ObjectPool.java @@ -61,12 +61,12 @@ public ObjectPool(int capacity, Construct construct, boolean fair) { /** * Gets an object from the pool. If the pool is empty, it will create and return - * a new instance instead of blocking until a new - * @return an object from the pool + * a new instance instead of blocking until an object is available. + * @return an object from the pool if available. Otherwise, a new instance of the object. */ public E get() { if (objects.isEmpty()) { - // Rather than block, return a newly a newly created instance instead + // Rather than block, return a newly created instance instead return construct.instance(); } @@ -85,8 +85,8 @@ public E get() { * @return whether the element got put back or not */ public boolean put(E element) { - // Use offer instead of put (which blocks until there is space), because a full - // ObjectPool is a fine state to be in. We can get into this state because the + // Use `offer` instead of `put` (which blocks until there is space), because a full + // ObjectPool is a good state to be in already. We can get into this state because the // get() function will create a new instance instead of waiting for an object // to be in the pool. return objects.offer(element); diff --git a/m3/src/main/java/com/uber/m3/util/TCalcTransport.java b/m3/src/main/java/com/uber/m3/util/TCalcTransport.java index 18e1f78..dc2afbb 100644 --- a/m3/src/main/java/com/uber/m3/util/TCalcTransport.java +++ b/m3/src/main/java/com/uber/m3/util/TCalcTransport.java @@ -23,11 +23,13 @@ import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; +import java.util.concurrent.atomic.AtomicInteger; + /** - * Dummy transport used for calculating size only + * Dummy transport used for calculating size of metrics only. */ public class TCalcTransport extends TTransport { - private int count = 0; + private AtomicInteger size = new AtomicInteger(0); @Override public boolean isOpen() { @@ -49,22 +51,30 @@ public int read(byte[] bytes, int i, int i1) throws TTransportException { @Override public void write(byte[] bytes, int i, int len) throws TTransportException { - count += len; + size.getAndAdd(len); } - public int getCountAndReset() { - int currentCount = count; - - resetCount(); - - return currentCount; + /** + * Returns the size that has been written to this transport since the last reset + * and resets the size. + * @return the size that has been written to this transport since the last reset + */ + public int getSizeAndReset() { + return size.getAndSet(0); } - public int getCount() { - return count; + /** + * Returns the size that has been written to this transport since the last reset. + * @return the size that has been written to this transport since the last reset + */ + public int getSize() { + return size.get(); } - public void resetCount() { - count = 0; + /** + * Resets the size that has been written to this transport since the last reset. + */ + public void resetSize() { + size.set(0); } } diff --git a/m3/src/main/java/com/uber/m3/util/TUdpClient.java b/m3/src/main/java/com/uber/m3/util/TUdpClient.java index d4338d5..85e22c3 100644 --- a/m3/src/main/java/com/uber/m3/util/TUdpClient.java +++ b/m3/src/main/java/com/uber/m3/util/TUdpClient.java @@ -26,13 +26,25 @@ import java.net.SocketException; /** - * A client for sending data via Thrift UDP + * A client for sending data via Thrift UDP. */ public class TUdpClient extends TUdpTransport implements AutoCloseable { + /** + * Constructs a UDP client with the given host and port. + * @param host the host for this transport + * @param port the port for this transport + * @throws SocketException + */ public TUdpClient(String host, int port) throws SocketException { super(host, port); } + /** + * Constructs a UDP client with the given host and port. + * @param hostPort a string containing both host and port information in the + * form "HOST:PORT" + * @throws SocketException + */ public TUdpClient(String hostPort) throws SocketException { super(hostPort); } diff --git a/m3/src/main/java/com/uber/m3/util/TUdpServer.java b/m3/src/main/java/com/uber/m3/util/TUdpServer.java index 4b03b7b..ef2ab9d 100644 --- a/m3/src/main/java/com/uber/m3/util/TUdpServer.java +++ b/m3/src/main/java/com/uber/m3/util/TUdpServer.java @@ -26,15 +26,28 @@ import java.net.SocketException; /** - * A server for receiving data via Thrift UDP + * A server for receiving data via Thrift UDP. */ public class TUdpServer extends TUdpTransport implements AutoCloseable { private int timeoutMillis; + /** + * Constructs a UDP server with the given host and port. Defaults to zero timeout. + * @param host the host for this transport + * @param port the port for this transport + * @throws SocketException + */ public TUdpServer(String host, int port) throws SocketException { this(host, port, 0); } + /** + * Constructs a UDP server with the given host and port. + * @param host the host for this transport + * @param port the port for this transport + * @param timeoutMillis the timeout in milliseconds + * @throws SocketException + */ public TUdpServer(String host, int port, int timeoutMillis) throws SocketException { super(host, port); diff --git a/m3/src/main/java/com/uber/m3/util/TUdpTransport.java b/m3/src/main/java/com/uber/m3/util/TUdpTransport.java index 5d2965c..752b801 100644 --- a/m3/src/main/java/com/uber/m3/util/TUdpTransport.java +++ b/m3/src/main/java/com/uber/m3/util/TUdpTransport.java @@ -30,7 +30,7 @@ import java.nio.ByteBuffer; /** - * Abstract class that supports Thrift UDP functionality + * Abstract class that supports Thrift UDP functionality. */ public abstract class TUdpTransport extends TTransport implements AutoCloseable { public static final int MAX_PACKET_SIZE = 14400; From c9932e5d2a7655c57f1db14a0e13bd0781d6572e Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Tue, 12 Sep 2017 13:08:15 -0400 Subject: [PATCH 19/44] Fix ObjectPool and SizedMetric bug; add some tests --- .../com/uber/m3/tally/m3/SizedMetric.java | 7 +- .../java/com/uber/m3/util/ObjectPool.java | 2 +- ...ReporterTest.java => SizedMetricTest.java} | 32 +++--- .../java/com/uber/m3/util/ObjectPoolTest.java | 98 +++++++++++++++++++ 4 files changed, 121 insertions(+), 18 deletions(-) rename m3/src/test/java/com/uber/m3/tally/m3/{M3ReporterTest.java => SizedMetricTest.java} (64%) create mode 100644 m3/src/test/java/com/uber/m3/util/ObjectPoolTest.java diff --git a/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java b/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java index f70c573..9bda64a 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java @@ -36,7 +36,12 @@ public class SizedMetric extends Metric { */ public SizedMetric(Metric metric, int size) { // Recreate metric to prevent underlying metric from being modified - this.metric = new Metric(metric); + if (metric == null) { + this.metric = null; + } else { + this.metric = new Metric(metric); + } + this.size = size; } diff --git a/m3/src/main/java/com/uber/m3/util/ObjectPool.java b/m3/src/main/java/com/uber/m3/util/ObjectPool.java index 1faec7e..17b44d1 100644 --- a/m3/src/main/java/com/uber/m3/util/ObjectPool.java +++ b/m3/src/main/java/com/uber/m3/util/ObjectPool.java @@ -54,7 +54,7 @@ public ObjectPool(int capacity, Construct construct, boolean fair) { this.construct = construct; - for (int i = 0; i < objects.size(); i++) { + for (int i = 0; i < capacity; i++) { objects.add(construct.instance()); } } diff --git a/m3/src/test/java/com/uber/m3/tally/m3/M3ReporterTest.java b/m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java similarity index 64% rename from m3/src/test/java/com/uber/m3/tally/m3/M3ReporterTest.java rename to m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java index 260c3d9..e1c77d4 100644 --- a/m3/src/test/java/com/uber/m3/tally/m3/M3ReporterTest.java +++ b/m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java @@ -20,29 +20,29 @@ package com.uber.m3.tally.m3; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; +import com.uber.m3.thrift.generated.Metric; import org.junit.Test; -public class M3ReporterTest { - @BeforeClass - public static void start() { +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertEquals; - } - - @Before - public void setUp() { - - } +public class SizedMetricTest { + @Test + public void simpleSizedMetric() { + Metric metric = new Metric(); - @AfterClass - public static void stop() { + SizedMetric sizedMetric = new SizedMetric(metric, 1); + assertFalse(sizedMetric.getMetric() == metric); + assertEquals(sizedMetric.getMetric(), metric); + assertEquals(sizedMetric.getSize(), 1); } @Test - public void capabilities() { + public void nullSizedMetric() { + SizedMetric sizedMetric = new SizedMetric(null, 9); + assertEquals(sizedMetric.getMetric(), null); + assertEquals(sizedMetric.getSize(), 9); } -} +} \ No newline at end of file diff --git a/m3/src/test/java/com/uber/m3/util/ObjectPoolTest.java b/m3/src/test/java/com/uber/m3/util/ObjectPoolTest.java new file mode 100644 index 0000000..fa3c116 --- /dev/null +++ b/m3/src/test/java/com/uber/m3/util/ObjectPoolTest.java @@ -0,0 +1,98 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.util; + +import org.junit.Assert; +import org.junit.Test; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +public class ObjectPoolTest { + private final Construct OBJECT_CONSTRUCT = new Construct() { + @Override + public Object instance() { + return new Object(); + } + }; + + @Test(expected = IllegalArgumentException.class) + public void negativeCapacityPool() { + ObjectPool pool = new ObjectPool<>(-3, OBJECT_CONSTRUCT); + } + + @Test(expected = IllegalArgumentException.class) + public void emptyPool() { + ObjectPool pool = new ObjectPool<>(0, OBJECT_CONSTRUCT); + } + + @Test + public void oneCapacityPool() { + ObjectPool pool = new ObjectPool<>(1, OBJECT_CONSTRUCT); + + Object obj1 = pool.get(); + // Pool should create new instances when pool is empty + Object obj2 = pool.get(); + Object obj3 = pool.get(); + + assertNotNull(obj1); + assertNotNull(obj2); + assertNotNull(obj3); + + // Should be able to put obj1 back to the pool because it's empty now + assertTrue(pool.put(obj1)); + // Should not be able to put obj2 back to the pool because it's full now + assertFalse(pool.put(obj2)); + + // Purposely don't use equals or assertEqual because we want to test that + // the object we get back is exactly the same (as opposed to sematically + // the same) as the one we put back. + assertTrue(obj1 == pool.get()); + } + + @Test + public void twoCapacityPool() { + ObjectPool pool = new ObjectPool<>(2, OBJECT_CONSTRUCT); + + // Should not be able to put a new Object in the pool because it's full now + assertFalse(pool.put(new Object())); + + Object obj1 = pool.get(); + Object obj2 = pool.get(); + // Pool should create new instances when pool is empty + Object obj3 = pool.get(); + + assertNotNull(obj1); + assertNotNull(obj2); + assertNotNull(obj3); + + // Should be able to put obj2 and obj1 back to the pool because it's empty now + assertTrue(pool.put(obj2)); + assertTrue(pool.put(obj1)); + // Should not be able to put obj3 back to the pool because it's full now + assertFalse(pool.put(obj3)); + + // Make sure we get exactly the same objects in the same order that we put them back in + assertTrue(obj2 == pool.get()); + assertTrue(obj1 == pool.get()); + } +} \ No newline at end of file From f7186dd3d9b2ce400485cb1da45bcec8f6a360e5 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Tue, 12 Sep 2017 13:09:55 -0400 Subject: [PATCH 20/44] Fix checkstyle --- m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java | 2 +- m3/src/test/java/com/uber/m3/util/ObjectPoolTest.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java b/m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java index e1c77d4..7f59600 100644 --- a/m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java +++ b/m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java @@ -45,4 +45,4 @@ public void nullSizedMetric() { assertEquals(sizedMetric.getMetric(), null); assertEquals(sizedMetric.getSize(), 9); } -} \ No newline at end of file +} diff --git a/m3/src/test/java/com/uber/m3/util/ObjectPoolTest.java b/m3/src/test/java/com/uber/m3/util/ObjectPoolTest.java index fa3c116..5ac6f78 100644 --- a/m3/src/test/java/com/uber/m3/util/ObjectPoolTest.java +++ b/m3/src/test/java/com/uber/m3/util/ObjectPoolTest.java @@ -20,7 +20,6 @@ package com.uber.m3.util; -import org.junit.Assert; import org.junit.Test; import static org.junit.Assert.assertNotNull; @@ -95,4 +94,4 @@ public void twoCapacityPool() { assertTrue(obj2 == pool.get()); assertTrue(obj1 == pool.get()); } -} \ No newline at end of file +} From 880147a535d0737ea6ffb516a1c19320c7adaff5 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Wed, 13 Sep 2017 14:46:27 -0400 Subject: [PATCH 21/44] Clarify some design choices in comments --- .../src/main/java/com/uber/m3/tally/ScopeImpl.java | 4 ++-- .../src/main/java/com/uber/m3/tally/TimerImpl.java | 4 ++++ .../main/java/com/uber/m3/tally/m3/M3Reporter.java | 14 +++++++------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/com/uber/m3/tally/ScopeImpl.java b/core/src/main/java/com/uber/m3/tally/ScopeImpl.java index e83b20e..02179e0 100644 --- a/core/src/main/java/com/uber/m3/tally/ScopeImpl.java +++ b/core/src/main/java/com/uber/m3/tally/ScopeImpl.java @@ -46,8 +46,8 @@ class ScopeImpl implements Scope { // ConcurrentHashMap nearly always allowing read operations seems like a good // performance upside to the consequence of reporting a newly-made metric in - // the middle of looping and reporting through all metrics. Therefore, locks - // are generally only used when having to allocate new metrics. + // the middle of looping and reporting through all metrics. Therefore, we only + // synchronize on these maps when having to allocate new metrics. private final Map counters = new ConcurrentHashMap<>(); private final Map gauges = new ConcurrentHashMap<>(); private final Map timers = new ConcurrentHashMap<>(); diff --git a/core/src/main/java/com/uber/m3/tally/TimerImpl.java b/core/src/main/java/com/uber/m3/tally/TimerImpl.java index e900d79..10261ce 100644 --- a/core/src/main/java/com/uber/m3/tally/TimerImpl.java +++ b/core/src/main/java/com/uber/m3/tally/TimerImpl.java @@ -94,6 +94,10 @@ Duration[] snapshot() { } static class Values { + // Using a ReadWriteLock here to protect against multithreaded reads + // and writes separately. In other places, synchronized blocks are used + // instead as this separation is not needed e.g. we only lock a + // ConcurrentHashMap when doing writes and not for reads. private final ReadWriteLock rwlock = new ReentrantReadWriteLock(); private List values = new ArrayList<>(); diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 4589c55..7df6ab9 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -126,7 +126,7 @@ private M3Reporter(Builder builder) { client = new M3.Client(protocolFactory.getProtocol(transport)); resourcePool = new ResourcePool(builder.maxQueueSize, protocolFactory); - Set commonTags = addMetricTagsToSet(builder.commonTags); + Set commonTags = toMetricTagsToSet(builder.commonTags); // Set and ensure required tags if (!builder.commonTags.containsKey(SERVICE_TAG)) { @@ -143,10 +143,8 @@ private M3Reporter(Builder builder) { commonTags.add(createMetricTag(ENV_TAG, builder.env)); } - if (builder.includeHost) { - if (!builder.commonTags.containsKey(HOST_TAG)) { - commonTags.add(createMetricTag(HOST_TAG, InetAddress.getLocalHost().getHostName())); - } + if (builder.includeHost && !builder.commonTags.containsKey(HOST_TAG)) { + commonTags.add(createMetricTag(HOST_TAG, InetAddress.getLocalHost().getHostName())); } metricBatch = resourcePool.getMetricBatch(); @@ -309,7 +307,7 @@ private Metric newMetric(String name, Map tags, MetricType metri Metric metric = resourcePool.getMetric(); metric.setName(name); - metric.setTags(addMetricTagsToSet(tags)); + metric.setTags(toMetricTagsToSet(tags)); metric.setTimestamp(Long.MAX_VALUE); @@ -338,7 +336,7 @@ private Metric newMetric(String name, Map tags, MetricType metri return metric; } - private Set addMetricTagsToSet(Map tags) { + private Set toMetricTagsToSet(Map tags) { Set metricTagSet = resourcePool.getTagList(); if (tags == null) { @@ -405,6 +403,8 @@ private void reportCopyMetric( timerValue.setI64Value(iValue); metricCopy.getMetricValue().setTimer(timerValue); break; + default: + throw new IllegalArgumentException("Unsupported metric type: " + metricType); } try { From b0c2ecaaaa7b2370b911aebafa53d1e8bb0c06a9 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Wed, 13 Sep 2017 15:07:45 -0400 Subject: [PATCH 22/44] Put generated sources in source dir to check into git --- checkstyles/suppressions.xml | 2 +- m3/build.gradle | 4 + .../uber/m3/thrift/generated/CountValue.java | 384 +++++++++ .../uber/m3/thrift/generated/GaugeValue.java | 479 +++++++++++ .../java/com/uber/m3/thrift/generated/M3.java | 533 +++++++++++++ .../com/uber/m3/thrift/generated/Metric.java | 752 ++++++++++++++++++ .../uber/m3/thrift/generated/MetricBatch.java | 601 ++++++++++++++ .../uber/m3/thrift/generated/MetricTag.java | 492 ++++++++++++ .../uber/m3/thrift/generated/MetricValue.java | 607 ++++++++++++++ .../uber/m3/thrift/generated/TimerValue.java | 479 +++++++++++ .../java/com/uber/m3/util/TUdpClient.java | 4 +- .../java/com/uber/m3/util/TUdpServer.java | 4 +- 12 files changed, 4336 insertions(+), 5 deletions(-) create mode 100644 m3/src/generated/java/com/uber/m3/thrift/generated/CountValue.java create mode 100644 m3/src/generated/java/com/uber/m3/thrift/generated/GaugeValue.java create mode 100644 m3/src/generated/java/com/uber/m3/thrift/generated/M3.java create mode 100644 m3/src/generated/java/com/uber/m3/thrift/generated/Metric.java create mode 100644 m3/src/generated/java/com/uber/m3/thrift/generated/MetricBatch.java create mode 100644 m3/src/generated/java/com/uber/m3/thrift/generated/MetricTag.java create mode 100644 m3/src/generated/java/com/uber/m3/thrift/generated/MetricValue.java create mode 100644 m3/src/generated/java/com/uber/m3/thrift/generated/TimerValue.java diff --git a/checkstyles/suppressions.xml b/checkstyles/suppressions.xml index c1dd743..94b9a4f 100644 --- a/checkstyles/suppressions.xml +++ b/checkstyles/suppressions.xml @@ -4,5 +4,5 @@ "http://www.puppycrawl.com/dtds/suppressions_1_1.dtd"> - + diff --git a/m3/build.gradle b/m3/build.gradle index e66ee73..ed69d9d 100644 --- a/m3/build.gradle +++ b/m3/build.gradle @@ -31,4 +31,8 @@ dependencies { compileThrift { generator 'java', 'private-members' + + outputDir "${projectDir}/src/generated/java" + + createGenFolder false } diff --git a/m3/src/generated/java/com/uber/m3/thrift/generated/CountValue.java b/m3/src/generated/java/com/uber/m3/thrift/generated/CountValue.java new file mode 100644 index 0000000..8ce5dfa --- /dev/null +++ b/m3/src/generated/java/com/uber/m3/thrift/generated/CountValue.java @@ -0,0 +1,384 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package com.uber.m3.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Different types of count values + */ +public class CountValue implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CountValue"); + + private static final org.apache.thrift.protocol.TField I64_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("i64Value", org.apache.thrift.protocol.TType.I64, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new CountValueStandardSchemeFactory()); + schemes.put(TupleScheme.class, new CountValueTupleSchemeFactory()); + } + + public long i64Value; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + I64_VALUE((short)1, "i64Value"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // I64_VALUE + return I64_VALUE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __I64VALUE_ISSET_ID = 0; + private byte __isset_bitfield = 0; + private _Fields optionals[] = {_Fields.I64_VALUE}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.I64_VALUE, new org.apache.thrift.meta_data.FieldMetaData("i64Value", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CountValue.class, metaDataMap); + } + + public CountValue() { + } + + /** + * Performs a deep copy on other. + */ + public CountValue(CountValue other) { + __isset_bitfield = other.__isset_bitfield; + this.i64Value = other.i64Value; + } + + public CountValue deepCopy() { + return new CountValue(this); + } + + @Override + public void clear() { + setI64ValueIsSet(false); + this.i64Value = 0; + } + + public long getI64Value() { + return this.i64Value; + } + + public CountValue setI64Value(long i64Value) { + this.i64Value = i64Value; + setI64ValueIsSet(true); + return this; + } + + public void unsetI64Value() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __I64VALUE_ISSET_ID); + } + + /** Returns true if field i64Value is set (has been assigned a value) and false otherwise */ + public boolean isSetI64Value() { + return EncodingUtils.testBit(__isset_bitfield, __I64VALUE_ISSET_ID); + } + + public void setI64ValueIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __I64VALUE_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case I64_VALUE: + if (value == null) { + unsetI64Value(); + } else { + setI64Value((Long)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case I64_VALUE: + return Long.valueOf(getI64Value()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case I64_VALUE: + return isSetI64Value(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof CountValue) + return this.equals((CountValue)that); + return false; + } + + public boolean equals(CountValue that) { + if (that == null) + return false; + + boolean this_present_i64Value = true && this.isSetI64Value(); + boolean that_present_i64Value = true && that.isSetI64Value(); + if (this_present_i64Value || that_present_i64Value) { + if (!(this_present_i64Value && that_present_i64Value)) + return false; + if (this.i64Value != that.i64Value) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(CountValue other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + CountValue typedOther = (CountValue)other; + + lastComparison = Boolean.valueOf(isSetI64Value()).compareTo(typedOther.isSetI64Value()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetI64Value()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.i64Value, typedOther.i64Value); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("CountValue("); + boolean first = true; + + if (isSetI64Value()) { + sb.append("i64Value:"); + sb.append(this.i64Value); + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class CountValueStandardSchemeFactory implements SchemeFactory { + public CountValueStandardScheme getScheme() { + return new CountValueStandardScheme(); + } + } + + private static class CountValueStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, CountValue struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // I64_VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.i64Value = iprot.readI64(); + struct.setI64ValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, CountValue struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.isSetI64Value()) { + oprot.writeFieldBegin(I64_VALUE_FIELD_DESC); + oprot.writeI64(struct.i64Value); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class CountValueTupleSchemeFactory implements SchemeFactory { + public CountValueTupleScheme getScheme() { + return new CountValueTupleScheme(); + } + } + + private static class CountValueTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, CountValue struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetI64Value()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetI64Value()) { + oprot.writeI64(struct.i64Value); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, CountValue struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.i64Value = iprot.readI64(); + struct.setI64ValueIsSet(true); + } + } + } + +} + diff --git a/m3/src/generated/java/com/uber/m3/thrift/generated/GaugeValue.java b/m3/src/generated/java/com/uber/m3/thrift/generated/GaugeValue.java new file mode 100644 index 0000000..5d313ea --- /dev/null +++ b/m3/src/generated/java/com/uber/m3/thrift/generated/GaugeValue.java @@ -0,0 +1,479 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package com.uber.m3.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Different types of gauge values + */ +public class GaugeValue implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GaugeValue"); + + private static final org.apache.thrift.protocol.TField I64_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("i64Value", org.apache.thrift.protocol.TType.I64, (short)1); + private static final org.apache.thrift.protocol.TField D_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("dValue", org.apache.thrift.protocol.TType.DOUBLE, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new GaugeValueStandardSchemeFactory()); + schemes.put(TupleScheme.class, new GaugeValueTupleSchemeFactory()); + } + + public long i64Value; // optional + public double dValue; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + I64_VALUE((short)1, "i64Value"), + D_VALUE((short)2, "dValue"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // I64_VALUE + return I64_VALUE; + case 2: // D_VALUE + return D_VALUE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __I64VALUE_ISSET_ID = 0; + private static final int __DVALUE_ISSET_ID = 1; + private byte __isset_bitfield = 0; + private _Fields optionals[] = {_Fields.I64_VALUE,_Fields.D_VALUE}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.I64_VALUE, new org.apache.thrift.meta_data.FieldMetaData("i64Value", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.D_VALUE, new org.apache.thrift.meta_data.FieldMetaData("dValue", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GaugeValue.class, metaDataMap); + } + + public GaugeValue() { + } + + /** + * Performs a deep copy on other. + */ + public GaugeValue(GaugeValue other) { + __isset_bitfield = other.__isset_bitfield; + this.i64Value = other.i64Value; + this.dValue = other.dValue; + } + + public GaugeValue deepCopy() { + return new GaugeValue(this); + } + + @Override + public void clear() { + setI64ValueIsSet(false); + this.i64Value = 0; + setDValueIsSet(false); + this.dValue = 0.0; + } + + public long getI64Value() { + return this.i64Value; + } + + public GaugeValue setI64Value(long i64Value) { + this.i64Value = i64Value; + setI64ValueIsSet(true); + return this; + } + + public void unsetI64Value() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __I64VALUE_ISSET_ID); + } + + /** Returns true if field i64Value is set (has been assigned a value) and false otherwise */ + public boolean isSetI64Value() { + return EncodingUtils.testBit(__isset_bitfield, __I64VALUE_ISSET_ID); + } + + public void setI64ValueIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __I64VALUE_ISSET_ID, value); + } + + public double getDValue() { + return this.dValue; + } + + public GaugeValue setDValue(double dValue) { + this.dValue = dValue; + setDValueIsSet(true); + return this; + } + + public void unsetDValue() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __DVALUE_ISSET_ID); + } + + /** Returns true if field dValue is set (has been assigned a value) and false otherwise */ + public boolean isSetDValue() { + return EncodingUtils.testBit(__isset_bitfield, __DVALUE_ISSET_ID); + } + + public void setDValueIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __DVALUE_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case I64_VALUE: + if (value == null) { + unsetI64Value(); + } else { + setI64Value((Long)value); + } + break; + + case D_VALUE: + if (value == null) { + unsetDValue(); + } else { + setDValue((Double)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case I64_VALUE: + return Long.valueOf(getI64Value()); + + case D_VALUE: + return Double.valueOf(getDValue()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case I64_VALUE: + return isSetI64Value(); + case D_VALUE: + return isSetDValue(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof GaugeValue) + return this.equals((GaugeValue)that); + return false; + } + + public boolean equals(GaugeValue that) { + if (that == null) + return false; + + boolean this_present_i64Value = true && this.isSetI64Value(); + boolean that_present_i64Value = true && that.isSetI64Value(); + if (this_present_i64Value || that_present_i64Value) { + if (!(this_present_i64Value && that_present_i64Value)) + return false; + if (this.i64Value != that.i64Value) + return false; + } + + boolean this_present_dValue = true && this.isSetDValue(); + boolean that_present_dValue = true && that.isSetDValue(); + if (this_present_dValue || that_present_dValue) { + if (!(this_present_dValue && that_present_dValue)) + return false; + if (this.dValue != that.dValue) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(GaugeValue other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + GaugeValue typedOther = (GaugeValue)other; + + lastComparison = Boolean.valueOf(isSetI64Value()).compareTo(typedOther.isSetI64Value()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetI64Value()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.i64Value, typedOther.i64Value); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetDValue()).compareTo(typedOther.isSetDValue()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetDValue()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.dValue, typedOther.dValue); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("GaugeValue("); + boolean first = true; + + if (isSetI64Value()) { + sb.append("i64Value:"); + sb.append(this.i64Value); + first = false; + } + if (isSetDValue()) { + if (!first) sb.append(", "); + sb.append("dValue:"); + sb.append(this.dValue); + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class GaugeValueStandardSchemeFactory implements SchemeFactory { + public GaugeValueStandardScheme getScheme() { + return new GaugeValueStandardScheme(); + } + } + + private static class GaugeValueStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, GaugeValue struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // I64_VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.i64Value = iprot.readI64(); + struct.setI64ValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // D_VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) { + struct.dValue = iprot.readDouble(); + struct.setDValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, GaugeValue struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.isSetI64Value()) { + oprot.writeFieldBegin(I64_VALUE_FIELD_DESC); + oprot.writeI64(struct.i64Value); + oprot.writeFieldEnd(); + } + if (struct.isSetDValue()) { + oprot.writeFieldBegin(D_VALUE_FIELD_DESC); + oprot.writeDouble(struct.dValue); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class GaugeValueTupleSchemeFactory implements SchemeFactory { + public GaugeValueTupleScheme getScheme() { + return new GaugeValueTupleScheme(); + } + } + + private static class GaugeValueTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, GaugeValue struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetI64Value()) { + optionals.set(0); + } + if (struct.isSetDValue()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetI64Value()) { + oprot.writeI64(struct.i64Value); + } + if (struct.isSetDValue()) { + oprot.writeDouble(struct.dValue); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, GaugeValue struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.i64Value = iprot.readI64(); + struct.setI64ValueIsSet(true); + } + if (incoming.get(1)) { + struct.dValue = iprot.readDouble(); + struct.setDValueIsSet(true); + } + } + } + +} + diff --git a/m3/src/generated/java/com/uber/m3/thrift/generated/M3.java b/m3/src/generated/java/com/uber/m3/thrift/generated/M3.java new file mode 100644 index 0000000..d4547f3 --- /dev/null +++ b/m3/src/generated/java/com/uber/m3/thrift/generated/M3.java @@ -0,0 +1,533 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package com.uber.m3.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class M3 { + + /** + * M3 Metrics Service + */ + public interface Iface { + + /** + * Emits a batch of metrics. + * + * @param batch + */ + public void emitMetricBatch(MetricBatch batch) throws org.apache.thrift.TException; + + } + + public interface AsyncIface { + + public void emitMetricBatch(MetricBatch batch, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + } + + public static class Client extends org.apache.thrift.TServiceClient implements Iface { + public static class Factory implements org.apache.thrift.TServiceClientFactory { + public Factory() {} + public Client getClient(org.apache.thrift.protocol.TProtocol prot) { + return new Client(prot); + } + public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) { + return new Client(iprot, oprot); + } + } + + public Client(org.apache.thrift.protocol.TProtocol prot) + { + super(prot, prot); + } + + public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) { + super(iprot, oprot); + } + + public void emitMetricBatch(MetricBatch batch) throws org.apache.thrift.TException + { + send_emitMetricBatch(batch); + } + + public void send_emitMetricBatch(MetricBatch batch) throws org.apache.thrift.TException + { + emitMetricBatch_args args = new emitMetricBatch_args(); + args.setBatch(batch); + sendBase("emitMetricBatch", args); + } + + } + public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface { + public static class Factory implements org.apache.thrift.async.TAsyncClientFactory { + private org.apache.thrift.async.TAsyncClientManager clientManager; + private org.apache.thrift.protocol.TProtocolFactory protocolFactory; + public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) { + this.clientManager = clientManager; + this.protocolFactory = protocolFactory; + } + public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) { + return new AsyncClient(protocolFactory, clientManager, transport); + } + } + + public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) { + super(protocolFactory, clientManager, transport); + } + + public void emitMetricBatch(MetricBatch batch, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + emitMetricBatch_call method_call = new emitMetricBatch_call(batch, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class emitMetricBatch_call extends org.apache.thrift.async.TAsyncMethodCall { + private MetricBatch batch; + public emitMetricBatch_call(MetricBatch batch, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, true); + this.batch = batch; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("emitMetricBatch", org.apache.thrift.protocol.TMessageType.CALL, 0)); + emitMetricBatch_args args = new emitMetricBatch_args(); + args.setBatch(batch); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + } + } + + } + + public static class Processor extends org.apache.thrift.TBaseProcessor implements org.apache.thrift.TProcessor { + private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName()); + public Processor(I iface) { + super(iface, getProcessMap(new HashMap>())); + } + + protected Processor(I iface, Map> processMap) { + super(iface, getProcessMap(processMap)); + } + + private static Map> getProcessMap(Map> processMap) { + processMap.put("emitMetricBatch", new emitMetricBatch()); + return processMap; + } + + public static class emitMetricBatch extends org.apache.thrift.ProcessFunction { + public emitMetricBatch() { + super("emitMetricBatch"); + } + + public emitMetricBatch_args getEmptyArgsInstance() { + return new emitMetricBatch_args(); + } + + protected boolean isOneway() { + return true; + } + + public org.apache.thrift.TBase getResult(I iface, emitMetricBatch_args args) throws org.apache.thrift.TException { + iface.emitMetricBatch(args.batch); + return null; + } + } + + } + + public static class emitMetricBatch_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("emitMetricBatch_args"); + + private static final org.apache.thrift.protocol.TField BATCH_FIELD_DESC = new org.apache.thrift.protocol.TField("batch", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new emitMetricBatch_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new emitMetricBatch_argsTupleSchemeFactory()); + } + + public MetricBatch batch; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + BATCH((short)1, "batch"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // BATCH + return BATCH; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.BATCH, new org.apache.thrift.meta_data.FieldMetaData("batch", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, MetricBatch.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(emitMetricBatch_args.class, metaDataMap); + } + + public emitMetricBatch_args() { + } + + public emitMetricBatch_args( + MetricBatch batch) + { + this(); + this.batch = batch; + } + + /** + * Performs a deep copy on other. + */ + public emitMetricBatch_args(emitMetricBatch_args other) { + if (other.isSetBatch()) { + this.batch = new MetricBatch(other.batch); + } + } + + public emitMetricBatch_args deepCopy() { + return new emitMetricBatch_args(this); + } + + @Override + public void clear() { + this.batch = null; + } + + public MetricBatch getBatch() { + return this.batch; + } + + public emitMetricBatch_args setBatch(MetricBatch batch) { + this.batch = batch; + return this; + } + + public void unsetBatch() { + this.batch = null; + } + + /** Returns true if field batch is set (has been assigned a value) and false otherwise */ + public boolean isSetBatch() { + return this.batch != null; + } + + public void setBatchIsSet(boolean value) { + if (!value) { + this.batch = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case BATCH: + if (value == null) { + unsetBatch(); + } else { + setBatch((MetricBatch)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case BATCH: + return getBatch(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case BATCH: + return isSetBatch(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof emitMetricBatch_args) + return this.equals((emitMetricBatch_args)that); + return false; + } + + public boolean equals(emitMetricBatch_args that) { + if (that == null) + return false; + + boolean this_present_batch = true && this.isSetBatch(); + boolean that_present_batch = true && that.isSetBatch(); + if (this_present_batch || that_present_batch) { + if (!(this_present_batch && that_present_batch)) + return false; + if (!this.batch.equals(that.batch)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(emitMetricBatch_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + emitMetricBatch_args typedOther = (emitMetricBatch_args)other; + + lastComparison = Boolean.valueOf(isSetBatch()).compareTo(typedOther.isSetBatch()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetBatch()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.batch, typedOther.batch); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("emitMetricBatch_args("); + boolean first = true; + + sb.append("batch:"); + if (this.batch == null) { + sb.append("null"); + } else { + sb.append(this.batch); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (batch != null) { + batch.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class emitMetricBatch_argsStandardSchemeFactory implements SchemeFactory { + public emitMetricBatch_argsStandardScheme getScheme() { + return new emitMetricBatch_argsStandardScheme(); + } + } + + private static class emitMetricBatch_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, emitMetricBatch_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // BATCH + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.batch = new MetricBatch(); + struct.batch.read(iprot); + struct.setBatchIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, emitMetricBatch_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.batch != null) { + oprot.writeFieldBegin(BATCH_FIELD_DESC); + struct.batch.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class emitMetricBatch_argsTupleSchemeFactory implements SchemeFactory { + public emitMetricBatch_argsTupleScheme getScheme() { + return new emitMetricBatch_argsTupleScheme(); + } + } + + private static class emitMetricBatch_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, emitMetricBatch_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetBatch()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetBatch()) { + struct.batch.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, emitMetricBatch_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.batch = new MetricBatch(); + struct.batch.read(iprot); + struct.setBatchIsSet(true); + } + } + } + + } + +} diff --git a/m3/src/generated/java/com/uber/m3/thrift/generated/Metric.java b/m3/src/generated/java/com/uber/m3/thrift/generated/Metric.java new file mode 100644 index 0000000..f2e0f8d --- /dev/null +++ b/m3/src/generated/java/com/uber/m3/thrift/generated/Metric.java @@ -0,0 +1,752 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package com.uber.m3.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The metric that is being emitted + */ +public class Metric implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("Metric"); + + private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField METRIC_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("metricValue", org.apache.thrift.protocol.TType.STRUCT, (short)2); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3); + private static final org.apache.thrift.protocol.TField TAGS_FIELD_DESC = new org.apache.thrift.protocol.TField("tags", org.apache.thrift.protocol.TType.SET, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new MetricStandardSchemeFactory()); + schemes.put(TupleScheme.class, new MetricTupleSchemeFactory()); + } + + public String name; // required + public MetricValue metricValue; // optional + public long timestamp; // optional + public Set tags; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + NAME((short)1, "name"), + METRIC_VALUE((short)2, "metricValue"), + TIMESTAMP((short)3, "timestamp"), + TAGS((short)4, "tags"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // NAME + return NAME; + case 2: // METRIC_VALUE + return METRIC_VALUE; + case 3: // TIMESTAMP + return TIMESTAMP; + case 4: // TAGS + return TAGS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private byte __isset_bitfield = 0; + private _Fields optionals[] = {_Fields.METRIC_VALUE,_Fields.TIMESTAMP,_Fields.TAGS}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.NAME, new org.apache.thrift.meta_data.FieldMetaData("name", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.METRIC_VALUE, new org.apache.thrift.meta_data.FieldMetaData("metricValue", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, MetricValue.class))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.TAGS, new org.apache.thrift.meta_data.FieldMetaData("tags", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.SetMetaData(org.apache.thrift.protocol.TType.SET, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, MetricTag.class)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(Metric.class, metaDataMap); + } + + public Metric() { + } + + public Metric( + String name) + { + this(); + this.name = name; + } + + /** + * Performs a deep copy on other. + */ + public Metric(Metric other) { + __isset_bitfield = other.__isset_bitfield; + if (other.isSetName()) { + this.name = other.name; + } + if (other.isSetMetricValue()) { + this.metricValue = new MetricValue(other.metricValue); + } + this.timestamp = other.timestamp; + if (other.isSetTags()) { + Set __this__tags = new HashSet(); + for (MetricTag other_element : other.tags) { + __this__tags.add(new MetricTag(other_element)); + } + this.tags = __this__tags; + } + } + + public Metric deepCopy() { + return new Metric(this); + } + + @Override + public void clear() { + this.name = null; + this.metricValue = null; + setTimestampIsSet(false); + this.timestamp = 0; + this.tags = null; + } + + public String getName() { + return this.name; + } + + public Metric setName(String name) { + this.name = name; + return this; + } + + public void unsetName() { + this.name = null; + } + + /** Returns true if field name is set (has been assigned a value) and false otherwise */ + public boolean isSetName() { + return this.name != null; + } + + public void setNameIsSet(boolean value) { + if (!value) { + this.name = null; + } + } + + public MetricValue getMetricValue() { + return this.metricValue; + } + + public Metric setMetricValue(MetricValue metricValue) { + this.metricValue = metricValue; + return this; + } + + public void unsetMetricValue() { + this.metricValue = null; + } + + /** Returns true if field metricValue is set (has been assigned a value) and false otherwise */ + public boolean isSetMetricValue() { + return this.metricValue != null; + } + + public void setMetricValueIsSet(boolean value) { + if (!value) { + this.metricValue = null; + } + } + + public long getTimestamp() { + return this.timestamp; + } + + public Metric setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return EncodingUtils.testBit(__isset_bitfield, __TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __TIMESTAMP_ISSET_ID, value); + } + + public int getTagsSize() { + return (this.tags == null) ? 0 : this.tags.size(); + } + + public java.util.Iterator getTagsIterator() { + return (this.tags == null) ? null : this.tags.iterator(); + } + + public void addToTags(MetricTag elem) { + if (this.tags == null) { + this.tags = new HashSet(); + } + this.tags.add(elem); + } + + public Set getTags() { + return this.tags; + } + + public Metric setTags(Set tags) { + this.tags = tags; + return this; + } + + public void unsetTags() { + this.tags = null; + } + + /** Returns true if field tags is set (has been assigned a value) and false otherwise */ + public boolean isSetTags() { + return this.tags != null; + } + + public void setTagsIsSet(boolean value) { + if (!value) { + this.tags = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case NAME: + if (value == null) { + unsetName(); + } else { + setName((String)value); + } + break; + + case METRIC_VALUE: + if (value == null) { + unsetMetricValue(); + } else { + setMetricValue((MetricValue)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case TAGS: + if (value == null) { + unsetTags(); + } else { + setTags((Set)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case NAME: + return getName(); + + case METRIC_VALUE: + return getMetricValue(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case TAGS: + return getTags(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case NAME: + return isSetName(); + case METRIC_VALUE: + return isSetMetricValue(); + case TIMESTAMP: + return isSetTimestamp(); + case TAGS: + return isSetTags(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof Metric) + return this.equals((Metric)that); + return false; + } + + public boolean equals(Metric that) { + if (that == null) + return false; + + boolean this_present_name = true && this.isSetName(); + boolean that_present_name = true && that.isSetName(); + if (this_present_name || that_present_name) { + if (!(this_present_name && that_present_name)) + return false; + if (!this.name.equals(that.name)) + return false; + } + + boolean this_present_metricValue = true && this.isSetMetricValue(); + boolean that_present_metricValue = true && that.isSetMetricValue(); + if (this_present_metricValue || that_present_metricValue) { + if (!(this_present_metricValue && that_present_metricValue)) + return false; + if (!this.metricValue.equals(that.metricValue)) + return false; + } + + boolean this_present_timestamp = true && this.isSetTimestamp(); + boolean that_present_timestamp = true && that.isSetTimestamp(); + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_tags = true && this.isSetTags(); + boolean that_present_tags = true && that.isSetTags(); + if (this_present_tags || that_present_tags) { + if (!(this_present_tags && that_present_tags)) + return false; + if (!this.tags.equals(that.tags)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(Metric other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + Metric typedOther = (Metric)other; + + lastComparison = Boolean.valueOf(isSetName()).compareTo(typedOther.isSetName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.name, typedOther.name); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMetricValue()).compareTo(typedOther.isSetMetricValue()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMetricValue()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.metricValue, typedOther.metricValue); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTags()).compareTo(typedOther.isSetTags()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTags()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tags, typedOther.tags); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("Metric("); + boolean first = true; + + sb.append("name:"); + if (this.name == null) { + sb.append("null"); + } else { + sb.append(this.name); + } + first = false; + if (isSetMetricValue()) { + if (!first) sb.append(", "); + sb.append("metricValue:"); + if (this.metricValue == null) { + sb.append("null"); + } else { + sb.append(this.metricValue); + } + first = false; + } + if (isSetTimestamp()) { + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + } + if (isSetTags()) { + if (!first) sb.append(", "); + sb.append("tags:"); + if (this.tags == null) { + sb.append("null"); + } else { + sb.append(this.tags); + } + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (metricValue != null) { + metricValue.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class MetricStandardSchemeFactory implements SchemeFactory { + public MetricStandardScheme getScheme() { + return new MetricStandardScheme(); + } + } + + private static class MetricStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, Metric struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.name = iprot.readString(); + struct.setNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // METRIC_VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.metricValue = new MetricValue(); + struct.metricValue.read(iprot); + struct.setMetricValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // TAGS + if (schemeField.type == org.apache.thrift.protocol.TType.SET) { + { + org.apache.thrift.protocol.TSet _set0 = iprot.readSetBegin(); + struct.tags = new HashSet(2*_set0.size); + for (int _i1 = 0; _i1 < _set0.size; ++_i1) + { + MetricTag _elem2; // required + _elem2 = new MetricTag(); + _elem2.read(iprot); + struct.tags.add(_elem2); + } + iprot.readSetEnd(); + } + struct.setTagsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, Metric struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.name != null) { + oprot.writeFieldBegin(NAME_FIELD_DESC); + oprot.writeString(struct.name); + oprot.writeFieldEnd(); + } + if (struct.metricValue != null) { + if (struct.isSetMetricValue()) { + oprot.writeFieldBegin(METRIC_VALUE_FIELD_DESC); + struct.metricValue.write(oprot); + oprot.writeFieldEnd(); + } + } + if (struct.isSetTimestamp()) { + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + } + if (struct.tags != null) { + if (struct.isSetTags()) { + oprot.writeFieldBegin(TAGS_FIELD_DESC); + { + oprot.writeSetBegin(new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRUCT, struct.tags.size())); + for (MetricTag _iter3 : struct.tags) + { + _iter3.write(oprot); + } + oprot.writeSetEnd(); + } + oprot.writeFieldEnd(); + } + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class MetricTupleSchemeFactory implements SchemeFactory { + public MetricTupleScheme getScheme() { + return new MetricTupleScheme(); + } + } + + private static class MetricTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, Metric struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetName()) { + optionals.set(0); + } + if (struct.isSetMetricValue()) { + optionals.set(1); + } + if (struct.isSetTimestamp()) { + optionals.set(2); + } + if (struct.isSetTags()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetName()) { + oprot.writeString(struct.name); + } + if (struct.isSetMetricValue()) { + struct.metricValue.write(oprot); + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetTags()) { + { + oprot.writeI32(struct.tags.size()); + for (MetricTag _iter4 : struct.tags) + { + _iter4.write(oprot); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, Metric struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.name = iprot.readString(); + struct.setNameIsSet(true); + } + if (incoming.get(1)) { + struct.metricValue = new MetricValue(); + struct.metricValue.read(iprot); + struct.setMetricValueIsSet(true); + } + if (incoming.get(2)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TSet _set5 = new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.tags = new HashSet(2*_set5.size); + for (int _i6 = 0; _i6 < _set5.size; ++_i6) + { + MetricTag _elem7; // required + _elem7 = new MetricTag(); + _elem7.read(iprot); + struct.tags.add(_elem7); + } + } + struct.setTagsIsSet(true); + } + } + } + +} + diff --git a/m3/src/generated/java/com/uber/m3/thrift/generated/MetricBatch.java b/m3/src/generated/java/com/uber/m3/thrift/generated/MetricBatch.java new file mode 100644 index 0000000..ecfd2fd --- /dev/null +++ b/m3/src/generated/java/com/uber/m3/thrift/generated/MetricBatch.java @@ -0,0 +1,601 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package com.uber.m3.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Structure that holds a group of metrics which share + * common properties like the cluster and service. + */ +public class MetricBatch implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("MetricBatch"); + + private static final org.apache.thrift.protocol.TField METRICS_FIELD_DESC = new org.apache.thrift.protocol.TField("metrics", org.apache.thrift.protocol.TType.LIST, (short)1); + private static final org.apache.thrift.protocol.TField COMMON_TAGS_FIELD_DESC = new org.apache.thrift.protocol.TField("commonTags", org.apache.thrift.protocol.TType.SET, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new MetricBatchStandardSchemeFactory()); + schemes.put(TupleScheme.class, new MetricBatchTupleSchemeFactory()); + } + + public List metrics; // required + public Set commonTags; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + METRICS((short)1, "metrics"), + COMMON_TAGS((short)2, "commonTags"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // METRICS + return METRICS; + case 2: // COMMON_TAGS + return COMMON_TAGS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private _Fields optionals[] = {_Fields.COMMON_TAGS}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.METRICS, new org.apache.thrift.meta_data.FieldMetaData("metrics", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Metric.class)))); + tmpMap.put(_Fields.COMMON_TAGS, new org.apache.thrift.meta_data.FieldMetaData("commonTags", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.SetMetaData(org.apache.thrift.protocol.TType.SET, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, MetricTag.class)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(MetricBatch.class, metaDataMap); + } + + public MetricBatch() { + } + + public MetricBatch( + List metrics) + { + this(); + this.metrics = metrics; + } + + /** + * Performs a deep copy on other. + */ + public MetricBatch(MetricBatch other) { + if (other.isSetMetrics()) { + List __this__metrics = new ArrayList(); + for (Metric other_element : other.metrics) { + __this__metrics.add(new Metric(other_element)); + } + this.metrics = __this__metrics; + } + if (other.isSetCommonTags()) { + Set __this__commonTags = new HashSet(); + for (MetricTag other_element : other.commonTags) { + __this__commonTags.add(new MetricTag(other_element)); + } + this.commonTags = __this__commonTags; + } + } + + public MetricBatch deepCopy() { + return new MetricBatch(this); + } + + @Override + public void clear() { + this.metrics = null; + this.commonTags = null; + } + + public int getMetricsSize() { + return (this.metrics == null) ? 0 : this.metrics.size(); + } + + public java.util.Iterator getMetricsIterator() { + return (this.metrics == null) ? null : this.metrics.iterator(); + } + + public void addToMetrics(Metric elem) { + if (this.metrics == null) { + this.metrics = new ArrayList(); + } + this.metrics.add(elem); + } + + public List getMetrics() { + return this.metrics; + } + + public MetricBatch setMetrics(List metrics) { + this.metrics = metrics; + return this; + } + + public void unsetMetrics() { + this.metrics = null; + } + + /** Returns true if field metrics is set (has been assigned a value) and false otherwise */ + public boolean isSetMetrics() { + return this.metrics != null; + } + + public void setMetricsIsSet(boolean value) { + if (!value) { + this.metrics = null; + } + } + + public int getCommonTagsSize() { + return (this.commonTags == null) ? 0 : this.commonTags.size(); + } + + public java.util.Iterator getCommonTagsIterator() { + return (this.commonTags == null) ? null : this.commonTags.iterator(); + } + + public void addToCommonTags(MetricTag elem) { + if (this.commonTags == null) { + this.commonTags = new HashSet(); + } + this.commonTags.add(elem); + } + + public Set getCommonTags() { + return this.commonTags; + } + + public MetricBatch setCommonTags(Set commonTags) { + this.commonTags = commonTags; + return this; + } + + public void unsetCommonTags() { + this.commonTags = null; + } + + /** Returns true if field commonTags is set (has been assigned a value) and false otherwise */ + public boolean isSetCommonTags() { + return this.commonTags != null; + } + + public void setCommonTagsIsSet(boolean value) { + if (!value) { + this.commonTags = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case METRICS: + if (value == null) { + unsetMetrics(); + } else { + setMetrics((List)value); + } + break; + + case COMMON_TAGS: + if (value == null) { + unsetCommonTags(); + } else { + setCommonTags((Set)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case METRICS: + return getMetrics(); + + case COMMON_TAGS: + return getCommonTags(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case METRICS: + return isSetMetrics(); + case COMMON_TAGS: + return isSetCommonTags(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof MetricBatch) + return this.equals((MetricBatch)that); + return false; + } + + public boolean equals(MetricBatch that) { + if (that == null) + return false; + + boolean this_present_metrics = true && this.isSetMetrics(); + boolean that_present_metrics = true && that.isSetMetrics(); + if (this_present_metrics || that_present_metrics) { + if (!(this_present_metrics && that_present_metrics)) + return false; + if (!this.metrics.equals(that.metrics)) + return false; + } + + boolean this_present_commonTags = true && this.isSetCommonTags(); + boolean that_present_commonTags = true && that.isSetCommonTags(); + if (this_present_commonTags || that_present_commonTags) { + if (!(this_present_commonTags && that_present_commonTags)) + return false; + if (!this.commonTags.equals(that.commonTags)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(MetricBatch other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + MetricBatch typedOther = (MetricBatch)other; + + lastComparison = Boolean.valueOf(isSetMetrics()).compareTo(typedOther.isSetMetrics()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMetrics()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.metrics, typedOther.metrics); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetCommonTags()).compareTo(typedOther.isSetCommonTags()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCommonTags()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.commonTags, typedOther.commonTags); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("MetricBatch("); + boolean first = true; + + sb.append("metrics:"); + if (this.metrics == null) { + sb.append("null"); + } else { + sb.append(this.metrics); + } + first = false; + if (isSetCommonTags()) { + if (!first) sb.append(", "); + sb.append("commonTags:"); + if (this.commonTags == null) { + sb.append("null"); + } else { + sb.append(this.commonTags); + } + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class MetricBatchStandardSchemeFactory implements SchemeFactory { + public MetricBatchStandardScheme getScheme() { + return new MetricBatchStandardScheme(); + } + } + + private static class MetricBatchStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, MetricBatch struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // METRICS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list8 = iprot.readListBegin(); + struct.metrics = new ArrayList(_list8.size); + for (int _i9 = 0; _i9 < _list8.size; ++_i9) + { + Metric _elem10; // required + _elem10 = new Metric(); + _elem10.read(iprot); + struct.metrics.add(_elem10); + } + iprot.readListEnd(); + } + struct.setMetricsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // COMMON_TAGS + if (schemeField.type == org.apache.thrift.protocol.TType.SET) { + { + org.apache.thrift.protocol.TSet _set11 = iprot.readSetBegin(); + struct.commonTags = new HashSet(2*_set11.size); + for (int _i12 = 0; _i12 < _set11.size; ++_i12) + { + MetricTag _elem13; // required + _elem13 = new MetricTag(); + _elem13.read(iprot); + struct.commonTags.add(_elem13); + } + iprot.readSetEnd(); + } + struct.setCommonTagsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, MetricBatch struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.metrics != null) { + oprot.writeFieldBegin(METRICS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.metrics.size())); + for (Metric _iter14 : struct.metrics) + { + _iter14.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.commonTags != null) { + if (struct.isSetCommonTags()) { + oprot.writeFieldBegin(COMMON_TAGS_FIELD_DESC); + { + oprot.writeSetBegin(new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRUCT, struct.commonTags.size())); + for (MetricTag _iter15 : struct.commonTags) + { + _iter15.write(oprot); + } + oprot.writeSetEnd(); + } + oprot.writeFieldEnd(); + } + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class MetricBatchTupleSchemeFactory implements SchemeFactory { + public MetricBatchTupleScheme getScheme() { + return new MetricBatchTupleScheme(); + } + } + + private static class MetricBatchTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, MetricBatch struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetMetrics()) { + optionals.set(0); + } + if (struct.isSetCommonTags()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetMetrics()) { + { + oprot.writeI32(struct.metrics.size()); + for (Metric _iter16 : struct.metrics) + { + _iter16.write(oprot); + } + } + } + if (struct.isSetCommonTags()) { + { + oprot.writeI32(struct.commonTags.size()); + for (MetricTag _iter17 : struct.commonTags) + { + _iter17.write(oprot); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, MetricBatch struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list18 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.metrics = new ArrayList(_list18.size); + for (int _i19 = 0; _i19 < _list18.size; ++_i19) + { + Metric _elem20; // required + _elem20 = new Metric(); + _elem20.read(iprot); + struct.metrics.add(_elem20); + } + } + struct.setMetricsIsSet(true); + } + if (incoming.get(1)) { + { + org.apache.thrift.protocol.TSet _set21 = new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.commonTags = new HashSet(2*_set21.size); + for (int _i22 = 0; _i22 < _set21.size; ++_i22) + { + MetricTag _elem23; // required + _elem23 = new MetricTag(); + _elem23.read(iprot); + struct.commonTags.add(_elem23); + } + } + struct.setCommonTagsIsSet(true); + } + } + } + +} + diff --git a/m3/src/generated/java/com/uber/m3/thrift/generated/MetricTag.java b/m3/src/generated/java/com/uber/m3/thrift/generated/MetricTag.java new file mode 100644 index 0000000..96ec0b4 --- /dev/null +++ b/m3/src/generated/java/com/uber/m3/thrift/generated/MetricTag.java @@ -0,0 +1,492 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package com.uber.m3.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tags that can be applied to a metric + */ +public class MetricTag implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("MetricTag"); + + private static final org.apache.thrift.protocol.TField TAG_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tagName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField TAG_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("tagValue", org.apache.thrift.protocol.TType.STRING, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new MetricTagStandardSchemeFactory()); + schemes.put(TupleScheme.class, new MetricTagTupleSchemeFactory()); + } + + public String tagName; // required + public String tagValue; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + TAG_NAME((short)1, "tagName"), + TAG_VALUE((short)2, "tagValue"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TAG_NAME + return TAG_NAME; + case 2: // TAG_VALUE + return TAG_VALUE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private _Fields optionals[] = {_Fields.TAG_VALUE}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TAG_NAME, new org.apache.thrift.meta_data.FieldMetaData("tagName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.TAG_VALUE, new org.apache.thrift.meta_data.FieldMetaData("tagValue", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(MetricTag.class, metaDataMap); + } + + public MetricTag() { + } + + public MetricTag( + String tagName) + { + this(); + this.tagName = tagName; + } + + /** + * Performs a deep copy on other. + */ + public MetricTag(MetricTag other) { + if (other.isSetTagName()) { + this.tagName = other.tagName; + } + if (other.isSetTagValue()) { + this.tagValue = other.tagValue; + } + } + + public MetricTag deepCopy() { + return new MetricTag(this); + } + + @Override + public void clear() { + this.tagName = null; + this.tagValue = null; + } + + public String getTagName() { + return this.tagName; + } + + public MetricTag setTagName(String tagName) { + this.tagName = tagName; + return this; + } + + public void unsetTagName() { + this.tagName = null; + } + + /** Returns true if field tagName is set (has been assigned a value) and false otherwise */ + public boolean isSetTagName() { + return this.tagName != null; + } + + public void setTagNameIsSet(boolean value) { + if (!value) { + this.tagName = null; + } + } + + public String getTagValue() { + return this.tagValue; + } + + public MetricTag setTagValue(String tagValue) { + this.tagValue = tagValue; + return this; + } + + public void unsetTagValue() { + this.tagValue = null; + } + + /** Returns true if field tagValue is set (has been assigned a value) and false otherwise */ + public boolean isSetTagValue() { + return this.tagValue != null; + } + + public void setTagValueIsSet(boolean value) { + if (!value) { + this.tagValue = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TAG_NAME: + if (value == null) { + unsetTagName(); + } else { + setTagName((String)value); + } + break; + + case TAG_VALUE: + if (value == null) { + unsetTagValue(); + } else { + setTagValue((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TAG_NAME: + return getTagName(); + + case TAG_VALUE: + return getTagValue(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TAG_NAME: + return isSetTagName(); + case TAG_VALUE: + return isSetTagValue(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof MetricTag) + return this.equals((MetricTag)that); + return false; + } + + public boolean equals(MetricTag that) { + if (that == null) + return false; + + boolean this_present_tagName = true && this.isSetTagName(); + boolean that_present_tagName = true && that.isSetTagName(); + if (this_present_tagName || that_present_tagName) { + if (!(this_present_tagName && that_present_tagName)) + return false; + if (!this.tagName.equals(that.tagName)) + return false; + } + + boolean this_present_tagValue = true && this.isSetTagValue(); + boolean that_present_tagValue = true && that.isSetTagValue(); + if (this_present_tagValue || that_present_tagValue) { + if (!(this_present_tagValue && that_present_tagValue)) + return false; + if (!this.tagValue.equals(that.tagValue)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(MetricTag other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + MetricTag typedOther = (MetricTag)other; + + lastComparison = Boolean.valueOf(isSetTagName()).compareTo(typedOther.isSetTagName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTagName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tagName, typedOther.tagName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTagValue()).compareTo(typedOther.isSetTagValue()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTagValue()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tagValue, typedOther.tagValue); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("MetricTag("); + boolean first = true; + + sb.append("tagName:"); + if (this.tagName == null) { + sb.append("null"); + } else { + sb.append(this.tagName); + } + first = false; + if (isSetTagValue()) { + if (!first) sb.append(", "); + sb.append("tagValue:"); + if (this.tagValue == null) { + sb.append("null"); + } else { + sb.append(this.tagValue); + } + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class MetricTagStandardSchemeFactory implements SchemeFactory { + public MetricTagStandardScheme getScheme() { + return new MetricTagStandardScheme(); + } + } + + private static class MetricTagStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, MetricTag struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TAG_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tagName = iprot.readString(); + struct.setTagNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // TAG_VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tagValue = iprot.readString(); + struct.setTagValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, MetricTag struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tagName != null) { + oprot.writeFieldBegin(TAG_NAME_FIELD_DESC); + oprot.writeString(struct.tagName); + oprot.writeFieldEnd(); + } + if (struct.tagValue != null) { + if (struct.isSetTagValue()) { + oprot.writeFieldBegin(TAG_VALUE_FIELD_DESC); + oprot.writeString(struct.tagValue); + oprot.writeFieldEnd(); + } + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class MetricTagTupleSchemeFactory implements SchemeFactory { + public MetricTagTupleScheme getScheme() { + return new MetricTagTupleScheme(); + } + } + + private static class MetricTagTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, MetricTag struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTagName()) { + optionals.set(0); + } + if (struct.isSetTagValue()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetTagName()) { + oprot.writeString(struct.tagName); + } + if (struct.isSetTagValue()) { + oprot.writeString(struct.tagValue); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, MetricTag struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.tagName = iprot.readString(); + struct.setTagNameIsSet(true); + } + if (incoming.get(1)) { + struct.tagValue = iprot.readString(); + struct.setTagValueIsSet(true); + } + } + } + +} + diff --git a/m3/src/generated/java/com/uber/m3/thrift/generated/MetricValue.java b/m3/src/generated/java/com/uber/m3/thrift/generated/MetricValue.java new file mode 100644 index 0000000..d12c798 --- /dev/null +++ b/m3/src/generated/java/com/uber/m3/thrift/generated/MetricValue.java @@ -0,0 +1,607 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package com.uber.m3.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Different types of values that m3 emits. Each metric + * must contain one of these values + */ +public class MetricValue implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("MetricValue"); + + private static final org.apache.thrift.protocol.TField COUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("count", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField GAUGE_FIELD_DESC = new org.apache.thrift.protocol.TField("gauge", org.apache.thrift.protocol.TType.STRUCT, (short)2); + private static final org.apache.thrift.protocol.TField TIMER_FIELD_DESC = new org.apache.thrift.protocol.TField("timer", org.apache.thrift.protocol.TType.STRUCT, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new MetricValueStandardSchemeFactory()); + schemes.put(TupleScheme.class, new MetricValueTupleSchemeFactory()); + } + + public CountValue count; // optional + public GaugeValue gauge; // optional + public TimerValue timer; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + COUNT((short)1, "count"), + GAUGE((short)2, "gauge"), + TIMER((short)3, "timer"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // COUNT + return COUNT; + case 2: // GAUGE + return GAUGE; + case 3: // TIMER + return TIMER; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private _Fields optionals[] = {_Fields.COUNT,_Fields.GAUGE,_Fields.TIMER}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.COUNT, new org.apache.thrift.meta_data.FieldMetaData("count", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CountValue.class))); + tmpMap.put(_Fields.GAUGE, new org.apache.thrift.meta_data.FieldMetaData("gauge", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, GaugeValue.class))); + tmpMap.put(_Fields.TIMER, new org.apache.thrift.meta_data.FieldMetaData("timer", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TimerValue.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(MetricValue.class, metaDataMap); + } + + public MetricValue() { + } + + /** + * Performs a deep copy on other. + */ + public MetricValue(MetricValue other) { + if (other.isSetCount()) { + this.count = new CountValue(other.count); + } + if (other.isSetGauge()) { + this.gauge = new GaugeValue(other.gauge); + } + if (other.isSetTimer()) { + this.timer = new TimerValue(other.timer); + } + } + + public MetricValue deepCopy() { + return new MetricValue(this); + } + + @Override + public void clear() { + this.count = null; + this.gauge = null; + this.timer = null; + } + + public CountValue getCount() { + return this.count; + } + + public MetricValue setCount(CountValue count) { + this.count = count; + return this; + } + + public void unsetCount() { + this.count = null; + } + + /** Returns true if field count is set (has been assigned a value) and false otherwise */ + public boolean isSetCount() { + return this.count != null; + } + + public void setCountIsSet(boolean value) { + if (!value) { + this.count = null; + } + } + + public GaugeValue getGauge() { + return this.gauge; + } + + public MetricValue setGauge(GaugeValue gauge) { + this.gauge = gauge; + return this; + } + + public void unsetGauge() { + this.gauge = null; + } + + /** Returns true if field gauge is set (has been assigned a value) and false otherwise */ + public boolean isSetGauge() { + return this.gauge != null; + } + + public void setGaugeIsSet(boolean value) { + if (!value) { + this.gauge = null; + } + } + + public TimerValue getTimer() { + return this.timer; + } + + public MetricValue setTimer(TimerValue timer) { + this.timer = timer; + return this; + } + + public void unsetTimer() { + this.timer = null; + } + + /** Returns true if field timer is set (has been assigned a value) and false otherwise */ + public boolean isSetTimer() { + return this.timer != null; + } + + public void setTimerIsSet(boolean value) { + if (!value) { + this.timer = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case COUNT: + if (value == null) { + unsetCount(); + } else { + setCount((CountValue)value); + } + break; + + case GAUGE: + if (value == null) { + unsetGauge(); + } else { + setGauge((GaugeValue)value); + } + break; + + case TIMER: + if (value == null) { + unsetTimer(); + } else { + setTimer((TimerValue)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case COUNT: + return getCount(); + + case GAUGE: + return getGauge(); + + case TIMER: + return getTimer(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case COUNT: + return isSetCount(); + case GAUGE: + return isSetGauge(); + case TIMER: + return isSetTimer(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof MetricValue) + return this.equals((MetricValue)that); + return false; + } + + public boolean equals(MetricValue that) { + if (that == null) + return false; + + boolean this_present_count = true && this.isSetCount(); + boolean that_present_count = true && that.isSetCount(); + if (this_present_count || that_present_count) { + if (!(this_present_count && that_present_count)) + return false; + if (!this.count.equals(that.count)) + return false; + } + + boolean this_present_gauge = true && this.isSetGauge(); + boolean that_present_gauge = true && that.isSetGauge(); + if (this_present_gauge || that_present_gauge) { + if (!(this_present_gauge && that_present_gauge)) + return false; + if (!this.gauge.equals(that.gauge)) + return false; + } + + boolean this_present_timer = true && this.isSetTimer(); + boolean that_present_timer = true && that.isSetTimer(); + if (this_present_timer || that_present_timer) { + if (!(this_present_timer && that_present_timer)) + return false; + if (!this.timer.equals(that.timer)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(MetricValue other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + MetricValue typedOther = (MetricValue)other; + + lastComparison = Boolean.valueOf(isSetCount()).compareTo(typedOther.isSetCount()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCount()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.count, typedOther.count); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetGauge()).compareTo(typedOther.isSetGauge()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetGauge()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.gauge, typedOther.gauge); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimer()).compareTo(typedOther.isSetTimer()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimer()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timer, typedOther.timer); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("MetricValue("); + boolean first = true; + + if (isSetCount()) { + sb.append("count:"); + if (this.count == null) { + sb.append("null"); + } else { + sb.append(this.count); + } + first = false; + } + if (isSetGauge()) { + if (!first) sb.append(", "); + sb.append("gauge:"); + if (this.gauge == null) { + sb.append("null"); + } else { + sb.append(this.gauge); + } + first = false; + } + if (isSetTimer()) { + if (!first) sb.append(", "); + sb.append("timer:"); + if (this.timer == null) { + sb.append("null"); + } else { + sb.append(this.timer); + } + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (count != null) { + count.validate(); + } + if (gauge != null) { + gauge.validate(); + } + if (timer != null) { + timer.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class MetricValueStandardSchemeFactory implements SchemeFactory { + public MetricValueStandardScheme getScheme() { + return new MetricValueStandardScheme(); + } + } + + private static class MetricValueStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, MetricValue struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // COUNT + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.count = new CountValue(); + struct.count.read(iprot); + struct.setCountIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // GAUGE + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.gauge = new GaugeValue(); + struct.gauge.read(iprot); + struct.setGaugeIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // TIMER + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.timer = new TimerValue(); + struct.timer.read(iprot); + struct.setTimerIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, MetricValue struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.count != null) { + if (struct.isSetCount()) { + oprot.writeFieldBegin(COUNT_FIELD_DESC); + struct.count.write(oprot); + oprot.writeFieldEnd(); + } + } + if (struct.gauge != null) { + if (struct.isSetGauge()) { + oprot.writeFieldBegin(GAUGE_FIELD_DESC); + struct.gauge.write(oprot); + oprot.writeFieldEnd(); + } + } + if (struct.timer != null) { + if (struct.isSetTimer()) { + oprot.writeFieldBegin(TIMER_FIELD_DESC); + struct.timer.write(oprot); + oprot.writeFieldEnd(); + } + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class MetricValueTupleSchemeFactory implements SchemeFactory { + public MetricValueTupleScheme getScheme() { + return new MetricValueTupleScheme(); + } + } + + private static class MetricValueTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, MetricValue struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetCount()) { + optionals.set(0); + } + if (struct.isSetGauge()) { + optionals.set(1); + } + if (struct.isSetTimer()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetCount()) { + struct.count.write(oprot); + } + if (struct.isSetGauge()) { + struct.gauge.write(oprot); + } + if (struct.isSetTimer()) { + struct.timer.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, MetricValue struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.count = new CountValue(); + struct.count.read(iprot); + struct.setCountIsSet(true); + } + if (incoming.get(1)) { + struct.gauge = new GaugeValue(); + struct.gauge.read(iprot); + struct.setGaugeIsSet(true); + } + if (incoming.get(2)) { + struct.timer = new TimerValue(); + struct.timer.read(iprot); + struct.setTimerIsSet(true); + } + } + } + +} + diff --git a/m3/src/generated/java/com/uber/m3/thrift/generated/TimerValue.java b/m3/src/generated/java/com/uber/m3/thrift/generated/TimerValue.java new file mode 100644 index 0000000..547b974 --- /dev/null +++ b/m3/src/generated/java/com/uber/m3/thrift/generated/TimerValue.java @@ -0,0 +1,479 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package com.uber.m3.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Different types of timer values + */ +public class TimerValue implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TimerValue"); + + private static final org.apache.thrift.protocol.TField I64_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("i64Value", org.apache.thrift.protocol.TType.I64, (short)1); + private static final org.apache.thrift.protocol.TField D_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("dValue", org.apache.thrift.protocol.TType.DOUBLE, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TimerValueStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TimerValueTupleSchemeFactory()); + } + + public long i64Value; // optional + public double dValue; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + I64_VALUE((short)1, "i64Value"), + D_VALUE((short)2, "dValue"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // I64_VALUE + return I64_VALUE; + case 2: // D_VALUE + return D_VALUE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __I64VALUE_ISSET_ID = 0; + private static final int __DVALUE_ISSET_ID = 1; + private byte __isset_bitfield = 0; + private _Fields optionals[] = {_Fields.I64_VALUE,_Fields.D_VALUE}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.I64_VALUE, new org.apache.thrift.meta_data.FieldMetaData("i64Value", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.D_VALUE, new org.apache.thrift.meta_data.FieldMetaData("dValue", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TimerValue.class, metaDataMap); + } + + public TimerValue() { + } + + /** + * Performs a deep copy on other. + */ + public TimerValue(TimerValue other) { + __isset_bitfield = other.__isset_bitfield; + this.i64Value = other.i64Value; + this.dValue = other.dValue; + } + + public TimerValue deepCopy() { + return new TimerValue(this); + } + + @Override + public void clear() { + setI64ValueIsSet(false); + this.i64Value = 0; + setDValueIsSet(false); + this.dValue = 0.0; + } + + public long getI64Value() { + return this.i64Value; + } + + public TimerValue setI64Value(long i64Value) { + this.i64Value = i64Value; + setI64ValueIsSet(true); + return this; + } + + public void unsetI64Value() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __I64VALUE_ISSET_ID); + } + + /** Returns true if field i64Value is set (has been assigned a value) and false otherwise */ + public boolean isSetI64Value() { + return EncodingUtils.testBit(__isset_bitfield, __I64VALUE_ISSET_ID); + } + + public void setI64ValueIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __I64VALUE_ISSET_ID, value); + } + + public double getDValue() { + return this.dValue; + } + + public TimerValue setDValue(double dValue) { + this.dValue = dValue; + setDValueIsSet(true); + return this; + } + + public void unsetDValue() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __DVALUE_ISSET_ID); + } + + /** Returns true if field dValue is set (has been assigned a value) and false otherwise */ + public boolean isSetDValue() { + return EncodingUtils.testBit(__isset_bitfield, __DVALUE_ISSET_ID); + } + + public void setDValueIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __DVALUE_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case I64_VALUE: + if (value == null) { + unsetI64Value(); + } else { + setI64Value((Long)value); + } + break; + + case D_VALUE: + if (value == null) { + unsetDValue(); + } else { + setDValue((Double)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case I64_VALUE: + return Long.valueOf(getI64Value()); + + case D_VALUE: + return Double.valueOf(getDValue()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case I64_VALUE: + return isSetI64Value(); + case D_VALUE: + return isSetDValue(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TimerValue) + return this.equals((TimerValue)that); + return false; + } + + public boolean equals(TimerValue that) { + if (that == null) + return false; + + boolean this_present_i64Value = true && this.isSetI64Value(); + boolean that_present_i64Value = true && that.isSetI64Value(); + if (this_present_i64Value || that_present_i64Value) { + if (!(this_present_i64Value && that_present_i64Value)) + return false; + if (this.i64Value != that.i64Value) + return false; + } + + boolean this_present_dValue = true && this.isSetDValue(); + boolean that_present_dValue = true && that.isSetDValue(); + if (this_present_dValue || that_present_dValue) { + if (!(this_present_dValue && that_present_dValue)) + return false; + if (this.dValue != that.dValue) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TimerValue other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TimerValue typedOther = (TimerValue)other; + + lastComparison = Boolean.valueOf(isSetI64Value()).compareTo(typedOther.isSetI64Value()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetI64Value()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.i64Value, typedOther.i64Value); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetDValue()).compareTo(typedOther.isSetDValue()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetDValue()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.dValue, typedOther.dValue); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TimerValue("); + boolean first = true; + + if (isSetI64Value()) { + sb.append("i64Value:"); + sb.append(this.i64Value); + first = false; + } + if (isSetDValue()) { + if (!first) sb.append(", "); + sb.append("dValue:"); + sb.append(this.dValue); + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TimerValueStandardSchemeFactory implements SchemeFactory { + public TimerValueStandardScheme getScheme() { + return new TimerValueStandardScheme(); + } + } + + private static class TimerValueStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TimerValue struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // I64_VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.i64Value = iprot.readI64(); + struct.setI64ValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // D_VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) { + struct.dValue = iprot.readDouble(); + struct.setDValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TimerValue struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.isSetI64Value()) { + oprot.writeFieldBegin(I64_VALUE_FIELD_DESC); + oprot.writeI64(struct.i64Value); + oprot.writeFieldEnd(); + } + if (struct.isSetDValue()) { + oprot.writeFieldBegin(D_VALUE_FIELD_DESC); + oprot.writeDouble(struct.dValue); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TimerValueTupleSchemeFactory implements SchemeFactory { + public TimerValueTupleScheme getScheme() { + return new TimerValueTupleScheme(); + } + } + + private static class TimerValueTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TimerValue struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetI64Value()) { + optionals.set(0); + } + if (struct.isSetDValue()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetI64Value()) { + oprot.writeI64(struct.i64Value); + } + if (struct.isSetDValue()) { + oprot.writeDouble(struct.dValue); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TimerValue struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.i64Value = iprot.readI64(); + struct.setI64ValueIsSet(true); + } + if (incoming.get(1)) { + struct.dValue = iprot.readDouble(); + struct.setDValueIsSet(true); + } + } + } + +} + diff --git a/m3/src/main/java/com/uber/m3/util/TUdpClient.java b/m3/src/main/java/com/uber/m3/util/TUdpClient.java index 85e22c3..9bdfc4f 100644 --- a/m3/src/main/java/com/uber/m3/util/TUdpClient.java +++ b/m3/src/main/java/com/uber/m3/util/TUdpClient.java @@ -33,7 +33,7 @@ public class TUdpClient extends TUdpTransport implements AutoCloseable { * Constructs a UDP client with the given host and port. * @param host the host for this transport * @param port the port for this transport - * @throws SocketException + * @throws SocketException if the underlying socket cannot be opened */ public TUdpClient(String host, int port) throws SocketException { super(host, port); @@ -43,7 +43,7 @@ public TUdpClient(String host, int port) throws SocketException { * Constructs a UDP client with the given host and port. * @param hostPort a string containing both host and port information in the * form "HOST:PORT" - * @throws SocketException + * @throws SocketException if the underlying socket cannot be opened */ public TUdpClient(String hostPort) throws SocketException { super(hostPort); diff --git a/m3/src/main/java/com/uber/m3/util/TUdpServer.java b/m3/src/main/java/com/uber/m3/util/TUdpServer.java index ef2ab9d..a8a5bc7 100644 --- a/m3/src/main/java/com/uber/m3/util/TUdpServer.java +++ b/m3/src/main/java/com/uber/m3/util/TUdpServer.java @@ -35,7 +35,7 @@ public class TUdpServer extends TUdpTransport implements AutoCloseable { * Constructs a UDP server with the given host and port. Defaults to zero timeout. * @param host the host for this transport * @param port the port for this transport - * @throws SocketException + * @throws SocketException if the underlying socket cannot be opened */ public TUdpServer(String host, int port) throws SocketException { this(host, port, 0); @@ -46,7 +46,7 @@ public TUdpServer(String host, int port) throws SocketException { * @param host the host for this transport * @param port the port for this transport * @param timeoutMillis the timeout in milliseconds - * @throws SocketException + * @throws SocketException if the underlying socket cannot be opened */ public TUdpServer(String host, int port, int timeoutMillis) throws SocketException { super(host, port); From 8291cd1ae421dcefceeb6a2b284b74681ea03076 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Wed, 13 Sep 2017 15:28:00 -0400 Subject: [PATCH 23/44] Only compile thrift if explicitly set a property to; update DEVELOPER docs to reflect on this --- DEVELOPER.md | 8 +++++++- ci/before_install.sh | 10 ---------- m3/build.gradle | 4 ++++ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/DEVELOPER.md b/DEVELOPER.md index a41f7aa..8be7c8e 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -7,7 +7,7 @@ This document summarizes information relevant to Java tally contributors. ### Prerequisites In order to build this project, you must have: - JDK-7 or later -- Apache Thrift 0.9.x +- Apache Thrift 0.9.x -- only if you plan to make changes to Thrift files and recompile (regenerate) source files ### Building and testing Gradle is used to build tally. Run this from the top-level directory to build the project: @@ -20,6 +20,12 @@ To run the project's tests: ./gradlew check ``` +By default, the build does *not* compile Thrift files to generate sources. If you make changes to Thrift files and need +regenerate sources, make sure you have thrift 0.9.x installed and build with the `genThrift` property set, e.g. +```bash +./gradlew clean :tally-m3:build -PgenThrift +``` + ### Making changes The source code of tally is managed using GitHub, and as such, we use its features to, for example, track [issues](https://help.github.com/articles/about-issues/) and create diff --git a/ci/before_install.sh b/ci/before_install.sh index 42bc3a8..e26aa9d 100755 --- a/ci/before_install.sh +++ b/ci/before_install.sh @@ -10,13 +10,3 @@ wget "https://bouncycastle.org/download/${BCPROV_FILENAME}" sudo mv $BCPROV_FILENAME /usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext sudo perl -pi.bak -e 's/^(security\.provider\.)([0-9]+)/$1.($2+1)/ge' /etc/java-7-openjdk/security/java.security echo "security.provider.1=org.bouncycastle.jce.provider.BouncyCastleProvider" | sudo tee -a /etc/java-7-openjdk/security/java.security - -# Install Thrift 0.9.3 -sudo apt-get update -qq -sudo apt-get install libboost-dev libboost-test-dev libboost-program-options-dev libevent-dev automake libtool flex bison pkg-config g++ libssl-dev -pushd . -cd -wget http://www.us.apache.org/dist/thrift/0.9.3/thrift-0.9.3.tar.gz -tar xfz thrift-0.9.3.tar.gz -cd thrift-0.9.3 && ./configure && sudo make install -popd diff --git a/m3/build.gradle b/m3/build.gradle index ed69d9d..d90b4cf 100644 --- a/m3/build.gradle +++ b/m3/build.gradle @@ -36,3 +36,7 @@ compileThrift { createGenFolder false } + +compileThrift.onlyIf { + project.hasProperty('genThrift') +} From 21260a1aeb6c43ea115aaa2bc621be28a869ea0b Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Wed, 13 Sep 2017 16:01:03 -0400 Subject: [PATCH 24/44] Extract out function to get hostname --- .../java/com/uber/m3/tally/m3/M3Reporter.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 7df6ab9..7f64929 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -51,6 +51,8 @@ import org.apache.thrift.transport.TTransport; import java.net.InetAddress; +import java.net.SocketException; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -76,6 +78,7 @@ public class M3Reporter implements CachedStatsReporter, AutoCloseable { private static final String SERVICE_TAG = "service"; private static final String ENV_TAG = "env"; private static final String HOST_TAG = "host"; + private static final String DEFAULT_TAG_VALUE = "default"; private static final String DEFAULT_HISTOGRAM_BUCKET_ID_NAME = "bucketid"; private static final String DEFAULT_HISTOGRAM_BUCKET_NAME = "bucket"; @@ -144,7 +147,7 @@ private M3Reporter(Builder builder) { commonTags.add(createMetricTag(ENV_TAG, builder.env)); } if (builder.includeHost && !builder.commonTags.containsKey(HOST_TAG)) { - commonTags.add(createMetricTag(HOST_TAG, InetAddress.getLocalHost().getHostName())); + commonTags.add(createMetricTag(HOST_TAG, getHostName())); } metricBatch = resourcePool.getMetricBatch(); @@ -170,11 +173,19 @@ private M3Reporter(Builder builder) { executor = builder.executor; addAndRunProcessor(); - } catch (Exception e) { + } catch (SocketException | TException e) { throw new IllegalArgumentException("Exception creating M3Reporter", e); } } + private String getHostName() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + return DEFAULT_TAG_VALUE; + } + } + private void addAndRunProcessor() { phaser.register(); From 6cbbf4e28e531da2b525e6163573a0479af67a8c Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Thu, 14 Sep 2017 16:12:13 -0400 Subject: [PATCH 25/44] Have SizedMetric ObjectPools; warn instead of throw on InterruptedException --- .../java/com/uber/m3/tally/m3/M3Reporter.java | 59 ++++++++++++++----- .../com/uber/m3/tally/m3/ResourcePool.java | 28 +++++++++ .../com/uber/m3/tally/m3/SizedMetric.java | 35 +++++++++-- .../com/uber/m3/tally/m3/SizedMetricTest.java | 2 - 4 files changed, 100 insertions(+), 24 deletions(-) diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 7f64929..3bd759a 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -64,10 +64,15 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Phaser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** - * An M3 reporter. + * An M3 implementation of a {@link CachedStatsReporter}. */ public class M3Reporter implements CachedStatsReporter, AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(M3Reporter.class); + private static final int DEFAULT_METRIC_SIZE = 100; private static final int DEFAULT_MAX_QUEUE_SIZE = 4096; @@ -151,18 +156,10 @@ private M3Reporter(Builder builder) { } metricBatch = resourcePool.getMetricBatch(); - metricBatch.setCommonTags(commonTags); - metricBatch.setMetrics(new ArrayList()); - calcProtocol = resourcePool.getProtocol(); - metricBatch.write(calcProtocol); calc = (TCalcTransport) calcProtocol.getTransport(); - int numOverheadBytes = EMIT_METRIC_BATCH_OVERHEAD + calc.getSizeAndReset(); - freeBytes = builder.maxPacketSizeBytes - numOverheadBytes; - if (freeBytes <= 0) { - throw new IllegalArgumentException("Common tags serialized size exceeds packet size"); - } + freeBytes = calculateFreeBytes(builder.maxPacketSizeBytes, commonTags); bucketIdTagName = builder.histogramBucketIdName; bucketTagName = builder.histogramBucketName; @@ -173,15 +170,41 @@ private M3Reporter(Builder builder) { executor = builder.executor; addAndRunProcessor(); - } catch (SocketException | TException e) { + } catch (SocketException e) { throw new IllegalArgumentException("Exception creating M3Reporter", e); } } + private int calculateFreeBytes(int maxPacketSizeBytes, Set commonTags) { + metricBatch.setCommonTags(commonTags); + metricBatch.setMetrics(new ArrayList()); + + int size; + + try { + metricBatch.write(calcProtocol); + size = calc.getSizeAndReset(); + } catch (TException e) { + LOG.warn("Unable to calculate metric batch size. Defaulting to: " + DEFAULT_METRIC_SIZE); + size = DEFAULT_METRIC_SIZE; + } + + int numOverheadBytes = EMIT_METRIC_BATCH_OVERHEAD + size; + + freeBytes = maxPacketSizeBytes - numOverheadBytes; + + if (freeBytes <= 0) { + throw new IllegalArgumentException("Common tags serialized size exceeds packet size"); + } + + return freeBytes; + } + private String getHostName() { try { return InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException e) { + LOG.warn("Unable to determine hostname. Defaulting to: " + DEFAULT_TAG_VALUE); return DEFAULT_TAG_VALUE; } } @@ -273,7 +296,7 @@ public void flush() { try { // A null metric signifies to processors that a flush was called, // which causes the metric queue to be cleared - metricQueue.put(new SizedMetric(null, 0)); + metricQueue.put(resourcePool.getSizedMetric()); } catch (InterruptedException e) { throw new RuntimeException(e); } @@ -380,6 +403,7 @@ private int calculateSize(Metric metric) { return calc.getSizeAndReset(); } } catch (TException e) { + LOG.warn("Unable to calculate metric batch size. Defaulting to: " + DEFAULT_METRIC_SIZE); return DEFAULT_METRIC_SIZE; } } @@ -419,9 +443,11 @@ private void reportCopyMetric( } try { - metricQueue.put(new SizedMetric(metricCopy, size)); + metricQueue.put(resourcePool.getSizedMetric().setMetric(metricCopy).setSize(size)); } catch (InterruptedException e) { - throw new RuntimeException(e); + LOG.warn(String.format("Interrupted while putting %s: %s", + metricType, + metricCopy.getName())); } } @@ -459,6 +485,9 @@ public void run() { while (!metricQueue.isEmpty()) { SizedMetric sizedMetric = metricQueue.poll(); Metric metric = sizedMetric.getMetric(); + int size = sizedMetric.getSize(); + + resourcePool.releaseSizedMetric(sizedMetric); if (metric == null) { // Explicit flush requested @@ -470,8 +499,6 @@ public void run() { continue; } - int size = sizedMetric.getSize(); - if (bytes + size > freeBytes) { metrics = flush(metrics); bytes = 0; diff --git a/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java b/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java index 5c58539..42b4bf6 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java @@ -51,6 +51,7 @@ public class ResourcePool { private ObjectPool gaugeValuePool; private ObjectPool timerValuePool; private ObjectPool protocolPool; + private ObjectPool sizedMetricPool; /** * Creates a {@link ResourcePool}, initializing all its inner pools. @@ -113,6 +114,13 @@ public TProtocol instance() { return protocolFactory.getProtocol(new TCalcTransport()); } }); + + sizedMetricPool = new ObjectPool<>(maxQueueSize, new Construct() { + @Override + public SizedMetric instance() { + return new SizedMetric(); + } + }); } /** @@ -187,6 +195,14 @@ public TProtocol getProtocol() { return protocolPool.get(); } + /** + * Returns a {@link SizedMetric} from the pool. + * @return a {@link SizedMetric} from the pool + */ + public SizedMetric getSizedMetric() { + return sizedMetricPool.get(); + } + /** * Puts the given {@link TProtocol} back to the pool, reseting the * {@link TCalcTransport} internal count. @@ -295,4 +311,16 @@ public void releaseMetricValue(MetricValue metricValue) { metricValuePool.put(metricValue); } + + /** + * Puts the given {@link SizedMetric} back to the pool after reseting + * inner values. + * @param sizedMetric the {@link SizedMetric} to put back to the pool + */ + public void releaseSizedMetric(SizedMetric sizedMetric) { + sizedMetric.setMetric(null); + sizedMetric.setSize(0); + + sizedMetricPool.put(sizedMetric); + } } diff --git a/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java b/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java index 9bda64a..f7d1a6f 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java @@ -35,14 +35,37 @@ public class SizedMetric extends Metric { * @param size the size of this metric */ public SizedMetric(Metric metric, int size) { - // Recreate metric to prevent underlying metric from being modified - if (metric == null) { - this.metric = null; - } else { - this.metric = new Metric(metric); - } + this.metric = metric; + this.size = size; + } + + /** + * Creates a default (empty) {@link SizedMetric}. + */ + public SizedMetric() { + this(null, 0); + } + + /** + * Sets the underlying metric. + * @param metric the metric to set + * @return this {@link SizedMetric} after setting the metric + */ + public SizedMetric setMetric(Metric metric) { + this.metric = metric; + + return this; + } + /** + * Sets the underlying metric's size. + * @param size the size to set + * @return this {@link SizedMetric} after setting the size + */ + public SizedMetric setSize(int size) { this.size = size; + + return this; } /** diff --git a/m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java b/m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java index 7f59600..25d2461 100644 --- a/m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java +++ b/m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java @@ -23,7 +23,6 @@ import com.uber.m3.thrift.generated.Metric; import org.junit.Test; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertEquals; public class SizedMetricTest { @@ -33,7 +32,6 @@ public void simpleSizedMetric() { SizedMetric sizedMetric = new SizedMetric(metric, 1); - assertFalse(sizedMetric.getMetric() == metric); assertEquals(sizedMetric.getMetric(), metric); assertEquals(sizedMetric.getSize(), 1); } From 7ed0f756afd72bfb974a08fecf79afb95c7f0444 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Mon, 18 Sep 2017 11:37:27 -0400 Subject: [PATCH 26/44] Move thrift-specific classes into m3 package --- m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java | 2 -- m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java | 1 - .../java/com/uber/m3/{util => tally/m3}/TCalcTransport.java | 2 +- m3/src/main/java/com/uber/m3/{util => tally/m3}/TUdpClient.java | 2 +- m3/src/main/java/com/uber/m3/{util => tally/m3}/TUdpServer.java | 2 +- .../main/java/com/uber/m3/{util => tally/m3}/TUdpTransport.java | 2 +- 6 files changed, 4 insertions(+), 7 deletions(-) rename m3/src/main/java/com/uber/m3/{util => tally/m3}/TCalcTransport.java (98%) rename m3/src/main/java/com/uber/m3/{util => tally/m3}/TUdpClient.java (98%) rename m3/src/main/java/com/uber/m3/{util => tally/m3}/TUdpServer.java (98%) rename m3/src/main/java/com/uber/m3/{util => tally/m3}/TUdpTransport.java (99%) diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 3bd759a..440057d 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -41,8 +41,6 @@ import com.uber.m3.thrift.generated.TimerValue; import com.uber.m3.util.Duration; import com.uber.m3.util.ImmutableMap; -import com.uber.m3.util.TCalcTransport; -import com.uber.m3.util.TUdpClient; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TCompactProtocol; diff --git a/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java b/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java index 42b4bf6..ecceedc 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java @@ -29,7 +29,6 @@ import com.uber.m3.thrift.generated.TimerValue; import com.uber.m3.util.Construct; import com.uber.m3.util.ObjectPool; -import com.uber.m3.util.TCalcTransport; import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.protocol.TProtocolFactory; diff --git a/m3/src/main/java/com/uber/m3/util/TCalcTransport.java b/m3/src/main/java/com/uber/m3/tally/m3/TCalcTransport.java similarity index 98% rename from m3/src/main/java/com/uber/m3/util/TCalcTransport.java rename to m3/src/main/java/com/uber/m3/tally/m3/TCalcTransport.java index dc2afbb..c76e5e6 100644 --- a/m3/src/main/java/com/uber/m3/util/TCalcTransport.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/TCalcTransport.java @@ -18,7 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -package com.uber.m3.util; +package com.uber.m3.tally.m3; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; diff --git a/m3/src/main/java/com/uber/m3/util/TUdpClient.java b/m3/src/main/java/com/uber/m3/tally/m3/TUdpClient.java similarity index 98% rename from m3/src/main/java/com/uber/m3/util/TUdpClient.java rename to m3/src/main/java/com/uber/m3/tally/m3/TUdpClient.java index 9bdfc4f..cc2c83d 100644 --- a/m3/src/main/java/com/uber/m3/util/TUdpClient.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/TUdpClient.java @@ -18,7 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -package com.uber.m3.util; +package com.uber.m3.tally.m3; import org.apache.thrift.transport.TTransportException; diff --git a/m3/src/main/java/com/uber/m3/util/TUdpServer.java b/m3/src/main/java/com/uber/m3/tally/m3/TUdpServer.java similarity index 98% rename from m3/src/main/java/com/uber/m3/util/TUdpServer.java rename to m3/src/main/java/com/uber/m3/tally/m3/TUdpServer.java index a8a5bc7..cfdc939 100644 --- a/m3/src/main/java/com/uber/m3/util/TUdpServer.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/TUdpServer.java @@ -18,7 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -package com.uber.m3.util; +package com.uber.m3.tally.m3; import org.apache.thrift.transport.TTransportException; diff --git a/m3/src/main/java/com/uber/m3/util/TUdpTransport.java b/m3/src/main/java/com/uber/m3/tally/m3/TUdpTransport.java similarity index 99% rename from m3/src/main/java/com/uber/m3/util/TUdpTransport.java rename to m3/src/main/java/com/uber/m3/tally/m3/TUdpTransport.java index 752b801..904d67d 100644 --- a/m3/src/main/java/com/uber/m3/util/TUdpTransport.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/TUdpTransport.java @@ -18,7 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -package com.uber.m3.util; +package com.uber.m3.tally.m3; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; From db8015f3ae143575089c38e31ac615845fcfce28 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Sat, 23 Sep 2017 22:46:34 -0400 Subject: [PATCH 27/44] JMH profiling; Use Gradle 4.0; Benchmarking ResourcePool; Use TCompactProtocol; Retry emitting metrics on fail; Fix Processor race condition --- build.gradle | 7 +- gradle/wrapper/gradle-wrapper.jar | Bin 54783 -> 54708 bytes gradle/wrapper/gradle-wrapper.properties | 3 +- m3/build.gradle | 19 +- .../m3/tally/m3/ResourcePoolBenchmark.java | 83 +++++ .../java/com/uber/m3/tally/m3/M3Reporter.java | 287 +++++++++++------- .../com/uber/m3/tally/m3/ResourcePool.java | 147 ++++++--- .../com/uber/m3/tally/m3/SizedMetric.java | 2 + .../com/uber/m3/tally/m3/TUdpTransport.java | 160 ---------- .../com/uber/m3/tally/m3/ThriftProtocol.java | 29 -- .../tally/m3/{ => thrift}/TCalcTransport.java | 34 ++- .../m3/tally/m3/{ => thrift}/TUdpClient.java | 44 +-- .../m3/tally/m3/{ => thrift}/TUdpServer.java | 25 +- .../m3/tally/m3/thrift/TUdpTransport.java | 142 +++++++++ .../com/uber/m3/tally/m3/SizedMetricTest.java | 19 ++ 15 files changed, 626 insertions(+), 375 deletions(-) create mode 100644 m3/src/jmh/java/com/uber/m3/tally/m3/ResourcePoolBenchmark.java delete mode 100644 m3/src/main/java/com/uber/m3/tally/m3/TUdpTransport.java delete mode 100644 m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java rename m3/src/main/java/com/uber/m3/tally/m3/{ => thrift}/TCalcTransport.java (66%) rename m3/src/main/java/com/uber/m3/tally/m3/{ => thrift}/TUdpClient.java (64%) rename m3/src/main/java/com/uber/m3/tally/m3/{ => thrift}/TUdpServer.java (76%) create mode 100644 m3/src/main/java/com/uber/m3/tally/m3/thrift/TUdpTransport.java diff --git a/build.gradle b/build.gradle index 4469650..7268724 100644 --- a/build.gradle +++ b/build.gradle @@ -33,7 +33,7 @@ plugins { id 'signing' id 'checkstyle' id 'jacoco' - id 'net.researchgate.release' version '2.4.0' apply false + id 'net.researchgate.release' version '2.6.0' apply false id 'com.github.kt3k.coveralls' version '2.8.1' } @@ -165,7 +165,8 @@ subprojects { ext."signing.secretKeyRingFile" = "ci/${ext.secretKeyRingFileName}" task wrapper(type: Wrapper) { - gradleVersion = '3.5' + // Gradle version 4+ required for JMH benchmarking, otherwise v3.5+ is sufficient + gradleVersion = '4.0' } // ---------------------- @@ -256,7 +257,7 @@ coveralls { task jacocoRootReport(type: org.gradle.testing.jacoco.tasks.JacocoReport) { dependsOn = subprojects.test sourceDirectories = files(subprojects.sourceSets.main.allSource.srcDirs) - classDirectories = files(subprojects.sourceSets.main.output.classesDir) + classDirectories = files(subprojects.sourceSets.main.output.classesDirs) executionData = files(subprojects.jacocoTestReport.executionData) reports { diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index dcdd34d9ee44425d0ea9ff5cc7505dbed7d701ca..736fb7d3f94c051b359fc7ae7212d351bc094bdd 100644 GIT binary patch delta 17061 zcmY(qV{qn8u)rJJwrv}mjcwbu`QK!dCr&oD&5ezXH|ECJ*tYN9w@%%2&s0rS&vZ}M zeCV2)uKD$66=YKt-)xD?l$Tt$@3XqEIHjn6>X@JeT;%ue%*@4SvlL32Fs9kwru zqPBCln?X%_j#y2u=-IXq; zEbYA$)}yEFVB|V3K+Sk4-Z<)oa(AL70K1}A-8chEa zF=TJ_R?>JcYnJHDH8dm;A(}WzI(`rh(-SJnepU9wx$@?VEE%XfVvP4B|CR#!dGurt z3Fu$*iv0!%z=;9-zn52!%}=mhPJc=#zzPshJdAJQYd9&bNE&IHThX&wR5#(ldqp+CAAR%J8@`6wVd;0P$F zxD1an%M;W=EOwrzc0Oq|CrGNS1Rk@$0mC5z#Zd0q;^^iMn?;FemPLF#+*42V*&&$r zODQ`Fr@0{-Lx;to$&QEg?pc(1|DLyGb4H*pd$%`TFxqh6eX$gl{b5qe5z;WxT zKpfH>br?n&RRgjn8=+)913#g*MEfE?|}8*`z{lI-6_*8=Xg(HxN%%Hf5K>kA@WYbDCXB>O?x2G+9e4o z;lyR8M<*(^ddsS6BL;j00((<-vWo1ot?2RIW3BO&d^oB>X2an=MFt@vzyV9V^+3cZ zX~|(!>99oJ@W*qs z=RD`&SJ1Qdf2;+!mfL<^TtsNAz^aMmY{@gf0XYJTCHp+dGrkWVQF9&c_Qkx8k_P#I z+{B%IS@=YJ;}qsb+%9T&H^1$dBqmn2hgQO6L2Tz@^#8b_6g-s!_%tpE)e!e|2YhnI zikY@ZSTna%$2N6>tSot?E!hHd1&us}NvOLV1fdmzn`^MW+4j~$e?Y9^6y@SP{; z)LdEiO5NcbhZWP59{kh?8`0f4H+T<_HcPL0wCbJYLc1iMR%iY6sw~oR@3x zibUK)?CexC8b}+vlI8Fc2sjZ&O>?st!z>lJD|;*9itH3|_GSxMw1eq1W~F@e!;0+W zAO%-Bf(@py4EoJbTt)P>N!wZbD9#4&UWciM?H<(hC;YeSFDfq#j#02f;t*-d_i?dE zaYS)(W$;zNQ9=@}bPn5qLsuVIWY@{jlN_a+xo7iRwsvEvkms6^8+yQ3JU5<5B6B1uadk4$j zZF&aj{;j&W6JIM=QTuVN!7A6XK96ICLW`?^hrhqTYKL&Q_Iqnlu|0yZg-nB-l9^FO zoj_Ytmu+)sOHx)EjHN^0b&WiAE!|Kb#3?gCQ>phgG4jXj6XMB@j^?+HZ~9RjV{*=v zj<2xiwv0(T-Szc*BzjV14{Y+cl)K7yroP3av4n_eQ);o;nSyyTezuw`IW#?iX(eLt z`)az)rJk0KxVAM0drZfPht|oncuIh5Y<+>kU;{>F$qfR!ta;1|sq9o+zi#(1no%t?+pRk-$U1r1)7v2rZMPCtHZ2kez*hvu`z=)I%4O zmIl_YJ60oK3Y*3fKPTs^Q5sPgq=;QgZ@iE0rdc^(JkoRSAENU^Je)~>T{p&dbuY3#;$(B?KXDyZ6Tv0tH;HPoY)q*kXvk+}GM_b9A z8Q`LVp>>-x`dpEmmCB7yjF*+!Y-u#REV@(rZs42@&$`uEZ0z}r+4KUMa7t=O+#LzIXEC zl2A9-Azfix3~{u#??9Zh5NF{wz6;XnqcH_)(!HZ<{N-;9Df39fj*qO^Hpex6mM{@zGKN1nJ?sr&yt4oC zK*3UP_}7L=4C<-I)*hEQCmM~l+>t~zRMVkohW^8U*?t^-$NxI;#Ji1vTBbY~OS8mJ zO{W>-+Qw)Q(;%UD-O)*3LR0@$Q=`?>NHU)pqbl^Y0xEZl)D!yi9VC9fymyMr~mnfyo9=rr`lM25_!Qm z933~hyc@<{HqCEnON`K?5Pn0hC1ZDd9LERAncQjL+Z zLCeEJ^FpPwqx`wvx`6iWruEF`gRDlle&e#MoZ_m#9Y1DAaNS!IFJYSgiAY!rngFGr zr@JD~`805v6BYh&8UyA0B_68r=%g#SWs0dIZ)7nSG22_|1~Ua5Uh?m`~ry?w+kB3~@)Hd0Sa;}Zf&K8|ZZj{DXx{Jz$2 za690p{c0BgFL%WVqYrG*HvAmtgvMas?lXvA{vO1dqmPIM?Rt=U&fh& z71LPXg&N{jJ3{wTPuqkyQ-_rL4p2TeMtcq4I-`M3-1_q$IjsDk2gh{`$5-;#*L2z@ruc-R&5XRsc*ScX);NnbIemTy6a&?`$$CVNF0vI=mX@Xrng^6P1%=X3bujmbr(m`rOW4H1=R zDKCo{F0B(kz`nm8V;EmDF*K?dBs_AiUIXqYFC6$^wUd8V?Xjx*HuwA%rcRs!U6y?n z-)@v^hb?y7hFy7p>PH<_W^))-rW0BzXM9Af3)T_(rObv?mS%pf5-QA$5jAxGpwq>^ z&?j{oV>qyo!m%tO-ke+Vm2|)PoqaQ48oh<|koXDSGntY<>_!@X8>=zCsY*5;P-7>i zo`~qKjKk;?v-oq4OMQmBg6NjF(Kc)pk({TNC>pD)Nugnyxq=eg@z!f3)Bhxzw1fMB z92CKZZUh4cwvPw~#`!Og#7U|iAp=sh^xZKJ@q{1zA$uB zN%d)vxi`4(hJIa~3-9)6uBz4gCa)|jzgq0*C$|c6o6kC+n9s1HK}VYoHE8DSb}H-5 zPisF`wDwbd3FvG{KyAaEeiG~nKMdSD^w$R7FQrK#`9jwOa{Jp+gd*X&6#zTFpOkU* z$j0*tgZ$0DVl|~}*M*tJYP7fuSDxGk8UxhGtgQoTaC=BT=N}+aqe<}=J6rPW%-;c%I(bnKgGXO@TZlz3H zTCWGDLg|gR)cSXCD`(ZtDa0Aw2l{_1?;1yH)3W_0dl*I>q~lWM8UQWARs(yj7pTmp ze#qci53KNW%anDOrYhK$BxYZyii_7d)PLH`7xuXnrp@;5wvil z*K)TIf)rwDUh?j#r*dUD0w!3u45|&-#F&j3VNny6vlADe-=*^npQB?OjPyjDUnLIc zWrTb7Q@iD`0IC+p6=11d)0?Y|dzPis2vbaCRi(w*QWmFD-g2dt{Ul*@benQvNtZ$w z*})B@kx^%D$XNcmKuVf1QKNX5Fs)JfTNXswD9rd>mcw=3i)bjV#RHVGYEX}3WwCi} zcjK^;8a&`}FMNk?Bq2%5DkvwfA3U;{V9JGQw86@K+WtCw1`w)^!GK%_egzS+u|+^a zNZH%K^n^Jh3iZ5%5!b9K>At}9e7_`kA~7OY24&B61#AE5L+f$=3)^PhsOJ%SPl!yPsA4ERGkKYV4-|0}P6Ete=fFWV4lsajj zO3b!7^LiuCUc_q;V+2QIZTFNC+f>$7j3WPDN@0`(&xsSZo-|b*G(sS$q>F9jBx}8B4`j&hZ!u$P`kmkF#nM1844`%$ z$(USru$qUF7bipIJu`QU~giuC*U0nQSr`?lP6d` zqd!WHc#d!dd9h4tA%FrruW7^hrp2>7X6lS=O8fL)iOGb9G28@^f|+v}T%NJbDda5J znz}}Qpenj${km0{fN0>Vgt8{nAlS{U0OvPC)q!4ZS^4xwV`(&7MBQY59_eY>6TOo- zf~^G^+b=^T0YA;@za84m$WyMH$mHYKWZv`1w(5CT!GdUzYoZbkCL`fC{-PQpSd~Nk z5AN&VQO<^F&Iaa%*eSUKNQ@(bAVZxb+SKCA00RKsxDab&a7-p25p9}@LZrJlc#tA- z-aFrQW%G;ljoDM2r@6ksYPA3mer5GjtX&$7V ztZwiL@*K&`kfdZ@^8(e82GW(gtazH%fvfthu0H7cWw8FQ7k{q}C{k?xMjCLWIFMl@ z03uKR48FK)YVasrvtG9>ExoUQF)hvN{@kOyXs{40uXIMAzB8H-wS{j$@Cy$?a&pON z6evOj9FdiL(l535C>*|L&r;Z|TYr2;DogUZFEjI0;aj3(X1MekPfj#dF4xq~ z9A~}PhvD2tP%4dH+Y>^$H%YP@kZSEH6HP*aZ+eFFAtod}TFCJM63RWCo;BA1cvHDY z%6+?JFQ>Pw$N|K+L+LWv`?3MS{CAd@Rukp@_;Dg-(BYx@gcQ~MU|E&2iuxzhq0W=W zM@((7|1#|r%g)fFFHxBZ?uOLC%MUnXEh5(IHwtH#tCg;GvmU}S7d^>o33@c^PgoUs zNGL2YFgQ3cFfcJNs0zOg0|>Cj3etQCCz~Q?C8wHXW z7^_2ALg;enLOn8`1)+nvnao#|=Q>HMoCKv9D^Dt7*{l{1nhj1_T3P*-*rXp*O~*=) zn~WR&@t>W7;|(6)fOlxojn~cArB|cex9Nd+@MSWdie*rzro*88-i=y&mqMQ{HZ_Jp zNE)M080pWuT}`(~JM5o*fN;?I>4-gcmy%@4U}gX5{x+dT7rL|Bl@&HO$`dm~%Wh5L z;o9DfS9_QEP+*&JA0)!&@kl4t(FQ~6t}w!8l^OPCnHl~RosY^8!RWq-Q1RdQpnxxu z#3JPs(eTgQ^l7N0`=+9$Z$1)Xr14|#<_J@jK18Hi*XRKnLnBwN0CwIlOazgNzv>Jv ze`T@rhOOV8s*9hqN%OR?pr4;5eV>wr-V!bDl1YtIV|sQ*w6OyeSKV3-dM-wYZypJQ z0@R0EuN4 zxQhG&WR!>YHlbgt>vi-@3{7sT*LZKQWs`V#WR6DBZO=#u!(+w@t#0`6Wkci<;bD|Ym?`UNX&K_hM!4{O=?>bk z&EJG1R^6m-fTM^WGQ(e~EN*AxI`uw2Ff8XDKIcUZf|F3^OtD!-MQSV2U4>}%sIuDo zUptftW@9WdfLPp*j}@J{b3@vq;U@k+FTZ69P@%LIr{jjta*(P|B3!wQRKIPi;EjjX z@;r0kEZ`PA?<&Yv=&%qcg0MH2wkA2n_{XimVU?(vfp?23ykj}-i604aYX)|umRQL< zuCRR-u$74%b5=Ci+^h40?02ER3bUOyMz3S`zsXh-PYv!+zDq4c#Om{NWd-)o5T3GT z(?}i|bJG^y?uQDPWNj~u?-YhOCO~i-M4xiS>}Vmk{ z0s)9bz}x{6d)7!g7C}&?G5u3BXzOq5AU}#euDBRAk%rifydiioSkBgpZ1CbHAC4Av0Dl2Ts%2GmjLTkwBZviGGULhI)Rwk{ zYy3KoH68Lnp;EmdzZrw@2Fp@rxt3e(?Rkkvx<8@)5eTB1n3iohj(d6_!Dgv^p*)H5 zLW4P52VD2#>!|V>iIQ@)K?p@AEO1E;`$8!uwLvq*g(zx|RbRWrkKEC;4~4lX>X`Kc zK!dV8q9n6T=HyiHD<^lo0u%1?VgYSZDK-K7Q5+N|IaciH@g1Kqp@3QG*V+#9O`-xy zdQSAHn?%9tvRIXkRS(1QYOZs^fQ1OXOdWD9xkz%w>cpK8)6T>!btU@gn^`+ocJVcP zkVEp1XpGDY3=Q%mqJm_CpQ*Kx+-5a2z+$98@c8e6bMAuT4TrDCu9O6V7@SgH>-YGd zTImXnFp=Bh&R2@M+1!Y&Kpu_4T<4hDM$ud?s_2TDI;@$xaUl)^nQc(5jAq}fqZW*#XcU+RYic5Qp1kq{w{&f{wa0`BTJeb>>u zSWO{1x(gyomwO_XC?S@!7)NAie|`zeYOj@hi2JZMF0cDAX?TWly>(-sM!^j`_gNXX zvK5OKOXx@a`lashsj#pH_|A_mAf}ro?0ERnSDZRm_viK#AF6g$p^# zP%WP*XczW*%nigUG85G{m7Tbi=$=!bmY`0>3wcOmap)G?q&N+Ak;^>F;d_i@I_B#u z+tUa$qILlP+~3%pG+n#~zxKsrqHc?1xITuYtQTiLNl@EHe*m1KZVyvn*C$Mx4aIN9 z2ih$Gw#HE03qsdisqG~i0Nuc_cDN&*h*;a`t6loCl%z~IZK?;{{kgtv181( z(X)mR-yR;FeZksvk~GZj+dk(BocHaBWnVd|cnR}4yy*9IAK1H%L2>^)3|pe>G+z|t zG0Wt&8paQQfkYP935#8Z`4vtEs>>U=;$EK=$A28~^-p<)^nFDDE>I878D+`$AVH`1 zp`qlk_W8xG5IbUo`)EsQgl!PKE`Nxij`$|FI3(D=GGRo|V`(cb?dh_-b>EWf##v7d@oA0OQ+G{D<_8SDAv2S6+X7QF*6Oi@Tgh$#lh~ z&H-V(9Y#yRY#-{Yx(+wdowCV26iaSDfty~|PzQdeH~BA5=8dkn)WumtUr3j`941)aR94$#ELfOGnQ=zNM892myOK!=CZ$xU( zDK#2#4Y}sC9Y;Uf<~4S9N-x^uj_>un+rv5m5klT$+N48m=%0pU-hAOisVFx4LR&d2 zndZC@cO zuUma+?4D1JHno!o??7jf|BWJ;1o14d~o<4d~*C2Dqvhz}0(gHyr zlttq*q&sGV++sLyTbiJcb(C#m)+RR^^z*NUrlk(pa&X#KEyUG1p8b&%u zJTixf8Ld`X4r*>0$TZw)~lf>Bc?rhBbLsZ1PWn z5;c&-x*iu!0T9?NR)%o`YZ-%r6p9PB*6R6!(T2jOW$a$^G56gFK%n(#Aoc2z-SGv=D{fnq<;=jP;loe!6)8WQH-;i)YzOFOg^Cn~F>Zx38b5vo zz_{Ftt?JdZEG}Bx2cB02{&YU}|LIJB30OHfIVFny+Dmh`>e;XUbnm$h z^u2dh7M7Pld@e$}cl|;jnVHY!3(SSy%M+RlYy=aLlC3yB|0YxrpWunzY@u}8?mC_EP89W1*MI;KxO;0iko zw?TpbblHS^+;j8gSH1`U%Wp+7 zBSzv$a)IzA*+!AU!&{gz{jW`+5m?_MAzoQ^{GG`z$!$a|rxaOz@us-eu<{I64Ilpq zy-;UPW2`;m8`pZS6ogZ9z;2%>Y)!l=b}fVmz5#s?C83owzH=F?h6Vmc$yZ7;J-z=L z>2H4aW2OWSX2geKyjplGBgj)JXa;Z~Ar2LZ^(Zzec(l+u7%=4^7EK$Re~Fa~#}KT~(- zT3uyUwpjH)s6+VL=4(9L(rFZQ!hjUjXvdB0kWc zbb1Cc2V)lLr&42oK&*%KJ!oJZOk(&rjmj{T7|rgUa`T7(eo=hWarPpR)z|Whm`CIkBG}!aC-guT2C0JZAsj2>P=fuYS~MeCT10$| zTRlsK*gUCCGI^vbHK!P6(pPMko3DRc4R8p)G&nYsCxC%)V&#lh4=uC|8^^ zHfqP;(C0%0FgLLA;rNtHm`D@7_>4Rvnr59hDff<0TS0B<$Xh>U5OIPB2*r!_t`V+Wt{fY1~@v z3zz6vbbb&|;EJ@DZgZnIG1cZ&#$;ZNC^dh_HriBd%6kW)(W2wpZ8|hzq4q6Cg+S4< zu};AXU<(PtC6u!GULd`mo!&$lnVbV&IN;~mA9dVvu5cxtFz}_U6Zx5My!^B#)M81K zp7TVlKUXSgMCk{*In>mWWr(vR4AfNAFrSWIg%h5rzua#{D0>X{8T`Yb1!^;P;W#?= zdJ5B;NuNIoPVBqOY5C1Ri#-^z6sWR1bu)0;fkM!p^7w#SdO4N)R1moDHMYZLh*>JQ zjq>IJ{u4TRa#obF)z@qO-1_A62m?wd{Xl$UKCj_9yS-u<`~)g!X$!Y+P*Z3Lp1SXe zhoQ;zS%IoJ^&lSxD+`R~Y4mYYc8gXO7DBlIN%QtHa)AQj*^RW6@9C9tLhebe8Z3(@ z0CV1V8er(z8p<0D6B>d(bJ;AL$q1i?tZwa?z%JvTG>sV zjYXCb3pYuNS~#tI$Lo?^nYr53^BR7+fRV~EA{C{$$DgYsn6!k`0fV{!Y!F)ZX$N&o1e0*3pqP&7NFmm1o8P$d*HM zMsDWE zl*_+@33^@jFkzQ!&slHGDVGP`3|j+_a_)I!kI^uwJmyj~`kaOV zwM9Qt4e4xdqmyo%W%_08u0v}I-kLl)Ymcu*6dLPRXsEI@ZIO?2v>P$?U>n-3?#zP#oGCJ!D+DI%}qeZBCM2-^-T{fgq{apwF=33lerxdj#VKQ=?7J5MTl z3&ljq9Nzz&$`1#r$Lh|Xg?DWh6#SLpfuW!|oW+o1JlNRKhYL^Ymy17gQ|_tz5i zIVqdulenXnWYHrRQ-X+M$*Z*FG33$WgpAR4>WNlNdl0KUf;@rn zHO?>BlKBfaVCRcZWX`?^e^Ig1+1tds+)%yeeI`j0rnp-SsqUsYJ?9GAX^X1}k_Vsd z3jbhCL62j#ztBDXU82GTb<%+)TLCHyp?OS9C**L6Oy8)g255OSVO3bRbJln*RY5|% zh5VY!E$K|$5=``^PSPUpb4fgNyYb8-{$IMG-&5!Y{Is~sj0-o6-yN3?^mSvU{N`0F zEH`4%wwE>fQV@bnafHryyRf_E9Xup)q)|44>6+}Os$Ee*O~MPN)B|riIc{qO%%OQPpZ8@a^TBe05>fQzRjgGscv4 zR_U(?M%xhuZi|2LA4tjGSRV9CHH^G?RE&A7!e>|vp-ruWB`z@YYGmngd~W|H(AnPm zRw{vtL5mP;O0wcE9_-z7t`2ZYOBTf!gus;FzL*)yzL*tm*}Mtk;hR5amu4yFh)U7ueQKF5 z%i!&26&zUnLxy)P)nY}@F3UnP4SW@9;qw6lJ~kP3z488F7bi*IeK72Y`Gv#sqghqA z)UP8PDz)FLQ@FlYl08F6PjR!f#$+=d15zFE82fYhcgP+18ef}2oq_`cb@ZiL=ZI$< zVD7q-F8Zt6xsrM!TfqVmtKM~dc^Ch7otzBB9wD0Ut%lf!(|39OC~Zweor$s9;`l6% zyz{7|X`RVyOtaDgxxd;TZ~z6L7ezoV+~EsC!CrNsv}ol~*$VfJo6;$mEq9bNpj*d= zha%)p%A77CJrLagcZU&mo}SxxyDKR3v^V@_U~CZj-Qy&+j>7P^IZl8+ak%!%PehLp3{H9=mBeN5#q>dxC4yYMU zu%>kLeiS+%cC9T$%Kcg2rsRJ+d!?t7v!3u{76zjvH>*1l^Mg}U$Tw_?e;BBSPA+yf z42@_{P38n_jR+1Fx_Fzu)pBmFB(*!IvH;FkPFZA1g36fH$~^um&t4 z5@a`jk!>R+lGi3mKLx)|JBXxcMC=m&t}+h0_z=yn!SkdnxJhaH7es$^!Hr}kV-;Th z2?0=&AJmuAg!fISt;8f@C&FlR?nHWICv|asEOqhAGHd0+o`{q@l(PJLxb2F|#FS^Y z8#JDgV}mFYAqWax%pAy$&HpNi~pOty;vdtjawR1Kb%?W*p&8fi?208V{a90)#7! z{C99#nKBiAVR`AgAOe?~MBF-PtjZ>$&1b&%x%aq&t_s zetn8%Zz64b1)t_*Cro=$-@j`t=o!_yeVZ~GpZxBjWd9ex((xXm=g0lVN?1`A26LGO;4F~WJmLvLw zkV?JDq;Hr&a-y}OndXTCJ1iw+l4W6_FEwbj0oqp*qaS+m)uwXXi_z$IV+k<}ymQXs zU`?7ctuG|{$_=dpm!j~IGXD3i>mE9;1FkCis=vv*ziItzJ#Z}a4Ig#ZOd=5Z;OEzg zR`Py-v_oymo}bbRmy&}*5p>I_0;mI<@jLIXIxlEu=4dWL5M({5f8SB+Ft)rPvxOsu zagV8W-t8PjS?3x5wuM{GR(&%=TW9i%H40C&x1oKH*!-48I=oIguRP=Z+t+_S_+bQp z8c-1R6cWwY><2Cl<3P@HYbZ_bdmN12u1-T~zk>Yw;s!jf&5Iw^_0@gF{6!_*Nb(In z!MB1HjZr6KPrjolM3pC{?`ErbnxZ3C`9F_7*!bh<-Ke6VT!k?GGo zsou10(6Mj}Qok9pTXjqKhA!`-lO)ar>bMh3d@@S!?c=*Mml+Elj3rck+|@3@C}BYH zBCCa;dFQY+bWjOrCFUx28T7LaE8$+u3YN=oRs4v@^EfW!Q+Zh=qXxUaSVT96^f(67 zr332^ItJuGkdPLL>*}vI*@$l%I{XcnN0a>?X&x{+tu9x;#cZ003DftfJlA|dKTu0p zI#9T0J&Rg?d$YfgD28sD+K8YEXnh%PJQBBeT15S=f7b338$BrU;B(n z7Z?)oq8um5`-|kZZX-yhH`uY&JnNlV3-(JZtO@Xe*AiC4fzuuNLEas<4wVV7mEq~)LtW(p9t6}CdNQNQ`;UofO3Wj}fPpT9E@@o>o5e+p~UifIAi0{$@ zI+fSSy6uUDU9wddP-$E^pR^vtoqR54(VqyA?a}sg$7Fx8C^oA$Tu?2V5Zd&Dr#@#s z{bsL8`D6t?r2+XYU*T#>Sz0*GB8~^*JGtV$RMtME5;D5>S3fi_ug7~bV&e9n%1Hf6 zXU9`S&OCV+Q=JX#@YOCTuU#>Hn@#biNMoj|%;E0Gw@NTmV*T!b({4pM#?vQwV~|Wc zDZ;kzFq;At7}y~~l9-JIpiQWW^`XRJYLf02S&UtP{bM1tco4CqS#~yAjSv}E5!%-N zj0(Sh#)4)_f$a@`%W&M=cr8uUKwdlVo%jRAGf!3qlV&cNS~uJMAvf^(Vbl6DL-h0G z92M-chPHNYvz#4KW1ZJVnss^n_IQoCeov?L9xWfIYUa3+flJuE7neP zR(}1gvVasrojVB$*tNWuII?N9vIkAEp+{PwCW}+6s@}O#SZ8G#(ag%m9~u79YKqb` zOTW0DV93-P-%**Jd2Djqp!&W2+qRB)rUh%B!axWelKXLj?a|IRkw>p~D$$98(zbqC z0epZMm0-!DLY#3$L26@>A=0xXH72*P!j1cT)GrQ=;$q}+YtC|P6hg==e8@Pw%UT(g%-qEF9Y~9(|7tS@R z8d-3XLZv4MO%1inPM(0zdBDRlkDpTA<15Ubof8hqLSasfpDt3#=ShQ6cZvY65%0gN z)CuR0KfzB`pt@JdXs^YhD^eCzKkxMqjK#-qOh>PAIAaTljvsOAB1RlJr-dWT)W(UfD!gi zknterEmJb-)?Inx^*Y8e(PfuCBNQuiqeZ(qZNWbPJdOIKJYt_h;c%Y?6tfOc7XRT^ zPowsEpHpWEWM6H22@u~|Kfja1 z&as2=d1{Hpt)9^BJ=%UegnyS>f6vFac$_l6zrZF>Hd8R!qsVqk{4(lp4(%TrOo|L z-SFE{4zd0TQGpLmSg@!?ZTwa~t0(mjmQ&~JSnvOfsw8bq5HvV?VIl0Lag_)_fPpPS zC4CE41ys?5y`Ob-aHE?fYbt86nT@^Vjbnn*QU)zB%@l23f`Sn@tyUbp(uR9?fWJd& znNKDG57PLDEQB%8LxX4+6FI`}R#RtZkEF&u5H^1Yzr`g6#|WtT6vmYaplG&L7gZLO z7ES$dlGO$|f+X7Arhc6Hi1*darqucf=G=t=!0;=IB!Z#yuN?OadC&OoqgUNXcn<)r>m#~9Iw#CTkwmp|_M;4b4#iey1rS(E{T)fLG zdv=4!-sT9#XugX?I4umDw)^+}Y}lO#7XEH$H!hWTEguUrT*-0w_A)h= zv5titewz!V5xi>M{X1&ylqnYdznC$l>lBCv%zkebbJRyN6LXEFeZF!4=6Q%_%X%dd z+a~W#cs#p*_-@bYPoKa3v0gT+r?I1N01&wbJ_P(SK}w|9mb2EWp``~Byh%p37b_*vEBhAi0{<;$(faQ^G&}?;Deg#>(ieNSG=B}VbVc1^KM(9^t(+vdn%T5$b2aJx)m0Kl^@Pfx_n>=3 ziJS1t_m*FX3RQ$@x~{jRKkj1gPm=__8O|WRb60KSiA68fy!;H+3%0ZeqPp>oqnX{9 zm)29YdAD@Co_7tmaog;n@sa+KUCi)5@fRyw@McsPR--$$V7CJ!q zp<*_~;fQu4cncF>dhb~Za6$D`Pc4f4VHX%&9~uu|o6~w8e;wOBT5Rg%+jMqN+d)U) z;Sd%h6BK&q!v)6^x$5q43D~y($OeSNuVUfADeum9zj}qK7N($mV_%0g7PnKicuUEq zJ~qe^6yJf^q4kQ}^S(3ux|(x|;TVA{M@QxL6)PHp8y*Ga4DWnWC*D>FT1Xl9c8VXdO%+%94!fRwu3 z8?a}QqeNbbdTN+=jpmYLi!ptbeJ8{k$NI%Fudz+~kn0aqEQRiV1dCcVmTc5Q1W}+{ zluz)wbE9&~1r!3_yW<}4FS0eElUPD&|AjS7l1C;vaIBYqQ)JQ{p;J)uTkQ)RJ$WJx zjUoYi-;X0lx+D&ZPI<4^K>%85Q)oFFSB8r{_q`~_xdl6=kekG7GyZwgf2+&4(YQD8 z4Jkj=6A^wwzwXYA!r81ZBv`Urh^M}|Rd$soRWTV4rX{(m-w~wKQ#Wm*(Ir00rlwvS zuxUpE$b6RYx%}qTyFKY&!R?vzTCxq5W-yVt%ncoLfBb6P9vihJgx&)uoJX$fX?Uzn zgbtPHlEppkyo_v2BrsPiI7Z{3-WJ}mjJORuH@%TvF=4Vu867osXg{)}u^i_@=mo2L zagSArwJW(Iml=rq*UmpI6OG6>NQUp>CA{7ZsF}IwtFI2X5SOhs zn;gUhl?C!^Nsv`Jk!^elJJEPR>3bJ}GjyeigfGTdTk_>}csrbUORb(~0$pcy46&^N zpENcsM8`b+MMotwE|YiBeH_Bv(FZ#?!U~WEys6cu4GxK191*MrAplvJdFUee`5t|- zjItM4TdeERcHx7^_bp1PAd@nm;Ph*p(rXkc2mfnr`;FhZeVJ@i=!fTL?OM}lysLR8 zU1n(!xbVU=Jq$vM0K%0j(k;8iGv5L51a%XDK`j!dnHvdL;~C`zZ|jJmjy_7fP9BTd zu4&{I3GP@z3BBIQ;)tbDy+0YyxqN znC4G(yf#)O%P5eJ|kwNs(7#OS(b^uu# z8Ix_#NKTH+0M>j!H@rL}#dJ1j^2W6{dkbQM(CmWjU>%uK44 z0m;fBnsW>cZYVAbZUigbeLjHcV>6hkf5DwexChLfb-|5Mbn?Rs0Wv5xIEn!rlfX*z zFFG=cP2PAhl}UIyRDgZ5<0TmxLn8|yRm~6CmKORs@CYnq!%h{c-RQ`Pkaza09g=5I33mo z>6Mg@*3@W2ga77o^ua8TJnD%+=AuGo<6r&D5OipT%VK)#h2^e{X6e{Q`HW_V^z5^E ziqGv0(TXJrH<;5Y-mi<#@8|Dp{@Yha-*Z+FcR+X~BucBX5fA z27qWWC%!bM2IochL!bVOpa%Qpr;q$dROYKB_1T5X@deJ8G|v4@{VV3$M}^&&JjXXf zhU4h5V4?7L*+CCn<{;qWr8wq$?uH)h{Wc-S@t(GID+sAxQ=Bl|224_12vb-zD5w}# zAC=u@d{(@QK@Q$=K`e{-T>K9M4z+sa96(G9RlZ%TBGN*>@zf+Mz`4!<6ItIb&V_Rx z#W|9JoHpkK!P(&kJuV7>cx01*0f zm>>c^8+&&#Mj}9IJS(k^+Wa8yV#;;KLxaTpx8YsotLMfAflA~%I9rTlC3gk|TviK% z&KC7V+F8u2!F<|YecuuEhAgGNWUlk}FH1h&ECQKg-;MZwURr3uLjxC;>Le_B(vr?u zUunuSLZnk${h`XFGMSoq^2 zimkr`V~JH{HX0PPmueBWiC0hJ?1d++VY|_JEl>6*h{4 zqVYn6_$mQlbJsRDFQP1@7Yi#tJ+g{hbLQIZPX`Ga z;?danhzE$4^}}*^)@~u$xm8qbfWW~*a5KB))eo2>i)F@!2+#dhRTs*_QW=-7wo$>; z#Clt6+jWBH1kj8LzK(`XZ@{LX3TjnjFu~u3`LVJ)tE)<^%p{I6CT!IqW-vN^u2Z=V zIs~A``gVhNZ<@+WcUKdr1C6=uLDbbrYNzo=j@wxqJh^K&e^!3;-%Xq|d0k{otESCa zT?1GJxok>cR9$Ardl09QXIWvP4MQGHf3Z+&OAu4x{^)c&3WEB32w)M^bD7DogoS#d zWhBy}waSGp`C7YeN*P@)r({sJdZa2SC0w-^O>09(Igt2-!j6*I*1_H(uExVQIXU7N zF=rNSD!W6wYyCP{@@ z{O^Wt!5wzS3;#wvQ~L#&!+P;EwN;u^$ z1uEg-Lh&m7UyE02Gw~c5ZC>Sc{mJrq80B^{^~?Av1NlwU2ygM&aKb)T3WMHA*C4hY zRZj!dcRA;rRmTXheLqz^w6641R`DLL!tfueco>p$&T2SM4gk;IB?@!=80d+#ACC0*a0}|5G3uLB4S&S&6rJiIvLF+TK5IA;zZ{ib0TBW3DjO=mJn8k?1Z28AV~61u5iq9ugPCe4USpE?k5>BJ`|m7ci88}V(|l+6Ciev%(`Z1K z6jc&q7+txBY^iOAa(TDMvWUWPY}H8^4H?t2#oDkGb3haIbR?|jVPd@_t;^zx%oH`J zrK*9tb>E!WINGm3UZ|KHo=s{0HWNQ={eY%25p);SKSSLyN%LodQU$K?5w?qluc%i#4;^+Nt?2N&v+i&^obPP059dEqoBX2@b0lW z8$dwa=2SR?s(pXGlY}TgGc%`)HR#M$>=Sx=1+dD;g+xDCVM(6`SiWhFMswIfrVa*;+6g>^|@g)QS@grMadu|C`g)nflPm zYR=g4^gv_tnw`gfwKHTIw2ZEhj0I~<0gy+Zj@ql2pNn)*#~QljFZvmK7rySi9Oe-SbKrObJC;#ENB@tRG3P~PyH#dIQ_7WKK-a%!?n zcm3l6ba_0j@0&1VQ#7o)S>>!A>X@;cMV!}g1)dL3KQ3El$hUeZu#`u|-_`K(1?XT* zAt3*4oVH6l8nJSC90B{#0cU40_4GZk8fb!D(ZBR<1d-i9>mgHA=^*2}%+sRlzde7vYd%4+@b zn6AeT7M)Ms7}k@+Z?e{g&0@?#< z&A7+10TDBdEatzH-t0a%*9LVPh;m;_QfY!4tnj!qV4xd=v=e!)+7nWt91(Obt*%~9 znWRpO9($zDH-`S2(J^HoS!!e36LQ`>Z41>X4!VM5chMSxgxN^mm62Ds1(31dpmO9r zcjjF-`AK3mL~*Isd^s&mw_e|7HDS>-MYD9DNMBD`1>=;=Sk$Q=nv~SLUh9X?&Z!Il zlhC8ff8v=W*F8rE{Pdj-r*Ng+vWQ`<(cN|K3}587)S!I}iJ?2MFBRNc-mVJ^adG~w zvZ;T0t+AiaoxHDsa|6Cz2~cC(h4J&x6dd*QXEnVP)K=1P%ha+e{NmX@^a{LMnM)l? z&z^-3I^di|E7@;~v?2%(s<%3WZfueSjntA8#k3XLNvvhi!cK%FF1!`Cv30kLU>7 z+xCf{RAs|x6`j97wefKNLLr(tbNLj$EpVj|?Mg|$08;S?5uNaA9Qu`_8u$Yggah~J zB)kISg~&#i_B`CYU&Js9$gn&iC~FefDYO-n>Lft(Q9KoQA%_q4pwqTW42$D=Zn=hg zm^Io*)W{r4gAqB^HOm8t%o`55(3`2UA?zMMb6y_t`f6zeHr*Hy)z1&Z{>OjEr85p5!L<2!+e z7dN_G345#Peqa|jPF&OOg$EQ4<{FNZl9^|IwRMz{tXXYghU5a%493;s_>kA;V%w6c z+cWbYKZ+j~$m_P%*(_2-NEOW+4^9^I%vp%50p(hnV>^OJQ*R{ZGGtk>f}Cc>O4$K|3oii zcf^!cnAiG-Sn$jK5$GRyE8x4%CbbhB23{Z7&`SP-JZeGdw_e6O(*z=a2d&GbVtdQm z(~t*>j+V1B%B(WhV-{xew@w>`CFZE53kGYuF_-qU+2Y~2y#n;5`dI&U(C3~%e* zhQ@i4c~bycFg_buxX7pD-VXe=n?~y{I4ZfMwH5QW;HYjtl#x!WJ!4dzz|IWo@Z3`I zn)G?O7~-TbVbZJvJhm&8>PoCS?(7P9Xg?F!FQkrUXmzI>k2Y*}c%pvZb5xl{W(%fd z-B(4BHfZ`*XaN154x8#1N;fnQTdzjzT(p-jP?G_+Q_?v?Kn)KfrsSCY+OEn3+;I2k zUiW(Q8G241!**c~NA8$At^{rL$F{YJdORxAJDsT~OzeGY5tq)pz}~-7vqf8Xok#8S zWuXXILxD1hdeJfA+L0pS7E9I^JCeJ|R+?WVq#Ma8Gphpeo-iskj&0~QI75WtS9-T@ zep>+iDQo&h+G-E+1czv^Ml~F61>Y?4gr;5!OQ|Gk@9Cz%5XH~% zITZ)f{8&Jxinsy`UfLBmcH-zD^oBE<;rcwbhK=r$PG*wlNX3LDZZjNSam9B)>-`gn_;{+^UeK>Lm?-79g!=+;Poj_s6xX8{3YXu&!-xy4ki_WMr zW~+DQHlnxSB4;-|sccRg^U`gHgpH!4JREJ$Nv|{#bZ4=Da;hNWX|2XxHTYZDX$#IrNDPpA!n<#1r$OPMftK z2k>1reB`WNix#`VMlQu6EpQ)rj@{)rYaI6@?n;FxUfKwdtX&r#NfrFwY$3>a+(M}bTy4S&@L^KA3nk7`{snl+7 zv7%)(@zCu#=mkz!?8!j0#R`}M@6}s*%)Dr?X@lF0HMq+>%<(H32bhF65BdGXm6;KD zJ3Y>LHa$dYMXw=U$d%i*XjK!=WqCs4@fglAqw?><%3JY5MOxR6h2X0C7E8dyUlCe6 z-|4sfDjyVg0X)2cn)g`4fkWG1!OkuTCM*fEPBd=8sqA=EG zL`w(>r#=OVYg^OpKtrSn0RA?bY&Kck2I=K8rCB zyGV!B``#diZPmtDldQy_zt9dZYWF-H)8py($E4+Te{FcMrAU#dd(Z(=eFd$|6*Q$=7+L&F2boMj#gh}`!YS0|4B{C{I_g4vdlYV9* z?#X=;D#?O@qk%NU5Y~Xgw}cyAO=KHg7{6!0$HA(&3dK5zBP;deE<5p4*>axL1n&Dc=1&Yf7q($b-y>Sj#D zJrqZG#BF?+BZ-8ur8fu4Z2-AGZuHoL0UCk8*zy-FfI#&=s)tA)CAyEy5C>n`{=!QX zVNMcT&i%nXOgE8W4;5lgLa1RvRQBD$M8cfr$4hx&jkNCHLoS5a&_+RcX_KgtvHoYAf6WwLUSmYHRsX50evU%Q z0+(x0-(@W}^4jhRxtI`1;G5JWRF2&F!GQz|pmSoIz{!2fi%qz|$N39}u8ddBs-9k7 zF($z#BhQwkY99ue2_8$68v4oDan~|0#RCt38+mq_J72W)UPi)-< zAmMTvW{@^+DvQbeR8(a}#|M_;&eI85cIm*Xz#D0Ld|ImZL{dZLx)xek;Y5sI5+$|H z65g{6``v3tclH#cifp=`N#koqw(}FT;@m>g^hp6y!)=fYr)E;LW+wKCL^DGVJIM{g zDzH<7t2f9v_~$#%yCNmsr<#WG{)?#`K=(CE9vf4=eIA|Fj7*h7;;}KRv%;fa*y%5r zx-Rvf^u8t? zb54&t?`gw=>)BkoFW5c-lwB|Kf-P!a1HNU~0)E7awPG%qAtA(Y*Sei%Co2jE@U6~M zh`TVj$u(R!dn5ZH(^td(0og-7?HrJQa8@duWB)2xog1>V_l)!9FG%wO3=uqFlq*kz zZPHbYcDp#h`HJYVe&PMfBM#teYd}qXL2WcW0)ods(qH{3wd86Or z(15D*FH6oKU|Y$vOQ()ZxDpY6VN)iHV9#U9?^j$BgLbgQZcE-%s79xnB1bN#N^-@S z81k{{BpGK@W}rb+E(;eY$ql2CSBo5q(8>?TsY_JIIOv|=VSSxXQP7$Lw8rx%+JK|i z$)RFO`}#^+7gHohqJ#3TKZx}=ijYEXAzP9JkudWo;gnd4N=})Ls&fubBlD!Q9pph} zY$EfU2idF?*<@H0SzpJSN({~q%GAloz*(C#p<%*paC(}~w~aUWG)m}A&OW6Q(PHa5s3U?+|v{W6z!=uTXW{UXtO~b$FMu{6D33TU5jgQ=iiWcxIoRpL{ zxg~bbJ=r4g7%HRaA3eRf#fqvlv>!p;UR&ARr-CJ+X(l>6hQ1bZ zuLB_wlV5$^IaSFiMsjG5cY`f^`{C+;X)4}rb2BrJ{WWi3t0d4z2-TU#iam3ok}meB zt#U9he!$tVIgymPr?tqnuXpC+nCFSC@<-3za^0r4Eq2reMAXSjR4wfJ;6>wZQVHB8!X`RkY*qOi2s!NVn)pYNkQW#bEb!C&54ZX=Pn_ymlhG8_W?O|Qr06Cm+ zEdAQw&;^_*T!7XEm`&F15Deco5)|}coFwqlHx_*nA$uGh@EmvqOEMvN&U!$&)cvgT z4ZQ@fAI@|FJfg!LJsP;4U@!Vk&o(zs-|NZHW>I&QydLrkUayU7dfu3lkua-rgod>J za&;%V>(CWOw{%+FD(1~9*}wu`WtSU<{cgsr&LO18F>pNm4W1rJ zsk~GP07n;zEK!nA9Kxm>%cU3v7EOKS0uIOmT^cIe%m&C9Y&8DZ44@?={**1Q4YV~n zBGZs3Vhdgx5dH>VmfPK@t|TvU!>-J!^F{>KTbY}>u(qb(Qjf)9GlIaWL&vSfm0wjX z(f=d=Fol5?l-+OM3cum>GyRstZUkE|{FhY|z?bvyOQ?tB)h>U(>L5 z5@DRFmJ~PQuEbzi6CuUE)a(i8O{_G-(4iC5NPLipb`_q$$Zknvl}~2T*cIF(PXI7# zgjzR7h_mO_>(>AtaSg;rRS{*067MEbX@<}z+37Y8=3**Xy0a8FI@r2$FhaXoiTxr9 zAQaspAEmY@W=*2(j{Fj)b8)IdrQX){D|>39dOGuxrs)&3zLk6;e`%XO;oZPMTRj@W9T*L7L{FgagPpXUb{^q@5Xg%i` zmUJ+BY`Y#zS#l<)qHE<#EQ8fq!SD(Sz)jSzYL+V0*0)OfiiGaA%_TE(bCAy1g7xny(|9IM+0_Icie38HyMhlhFiS;&pY+Y^EWn8t-&4Qa~!LQTwlilRH}x@ zGpEGWvBj;WB`JA#PDmcs)rBr&9MhV?)!Bi9>S)W1BlvTex9O{<88441@0s{_08Qg; z-~5>U6QdJ4uqR{3`_gbEd+LGmMO|cSLJPzuRjML2!ZvL>`oTWcpgU06(Rlr@asOZQ zdKp|?s=WB*ZQ%{`Xq%z?8=rnncq3F^(W^{yEO{kJ+|rG&n2Z>1|T zTF!UYQ=mYO*tgta)@36VakRM{&377{mbD@7*jB$j!?QiZ^Bpx@oHaJNoDq{?ti$-8 z%3l}cP8*bqp>MDTe0n%2cu0G~bbM40kV|S15TgI8SUGlNfZ9=$ASG)WZ<-zvxCquV za<~~HFAfsCblfUZTrC#TEyXXOUK;s!n<&bK2BC(|mBg0)5WbBydJ=eag#A`ivqnW# zQ&aQuQ`1vdchU7f>-=u}ZNVbV07}v0^StfIees=n?PF;0`VO2yngjh}68gLZO2iK{ z%4f3$@Ua12=S%On8HCLF4ECT2_FJ;ymHc7N?DE-ye`F5=i>P64&=?SAtN7H@bNd%S z+0sfTKCoeC9_}kaW{%_fOCYRpW}t7*_%yv_3b{r1Jb$E59f0YLoBt?3aCxnaD^^J_ z_o+U{2|juE4Cse7AI=AvIvASo8Jv+lYLgq}R~-gS!zeMYym_|qX&&CLb+YuH+_e4Z zT%zuz5t`_QHGQ0ClUkrc(|`@W|LBzPN=Fkg+?#o6Tbr>hDj*j+x#O{3{`msBo$F-l zzkK5%=$LpyVOFo!e6fr%y^#H8 zwG%+nzuCDXXTWdybCdec{m7XUH-D+fZ<~513<_193nN!S3q_tU$NPoFS}qv%$NE_G zycP>{*$qRmH`pTSuY*lW*f;UH39t|CZV`c+xE9(WN=+&sbX4Xt`MOkxWywY{5k)2l zTpZ06dzZkqw0*+4fRVhTKBNfrX}gN?9wCq}rfaj~`o zY6gzVL4T2CL;D4hNh3c=;z=vLRu2Eq$l^0!{CKne)uI{>5-)wp&}^y7C6jb|-Dm)A z@bCcx2=@*Qsf8A88Uw>8c$a`S5H(W2Ic-dr5b;6|H9b0YJ0Yt1;#dOy&whYMD8^c4 zT8K{xPnSl^kC`c_Bf9QzF(TjGfkk#%MnI^1S_F%McbqQ0cj-4-^5&R$+3jY2}~ zF{DyBxJLN*{i#Fo5M|@tCUZrvHu5xlNE)(@mgSN~_tI&+<0=Z*oVS^_$lgpJJ=Ud^ zgwZx#z)50Ep~^7^f3`zcY&iS}dmk{1<292?#H5&&Kkql0{J?pQHY^=z7(75CCA-$l zyo9TZYM_968Q3VKxss1%p4zvPeR+vidSa&=Y-M{L`phnCIu52P30hocTBu`Q0~(F3 z->!}c>$ogm&>*;>fxGW)Wc5Y$Ib8nl(>1i#Z9j{OUp#lvI*H36SE{;O+3QhQyjJ^| z$%1nC$#1 zJR#qgZV+v`&8&HTJjG8YXqNmy{;8rwe~1ujr{0xrJuMz*hilL2C>Co>G$I6Cv2h(@ zqqROa)n`gCj+8E@aIL6A%&i}cxNu{&AwvN*3!1mG$p|(l`Gfqxy9$^cz;&g+JRA%{ zkD;uBL$Y74IIvJ_pz{vo48()v;lCOk(U#D6947Cmmcc;|MR{UNXLApXIwf`LWmWDG(rtwqX`ai$Hh&Wj|aYVG@Zv&<$S2Uu3SE zXT&#`KQ`W3OE57@>saoCd9C{D z&@Gwnt=_H3)@pe4^VI$lbRd$G%rqH!iFGagEaBcRSY$0&#L7V0#DEHK1vO7=6DIGY zk&P5%-sCgY!lWTu7PJC^z1V#_4js{rBr^0tz-~VNYc_DQ^UG!tYv5z0$uPnpiSY2# zq!x?@0Em2Mwim2MqTYtC4GYcBQ2dOIDL z0hfh?hbB-(rh#G$CZT^K*7T*+(kzSEbm2v3W<5u+Rx8>SvZSw%(g5qiic{at4jWC7 zexlX?ViD=vpmB9k#WYs5-$Q3}cY8#TnM(xhK?MHL6V zv<9)B`Xh-DgH0g+j@VCaZ$!;c4-b&~vsN9kWQu2D@s0d<(jPzLQ4Wh`rz!AJSCOy$@I9u=Bs~jOqinSQO#0BiG ztQr+YGTc4~kL4U*Gylir#3tS47hutRMyevK(dF-Bs-kR``MDk<1xI}yraM{ut*M4Q ze-ID~-+5%uu3E|IjZ)8Sw-@fw{?i1177#Ge*LexfNetAR$DBtdFz!2SRyUWQrCLK> zrlSVT?)5HWDunOS(*to6!bw?yzOt!~Lo0BG$!JL&ZTnN92Qn#3_9Fo|02JPdGZ!!G z&(+to_oVTIi*t@G$Dsw6C_?RRQEJjTtgAXFfd{vB>otMFsJw3bN^iYKY8Rl~Uvkvp zBc{Bc47jFIQ60W-=2Cs(%4+kMhf-IWg;}RHwu*zK(y}QYIBn8T1uXuA{HrAOGv61) zQ#ab~39_7%bEJmIZJP7%0Jx2Tn2}ral5W1X6l9@|6#{cp2EDszG0}FTKTtYY(|n(U z&-zBnAX(>4T6`fY4Et2tf&nz zlc##WUh{dgDpq%E0k|WZi;HF(MuYT=w>K*+8b>h`v9rFmK)9sXT5obp@n6k3`X|XeV=| z7~vDVQiqc?@hFNq@0ec4D-af5L^(onXB9{Z*SWQ@^T%9y0A{JO4W;0Ymb1}V)W5Cf zQ%)zuo-#%UpE9J9Onbuh2p;@`yCVqVkD(x>`hJ-Yfyiq`7UvdyIOD{ogg0&H{`QBl z{dL4#wjBE9EWCpx??8+A7TA`AuKq`jO-MDnXkSudOdAq^Lbwg3USrO+Jmm1+o_mFB zlf|XFux8Fc3XmvkIk$2~p#(i^(vJP~Xk^d5CDJ0=a5j67(2m-0JPY>*x$W~HU-@Km zq_6eozPtdDGmp_f{sN|iAa8e_IDJ`zS!zL-Pqqsc_LLf6?-0*vNjOysbBr(>RkTfV ztv~Xmf<5_&COnq*9_y>cE!1F>cXZh-46{U&UGzl)0L(s5cCi6S@L}H~J|e4mL`s&9 zA?_iqVxAqw5osXqf1WUkhf z6<@K%GKA#0&!#OR{0chbo3{pYn4#yo5<4Y+rK$lPFq51P^gRsm|5H&!|{sr~%0+bPOU_pAxN-SXDV%hcJ2a3DYy1?_x z#0G31oT}T?O3fHQm|grU5~#DXLm+}|O_yNR3=O!Y*qgR#gg-xct&l~QnO5Xk7SrS)9;XDfynW~Vh3qRUwUCX z0FcWas}X-hUmBBJ8K{I6)V|EiqHLvstp#e%i7~i5!Uc2rEwKBCjvsGXlJE{{BT48#x)Vgg zP7U7E6kW!yywZim%P|H`kuLIbi}0>4;1n+`3E@U1_tTG8ic%GVUsSTrkH+b%GP?cj z=Ir?LdjGkf6lwcCE3aQB+qYg@YHgP`>=@0*n9-0m!TUx6~3DH8i5BNS^+yBz|N$h zy7vn9fbGmi0Jg{{Wc$2uPHbb1o9|}N0{A4-XOhAtQ*AWq8`cq@Y$>Z+{su+=nbbdp zu=wX|`Nw)agr_)4e=z;Kb;N)>t-*cPN~8`*N4?$gh9qlBUw`DS3sgHMaXunZ?C3r7 zA@WZq}$qG@oqmbOS=pdgz6&DB( zV&8V`?kBU>8Xq5|9rwt zIjl%H`1p(%v@n}l<*bH7*pyaz&_B(Y}zVuE%2g5leP*ICUW0LNKfj>cby zEVNC|4nrxuemZ+Gq^@HHh44z=zr>A4K!Sh$Ez!Fcp6Qc2MPKKO0TrB?N6WRRQ6j~W zbEYcvs9?+9t{b3{beK@}A@ENOoH9pu=Tke#t9h)<)DD;5ZFIpkNMgkVHT-j!1b>Jy z{)o4E6|dE|2r(eSb^A>M2Y5iy>WVV*fS~Y3r@mw-`$(}SM#Id*_@fVP(IwmCzj}a7 zkTA=y^$T+HTUPhb9Qzl2f#KTmczngzV)bugE@`Ao{CtJA>vlXdH(4(ZGJaf-wnXEo zEQC$c7S%c{6>37|Qx+lS+=nURe2xx{RPRhO>taN8#@|(OD~A4yFZ#^W;xBB0?E`SALuP3rYL_Fr>Q`1O-zc9q#WXTD%#Ev`54fxDyv=Jn zCQXGeUD&*TAu6+?O#zIhFqbfK834p`g3;QacwbO!3vEsC2LRBYlr7u-0s&HEVvIHs z$q_V*DE6{lzvo}ZX*v>4w*Z$jAfkl%T-@zp@0g9>2i-O-PSDZUAcIw6%G!p|Z$1@=onj zOowuL0C{G$(;XEKWukn$;+R`XdZaS)2Pk zeL~aw>uM-BYe%8)Hk6~j*i}9;8oBp^4?Pa9oj(=hr@!ZRR4oh;FQm$E2PR8H`9t-n zxy2&|MaK|jrUf1ThGu1LZ7PmP{5^*7TMgIWN;{Z725xy;Ci4cMPaDT(Ft27kIY$X| z)0H1NG59V|cW_Jnnx3iRgJ}BkE!f~I)kX;8L9wr91P%g%0}B+g5(E6kQAhhifRkfM zI}Xfar>K&KLkESgYPP0llUZ0a8WS%Ix2ARXhfYJYaIy#!`~GuXedKL;0c)=(tC{nO z|ApX|6F_8ETcq8;ulKZOdF=>z&-8scs4)cmbb=6+;tJg$NhZ%6!r!ixYK>GRuOyi& z$+?~5hsLgy)*2Z`;sD&fB6SNTb3XsPDFR1qhOFfea`qXb1<>`QJ)Ax&h8mUx$JV&k)o0`rt@!K^ofWkGGg zl4PPCjXAEvm(*gXO1+vHp2r#$ zN}A?-Bov;4-i0=^UR<0a<{Y1^p=zs*M~C%Rtx=u^&W$Dj-3+!~ zY@!nsiMm(AV;Zuyash0fbNToGNx?FY0V@xK<(wR>?`ga>$PkeH$?{mq`;Zw0{g4cJJy z<)FZlP+U*CWhgeEG}0L~#l99rj!#h*QXZx=RIa1VAJlvjvtoI=6%*5zfM|yjl|bJa zXX>lzrN`#5fs1qYzcyQR9?iJMo^~CgB@OiC2`Hh_MSH?!x?27_E0b2FBffO?6mF5~ zr{9$4f(!6FomME!>A#0L7+KQXR;@JE5GTAtG?)uyBuk@HQX2& z`eG}qovqkcQ{5*<`dbNtJ+-gt(5lg*t%yov68|yDQrp4*^ z5cf?i)UYFe)AcN!0{@o$%W=-1PZ2Ogin2%Eexl;u>#73(`&JMrP)2wK!G&{6Peq<< z<7)FpnRSD6MhV}ZzJ`M=$KH%B5YQ$dxNAiB2{K76 zULU}9K?gN}6=p6rej7?Xm+8u)(<2V!`HRx?Sv4opr3W64g9zU^%=mL2*(@ZdR^$&W z9OGUawH#Ey9;5_@ukg+cR^*)P1C*(Oo!}7JRHv{ZSO7HfDJlWyjfU;nHD;aVVc$h= zw@_l>Diks*>|lTBplHO+6;i!qP^4t7SeQ8mQjufSY_Jo0iWP$q+%pwKO4f_Z)r-r6 z<2_jDBc67L%&ordNq{Vdv`f}7sIuhA3^O zLE~rjq1!cyZ+_7pib!VcD0(v?ik2}EA+r%#c>V#IX|ZPSo-*9C57-?_Nq;f&`Inck z>2T#>AyGwBy-%jwPNw;PKb_P#?)+f8;R6FxWRW@}58&@ggjaKIwaInZEer4sF_Yz{ z;W7zJqNRkpZRSmc!__|;{ksJV3T~rr@nn^LR zhX1GS!3yGSA2w+7rkM^lhqbhuik~`?DiY=XtDUP?lJ~9+N*9Az){uu2jim)Rfq8%b zD#HVxItsHm$w6R1m9%^-H10JKl4@ywl~A3@e_#{}aaD}QN0w_ssp`>p!=c)EBQ>;b zdne5wf7mSJjB`vRa4;_0>{eij?_)>(X_d+aPMy{vV{U?~3z^N_`LV-nXp+M@Sld)i z8nX3h!)zxJd%SuTg}(x97Z?CMMf@!MnQoAa9dU6gjq*o;b37lB38C68KTDc7k`A6Q z{9N;f@zBZ4TK=3!ai8b)7|$R6FfRvM8jCy@3ZF8gW{DArcl|YLa9ZyDb}f*)2AMCI zOp$bem8lK8UgIPUx#Fo=8H)DE8OW#bo5>lr1$m+?KmB)px+8J{VO*8;vLBIh$u~qo zLlP$AbU-AE_U2Gmtr!OH;Vqt2xQWaAbkPdn|X+57I=DcP!3+ z0j@Cv(U`w}Idm>%I}Nvg$Pu9NY%%Jl{!OwCAYp6Y-dwZWO7G$`pD6#Xv?YK7Ril%|Ke~;KmJF92ytJfYsF= zD|yczz(VHo>I+$CMtBbAi#{;_J0luw>$MU;wGg{>m3?Y+1@F(bXwj)JyI#oTcAe#_ z#+U=1ZOri+!XK4px-IIo^o#Q#t@2d`{VLw|ys`@R1tsW>*-BA@W8iUXJF6$KV3cU- z10Uiot|OS}hHjYj&V0Yox2=~6`Fb{xab6!60Qrnlif2ukZeydNU*#2Jyc8J_O{Ch! zaj@MNZ*U)xh2saax6$ubK08bg0j-Ac;##Loi{`mRaSdvAT8-l-d!No%ko1k;bcS90 zRs)B7tG!NU(`Lh&mELW7(>&@PlykBO!*Z&51F4JBjID9KiXxo6kwJ@Vzx$zzvQ0w; z*u!@_ni65Q8w#Pbt~O}a+j4=Q;hN($jU6IFi`wNM@X%Z4a!AZ`x^8QAugi6T6ICuM zz=(B(rP0@a4kThL{cQ8m#Z`0LPH5RAH!{X@h;>YNE%s*+{v&KXqn>{`2d_CP89mfA z2Xj)b3S=}5=yvR4rC|MBPH@4k$CFmXZ>h`%u( zc#aT_^65v`XhLK4WG1omk6Tz?$vet+2I8w}_tNDd*us}?ZwE2HKR)Xb%aP!i%v7^k z;C->}cub+nZ4XefpTNppXz9&+?rf7NpORCP1G4Zmd!DU^8&|?HkxnVa8NGZ-J0NJclf6jC@K`o;b-AW<{vqT|sXWQ*>2}H^5i#l^ z0rY31Mj6D(If6#B;|^OUSY=WpUQ2v{YMpIdn!kj zwg#kNK*{%3jeV&7mrHBY$%v_)WG4hA#3M|yf_UAUBQqQ(g zlfT|ZxQsU+=k|F*(_4%?iMVJ53n}^H{dFjzJ!tbA1Uux(d;nQkHN5_jv3yEs{g{xv zr9F7HSY^0-94Vc8Nt!GbSR z`9T3VMVy6{{>fCW^MG}`N+mr2W59#435(#V2Ra>V!v&|$_$9;&$KoxkkA+|4kHkI( zzc#!cv}}j*dYYNF4w7Lj__rLw2vijq&8VtWe2x{E6Yhc1FY!CEP(fk|XQL9QfYhfS z?N7)$cHXzwTkXmGf0*;KNT+qzejQ~KS2uNwcuz1OGa`BAE?N8Je2Mk~B$+2i;*Jhd zk5EVNrO{r|G(0KD8kZe#kX5$HibWE5TSW;dT`ER*8RjNPLcI^I%kck+RKt1-<)Z9d z@^0|oNIn^uR?lA_S?!HVv=gWlLZ=cEGc=gk*_HR?>-iFZkIAfCkghPpWkxY~%zOy9 z4XLp91g2)JXoMm^3J&>Ui6R$?wv2u##=Sx`%w`f%f>)Xo%K{$;IN_FAAwZO5e?l#L z&N~7*2NnNQXP2D?mV4LH_qKkM)23 zfH2rZKR-2cFdg#Up?B#^)!LNws!2om1^LyG+$&wppUB>$f?CNOmh2}n!- zzax>;{|742BLYMJb%W0Ezd=NX|3EH=|3JL|hq3VgbV17gpQ9W*{C`m*rT%vmKdJw~ zYAHCN;}{Jnvh2T05}<-C)_=pHl1GC5pIEpF9FrCVxEbB8}+w0X>Z{6Ufq>9B?Ls=~2$)g=I36 zubpvZs>lN~|6Gxu9CB8Q=|(=7+jG{8X zD&EOEfLxVYup+-&X^?+qP+EN`7H2eq6;5hYot%3~VDhr_QcPkkU?GDRnaO8?LV_2h zm@In0LID?CnXLLJ8}{3SWTfHMJaGGsIf|*kIJ*eUx>JElH78HFXwBp^9n6fF&I5K2 zywC);LTyoW>dyizw4NmfbdMI3^jxsT{Bs3?Hfk{y%?ER3E_*O-STwnDk;de{%UVp) zTPO2f0Y=zXHgM#A+k+gr#(RO15lof`CUYIIncR0qVDgqLQcS#uCSSOc%Jksy()); + } + + System.gc(); + } + } +} diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 440057d..ffae404 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -31,6 +31,8 @@ import com.uber.m3.tally.CachedTimer; import com.uber.m3.tally.Capabilities; import com.uber.m3.tally.CapableOf; +import com.uber.m3.tally.m3.thrift.TCalcTransport; +import com.uber.m3.tally.m3.thrift.TUdpClient; import com.uber.m3.thrift.generated.CountValue; import com.uber.m3.thrift.generated.GaugeValue; import com.uber.m3.thrift.generated.M3; @@ -42,13 +44,13 @@ import com.uber.m3.util.Duration; import com.uber.m3.util.ImmutableMap; import org.apache.thrift.TException; -import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TCompactProtocol; import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.protocol.TProtocolFactory; import org.apache.thrift.transport.TTransport; import java.net.InetAddress; +import java.net.SocketAddress; import java.net.SocketException; import java.net.UnknownHostException; import java.util.ArrayList; @@ -61,7 +63,10 @@ import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Phaser; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.thrift.transport.TTransportException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,8 +74,22 @@ * An M3 implementation of a {@link CachedStatsReporter}. */ public class M3Reporter implements CachedStatsReporter, AutoCloseable { + public static final String SERVICE_TAG = "service"; + public static final String ENV_TAG = "env"; + public static final String HOST_TAG = "host"; + public static final String DEFAULT_TAG_VALUE = "default"; + + public static final String DEFAULT_HISTOGRAM_BUCKET_ID_NAME = "bucketid"; + public static final String DEFAULT_HISTOGRAM_BUCKET_NAME = "bucket"; + public static final int DEFAULT_HISTOGRAM_BUCKET_TAG_PRECISION = 6; + private static final Logger LOG = LoggerFactory.getLogger(M3Reporter.class); + private static final int MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS = 1000; + + private static final int DEFAULT_FLUSH_TRY_COUNT = 2; + private static final int FLUSH_RETRY_DELAY_MILLIS = 200; + private static final int DEFAULT_METRIC_SIZE = 100; private static final int DEFAULT_MAX_QUEUE_SIZE = 4096; @@ -78,15 +97,6 @@ public class M3Reporter implements CachedStatsReporter, AutoCloseable { private static final int THREAD_POOL_SIZE = 10; - private static final String SERVICE_TAG = "service"; - private static final String ENV_TAG = "env"; - private static final String HOST_TAG = "host"; - private static final String DEFAULT_TAG_VALUE = "default"; - - private static final String DEFAULT_HISTOGRAM_BUCKET_ID_NAME = "bucketid"; - private static final String DEFAULT_HISTOGRAM_BUCKET_NAME = "bucket"; - private static final int DEFAULT_HISTOGRAM_BUCKET_TAG_PRECISION = 6; - private static final int EMIT_METRIC_BATCH_OVERHEAD = 19; private static final int MIN_METRIC_BUCKET_ID_TAG_LENGTH = 4; @@ -107,44 +117,50 @@ public class M3Reporter implements CachedStatsReporter, AutoCloseable { private String bucketValFmt; // Holds metrics to be flushed - private BlockingQueue metricQueue; + private final BlockingQueue metricQueue; // The service used to execute metric flushes - private ExecutorService executor; + private ExecutorService executor = Executors.newFixedThreadPool(THREAD_POOL_SIZE); // Used to keep track of how many processors are in progress in // order to wait for them to finish before closing private Phaser phaser = new Phaser(1); + private TTransport transport; + // Use inner Builder class to construct an M3Reporter private M3Reporter(Builder builder) { try { - // Builder verifies non-null, non-empty hostPorts - String[] hostPorts = builder.hostPorts; + // Builder verifies non-null, non-empty socketAddresses + SocketAddress[] socketAddresses = builder.socketAddresses; + + TProtocolFactory protocolFactory = new TCompactProtocol.Factory(); - TProtocolFactory protocolFactory; - if (builder.protocol == ThriftProtocol.COMPACT) { - protocolFactory = new TCompactProtocol.Factory(); + if (socketAddresses.length > 1) { + throw new UnsupportedOperationException( + "Support for a multiple SocketAddress transport not yet implemented." + ); } else { - protocolFactory = new TBinaryProtocol.Factory(); + transport = new TUdpClient(socketAddresses[0]); } - TTransport transport = new TUdpClient(hostPorts[0]); + transport.open(); client = new M3.Client(protocolFactory.getProtocol(transport)); - resourcePool = new ResourcePool(builder.maxQueueSize, protocolFactory); + + resourcePool = new ResourcePool(builder.maxQueueSize); Set commonTags = toMetricTagsToSet(builder.commonTags); // Set and ensure required tags if (!builder.commonTags.containsKey(SERVICE_TAG)) { if (builder.service == null || builder.service.isEmpty()) { - throw new IllegalArgumentException(String.format("%s common tag is required", SERVICE_TAG)); + throw new IllegalArgumentException(String.format("Common tag [%s] is required", SERVICE_TAG)); } commonTags.add(createMetricTag(SERVICE_TAG, builder.service)); } if (!builder.commonTags.containsKey(ENV_TAG)) { if (builder.env == null || builder.env.isEmpty()) { - throw new IllegalArgumentException(String.format("%s common tag is required", ENV_TAG)); + throw new IllegalArgumentException(String.format("Common tag [%s] is required", ENV_TAG)); } commonTags.add(createMetricTag(ENV_TAG, builder.env)); @@ -165,10 +181,8 @@ private M3Reporter(Builder builder) { metricQueue = new LinkedBlockingQueue<>(builder.maxQueueSize); - executor = builder.executor; - addAndRunProcessor(); - } catch (SocketException e) { + } catch (TTransportException | SocketException e) { throw new IllegalArgumentException("Exception creating M3Reporter", e); } } @@ -183,7 +197,7 @@ private int calculateFreeBytes(int maxPacketSizeBytes, Set commonTags metricBatch.write(calcProtocol); size = calc.getSizeAndReset(); } catch (TException e) { - LOG.warn("Unable to calculate metric batch size. Defaulting to: " + DEFAULT_METRIC_SIZE); + LOG.warn("Unable to calculate metric batch size. Defaulting to: %s", DEFAULT_METRIC_SIZE); size = DEFAULT_METRIC_SIZE; } @@ -202,7 +216,7 @@ private String getHostName() { try { return InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException e) { - LOG.warn("Unable to determine hostname. Defaulting to: " + DEFAULT_TAG_VALUE); + LOG.warn("Unable to determine hostname. Defaulting to: %s", DEFAULT_TAG_VALUE); return DEFAULT_TAG_VALUE; } } @@ -292,27 +306,60 @@ public Capabilities capabilities() { @Override public void flush() { try { - // A null metric signifies to processors that a flush was called, - // which causes the metric queue to be cleared - metricQueue.put(resourcePool.getSizedMetric()); + metricQueue.put(SizedMetric.FLUSH); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); } } - private List flush(List metrics) throws TException { + private void flush(List metrics) { + flush(metrics, DEFAULT_FLUSH_TRY_COUNT); + } + + private void flush(List metrics, int maxTries) { + if (metrics.isEmpty()) { + return; + } + TException exception = null; + int tries = 0; + boolean tryFlush = true; + synchronized (metricBatch) { metricBatch.setMetrics(metrics); - try { - client.emitMetricBatch(metricBatch); - } catch (TException e) { - // Store exception to throw later after clean up. - // Don't want to put everything in a finally block here, - // which will hold the lock unnecessarily long - exception = e; + while (tryFlush) { + try { + client.emitMetricBatch(metricBatch); + + tryFlush = false; + } catch (TException tException) { + tries++; + + if (tries < maxTries) { + try { + Thread.sleep(FLUSH_RETRY_DELAY_MILLIS); + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + + // If someone interrupts this thread, we should probably + // not still try to flush + tryFlush = false; + + exception = new TException("Interrupted while about to retry emitting metrics", tException); + } + } else { + tryFlush = false; + + // Store exception to throw later after clean up. + // Don't want to put everything in a finally block here, + // which will hold the lock unnecessarily long + exception = tException; + } + } } metricBatch.setMetrics(null); @@ -323,16 +370,39 @@ private List flush(List metrics) throws TException { metrics.clear(); if (exception != null) { - throw exception; + throw new RuntimeException( + String.format("Failed to flush metrics after %s tries", maxTries), + exception + ); } - - return metrics; } @Override public void close() { - // Wait (block) until all processors to return from flushing - phaser.arriveAndAwaitAdvance(); + // Important to use `shutdownNow` instead of `shutdown` to interrupt processor + // thread(s) or else they will block forever + executor.shutdownNow(); + + // Wait a maximum of `MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS` for all processors + // to return from flushing + try { + phaser.awaitAdvanceInterruptibly( + phaser.arrive(), + MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS, + TimeUnit.MILLISECONDS + ); + } catch (TimeoutException e) { + LOG.warn( + "M3Reporter closing before Processors complete after waiting timeout of %dms!", + MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS + ); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + + LOG.warn("M3Reporter closing before Processors complete due to being interrupted!"); + } + + transport.close(); } private Metric newMetric(String name, Map tags, MetricType metricType) { @@ -361,6 +431,8 @@ private Metric newMetric(String name, Map tags, MetricType metri timerValue.setI64Value(Long.MAX_VALUE); metricValue.setTimer(timerValue); break; + default: + throw new IllegalArgumentException("Unsupported metric type: " + metricType); } metric.setMetricValue(metricValue); @@ -413,6 +485,11 @@ private void reportCopyMetric( long iValue, double dValue ) { + if (executor.isShutdown()) { + LOG.warn("Scheduler is shutdown already, so no metrics can be flushed"); + return; + } + Metric metricCopy = resourcePool.getMetric(); metricCopy.setName(metric.getName()); metricCopy.setTags(metric.getTags()); @@ -443,6 +520,8 @@ private void reportCopyMetric( try { metricQueue.put(resourcePool.getSizedMetric().setMetric(metricCopy).setSize(size)); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.warn(String.format("Interrupted while putting %s: %s", metricType, metricCopy.getName())); @@ -474,48 +553,61 @@ private String durationBucketString(Duration bucketBound) { } private class Processor implements Runnable { + List metrics = new ArrayList<>(freeBytes / 10); + int metricsSize = 0; + @Override public void run() { try { - List metrics = new ArrayList<>(freeBytes / 10); - int bytes = 0; + while (!executor.isShutdown()) { + // This `take` call will block until there is an item on the queue to take. + // When this reporter is closed, shutdownNow will be called on the executor, + // which will interrupt this thread and proceed to the `InterruptedException` + // catch block. + SizedMetric sizedMetric = metricQueue.take(); + + flushMetricIteration(sizedMetric); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + flushRemainingMetrics(); - while (!metricQueue.isEmpty()) { - SizedMetric sizedMetric = metricQueue.poll(); - Metric metric = sizedMetric.getMetric(); - int size = sizedMetric.getSize(); + // Always arrive at phaser to prevent deadlock + phaser.arrive(); + } + } - resourcePool.releaseSizedMetric(sizedMetric); + private void flushMetricIteration(SizedMetric sizedMetric) { + Metric metric = sizedMetric.getMetric(); - if (metric == null) { - // Explicit flush requested - if (metrics.size() > 0) { - metrics.clear(); - bytes = 0; - } + if (sizedMetric == SizedMetric.FLUSH) { + flush(metrics); + resourcePool.releaseSizedMetric(sizedMetric); - continue; - } + return; + } - if (bytes + size > freeBytes) { - metrics = flush(metrics); - bytes = 0; - } + int size = sizedMetric.getSize(); - metrics.add(metric); - bytes += size; - } + resourcePool.releaseSizedMetric(sizedMetric); - if (metrics.size() > 0) { - // Final flush - flush(metrics); - } - } catch (TException e) { - throw new RuntimeException("Failed to emit metrics", e); - } finally { - // Always arrive at phaser to prevent deadlock - phaser.arrive(); + if (metricsSize + size > freeBytes) { + flush(metrics); + metricsSize = 0; + } + + metrics.add(metric); + + metricsSize += size; + } + + private void flushRemainingMetrics() { + while (!metricQueue.isEmpty()) { + flushMetricIteration(metricQueue.remove()); } + + flush(metrics); } } @@ -621,32 +713,37 @@ CachedHistogramBucket getMetric() { * Builder pattern to construct an {@link M3Reporter}. */ public static class Builder { - private String[] hostPorts; + private SocketAddress[] socketAddresses; private String service; private String env; // Non-generic EMPTY ImmutableMap will never contain any elements @SuppressWarnings("unchecked") private ImmutableMap commonTags = ImmutableMap.EMPTY; private boolean includeHost; - private ThriftProtocol protocol = ThriftProtocol.COMPACT; private int maxQueueSize = DEFAULT_MAX_QUEUE_SIZE; private int maxPacketSizeBytes = DEFAULT_MAX_PACKET_SIZE; private String histogramBucketIdName = DEFAULT_HISTOGRAM_BUCKET_ID_NAME; private String histogramBucketName = DEFAULT_HISTOGRAM_BUCKET_NAME; private int histogramBucketTagPrecision = DEFAULT_HISTOGRAM_BUCKET_TAG_PRECISION; - private ExecutorService executor = Executors.newFixedThreadPool(THREAD_POOL_SIZE); - /** - * Constructs a {@link Builder}. Having at least one host/port is required. - * @param hostPorts the array of host/ports + * Constructs a {@link Builder}. Having at least one {@code SocketAddress} is required. + * @param socketAddresses the array of {@code SocketAddress}es for this {@link M3Reporter} */ - public Builder(String[] hostPorts) { - if (hostPorts == null || hostPorts.length == 0) { - throw new IllegalArgumentException("Must specify at least one host port"); + public Builder(SocketAddress[] socketAddresses) { + if (socketAddresses == null || socketAddresses.length == 0) { + throw new IllegalArgumentException("Must specify at least one SocketAddress"); } - this.hostPorts = hostPorts; + this.socketAddresses = socketAddresses; + } + + /** + * Constructs a {@link Builder}. Having at least one {@code SocketAddress} is required. + * @param socketAddress the {@code SocketAddress} for this {@link M3Reporter} + */ + public Builder(SocketAddress socketAddress) { + this(new SocketAddress[]{socketAddress}); } /** @@ -693,17 +790,6 @@ public Builder includeHost(boolean includeHost) { return this; } - /** - * Configures the protocol of this {@link Builder}. - * @param protocol the value to set - * @return this {@link Builder} with the new value set - */ - public Builder protocol(ThriftProtocol protocol) { - this.protocol = protocol; - - return this; - } - /** * Configures the maximum queue size of this {@link Builder}. * @param maxQueueSize the value to set @@ -759,17 +845,6 @@ public Builder histogramBucketTagPrecision(int histogramBucketTagPrecision) { return this; } - /** - * Configures the executor service of this {@link Builder}. - * @param executor the value to set - * @return this {@link Builder} with the new value set - */ - public Builder executor(ExecutorService executor) { - this.executor = executor; - - return this; - } - /** * Builds and returns an {@link M3Reporter} with the configured paramters. * @return a new {@link M3Reporter} instance with the configured paramters diff --git a/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java b/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java index ecceedc..6c8aa91 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java @@ -20,6 +20,7 @@ package com.uber.m3.tally.m3; +import com.uber.m3.tally.m3.thrift.TCalcTransport; import com.uber.m3.thrift.generated.CountValue; import com.uber.m3.thrift.generated.GaugeValue; import com.uber.m3.thrift.generated.Metric; @@ -29,24 +30,25 @@ import com.uber.m3.thrift.generated.TimerValue; import com.uber.m3.util.Construct; import com.uber.m3.util.ObjectPool; +import org.apache.thrift.protocol.TCompactProtocol; import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.protocol.TProtocolFactory; import java.util.HashSet; import java.util.List; +import java.util.Set; /** * A collection of {@link ObjectPool}s used for the {@link M3Reporter}. */ public class ResourcePool { - private static final int BATCH_POOL_SIZE = 10; - private static final int PROTOCOL_POOL_SIZE = 10; + public static final int BATCH_POOL_SIZE = 10; + public static final int PROTOCOL_POOL_SIZE = 10; private ObjectPool metricBatchPool; private ObjectPool metricPool; private ObjectPool metricTagPool; private ObjectPool metricValuePool; - private ObjectPool counterValuePool; + private ObjectPool countValuePool; private ObjectPool gaugeValuePool; private ObjectPool timerValuePool; private ObjectPool protocolPool; @@ -55,9 +57,8 @@ public class ResourcePool { /** * Creates a {@link ResourcePool}, initializing all its inner pools. * @param maxQueueSize the maximum metric queue size - * @param protocolFactory a factory to create the {@link TProtocol} we want */ - ResourcePool(int maxQueueSize, final TProtocolFactory protocolFactory) { + ResourcePool(int maxQueueSize) { metricBatchPool = new ObjectPool<>(BATCH_POOL_SIZE, new Construct() { @Override public MetricBatch instance() { @@ -86,7 +87,7 @@ public MetricValue instance() { } }); - counterValuePool = new ObjectPool<>(maxQueueSize, new Construct() { + countValuePool = new ObjectPool<>(maxQueueSize, new Construct() { @Override public CountValue instance() { return new CountValue(); @@ -110,7 +111,7 @@ public TimerValue instance() { protocolPool = new ObjectPool<>(PROTOCOL_POOL_SIZE, new Construct() { @Override public TProtocol instance() { - return protocolFactory.getProtocol(new TCalcTransport()); + return new TCompactProtocol.Factory().getProtocol(new TCalcTransport()); } }); @@ -167,7 +168,7 @@ public MetricValue getMetricValue() { * @return a {@link CountValue} from the pool */ public CountValue getCountValue() { - return counterValuePool.get(); + return countValuePool.get(); } /** @@ -206,68 +207,79 @@ public SizedMetric getSizedMetric() { * Puts the given {@link TProtocol} back to the pool, reseting the * {@link TCalcTransport} internal count. * @param protocol the {@link TProtocol} to put back to the pool + * @return whether the {@link TProtocol} was put back into the pool */ - public void releaseProtocol(TProtocol protocol) { + public boolean releaseProtocol(TProtocol protocol) { ((TCalcTransport) protocol.getTransport()).resetSize(); - protocolPool.put(protocol); + return protocolPool.put(protocol); } /** * Puts the given {@link MetricBatch} back to the pool after reseting inner * values and releasing each of its internal {@link Metric}s as well. * @param metricBatch the {@link MetricBatch} to put back to the pool + * @return whether the {@link MetricBatch} was put back into the pool */ - public void releaseMetricBatch(MetricBatch metricBatch) { - metricBatch.setCommonTags(null); + public boolean releaseMetricBatch(MetricBatch metricBatch) { + metricBatch.setCommonTagsIsSet(false); - for (Metric metric : metricBatch.getMetrics()) { - releaseMetric(metric); + List metrics = metricBatch.getMetrics(); + + if (metrics != null) { + for (Metric metric : metrics) { + releaseMetric(metric); + } } - metricBatch.setMetrics(null); + metricBatch.setMetricsIsSet(false); - metricBatchPool.put(metricBatch); + return metricBatchPool.put(metricBatch); } /** * Puts the given {@link Metric} back to the pool after reseting its * internal attributes. * @param metric the {@link Metric} to put back to the pool + * @return whether the {@link Metric} was put back into the pool */ - public void releaseMetric(Metric metric) { - metric.setName(""); + public boolean releaseMetric(Metric metric) { + metric.setNameIsSet(false); - for (MetricTag tag : metric.getTags()) { - tag.setTagName(""); - tag.setTagValue(null); - metricTagPool.put(tag); + Set tags = metric.getTags(); + + if (tags != null) { + for (MetricTag tag : tags) { + tag.setTagNameIsSet(false); + tag.setTagValueIsSet(false); + metricTagPool.put(tag); + } } - metric.setTags(null); + metric.setTagsIsSet(false); - releaseShallowMetric(metric); + return releaseShallowMetric(metric); } /** * Puts a tag-less {@link Metric} back to the pool as well as * its {@link MetricValue}. * @param metric the {@link Metric} to put back to the pool + * @return whether the {@link Metric} was put back into the pool */ - public void releaseShallowMetric(Metric metric) { - metric.setTimestamp(0); + public boolean releaseShallowMetric(Metric metric) { + metric.setTimestampIsSet(false); MetricValue metricValue = metric.getMetricValue(); if (metricValue == null) { - metricPool.put(metric); - return; + return metricPool.put(metric); } releaseMetricValue(metricValue); - metric.setMetricValue(null); + metric.unsetMetricValue(); - metricPool.put(metric); + return metricPool.put(metric); } /** @@ -284,42 +296,95 @@ public void releaseShallowMetrics(List metrics) { * Puts the given {@link MetricValue} back to the pool after reseting its * internal attributes and values. * @param metricValue the {@link MetricValue} to put back to the pool + * @return whether the {@link MetricValue} was put back into the pool */ - public void releaseMetricValue(MetricValue metricValue) { + public boolean releaseMetricValue(MetricValue metricValue) { if (metricValue.isSetCount()) { - metricValue.getCount().setI64Value(0); + metricValue.getCount().setI64ValueIsSet(false); - counterValuePool.put(metricValue.getCount()); + countValuePool.put(metricValue.getCount()); metricValue.unsetCount(); } else if (metricValue.isSetGauge()) { - metricValue.getGauge().setI64Value(0); - metricValue.getGauge().setDValue(0); + metricValue.getGauge().setI64ValueIsSet(false); + metricValue.getGauge().setDValueIsSet(false); gaugeValuePool.put(metricValue.getGauge()); metricValue.unsetGauge(); } else if (metricValue.isSetTimer()) { - metricValue.getTimer().setI64Value(0); - metricValue.getTimer().setDValue(0); + metricValue.getTimer().setI64ValueIsSet(false); + metricValue.getTimer().setDValueIsSet(false); timerValuePool.put(metricValue.getTimer()); metricValue.unsetTimer(); } - metricValuePool.put(metricValue); + return metricValuePool.put(metricValue); } /** * Puts the given {@link SizedMetric} back to the pool after reseting * inner values. * @param sizedMetric the {@link SizedMetric} to put back to the pool + * @return whether the {@link SizedMetric} was put back into the pool */ - public void releaseSizedMetric(SizedMetric sizedMetric) { + public boolean releaseSizedMetric(SizedMetric sizedMetric) { sizedMetric.setMetric(null); sizedMetric.setSize(0); - sizedMetricPool.put(sizedMetric); + return sizedMetricPool.put(sizedMetric); + } + + /** + * Puts the given {@link MetricTag} back to the pool after reseting + * inner values. + * @param metricTag the {@link MetricTag} to put back to the pool + * @return whether the {@link MetricTag} was put back into the pool + */ + public boolean releaseMetricTag(MetricTag metricTag) { + metricTag.setTagNameIsSet(false); + metricTag.setTagValueIsSet(false); + + return metricTagPool.put(metricTag); + } + + /** + * Puts the given {@link CountValue} back to the pool after reseting + * inner values. + * @param countValue the {@link CountValue} to put back to the pool + * @return whether the {@link CountValue} was put back into the pool + */ + public boolean releaseCountValue(CountValue countValue) { + countValue.setI64ValueIsSet(false); + + return countValuePool.put(countValue); + } + + /** + * Puts the given {@link GaugeValue} back to the pool after reseting + * inner values. + * @param gaugeValue the {@link GaugeValue} to put back to the pool + * @return whether the {@link GaugeValue} was put back into the pool + */ + public boolean releaseGaugeValue(GaugeValue gaugeValue) { + gaugeValue.setDValueIsSet(false); + gaugeValue.setI64ValueIsSet(false); + + return gaugeValuePool.put(gaugeValue); + } + + /** + * Puts the given {@link TimerValue} back to the pool after reseting + * inner values. + * @param timerValue the {@link TimerValue} to put back to the pool + * @return whether the {@link TimerValue} was put back into the pool + */ + public boolean releaseTimerValue(TimerValue timerValue) { + timerValue.setDValueIsSet(false); + timerValue.setI64ValueIsSet(false); + + return timerValuePool.put(timerValue); } } diff --git a/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java b/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java index f7d1a6f..14a4ff2 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java @@ -26,6 +26,8 @@ * A metric along with its associated size. */ public class SizedMetric extends Metric { + public static final SizedMetric FLUSH = new SizedMetric(null, -1); + private Metric metric; private int size; diff --git a/m3/src/main/java/com/uber/m3/tally/m3/TUdpTransport.java b/m3/src/main/java/com/uber/m3/tally/m3/TUdpTransport.java deleted file mode 100644 index 904d67d..0000000 --- a/m3/src/main/java/com/uber/m3/tally/m3/TUdpTransport.java +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package com.uber.m3.tally.m3; - -import org.apache.thrift.transport.TTransport; -import org.apache.thrift.transport.TTransportException; - -import java.io.IOException; -import java.net.DatagramPacket; -import java.net.DatagramSocket; -import java.net.SocketException; -import java.nio.ByteBuffer; - -/** - * Abstract class that supports Thrift UDP functionality. - */ -public abstract class TUdpTransport extends TTransport implements AutoCloseable { - public static final int MAX_PACKET_SIZE = 14400; - public static final int MAX_BUFFER_SIZE = 65536; - - public final Object receiveLock = new Object(); - public final Object sendLock = new Object(); - - protected final DatagramSocket socket = new DatagramSocket(null); - protected byte[] receiveBuf; - protected int receiveOffSet = -1; - protected int receiveLength = 0; - protected ByteBuffer writeButter; - - protected String host; - protected int port; - - protected TUdpTransport(String host, int port) throws SocketException { - this.host = host; - this.port = port; - } - - protected TUdpTransport(String hostPort) throws SocketException { - try { - int colonIdx = hostPort.lastIndexOf(':'); - - if (colonIdx == -1) { - throw new IllegalArgumentException(); - } - - this.host = hostPort.substring(0, colonIdx); - this.port = Integer.parseInt(hostPort.substring(colonIdx + 1)); - } catch (Exception e) { - throw new IllegalArgumentException("Invalid host/port argument: " + hostPort, e); - } - } - - @Override - public boolean isOpen() { - return !socket.isClosed(); - } - - @Override - public abstract void open() throws TTransportException; - - @Override - public void close() { - socket.close(); - } - - @Override - public int getBytesRemainingInBuffer() { - return Integer.MAX_VALUE; - } - - @Override - public int read(byte[] bytes, int offset, int len) throws TTransportException { - synchronized (receiveLock) { - if (!isOpen()) { - throw new TTransportException(TTransportException.NOT_OPEN); - } - - if (receiveOffSet == -1) { - receiveBuf = new byte[MAX_BUFFER_SIZE]; - DatagramPacket dg = new DatagramPacket(receiveBuf, MAX_BUFFER_SIZE); - - try { - socket.receive(dg); - } catch (IOException e) { - throw new TTransportException("Error from underlying socket", e); - } - - receiveOffSet = 0; - receiveLength = dg.getLength(); - } - - int curDataSize = receiveLength - receiveOffSet; - - if (curDataSize <= len) { - System.arraycopy(receiveBuf, receiveOffSet, bytes, offset, curDataSize); - receiveOffSet = -1; - return curDataSize; - } else { - System.arraycopy(receiveBuf, receiveOffSet, bytes, offset, len); - receiveOffSet += len; - return len; - } - } - } - - @Override - public void write(byte[] bytes, int offset, int len) throws TTransportException { - synchronized (sendLock) { - if (!isOpen()) { - throw new TTransportException(TTransportException.NOT_OPEN); - } - if (writeButter == null) { - writeButter = ByteBuffer.allocate(MAX_BUFFER_SIZE); - } - if (writeButter.position() + len > MAX_PACKET_SIZE) { - throw new TTransportException("Message size too large: " + len + " > " + MAX_BUFFER_SIZE); - } - writeButter.put(bytes, offset, len); - } - } - - @Override - public void flush() throws TTransportException { - synchronized (sendLock) { - if (writeButter != null) { - byte[] bytes = new byte[MAX_BUFFER_SIZE]; - int len = writeButter.position(); - - writeButter.flip(); - writeButter.get(bytes, 0, len); - - try { - socket.send(new DatagramPacket(bytes, len)); - } catch (IOException e) { - throw new TTransportException("Cannot flush closed transport", e); - } finally { - writeButter = null; - } - } - } - } -} diff --git a/m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java b/m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java deleted file mode 100644 index 27f6e7c..0000000 --- a/m3/src/main/java/com/uber/m3/tally/m3/ThriftProtocol.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package com.uber.m3.tally.m3; - -/** - * Enum of Thrift protocols. - */ -public enum ThriftProtocol { - COMPACT, - BINARY -} diff --git a/m3/src/main/java/com/uber/m3/tally/m3/TCalcTransport.java b/m3/src/main/java/com/uber/m3/tally/m3/thrift/TCalcTransport.java similarity index 66% rename from m3/src/main/java/com/uber/m3/tally/m3/TCalcTransport.java rename to m3/src/main/java/com/uber/m3/tally/m3/thrift/TCalcTransport.java index c76e5e6..1bbf532 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/TCalcTransport.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/thrift/TCalcTransport.java @@ -18,7 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -package com.uber.m3.tally.m3; +package com.uber.m3.tally.m3.thrift; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; @@ -31,27 +31,53 @@ public class TCalcTransport extends TTransport { private AtomicInteger size = new AtomicInteger(0); + /** + * Dummy override to satisfy interface. Not used for our purposes. + * @return not used for our purposes + */ @Override public boolean isOpen() { return true; } + /** + * Dummy override to satisfy interface. Not used for our purposes. + * @throws TTransportException not used for our purposes + */ @Override public void open() throws TTransportException { } + /** + * Dummy override to satisfy interface. Not used for our purposes. + */ @Override public void close() { } + /** + * Dummy override to satisfy interface. Not used for our purposes. + * @param bytes not used for our purposes + * @param offset not used for our purposes + * @param length not used for our purposes + * @return not used for our purposes + * @throws TTransportException not used for our purposes + */ @Override - public int read(byte[] bytes, int i, int i1) throws TTransportException { + public int read(byte[] bytes, int offset, int length) throws TTransportException { return 0; } + /** + * Record the number of bytes being written to this transport. + * @param bytes not used for our purposes + * @param offset not used for our purposes + * @param length the length of bytes being written + * @throws TTransportException this will never throw + */ @Override - public void write(byte[] bytes, int i, int len) throws TTransportException { - size.getAndAdd(len); + public void write(byte[] bytes, int offset, int length) throws TTransportException { + size.getAndAdd(length); } /** diff --git a/m3/src/main/java/com/uber/m3/tally/m3/TUdpClient.java b/m3/src/main/java/com/uber/m3/tally/m3/thrift/TUdpClient.java similarity index 64% rename from m3/src/main/java/com/uber/m3/tally/m3/TUdpClient.java rename to m3/src/main/java/com/uber/m3/tally/m3/thrift/TUdpClient.java index cc2c83d..b9446ef 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/TUdpClient.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/thrift/TUdpClient.java @@ -18,11 +18,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -package com.uber.m3.tally.m3; +package com.uber.m3.tally.m3.thrift; import org.apache.thrift.transport.TTransportException; -import java.net.InetSocketAddress; +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.SocketAddress; import java.net.SocketException; /** @@ -31,30 +33,38 @@ public class TUdpClient extends TUdpTransport implements AutoCloseable { /** * Constructs a UDP client with the given host and port. - * @param host the host for this transport - * @param port the port for this transport + * @param socketAddress the {@code SocketAddress} for this transport * @throws SocketException if the underlying socket cannot be opened */ - public TUdpClient(String host, int port) throws SocketException { - super(host, port); - } - - /** - * Constructs a UDP client with the given host and port. - * @param hostPort a string containing both host and port information in the - * form "HOST:PORT" - * @throws SocketException if the underlying socket cannot be opened - */ - public TUdpClient(String hostPort) throws SocketException { - super(hostPort); + public TUdpClient(SocketAddress socketAddress) throws SocketException { + super(socketAddress); } @Override public void open() throws TTransportException { try { - socket.connect(new InetSocketAddress(host, port)); + socket.connect(socketAddress); } catch (SocketException e) { throw new TTransportException("Error opening transport", e); } } + + @Override + public void flush() throws TTransportException { + synchronized (sendLock) { + byte[] bytes = new byte[MAX_BUFFER_SIZE]; + int length = writeBuffer.position(); + + writeBuffer.flip(); + writeBuffer.get(bytes, 0, length); + + try { + socket.send(new DatagramPacket(bytes, length)); + } catch (IOException e) { + throw new TTransportException("Cannot flush closed transport", e); + } finally { + writeBuffer.clear(); + } + } + } } diff --git a/m3/src/main/java/com/uber/m3/tally/m3/TUdpServer.java b/m3/src/main/java/com/uber/m3/tally/m3/thrift/TUdpServer.java similarity index 76% rename from m3/src/main/java/com/uber/m3/tally/m3/TUdpServer.java rename to m3/src/main/java/com/uber/m3/tally/m3/thrift/TUdpServer.java index cfdc939..0e6ca72 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/TUdpServer.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/thrift/TUdpServer.java @@ -18,11 +18,11 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -package com.uber.m3.tally.m3; +package com.uber.m3.tally.m3.thrift; import org.apache.thrift.transport.TTransportException; -import java.net.InetSocketAddress; +import java.net.SocketAddress; import java.net.SocketException; /** @@ -33,23 +33,21 @@ public class TUdpServer extends TUdpTransport implements AutoCloseable { /** * Constructs a UDP server with the given host and port. Defaults to zero timeout. - * @param host the host for this transport - * @param port the port for this transport + * @param socketAddress the {@code SocketAddress} for this transport * @throws SocketException if the underlying socket cannot be opened */ - public TUdpServer(String host, int port) throws SocketException { - this(host, port, 0); + public TUdpServer(SocketAddress socketAddress) throws SocketException { + this(socketAddress, 0); } /** * Constructs a UDP server with the given host and port. - * @param host the host for this transport - * @param port the port for this transport + * @param socketAddress the {@code SocketAddress} for this transport * @param timeoutMillis the timeout in milliseconds * @throws SocketException if the underlying socket cannot be opened */ - public TUdpServer(String host, int port, int timeoutMillis) throws SocketException { - super(host, port); + public TUdpServer(SocketAddress socketAddress, int timeoutMillis) throws SocketException { + super(socketAddress); this.timeoutMillis = timeoutMillis; } @@ -57,10 +55,15 @@ public TUdpServer(String host, int port, int timeoutMillis) throws SocketExcepti @Override public void open() throws TTransportException { try { - socket.bind(new InetSocketAddress(host, port)); + socket.bind(socketAddress); socket.setSoTimeout(timeoutMillis); } catch (SocketException e) { throw new TTransportException("Error opening transport", e); } } + + @Override + public void flush() throws TTransportException { + // No-op -- UDP server does not write responses back + } } diff --git a/m3/src/main/java/com/uber/m3/tally/m3/thrift/TUdpTransport.java b/m3/src/main/java/com/uber/m3/tally/m3/thrift/TUdpTransport.java new file mode 100644 index 0000000..a51f56d --- /dev/null +++ b/m3/src/main/java/com/uber/m3/tally/m3/thrift/TUdpTransport.java @@ -0,0 +1,142 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.tally.m3.thrift; + +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.SocketAddress; +import java.net.SocketException; +import java.nio.ByteBuffer; + +/** + * Abstract class that supports Thrift UDP functionality. + */ +public abstract class TUdpTransport extends TTransport implements AutoCloseable { + public static final int MAX_BUFFER_SIZE = 65536; + + public final Object receiveLock = new Object(); + public final Object sendLock = new Object(); + + protected final DatagramSocket socket = new DatagramSocket(null); + protected ByteBuffer receiveBuffer; + protected ByteBuffer writeBuffer; + + protected SocketAddress socketAddress; + + protected TUdpTransport(SocketAddress socketAddress) throws SocketException { + this.socketAddress = socketAddress; + + writeBuffer = ByteBuffer.allocate(MAX_BUFFER_SIZE); + + receiveBuffer = ByteBuffer.allocate(MAX_BUFFER_SIZE); + receiveBuffer.flip(); + } + + @Override + public boolean isOpen() { + return !socket.isClosed(); + } + + @Override + public abstract void open() throws TTransportException; + + @Override + public void close() { + socket.close(); + } + + @Override + public int read(byte[] bytes, int offset, int length) throws TTransportException { + if (!isOpen()) { + throw new TTransportException(TTransportException.NOT_OPEN); + } + + synchronized (receiveLock) { + if (!receiveBuffer.hasRemaining()) { + // Use ByteBuffer's backing array and manually set the position and limit to + // avoid having to re-copy contents to a new array via `get` + DatagramPacket packet = new DatagramPacket(receiveBuffer.array(), MAX_BUFFER_SIZE); + + try { + socket.receive(packet); + } catch (IOException e) { + throw new TTransportException("Error from underlying socket", e); + } + + receiveBuffer.position(0); + receiveBuffer.limit(packet.getLength()); + } + + length = Math.min(length, receiveBuffer.remaining()); + + receiveBuffer.get(bytes, offset, length); + + return length; + } + } + + @Override + public void write(byte[] bytes, int offset, int length) throws TTransportException { + if (!isOpen()) { + throw new TTransportException(TTransportException.NOT_OPEN); + } + + synchronized (sendLock) { + if (writeBuffer.position() + length > MAX_BUFFER_SIZE) { + throw new TTransportException( + String.format("Message size too large: %d is greater than available size %d", + length, + MAX_BUFFER_SIZE - writeBuffer.position() + ) + ); + } + + writeBuffer.put(bytes, offset, length); + } + } + + @Override + public abstract void flush() throws TTransportException; + + @Override + public int getBytesRemainingInBuffer() { + return receiveBuffer.remaining(); + } + + @Override + public byte[] getBuffer() { + return receiveBuffer.array(); + } + + @Override + public int getBufferPosition() { + return receiveBuffer.position(); + } + + @Override + public void consumeBuffer(int length) { + receiveBuffer.position(receiveBuffer.position() + length); + } +} diff --git a/m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java b/m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java index 25d2461..0f8e175 100644 --- a/m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java +++ b/m3/src/test/java/com/uber/m3/tally/m3/SizedMetricTest.java @@ -43,4 +43,23 @@ public void nullSizedMetric() { assertEquals(sizedMetric.getMetric(), null); assertEquals(sizedMetric.getSize(), 9); } + + @Test + public void setters() { + SizedMetric sizedMetric = new SizedMetric(); + + sizedMetric.setSize(9); + + assertEquals(9, sizedMetric.getSize()); + + sizedMetric.setSize(11); + sizedMetric.setSize(12); + + assertEquals(12, sizedMetric.getSize()); + + Metric metric1 = new Metric(); + sizedMetric.setMetric(metric1); + + assertEquals(metric1, sizedMetric.getMetric()); + } } From 7221592f1ce8b4a3c7b3c667d5ce1ccc5d0b5d30 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Sat, 23 Sep 2017 23:03:32 -0400 Subject: [PATCH 28/44] Remove ResourcePool usage --- .../m3/tally/m3/ResourcePoolBenchmark.java | 83 ---- .../java/com/uber/m3/tally/m3/M3Reporter.java | 45 +- .../com/uber/m3/tally/m3/ResourcePool.java | 390 ------------------ 3 files changed, 16 insertions(+), 502 deletions(-) delete mode 100644 m3/src/jmh/java/com/uber/m3/tally/m3/ResourcePoolBenchmark.java delete mode 100644 m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java diff --git a/m3/src/jmh/java/com/uber/m3/tally/m3/ResourcePoolBenchmark.java b/m3/src/jmh/java/com/uber/m3/tally/m3/ResourcePoolBenchmark.java deleted file mode 100644 index 5adfcc4..0000000 --- a/m3/src/jmh/java/com/uber/m3/tally/m3/ResourcePoolBenchmark.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package com.uber.m3.tally.m3; - -import com.uber.m3.tally.m3.thrift.TCalcTransport; -import com.uber.m3.thrift.generated.CountValue; -import com.uber.m3.thrift.generated.GaugeValue; -import com.uber.m3.thrift.generated.Metric; -import com.uber.m3.thrift.generated.MetricBatch; -import com.uber.m3.thrift.generated.MetricTag; -import com.uber.m3.thrift.generated.MetricValue; -import com.uber.m3.thrift.generated.TimerValue; -import org.apache.thrift.protocol.TCompactProtocol; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.infra.Blackhole; - -import java.util.HashSet; - -public class ResourcePoolBenchmark { - static final int NUM_OUTER_ITERATIONS = 5; - static final int NUM_INNER_ITERATIONS = 10000; - - @Benchmark - public void usingResourcePool(Blackhole sink) { - ResourcePool pool = new ResourcePool(ResourcePool.BATCH_POOL_SIZE); - - for (int iterations = 0; iterations < NUM_OUTER_ITERATIONS; iterations++) { - // Get and release from the pool right away, then suggest a GC run - for (int innerIterations = 0; innerIterations < NUM_INNER_ITERATIONS; innerIterations++) { - sink.consume(pool.releaseMetric(pool.getMetric())); - sink.consume(pool.releaseMetricTag(pool.getMetricTag())); - sink.consume(pool.releaseMetricValue(pool.getMetricValue())); - sink.consume(pool.releaseMetricBatch(pool.getMetricBatch())); - sink.consume(pool.releaseProtocol(pool.getProtocol())); - sink.consume(pool.releaseSizedMetric(pool.getSizedMetric())); - sink.consume(pool.releaseCountValue(pool.getCountValue())); - sink.consume(pool.releaseGaugeValue(pool.getGaugeValue())); - sink.consume(pool.releaseTimerValue(pool.getTimerValue())); - sink.consume(pool.getTagList()); - } - - System.gc(); - } - } - - @Benchmark - public void usingNaiveObjectCreation(Blackhole sink) { - for (int iterations = 0; iterations < NUM_OUTER_ITERATIONS; iterations++) { - for (int innerIterations = 0; innerIterations < NUM_INNER_ITERATIONS; innerIterations++) { - sink.consume(new Metric()); - sink.consume(new MetricTag()); - sink.consume(new MetricValue()); - sink.consume(new MetricBatch()); - sink.consume(new TCompactProtocol.Factory().getProtocol(new TCalcTransport())); - sink.consume(new SizedMetric()); - sink.consume(new CountValue()); - sink.consume(new GaugeValue()); - sink.consume(new TimerValue()); - sink.consume(new HashSet<>()); - } - - System.gc(); - } - } -} diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index ffae404..169d7ab 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -55,6 +55,7 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -110,8 +111,6 @@ public class M3Reporter implements CachedStatsReporter, AutoCloseable { private int freeBytes; - private ResourcePool resourcePool; - private String bucketIdTagName; private String bucketTagName; private String bucketValFmt; @@ -146,8 +145,6 @@ private M3Reporter(Builder builder) { client = new M3.Client(protocolFactory.getProtocol(transport)); - resourcePool = new ResourcePool(builder.maxQueueSize); - Set commonTags = toMetricTagsToSet(builder.commonTags); // Set and ensure required tags @@ -169,8 +166,8 @@ private M3Reporter(Builder builder) { commonTags.add(createMetricTag(HOST_TAG, getHostName())); } - metricBatch = resourcePool.getMetricBatch(); - calcProtocol = resourcePool.getProtocol(); + metricBatch = new MetricBatch(); + calcProtocol = new TCompactProtocol.Factory().getProtocol(new TCalcTransport()); calc = (TCalcTransport) calcProtocol.getTransport(); freeBytes = calculateFreeBytes(builder.maxPacketSizeBytes, commonTags); @@ -365,8 +362,6 @@ private void flush(List metrics, int maxTries) { metricBatch.setMetrics(null); } - resourcePool.releaseShallowMetrics(metrics); - metrics.clear(); if (exception != null) { @@ -406,28 +401,25 @@ public void close() { } private Metric newMetric(String name, Map tags, MetricType metricType) { - Metric metric = resourcePool.getMetric(); - metric.setName(name); - + Metric metric = new Metric(name); metric.setTags(toMetricTagsToSet(tags)); - metric.setTimestamp(Long.MAX_VALUE); - MetricValue metricValue = resourcePool.getMetricValue(); + MetricValue metricValue = new MetricValue(); switch (metricType) { case COUNTER: - CountValue countValue = resourcePool.getCountValue(); + CountValue countValue = new CountValue(); countValue.setI64Value(Long.MAX_VALUE); metricValue.setCount(countValue); break; case GAUGE: - GaugeValue gaugeValue = resourcePool.getGaugeValue(); + GaugeValue gaugeValue = new GaugeValue(); gaugeValue.setDValue(Double.MAX_VALUE); metricValue.setGauge(gaugeValue); break; case TIMER: - TimerValue timerValue = resourcePool.getTimerValue(); + TimerValue timerValue = new TimerValue(); timerValue.setI64Value(Long.MAX_VALUE); metricValue.setTimer(timerValue); break; @@ -441,7 +433,7 @@ private Metric newMetric(String name, Map tags, MetricType metri } private Set toMetricTagsToSet(Map tags) { - Set metricTagSet = resourcePool.getTagList(); + Set metricTagSet = new HashSet<>(); if (tags == null) { return metricTagSet; @@ -455,8 +447,7 @@ private Set toMetricTagsToSet(Map tags) { } private MetricTag createMetricTag(String tagName, String tagValue) { - MetricTag metricTag = resourcePool.getMetricTag(); - metricTag.setTagName(tagName); + MetricTag metricTag = new MetricTag(tagName); if (tagValue != null && !tagValue.isEmpty()) { metricTag.setTagValue(tagValue); @@ -490,26 +481,25 @@ private void reportCopyMetric( return; } - Metric metricCopy = resourcePool.getMetric(); - metricCopy.setName(metric.getName()); + Metric metricCopy = new Metric(metric.getName()); metricCopy.setTags(metric.getTags()); // Unfortunately, there is no finer time resolution for Java 7 metricCopy.setTimestamp(System.currentTimeMillis() * Duration.NANOS_PER_MILLI); - metricCopy.setMetricValue(resourcePool.getMetricValue()); + metricCopy.setMetricValue(new MetricValue()); switch (metricType) { case COUNTER: - CountValue countValue = resourcePool.getCountValue(); + CountValue countValue = new CountValue(); countValue.setI64Value(iValue); metricCopy.getMetricValue().setCount(countValue); break; case GAUGE: - GaugeValue gaugeValue = resourcePool.getGaugeValue(); + GaugeValue gaugeValue = new GaugeValue(); gaugeValue.setDValue(dValue); metricCopy.getMetricValue().setGauge(gaugeValue); break; case TIMER: - TimerValue timerValue = resourcePool.getTimerValue(); + TimerValue timerValue = new TimerValue(); timerValue.setI64Value(iValue); metricCopy.getMetricValue().setTimer(timerValue); break; @@ -518,7 +508,7 @@ private void reportCopyMetric( } try { - metricQueue.put(resourcePool.getSizedMetric().setMetric(metricCopy).setSize(size)); + metricQueue.put(new SizedMetric(metricCopy, size)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -583,15 +573,12 @@ private void flushMetricIteration(SizedMetric sizedMetric) { if (sizedMetric == SizedMetric.FLUSH) { flush(metrics); - resourcePool.releaseSizedMetric(sizedMetric); return; } int size = sizedMetric.getSize(); - resourcePool.releaseSizedMetric(sizedMetric); - if (metricsSize + size > freeBytes) { flush(metrics); metricsSize = 0; diff --git a/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java b/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java deleted file mode 100644 index 6c8aa91..0000000 --- a/m3/src/main/java/com/uber/m3/tally/m3/ResourcePool.java +++ /dev/null @@ -1,390 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package com.uber.m3.tally.m3; - -import com.uber.m3.tally.m3.thrift.TCalcTransport; -import com.uber.m3.thrift.generated.CountValue; -import com.uber.m3.thrift.generated.GaugeValue; -import com.uber.m3.thrift.generated.Metric; -import com.uber.m3.thrift.generated.MetricBatch; -import com.uber.m3.thrift.generated.MetricTag; -import com.uber.m3.thrift.generated.MetricValue; -import com.uber.m3.thrift.generated.TimerValue; -import com.uber.m3.util.Construct; -import com.uber.m3.util.ObjectPool; -import org.apache.thrift.protocol.TCompactProtocol; -import org.apache.thrift.protocol.TProtocol; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * A collection of {@link ObjectPool}s used for the {@link M3Reporter}. - */ -public class ResourcePool { - public static final int BATCH_POOL_SIZE = 10; - public static final int PROTOCOL_POOL_SIZE = 10; - - private ObjectPool metricBatchPool; - private ObjectPool metricPool; - private ObjectPool metricTagPool; - private ObjectPool metricValuePool; - private ObjectPool countValuePool; - private ObjectPool gaugeValuePool; - private ObjectPool timerValuePool; - private ObjectPool protocolPool; - private ObjectPool sizedMetricPool; - - /** - * Creates a {@link ResourcePool}, initializing all its inner pools. - * @param maxQueueSize the maximum metric queue size - */ - ResourcePool(int maxQueueSize) { - metricBatchPool = new ObjectPool<>(BATCH_POOL_SIZE, new Construct() { - @Override - public MetricBatch instance() { - return new MetricBatch(); - } - }); - - metricPool = new ObjectPool<>(maxQueueSize, new Construct() { - @Override - public Metric instance() { - return new Metric(); - } - }); - - metricTagPool = new ObjectPool<>(maxQueueSize, new Construct() { - @Override - public MetricTag instance() { - return new MetricTag(); - } - }); - - metricValuePool = new ObjectPool<>(maxQueueSize, new Construct() { - @Override - public MetricValue instance() { - return new MetricValue(); - } - }); - - countValuePool = new ObjectPool<>(maxQueueSize, new Construct() { - @Override - public CountValue instance() { - return new CountValue(); - } - }); - - gaugeValuePool = new ObjectPool<>(maxQueueSize, new Construct() { - @Override - public GaugeValue instance() { - return new GaugeValue(); - } - }); - - timerValuePool = new ObjectPool<>(maxQueueSize, new Construct() { - @Override - public TimerValue instance() { - return new TimerValue(); - } - }); - - protocolPool = new ObjectPool<>(PROTOCOL_POOL_SIZE, new Construct() { - @Override - public TProtocol instance() { - return new TCompactProtocol.Factory().getProtocol(new TCalcTransport()); - } - }); - - sizedMetricPool = new ObjectPool<>(maxQueueSize, new Construct() { - @Override - public SizedMetric instance() { - return new SizedMetric(); - } - }); - } - - /** - * Returns a {@link MetricBatch} from the pool. - * @return a {@link MetricBatch} from the pool - */ - public MetricBatch getMetricBatch() { - return metricBatchPool.get(); - } - - /** - * Returns a {@link Metric} from the pool. - * @return a {@link Metric} from the pool - */ - public Metric getMetric() { - return metricPool.get(); - } - - /** - * Returns an empty {@code Set} of {@link MetricTag}s. - * @return an empty {@code Set} of {@link MetricTag}s - */ - public HashSet getTagList() { - return new HashSet<>(); - } - - /** - * Returns a {@link MetricTag} from the pool. - * @return a {@link MetricTag} from the pool - */ - public MetricTag getMetricTag() { - return metricTagPool.get(); - } - - /** - * Returns a {@link MetricValue} from the pool. - * @return a {@link MetricValue} from the pool - */ - public MetricValue getMetricValue() { - return metricValuePool.get(); - } - - /** - * Returns a {@link CountValue} from the pool. - * @return a {@link CountValue} from the pool - */ - public CountValue getCountValue() { - return countValuePool.get(); - } - - /** - * Returns a {@link GaugeValue} from the pool. - * @return a {@link GaugeValue} from the pool - */ - public GaugeValue getGaugeValue() { - return gaugeValuePool.get(); - } - - /** - * Returns a {@link TimerValue} from the pool. - * @return a {@link TimerValue} from the pool - */ - public TimerValue getTimerValue() { - return timerValuePool.get(); - } - - /** - * Returns a {@link TProtocol} from the pool. - * @return a {@link TProtocol} from the pool - */ - public TProtocol getProtocol() { - return protocolPool.get(); - } - - /** - * Returns a {@link SizedMetric} from the pool. - * @return a {@link SizedMetric} from the pool - */ - public SizedMetric getSizedMetric() { - return sizedMetricPool.get(); - } - - /** - * Puts the given {@link TProtocol} back to the pool, reseting the - * {@link TCalcTransport} internal count. - * @param protocol the {@link TProtocol} to put back to the pool - * @return whether the {@link TProtocol} was put back into the pool - */ - public boolean releaseProtocol(TProtocol protocol) { - ((TCalcTransport) protocol.getTransport()).resetSize(); - - return protocolPool.put(protocol); - } - - /** - * Puts the given {@link MetricBatch} back to the pool after reseting inner - * values and releasing each of its internal {@link Metric}s as well. - * @param metricBatch the {@link MetricBatch} to put back to the pool - * @return whether the {@link MetricBatch} was put back into the pool - */ - public boolean releaseMetricBatch(MetricBatch metricBatch) { - metricBatch.setCommonTagsIsSet(false); - - List metrics = metricBatch.getMetrics(); - - if (metrics != null) { - for (Metric metric : metrics) { - releaseMetric(metric); - } - } - - metricBatch.setMetricsIsSet(false); - - return metricBatchPool.put(metricBatch); - } - - /** - * Puts the given {@link Metric} back to the pool after reseting its - * internal attributes. - * @param metric the {@link Metric} to put back to the pool - * @return whether the {@link Metric} was put back into the pool - */ - public boolean releaseMetric(Metric metric) { - metric.setNameIsSet(false); - - Set tags = metric.getTags(); - - if (tags != null) { - for (MetricTag tag : tags) { - tag.setTagNameIsSet(false); - tag.setTagValueIsSet(false); - metricTagPool.put(tag); - } - } - - metric.setTagsIsSet(false); - - return releaseShallowMetric(metric); - } - - /** - * Puts a tag-less {@link Metric} back to the pool as well as - * its {@link MetricValue}. - * @param metric the {@link Metric} to put back to the pool - * @return whether the {@link Metric} was put back into the pool - */ - public boolean releaseShallowMetric(Metric metric) { - metric.setTimestampIsSet(false); - - MetricValue metricValue = metric.getMetricValue(); - - if (metricValue == null) { - return metricPool.put(metric); - } - - releaseMetricValue(metricValue); - metric.unsetMetricValue(); - - return metricPool.put(metric); - } - - /** - * Puts a list of metrics back to the pool. - * @param metrics the list of metrics to put back to the pool - */ - public void releaseShallowMetrics(List metrics) { - for (Metric metric : metrics) { - releaseShallowMetric(metric); - } - } - - /** - * Puts the given {@link MetricValue} back to the pool after reseting its - * internal attributes and values. - * @param metricValue the {@link MetricValue} to put back to the pool - * @return whether the {@link MetricValue} was put back into the pool - */ - public boolean releaseMetricValue(MetricValue metricValue) { - if (metricValue.isSetCount()) { - metricValue.getCount().setI64ValueIsSet(false); - - countValuePool.put(metricValue.getCount()); - - metricValue.unsetCount(); - } else if (metricValue.isSetGauge()) { - metricValue.getGauge().setI64ValueIsSet(false); - metricValue.getGauge().setDValueIsSet(false); - - gaugeValuePool.put(metricValue.getGauge()); - - metricValue.unsetGauge(); - } else if (metricValue.isSetTimer()) { - metricValue.getTimer().setI64ValueIsSet(false); - metricValue.getTimer().setDValueIsSet(false); - - timerValuePool.put(metricValue.getTimer()); - - metricValue.unsetTimer(); - } - - return metricValuePool.put(metricValue); - } - - /** - * Puts the given {@link SizedMetric} back to the pool after reseting - * inner values. - * @param sizedMetric the {@link SizedMetric} to put back to the pool - * @return whether the {@link SizedMetric} was put back into the pool - */ - public boolean releaseSizedMetric(SizedMetric sizedMetric) { - sizedMetric.setMetric(null); - sizedMetric.setSize(0); - - return sizedMetricPool.put(sizedMetric); - } - - /** - * Puts the given {@link MetricTag} back to the pool after reseting - * inner values. - * @param metricTag the {@link MetricTag} to put back to the pool - * @return whether the {@link MetricTag} was put back into the pool - */ - public boolean releaseMetricTag(MetricTag metricTag) { - metricTag.setTagNameIsSet(false); - metricTag.setTagValueIsSet(false); - - return metricTagPool.put(metricTag); - } - - /** - * Puts the given {@link CountValue} back to the pool after reseting - * inner values. - * @param countValue the {@link CountValue} to put back to the pool - * @return whether the {@link CountValue} was put back into the pool - */ - public boolean releaseCountValue(CountValue countValue) { - countValue.setI64ValueIsSet(false); - - return countValuePool.put(countValue); - } - - /** - * Puts the given {@link GaugeValue} back to the pool after reseting - * inner values. - * @param gaugeValue the {@link GaugeValue} to put back to the pool - * @return whether the {@link GaugeValue} was put back into the pool - */ - public boolean releaseGaugeValue(GaugeValue gaugeValue) { - gaugeValue.setDValueIsSet(false); - gaugeValue.setI64ValueIsSet(false); - - return gaugeValuePool.put(gaugeValue); - } - - /** - * Puts the given {@link TimerValue} back to the pool after reseting - * inner values. - * @param timerValue the {@link TimerValue} to put back to the pool - * @return whether the {@link TimerValue} was put back into the pool - */ - public boolean releaseTimerValue(TimerValue timerValue) { - timerValue.setDValueIsSet(false); - timerValue.setI64ValueIsSet(false); - - return timerValuePool.put(timerValue); - } -} From 488c8911e481e17f551ce8e16b1c2fac78300676 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Sun, 24 Sep 2017 00:19:29 -0400 Subject: [PATCH 29/44] Use isOpen flag on M3Reporter --- .../java/com/uber/m3/tally/m3/M3Reporter.java | 127 ++++++++++-------- 1 file changed, 70 insertions(+), 57 deletions(-) diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 169d7ab..252a5a1 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -123,6 +123,9 @@ public class M3Reporter implements CachedStatsReporter, AutoCloseable { // order to wait for them to finish before closing private Phaser phaser = new Phaser(1); + private final Object isOpenLock = new Object(); + private boolean isOpen = true; + private TTransport transport; // Use inner Builder class to construct an M3Reporter @@ -374,30 +377,38 @@ private void flush(List metrics, int maxTries) { @Override public void close() { - // Important to use `shutdownNow` instead of `shutdown` to interrupt processor - // thread(s) or else they will block forever - executor.shutdownNow(); + synchronized (isOpenLock) { + if (!isOpen) { + return; + } - // Wait a maximum of `MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS` for all processors - // to return from flushing - try { - phaser.awaitAdvanceInterruptibly( - phaser.arrive(), - MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS, - TimeUnit.MILLISECONDS - ); - } catch (TimeoutException e) { - LOG.warn( - "M3Reporter closing before Processors complete after waiting timeout of %dms!", - MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS - ); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + // Important to use `shutdownNow` instead of `shutdown` to interrupt processor + // thread(s) or else they will block forever + executor.shutdownNow(); - LOG.warn("M3Reporter closing before Processors complete due to being interrupted!"); - } + // Wait a maximum of `MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS` for all processors + // to return from flushing + try { + phaser.awaitAdvanceInterruptibly( + phaser.arrive(), + MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS, + TimeUnit.MILLISECONDS + ); + } catch (TimeoutException e) { + LOG.warn( + "M3Reporter closing before Processors complete after waiting timeout of %dms!", + MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS + ); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + + LOG.warn("M3Reporter closing before Processors complete due to being interrupted!"); + } + + transport.close(); - transport.close(); + isOpen = false; + } } private Metric newMetric(String name, Map tags, MetricType metricType) { @@ -476,45 +487,47 @@ private void reportCopyMetric( long iValue, double dValue ) { - if (executor.isShutdown()) { - LOG.warn("Scheduler is shutdown already, so no metrics can be flushed"); - return; - } - - Metric metricCopy = new Metric(metric.getName()); - metricCopy.setTags(metric.getTags()); - // Unfortunately, there is no finer time resolution for Java 7 - metricCopy.setTimestamp(System.currentTimeMillis() * Duration.NANOS_PER_MILLI); - metricCopy.setMetricValue(new MetricValue()); + synchronized (isOpenLock) { + if (!isOpen) { + LOG.warn("Reporter already closed; no more metrics can be flushed"); + return; + } - switch (metricType) { - case COUNTER: - CountValue countValue = new CountValue(); - countValue.setI64Value(iValue); - metricCopy.getMetricValue().setCount(countValue); - break; - case GAUGE: - GaugeValue gaugeValue = new GaugeValue(); - gaugeValue.setDValue(dValue); - metricCopy.getMetricValue().setGauge(gaugeValue); - break; - case TIMER: - TimerValue timerValue = new TimerValue(); - timerValue.setI64Value(iValue); - metricCopy.getMetricValue().setTimer(timerValue); - break; - default: - throw new IllegalArgumentException("Unsupported metric type: " + metricType); - } + Metric metricCopy = new Metric(metric.getName()); + metricCopy.setTags(metric.getTags()); + // Unfortunately, there is no finer time resolution for Java 7 + metricCopy.setTimestamp(System.currentTimeMillis() * Duration.NANOS_PER_MILLI); + metricCopy.setMetricValue(new MetricValue()); + + switch (metricType) { + case COUNTER: + CountValue countValue = new CountValue(); + countValue.setI64Value(iValue); + metricCopy.getMetricValue().setCount(countValue); + break; + case GAUGE: + GaugeValue gaugeValue = new GaugeValue(); + gaugeValue.setDValue(dValue); + metricCopy.getMetricValue().setGauge(gaugeValue); + break; + case TIMER: + TimerValue timerValue = new TimerValue(); + timerValue.setI64Value(iValue); + metricCopy.getMetricValue().setTimer(timerValue); + break; + default: + throw new IllegalArgumentException("Unsupported metric type: " + metricType); + } - try { - metricQueue.put(new SizedMetric(metricCopy, size)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + try { + metricQueue.put(new SizedMetric(metricCopy, size)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); - LOG.warn(String.format("Interrupted while putting %s: %s", - metricType, - metricCopy.getName())); + LOG.warn(String.format("Interrupted while putting %s: %s", + metricType, + metricCopy.getName())); + } } } From 0bc003fbe4648d7ee992fc18e88ca59d2fd1aeee Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Tue, 26 Sep 2017 10:55:26 -0400 Subject: [PATCH 30/44] Make Duration toString similar to Go's; remove Cached* usage --- .../com/uber/m3/tally/AbstractBuckets.java | 11 +- .../com/uber/m3/tally/DurationBuckets.java | 6 +- .../java/com/uber/m3/tally/ValueBuckets.java | 6 +- .../main/java/com/uber/m3/util/Duration.java | 51 ++++- .../java/com/uber/m3/tally/m3/M3Reporter.java | 206 ++++++++++++++++-- 5 files changed, 250 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/com/uber/m3/tally/AbstractBuckets.java b/core/src/main/java/com/uber/m3/tally/AbstractBuckets.java index dd43be2..8841775 100644 --- a/core/src/main/java/com/uber/m3/tally/AbstractBuckets.java +++ b/core/src/main/java/com/uber/m3/tally/AbstractBuckets.java @@ -22,6 +22,7 @@ import com.uber.m3.util.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; @@ -35,7 +36,15 @@ public abstract class AbstractBuckets implements Buckets { protected List buckets; AbstractBuckets(T[] buckets) { - this.buckets = Arrays.asList(buckets); + if (buckets == null) { + this.buckets = new ArrayList<>(); + } else { + this.buckets = Arrays.asList(buckets); + } + } + + AbstractBuckets() { + this(null); } @Override diff --git a/core/src/main/java/com/uber/m3/tally/DurationBuckets.java b/core/src/main/java/com/uber/m3/tally/DurationBuckets.java index 23485eb..8068934 100644 --- a/core/src/main/java/com/uber/m3/tally/DurationBuckets.java +++ b/core/src/main/java/com/uber/m3/tally/DurationBuckets.java @@ -26,10 +26,14 @@ * {@link Buckets} implementation backed by {@link Duration}s. */ public class DurationBuckets extends AbstractBuckets { - DurationBuckets(Duration[] durations) { + public DurationBuckets(Duration[] durations) { super(durations); } + public DurationBuckets() { + super(); + } + @Override public Double[] asValues() { Double[] values = new Double[buckets.size()]; diff --git a/core/src/main/java/com/uber/m3/tally/ValueBuckets.java b/core/src/main/java/com/uber/m3/tally/ValueBuckets.java index e461b34..23acf9d 100644 --- a/core/src/main/java/com/uber/m3/tally/ValueBuckets.java +++ b/core/src/main/java/com/uber/m3/tally/ValueBuckets.java @@ -26,10 +26,14 @@ * {@link Buckets} implementation backed by {@code Double} values. */ public class ValueBuckets extends AbstractBuckets { - ValueBuckets(Double[] values) { + public ValueBuckets(Double[] values) { super(values); } + public ValueBuckets() { + super(); + } + @Override public Double[] asValues() { return buckets.toArray(new Double[buckets.size()]); diff --git a/core/src/main/java/com/uber/m3/util/Duration.java b/core/src/main/java/com/uber/m3/util/Duration.java index f1cb44c..fdd5e06 100644 --- a/core/src/main/java/com/uber/m3/util/Duration.java +++ b/core/src/main/java/com/uber/m3/util/Duration.java @@ -39,6 +39,11 @@ public class Duration implements Comparable { */ public static final Duration MAX_VALUE = new Duration(Long.MAX_VALUE); + /** + * The number of nanoseconds in a microsecond. + */ + public static final long NANOS_PER_MICRO = 1_000; + /** * The number of nanoseconds in a millisecond. */ @@ -193,13 +198,14 @@ public String toString() { boolean isNegative = nanos < 0; - long localNanos = Math.abs(nanos); + long nanosLocal = Math.abs(nanos); - long hours = localNanos / NANOS_PER_HOUR; - int minutes = (int) ((localNanos % NANOS_PER_HOUR) / NANOS_PER_MINUTE); - float seconds = (float) (localNanos % NANOS_PER_MINUTE) / (float) NANOS_PER_SECOND; + long hours = nanosLocal / NANOS_PER_HOUR; + int minutes = (int) ((nanosLocal % NANOS_PER_HOUR) / NANOS_PER_MINUTE); + long secondsInNanos = nanosLocal % NANOS_PER_MINUTE; + int nanoOffset = (int) (nanosLocal % NANOS_PER_SECOND); - StringBuilder buf = new StringBuilder(16); + StringBuilder buf = new StringBuilder(24); if (isNegative) { buf.append("-"); @@ -210,13 +216,44 @@ public String toString() { if (minutes != 0) { buf.append(minutes).append("m"); } - if (seconds != 0) { - buf.append(seconds).append("s"); + if (secondsInNanos / NANOS_PER_SECOND > 0) { + appendDurationSegment(buf, secondsInNanos, 9, "s"); + } else if (nanoOffset > 0) { + // If less than one second, print the highest unit that is >1 + if (nanoOffset / NANOS_PER_MILLI > 0) { + appendDurationSegment(buf, nanoOffset, 6, "ms"); + } else if (nanoOffset / NANOS_PER_MICRO > 0) { + appendDurationSegment(buf, nanoOffset % NANOS_PER_MILLI, 3, "µs"); + } else { + buf.append(nanoOffset).append("ns"); + } } return buf.toString(); } + private static void appendDurationSegment( + StringBuilder buf, + long durationPart, + int exponent, + String suffix + ) { + buf.append(durationPart); + + int decimalIdx = buf.length() - exponent; + + // Remove trailing 0s if they come after decimal + while (decimalIdx < buf.length() && buf.charAt(buf.length() - 1) == '0') { + buf.setLength(buf.length() - 1); + } + + if (decimalIdx < buf.length()) { + buf.insert(decimalIdx, '.'); + } + + buf.append(suffix); + } + @Override public boolean equals(Object other) { if (other == null) { diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 252a5a1..e755faf 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -31,6 +31,7 @@ import com.uber.m3.tally.CachedTimer; import com.uber.m3.tally.Capabilities; import com.uber.m3.tally.CapableOf; +import com.uber.m3.tally.StatsReporter; import com.uber.m3.tally.m3.thrift.TCalcTransport; import com.uber.m3.tally.m3.thrift.TUdpClient; import com.uber.m3.thrift.generated.CountValue; @@ -74,7 +75,7 @@ /** * An M3 implementation of a {@link CachedStatsReporter}. */ -public class M3Reporter implements CachedStatsReporter, AutoCloseable { +public class M3Reporter implements StatsReporter, AutoCloseable { public static final String SERVICE_TAG = "service"; public static final String ENV_TAG = "env"; public static final String HOST_TAG = "host"; @@ -148,7 +149,7 @@ private M3Reporter(Builder builder) { client = new M3.Client(protocolFactory.getProtocol(transport)); - Set commonTags = toMetricTagsToSet(builder.commonTags); + Set commonTags = toMetricTagSet(builder.commonTags); // Set and ensure required tags if (!builder.commonTags.containsKey(SERVICE_TAG)) { @@ -227,28 +228,12 @@ private void addAndRunProcessor() { executor.execute(new Processor()); } - @Override public CachedCounter allocateCounter(String name, Map tags) { Metric counter = newMetric(name, tags, MetricType.COUNTER); return new CachedMetric(counter, calculateSize(counter)); } - @Override - public CachedGauge allocateGauge(String name, Map tags) { - Metric gauge = newMetric(name, tags, MetricType.GAUGE); - - return new CachedMetric(gauge, calculateSize(gauge)); - } - - @Override - public CachedTimer allocateTimer(String name, Map tags) { - Metric timer = newMetric(name, tags, MetricType.TIMER); - - return new CachedMetric(timer, calculateSize(timer)); - } - - @Override public CachedHistogram allocateHistogram(String name, Map tags, Buckets buckets) { List cachedValueBuckets = new ArrayList<>(); List cachedDurationBuckets = new ArrayList<>(); @@ -379,6 +364,7 @@ private void flush(List metrics, int maxTries) { public void close() { synchronized (isOpenLock) { if (!isOpen) { + LOG.warn("M3Reporter already closed"); return; } @@ -413,7 +399,7 @@ public void close() { private Metric newMetric(String name, Map tags, MetricType metricType) { Metric metric = new Metric(name); - metric.setTags(toMetricTagsToSet(tags)); + metric.setTags(toMetricTagSet(tags)); metric.setTimestamp(Long.MAX_VALUE); MetricValue metricValue = new MetricValue(); @@ -443,7 +429,7 @@ private Metric newMetric(String name, Map tags, MetricType metri return metric; } - private Set toMetricTagsToSet(Map tags) { + private Set toMetricTagSet(Map tags) { Set metricTagSet = new HashSet<>(); if (tags == null) { @@ -544,6 +530,10 @@ private String valueBucketString(double bucketBound) { } private String durationBucketString(Duration bucketBound) { + if (Duration.ZERO.equals(bucketBound)) { + return "0"; + } + if (Duration.MAX_VALUE.equals(bucketBound)) { return "infinity"; } @@ -555,6 +545,182 @@ private String durationBucketString(Duration bucketBound) { return bucketBound.toString(); } + @Override + public void reportCounter(String name, Map tags, long value) { + synchronized (isOpenLock) { + if (!isOpen) { + LOG.error("Reporter already closed; no more metrics can be flushed"); + return; + } + + reportCounterInternal(name, tags, value); + } + } + + @Override + public void reportGauge(String name, Map tags, double value) { + synchronized (isOpenLock) { + if (!isOpen) { + LOG.error("Reporter already closed; no more metrics can be flushed"); + return; + } + + GaugeValue gaugeValue = new GaugeValue(); + gaugeValue.setDValue(value); + + MetricValue metricValue = new MetricValue(); + metricValue.setGauge(gaugeValue); + + Metric metric = newMetric(name, tags, metricValue); + + queueSizedMetric(new SizedMetric(metric, calculateSize(metric))); + } + } + + @Override + public void reportTimer(String name, Map tags, Duration interval) { + synchronized (isOpenLock) { + if (!isOpen) { + LOG.error("Reporter already closed; no more metrics can be flushed"); + return; + } + + TimerValue timerValue = new TimerValue(); + timerValue.setI64Value(interval.getNanos()); + + MetricValue metricValue = new MetricValue(); + metricValue.setTimer(timerValue); + + Metric metric = newMetric(name, tags, metricValue); + + queueSizedMetric(new SizedMetric(metric, calculateSize(metric))); + } + } + + @Override + public void reportHistogramValueSamples( + String name, + Map tags, + Buckets buckets, + double bucketLowerBound, + double bucketUpperBound, + long samples + ) { + synchronized (isOpenLock) { + if (!isOpen) { + LOG.error("Reporter already closed; no more metrics can be flushed"); + return; + } + // Append histogram bucket-specific tags + int bucketIdLen = String.valueOf(buckets.size()).length(); + bucketIdLen = Math.max(bucketIdLen, MIN_METRIC_BUCKET_ID_TAG_LENGTH); + + String bucketIdFmt = String.format("%%0%sd", bucketIdLen); + + BucketPair[] bucketPairs = BucketPairImpl.bucketPairs(buckets); + + for (int i = 0; i < bucketPairs.length; i++) { + // Look for the first pair with an upper bound greater than or equal + // to the given upper bound. + if (bucketPairs[i].upperBoundValue() > bucketUpperBound) { + String idTagValue = String.format(bucketIdFmt, i); + + tags.put(bucketIdTagName, idTagValue); + tags.put(bucketTagName, + String.format("%s-%s", + valueBucketString(bucketPairs[i].lowerBoundValue()), + valueBucketString(bucketPairs[i].upperBoundValue()) + ) + ); + + break; + } + } + + reportCounterInternal(name, tags, samples); + } + } + + @Override + public void reportHistogramDurationSamples( + String name, + Map tags, + Buckets buckets, + Duration bucketLowerBound, + Duration bucketUpperBound, + long samples + ) { + synchronized (isOpenLock) { + if (!isOpen) { + LOG.error("Reporter already closed; no more metrics can be flushed"); + return; + } + // Append histogram bucket-specific tags + int bucketIdLen = String.valueOf(buckets.size()).length(); + bucketIdLen = Math.max(bucketIdLen, MIN_METRIC_BUCKET_ID_TAG_LENGTH); + + String bucketIdFmt = String.format("%%0%sd", bucketIdLen); + + BucketPair[] bucketPairs = BucketPairImpl.bucketPairs(buckets); + + if (tags == null) { + tags = new HashMap<>(2, 1); + } + + for (int i = 0; i < bucketPairs.length; i++) { + // Look for the first pair with an upper bound greater than or equal + // to the given upper bound. + if (bucketPairs[i].upperBoundDuration().compareTo(bucketUpperBound) >= 0) { + String idTagValue = String.format(bucketIdFmt, i); + + tags.put(bucketIdTagName, idTagValue); + tags.put(bucketTagName, + String.format("%s-%s", + durationBucketString(bucketPairs[i].lowerBoundDuration()), + durationBucketString(bucketPairs[i].upperBoundDuration()) + ) + ); + + break; + } + } + + reportCounterInternal(name, tags, samples); + } + } + + // Relies on the calling function to provide guarantees of the reporter being open + private void reportCounterInternal(String name, Map tags, long value) { + CountValue countValue = new CountValue(); + countValue.setI64Value(value); + + MetricValue metricValue = new MetricValue(); + metricValue.setCount(countValue); + + Metric metric = newMetric(name, tags, metricValue); + + queueSizedMetric(new SizedMetric(metric, calculateSize(metric))); + } + + private Metric newMetric(String name, Map tags, MetricValue metricValue) { + Metric metric = new Metric(name); + metric.setTags(toMetricTagSet(tags)); + metric.setTimestamp(System.currentTimeMillis() * Duration.NANOS_PER_MILLI); + metric.setMetricValue(metricValue); + + return metric; + } + + private void queueSizedMetric(SizedMetric sizedMetric) { + try { + metricQueue.put(sizedMetric); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + + LOG.warn(String.format("Interrupted queueing metric: %s", sizedMetric.getMetric().getName())); + } + } + private class Processor implements Runnable { List metrics = new ArrayList<>(freeBytes / 10); int metricsSize = 0; From 0376dc730afb81436dccefdcf885efc992e4c0f0 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Tue, 26 Sep 2017 11:47:56 -0400 Subject: [PATCH 31/44] Remove dead ObjectPool and Cached* code --- .../java/com/uber/m3/tally/CachedCounter.java | 32 --- .../java/com/uber/m3/tally/CachedGauge.java | 32 --- .../com/uber/m3/tally/CachedHistogram.java | 44 ---- .../uber/m3/tally/CachedHistogramBucket.java | 32 --- .../uber/m3/tally/CachedStatsReporter.java | 37 ---- .../java/com/uber/m3/tally/CachedTimer.java | 34 --- .../java/com/uber/m3/tally/CounterImpl.java | 19 -- .../java/com/uber/m3/tally/GaugeImpl.java | 15 -- .../java/com/uber/m3/tally/HistogramImpl.java | 46 +---- .../java/com/uber/m3/tally/ScopeBuilder.java | 18 -- .../java/com/uber/m3/tally/ScopeImpl.java | 76 +------ .../java/com/uber/m3/tally/TimerImpl.java | 14 +- .../java/com/uber/m3/util/DurationTest.java | 8 +- .../java/com/uber/m3/tally/m3/M3Reporter.java | 193 +----------------- .../main/java/com/uber/m3/util/Construct.java | 33 --- .../java/com/uber/m3/util/ObjectPool.java | 94 --------- .../java/com/uber/m3/util/ObjectPoolTest.java | 97 --------- 17 files changed, 19 insertions(+), 805 deletions(-) delete mode 100644 core/src/main/java/com/uber/m3/tally/CachedCounter.java delete mode 100644 core/src/main/java/com/uber/m3/tally/CachedGauge.java delete mode 100644 core/src/main/java/com/uber/m3/tally/CachedHistogram.java delete mode 100644 core/src/main/java/com/uber/m3/tally/CachedHistogramBucket.java delete mode 100644 core/src/main/java/com/uber/m3/tally/CachedStatsReporter.java delete mode 100644 core/src/main/java/com/uber/m3/tally/CachedTimer.java delete mode 100644 m3/src/main/java/com/uber/m3/util/Construct.java delete mode 100644 m3/src/main/java/com/uber/m3/util/ObjectPool.java delete mode 100644 m3/src/test/java/com/uber/m3/util/ObjectPoolTest.java diff --git a/core/src/main/java/com/uber/m3/tally/CachedCounter.java b/core/src/main/java/com/uber/m3/tally/CachedCounter.java deleted file mode 100644 index 1b78018..0000000 --- a/core/src/main/java/com/uber/m3/tally/CachedCounter.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package com.uber.m3.tally; - -/** - * A cached count for reporting an individual {@link Counter}. - */ -public interface CachedCounter { - /** - * Reports a count for a {@link Counter}. - * @param value value to report - */ - void reportCount(long value); -} diff --git a/core/src/main/java/com/uber/m3/tally/CachedGauge.java b/core/src/main/java/com/uber/m3/tally/CachedGauge.java deleted file mode 100644 index f4e6a50..0000000 --- a/core/src/main/java/com/uber/m3/tally/CachedGauge.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package com.uber.m3.tally; - -/** - * A cached gauge for reporting an individual {@link Gauge}. - */ -public interface CachedGauge { - /** - * Reports a value for a {@link Gauge}. - * @param value value to report - */ - void reportGauge(double value); -} diff --git a/core/src/main/java/com/uber/m3/tally/CachedHistogram.java b/core/src/main/java/com/uber/m3/tally/CachedHistogram.java deleted file mode 100644 index 4d1d780..0000000 --- a/core/src/main/java/com/uber/m3/tally/CachedHistogram.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package com.uber.m3.tally; - -import com.uber.m3.util.Duration; - -/** - * Interface for reporting cached histogram samples to buckets. - */ -public interface CachedHistogram { - /** - * Returns a {@link CachedHistogramBucket} with the specificed bounds. - * @param bucketLowerBound the {@link CachedHistogramBucket}'s lower bound - * @param bucketUpperBound the {@link CachedHistogramBucket}'s upper bound - * @return a {@link CachedHistogramBucket} with the specificed bounds - */ - CachedHistogramBucket valueBucket(double bucketLowerBound, double bucketUpperBound); - - /** - * Returns a {@link CachedHistogramBucket} with the specificed bounds. - * @param bucketLowerBound the {@link CachedHistogramBucket}'s lower bound {@link Duration} - * @param bucketUpperBound the {@link CachedHistogramBucket}'s upper bound {@link Duration} - * @return a {@link CachedHistogramBucket} with the specificed bounds - */ - CachedHistogramBucket durationBucket(Duration bucketLowerBound, Duration bucketUpperBound); -} diff --git a/core/src/main/java/com/uber/m3/tally/CachedHistogramBucket.java b/core/src/main/java/com/uber/m3/tally/CachedHistogramBucket.java deleted file mode 100644 index 060d984..0000000 --- a/core/src/main/java/com/uber/m3/tally/CachedHistogramBucket.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package com.uber.m3.tally; - -/** - * Interface for reporting histogram samples to a specific bucket. - */ -public interface CachedHistogramBucket { - /** - * Report a sample to a specific bucket. - * @param value value to report - */ - void reportSamples(long value); -} diff --git a/core/src/main/java/com/uber/m3/tally/CachedStatsReporter.java b/core/src/main/java/com/uber/m3/tally/CachedStatsReporter.java deleted file mode 100644 index fcc118b..0000000 --- a/core/src/main/java/com/uber/m3/tally/CachedStatsReporter.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package com.uber.m3.tally; - -import java.util.Map; - -/** - * A backend for {@link Scope}s that pre allocates all metrics. - * This results in a more performant reporter. - */ -public interface CachedStatsReporter extends BaseStatsReporter { - CachedCounter allocateCounter(String name, Map tags); - - CachedGauge allocateGauge(String name, Map tags); - - CachedTimer allocateTimer(String name, Map tags); - - CachedHistogram allocateHistogram(String name, Map tags, Buckets buckets); -} diff --git a/core/src/main/java/com/uber/m3/tally/CachedTimer.java b/core/src/main/java/com/uber/m3/tally/CachedTimer.java deleted file mode 100644 index 8dd4f79..0000000 --- a/core/src/main/java/com/uber/m3/tally/CachedTimer.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package com.uber.m3.tally; - -import com.uber.m3.util.Duration; - -/** - * A cached timer for reporting an individual {@link Timer}. - */ -public interface CachedTimer { - /** - * Reports a {@link Duration} for a {@link Timer}. - * @param interval interval to report - */ - void reportTimer(Duration interval); -} diff --git a/core/src/main/java/com/uber/m3/tally/CounterImpl.java b/core/src/main/java/com/uber/m3/tally/CounterImpl.java index ad6a098..bb56e96 100644 --- a/core/src/main/java/com/uber/m3/tally/CounterImpl.java +++ b/core/src/main/java/com/uber/m3/tally/CounterImpl.java @@ -30,15 +30,6 @@ class CounterImpl implements Counter { private AtomicLong prev = new AtomicLong(0); private AtomicLong curr = new AtomicLong(0); - private CachedCounter cachedCount; - - CounterImpl(CachedCounter cachedCount) { - this.cachedCount = cachedCount; - } - - CounterImpl() { - this(null); - } @Override public void inc(long delta) { @@ -68,16 +59,6 @@ void report(String name, ImmutableMap tags, StatsReporter report reporter.reportCounter(name, tags, delta); } - void cachedReport() { - long delta = value(); - - if (delta == 0) { - return; - } - - cachedCount.reportCount(delta); - } - long snapshot() { return curr.get() - prev.get(); } diff --git a/core/src/main/java/com/uber/m3/tally/GaugeImpl.java b/core/src/main/java/com/uber/m3/tally/GaugeImpl.java index 9decdcc..e4b75de 100644 --- a/core/src/main/java/com/uber/m3/tally/GaugeImpl.java +++ b/core/src/main/java/com/uber/m3/tally/GaugeImpl.java @@ -31,15 +31,6 @@ class GaugeImpl implements Gauge { private AtomicBoolean updated = new AtomicBoolean(false); private AtomicLong curr = new AtomicLong(0); - private CachedGauge cachedGauge; - - GaugeImpl(CachedGauge cachedGauge) { - this.cachedGauge = cachedGauge; - } - - GaugeImpl() { - this(null); - } @Override public void update(double value) { @@ -57,12 +48,6 @@ void report(String name, ImmutableMap tags, StatsReporter report } } - void cachedReport() { - if (updated.getAndSet(false)) { - cachedGauge.reportGauge(value()); - } - } - double snapshot() { return value(); } diff --git a/core/src/main/java/com/uber/m3/tally/HistogramImpl.java b/core/src/main/java/com/uber/m3/tally/HistogramImpl.java index 9a59108..cf833c6 100644 --- a/core/src/main/java/com/uber/m3/tally/HistogramImpl.java +++ b/core/src/main/java/com/uber/m3/tally/HistogramImpl.java @@ -36,7 +36,6 @@ class HistogramImpl implements Histogram { private Type type; private String name; private ImmutableMap tags; - private StatsReporter reporter; private Buckets specification; private List buckets; private List lookupByValue; @@ -46,8 +45,7 @@ class HistogramImpl implements Histogram { String name, ImmutableMap tags, StatsReporter reporter, - Buckets buckets, - CachedHistogram cachedHistogram + Buckets buckets ) { if (buckets instanceof DurationBuckets) { type = Type.DURATION; @@ -60,7 +58,6 @@ class HistogramImpl implements Histogram { this.name = name; this.tags = tags; - this.reporter = reporter; specification = buckets; this.buckets = new ArrayList<>(pairsLen); lookupByValue = new ArrayList<>(pairsLen); @@ -71,21 +68,11 @@ class HistogramImpl implements Histogram { pair.lowerBoundValue(), pair.upperBoundValue(), pair.lowerBoundDuration(), - pair.upperBoundDuration(), - cachedHistogram + pair.upperBoundDuration() )); } } - HistogramImpl( - String name, - ImmutableMap tags, - StatsReporter reporter, - Buckets buckets - ) { - this(name, tags, reporter, buckets, null); - } - private void addBucket(HistogramBucket bucket) { buckets.add(bucket); lookupByValue.add(bucket.valueUpperBound); @@ -154,25 +141,6 @@ void report(String name, ImmutableMap tags, StatsReporter report } } - void cachedReport() { - for (HistogramBucket bucket : buckets) { - long samples = bucket.samples.value(); - - if (samples == 0) { - continue; - } - - switch (type) { - case VALUE: - bucket.cachedValueBucket.reportSamples(samples); - break; - case DURATION: - bucket.cachedDurationBucket.reportSamples(samples); - break; - } - } - } - Map snapshotValues() { if (type == Type.DURATION) { return null; @@ -207,26 +175,18 @@ class HistogramBucket { double valueUpperBound; Duration durationLowerBound; Duration durationUpperBound; - CachedHistogramBucket cachedValueBucket; - CachedHistogramBucket cachedDurationBucket; HistogramBucket( double valueLowerBound, double valueUpperBound, Duration durationLowerBound, - Duration durationUpperBound, - CachedHistogram cachedHistogram + Duration durationUpperBound ) { samples = new CounterImpl(); this.valueLowerBound = valueLowerBound; this.valueUpperBound = valueUpperBound; this.durationLowerBound = durationLowerBound; this.durationUpperBound = durationUpperBound; - - if (cachedHistogram != null) { - cachedValueBucket = cachedHistogram.valueBucket(valueLowerBound, valueUpperBound); - cachedDurationBucket = cachedHistogram.durationBucket(durationLowerBound, durationUpperBound); - } } } diff --git a/core/src/main/java/com/uber/m3/tally/ScopeBuilder.java b/core/src/main/java/com/uber/m3/tally/ScopeBuilder.java index a77bb92..9e0eefd 100644 --- a/core/src/main/java/com/uber/m3/tally/ScopeBuilder.java +++ b/core/src/main/java/com/uber/m3/tally/ScopeBuilder.java @@ -51,8 +51,6 @@ public class ScopeBuilder { }); StatsReporter reporter = null; - CachedStatsReporter cachedReporter = null; - BaseStatsReporter baseReporter = null; String prefix = ""; String separator = DEFAULT_SEPARATOR; ImmutableMap tags; @@ -78,16 +76,6 @@ public ScopeBuilder reporter(StatsReporter reporter) { return this; } - /** - * Update the cachedReporter - * @param cachedReporter value to update to - * @return Builder with new param updated - */ - public ScopeBuilder cachedReporter(CachedStatsReporter cachedReporter) { - this.cachedReporter = cachedReporter; - return this; - } - /** * Update the prefix * @param prefix value to update to @@ -143,12 +131,6 @@ public ScopeBuilder defaultBuckets(Buckets defaultBuckets) { // Private build method - clients should rely on `reportEvery` to create root scopes, and // a root scope's `tagged` and `subScope` functions to create subscopes. ScopeImpl build() { - if (reporter != null) { - baseReporter = reporter; - } else if (cachedReporter != null) { - baseReporter = cachedReporter; - } - return new ScopeImpl(scheduler, registry, this); } diff --git a/core/src/main/java/com/uber/m3/tally/ScopeImpl.java b/core/src/main/java/com/uber/m3/tally/ScopeImpl.java index 02179e0..e40f01c 100644 --- a/core/src/main/java/com/uber/m3/tally/ScopeImpl.java +++ b/core/src/main/java/com/uber/m3/tally/ScopeImpl.java @@ -34,8 +34,6 @@ */ class ScopeImpl implements Scope { private StatsReporter reporter; - private CachedStatsReporter cachedReporter; - private BaseStatsReporter baseReporter; private String prefix; private String separator; private ImmutableMap tags; @@ -59,8 +57,6 @@ class ScopeImpl implements Scope { this.registry = registry; this.reporter = builder.reporter; - this.cachedReporter = builder.cachedReporter; - this.baseReporter = builder.baseReporter; this.prefix = builder.prefix; this.separator = builder.separator; this.tags = builder.tags; @@ -77,13 +73,7 @@ public Counter counter(String name) { synchronized (counters) { if (!counters.containsKey(name)) { - CachedCounter cachedCounter = null; - - if (cachedReporter != null) { - cachedCounter = cachedReporter.allocateCounter(fullyQualifiedName(name), tags); - } - - counters.put(name, new CounterImpl(cachedCounter)); + counters.put(name, new CounterImpl()); } counter = counters.get(name); @@ -102,13 +92,7 @@ public Gauge gauge(String name) { synchronized (gauges) { if (!gauges.containsKey(name)) { - CachedGauge cachedGauge = null; - - if (cachedReporter != null) { - cachedGauge = cachedReporter.allocateGauge(fullyQualifiedName(name), tags); - } - - gauges.put(name, new GaugeImpl(cachedGauge)); + gauges.put(name, new GaugeImpl()); } gauge = gauges.get(name); @@ -127,15 +111,7 @@ public Timer timer(String name) { synchronized (timers) { if (!timers.containsKey(name)) { - CachedTimer cachedTimer = null; - - String fullName = fullyQualifiedName(name); - - if (cachedReporter != null) { - cachedTimer = cachedReporter.allocateTimer(fullName, tags); - } - - timers.put(name, new TimerImpl(fullName, tags, reporter, cachedTimer)); + timers.put(name, new TimerImpl(fullyQualifiedName(name), tags, reporter)); } timer = timers.get(name); @@ -158,15 +134,7 @@ public Histogram histogram(String name, Buckets buckets) { synchronized (histograms) { if (!histograms.containsKey(name)) { - CachedHistogram cachedHistogram = null; - - String fullName = fullyQualifiedName(name); - - if (cachedReporter != null) { - cachedHistogram = cachedReporter.allocateHistogram(fullName, tags, buckets); - } - - histograms.put(name, new HistogramImpl(fullName, tags, reporter, buckets, cachedHistogram)); + histograms.put(name, new HistogramImpl(fullyQualifiedName(name), tags, reporter, buckets)); } histogram = histograms.get(name); @@ -187,8 +155,8 @@ public Scope subScope(String name) { @Override public Capabilities capabilities() { - if (baseReporter != null) { - return baseReporter.capabilities(); + if (reporter != null) { + return reporter.capabilities(); } return CapableOf.NONE; @@ -205,7 +173,7 @@ public void close() { reportLoopIteration(); // Now that all metrics should be known to the reporter, close the reporter - baseReporter.close(); + reporter.close(); } /** @@ -231,29 +199,6 @@ void report(StatsReporter reporter) { reporter.flush(); } - /** - * Reports using the specified cachedReporter. - * @param cachedReporter the cachedReporter to report - */ - void cachedReport(CachedStatsReporter cachedReporter) { - for (CounterImpl counter : counters.values()) { - counter.cachedReport(); - } - - for (GaugeImpl gauge : gauges.values()) { - gauge.cachedReport(); - } - - // No operations on timers required here; they report directly to the StatsReporter - // i.e. they are not buffered - - for (HistogramImpl histogram : histograms.values()) { - histogram.cachedReport(); - } - - cachedReporter.flush(); - } - // Serializes a map to generate a key for a prefix/map combination // Non-generic EMPTY ImmutableMap will never contain any elements @SuppressWarnings("unchecked") @@ -392,7 +337,6 @@ private Scope subScopeHelper(String prefix, Map tags) { key, new ScopeBuilder(scheduler, registry) .reporter(reporter) - .cachedReporter(cachedReporter) .prefix(prefix) .separator(separator) .tags(mergedTags) @@ -416,12 +360,6 @@ private void reportLoopIteration() { subscope.report(reporter); } } - - if (cachedReporter != null) { - for (ScopeImpl subscope : subscopes) { - subscope.cachedReport(cachedReporter); - } - } } class ReportLoop implements Runnable { diff --git a/core/src/main/java/com/uber/m3/tally/TimerImpl.java b/core/src/main/java/com/uber/m3/tally/TimerImpl.java index 10261ce..b31be1c 100644 --- a/core/src/main/java/com/uber/m3/tally/TimerImpl.java +++ b/core/src/main/java/com/uber/m3/tally/TimerImpl.java @@ -37,13 +37,11 @@ class TimerImpl implements Timer, StopwatchRecorder { private String name; private ImmutableMap tags; private StatsReporter reporter; - private CachedTimer cachedTimer; private Values unreported = new Values(); - TimerImpl(String name, ImmutableMap tags, StatsReporter reporter, CachedTimer cachedTimer) { + TimerImpl(String name, ImmutableMap tags, StatsReporter reporter) { this.name = name; this.tags = tags; - this.cachedTimer = cachedTimer; if (reporter == null) { this.reporter = new NoReporterSink(); @@ -52,17 +50,9 @@ class TimerImpl implements Timer, StopwatchRecorder { } } - TimerImpl(String name, ImmutableMap tags, StatsReporter reporter) { - this(name, tags, reporter, null); - } - @Override public void record(Duration interval) { - if (cachedTimer != null) { - cachedTimer.reportTimer(interval); - } else { - reporter.reportTimer(name, tags, interval); - } + reporter.reportTimer(name, tags, interval); } @Override diff --git a/core/src/test/java/com/uber/m3/util/DurationTest.java b/core/src/test/java/com/uber/m3/util/DurationTest.java index f9875a6..d4eb43f 100644 --- a/core/src/test/java/com/uber/m3/util/DurationTest.java +++ b/core/src/test/java/com/uber/m3/util/DurationTest.java @@ -157,8 +157,12 @@ public void toStringTest() { assertEquals("1.234s", Duration.ofMillis(1234).toString()); assertEquals("2m3.456789s", Duration.ofSeconds(123.456789).toString()); assertEquals("26h30m", Duration.ofHours(26.5).toString()); - assertEquals("1.0E-9s", Duration.ofNanos(1).toString()); - assertEquals("1.2345678s", Duration.ofNanos(1234567800).toString()); + assertEquals("1ns", Duration.ofNanos(1).toString()); + assertEquals("1.03µs", Duration.ofNanos(1_030).toString()); + assertEquals("1.23456789s", Duration.ofNanos(1_234_567_890).toString()); + assertEquals("50ms", Duration.ofMillis(50).toString()); + assertEquals("200µs", Duration.ofNanos(200_000).toString()); + assertEquals("99.009µs", Duration.ofNanos(99_009).toString()); assertEquals("-1.25s", Duration.ofSeconds(-1.25).toString()); assertEquals("-34h17m36.7s", Duration.ofSeconds(-123456.7).toString()); diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index e755faf..0cb7924 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -23,12 +23,6 @@ import com.uber.m3.tally.BucketPair; import com.uber.m3.tally.BucketPairImpl; import com.uber.m3.tally.Buckets; -import com.uber.m3.tally.CachedCounter; -import com.uber.m3.tally.CachedGauge; -import com.uber.m3.tally.CachedHistogram; -import com.uber.m3.tally.CachedHistogramBucket; -import com.uber.m3.tally.CachedStatsReporter; -import com.uber.m3.tally.CachedTimer; import com.uber.m3.tally.Capabilities; import com.uber.m3.tally.CapableOf; import com.uber.m3.tally.StatsReporter; @@ -73,7 +67,7 @@ import org.slf4j.LoggerFactory; /** - * An M3 implementation of a {@link CachedStatsReporter}. + * An M3 implementation of a {@link StatsReporter}. */ public class M3Reporter implements StatsReporter, AutoCloseable { public static final String SERVICE_TAG = "service"; @@ -228,61 +222,6 @@ private void addAndRunProcessor() { executor.execute(new Processor()); } - public CachedCounter allocateCounter(String name, Map tags) { - Metric counter = newMetric(name, tags, MetricType.COUNTER); - - return new CachedMetric(counter, calculateSize(counter)); - } - - public CachedHistogram allocateHistogram(String name, Map tags, Buckets buckets) { - List cachedValueBuckets = new ArrayList<>(); - List cachedDurationBuckets = new ArrayList<>(); - - int bucketIdLen = String.valueOf(buckets.size()).length(); - bucketIdLen = Math.max(bucketIdLen, MIN_METRIC_BUCKET_ID_TAG_LENGTH); - - String bucketIdFmt = String.format("%%0%sd", bucketIdLen); - - BucketPair[] bucketPairs = BucketPairImpl.bucketPairs(buckets); - - for (int i = 0; i < bucketPairs.length; i++) { - Map valueTags = new HashMap<>(tags); - Map durationTags = new HashMap<>(tags); - - String idTagValue = String.format(bucketIdFmt, i); - - valueTags.put(bucketIdTagName, idTagValue); - valueTags.put(bucketTagName, String.format("%s-%s", - valueBucketString(bucketPairs[i].lowerBoundValue()), - valueBucketString(bucketPairs[i].upperBoundValue()))); - - cachedValueBuckets.add(new CachedHistogramBucketImpl( - bucketPairs[i].upperBoundValue(), - bucketPairs[i].upperBoundDuration(), - (CachedMetric) allocateCounter(name, valueTags) - )); - - durationTags.put(bucketIdTagName, idTagValue); - durationTags.put(bucketTagName, String.format("%s-%s", - durationBucketString(bucketPairs[i].lowerBoundDuration()), - durationBucketString(bucketPairs[i].upperBoundDuration()))); - - cachedDurationBuckets.add(new CachedHistogramBucketImpl( - bucketPairs[i].upperBoundValue(), - bucketPairs[i].upperBoundDuration(), - (CachedMetric) allocateCounter(name, durationTags) - )); - } - - return new CachedHistogramImpl( - name, - tags, - buckets, - cachedValueBuckets, - cachedDurationBuckets - ); - } - @Override public Capabilities capabilities() { return CapableOf.REPORTING_TAGGING; @@ -397,38 +336,6 @@ public void close() { } } - private Metric newMetric(String name, Map tags, MetricType metricType) { - Metric metric = new Metric(name); - metric.setTags(toMetricTagSet(tags)); - metric.setTimestamp(Long.MAX_VALUE); - - MetricValue metricValue = new MetricValue(); - - switch (metricType) { - case COUNTER: - CountValue countValue = new CountValue(); - countValue.setI64Value(Long.MAX_VALUE); - metricValue.setCount(countValue); - break; - case GAUGE: - GaugeValue gaugeValue = new GaugeValue(); - gaugeValue.setDValue(Double.MAX_VALUE); - metricValue.setGauge(gaugeValue); - break; - case TIMER: - TimerValue timerValue = new TimerValue(); - timerValue.setI64Value(Long.MAX_VALUE); - metricValue.setTimer(timerValue); - break; - default: - throw new IllegalArgumentException("Unsupported metric type: " + metricType); - } - - metric.setMetricValue(metricValue); - - return metric; - } - private Set toMetricTagSet(Map tags) { Set metricTagSet = new HashSet<>(); @@ -777,104 +684,6 @@ private void flushRemainingMetrics() { } } - private class CachedMetric implements CachedCounter, CachedGauge, CachedTimer, CachedHistogramBucket { - Metric metric; - int size; - - CachedMetric(Metric metric, int size) { - this.metric = metric; - this.size = size; - } - - @Override - public void reportCount(long value) { - reportCopyMetric(metric, size, MetricType.COUNTER, value, 0); - } - - @Override - public void reportGauge(double value) { - reportCopyMetric(metric, size, MetricType.GAUGE, 0, value); - } - - @Override - public void reportTimer(Duration interval) { - reportCopyMetric(metric, size, MetricType.TIMER, interval.getNanos(), 0); - } - - @Override - public void reportSamples(long value) { - reportCopyMetric(metric, size, MetricType.COUNTER, value, 0); - } - } - - private class CachedHistogramImpl implements CachedHistogram { - String name; - Map tags; - Buckets buckets; - List cachedValueBuckets; - List cachedDurationBuckets; - - CachedHistogramImpl( - String name, - Map tags, - Buckets buckets, - List cachedValueBuckets, - List cachedDurationBuckets - ) { - this.name = name; - this.tags = tags; - this.buckets = buckets; - this.cachedValueBuckets = cachedValueBuckets; - this.cachedDurationBuckets = cachedDurationBuckets; - } - - @Override - public CachedHistogramBucket valueBucket(double bucketLowerBound, double bucketUpperBound) { - for (CachedHistogramBucketImpl bucket : cachedValueBuckets) { - if (bucket.getValueUpperBound() >= bucketUpperBound) { - return bucket.getMetric(); - } - } - - return null; - } - - @Override - public CachedHistogramBucket durationBucket(Duration bucketLowerBound, Duration bucketUpperBound) { - for (CachedHistogramBucketImpl bucket : cachedValueBuckets) { - if (bucket.getDurationUpperBound().compareTo(bucketUpperBound) >= 0) { - return bucket.getMetric(); - } - } - - return null; - } - } - - private class CachedHistogramBucketImpl { - private double valueUpperBound; - private Duration durationUpperBound; - private CachedHistogramBucket metric; - - CachedHistogramBucketImpl(double valueUpperBound, Duration durationUpperBound, CachedMetric metric) { - this.valueUpperBound = valueUpperBound; - this.durationUpperBound = durationUpperBound; - this.metric = metric; - } - - double getValueUpperBound() { - return valueUpperBound; - } - - Duration getDurationUpperBound() { - return durationUpperBound; - } - - CachedHistogramBucket getMetric() { - return metric; - } - } - /** * Builder pattern to construct an {@link M3Reporter}. */ diff --git a/m3/src/main/java/com/uber/m3/util/Construct.java b/m3/src/main/java/com/uber/m3/util/Construct.java deleted file mode 100644 index 5be3290..0000000 --- a/m3/src/main/java/com/uber/m3/util/Construct.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package com.uber.m3.util; - -/** - * A wrapper around a constructor for the type E. - * @param the type this constructor instantiates - */ -public interface Construct { - /** - * Returns a new instance of E. - * @return a new instance of E - */ - E instance(); -} diff --git a/m3/src/main/java/com/uber/m3/util/ObjectPool.java b/m3/src/main/java/com/uber/m3/util/ObjectPool.java deleted file mode 100644 index 17b44d1..0000000 --- a/m3/src/main/java/com/uber/m3/util/ObjectPool.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package com.uber.m3.util; - -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; - -/** - * An object pool used to reduce the number of objects required to be allocated - * and subsequently garbage-collected. - * @param the type of object this is a pool of - */ -public class ObjectPool { - private BlockingQueue objects; - private Construct construct; - - /** - * Initializes an {@link ObjectPool} and preallocates objects in it up to capacity. - * Defaults fair-ness (FIFO for requests) to false. This improves throughput - * with the slight chance of starvation. - * @param capacity capacity of this pool - * @param construct the constructor of E - */ - public ObjectPool(int capacity, Construct construct) { - this(capacity, construct, false); - } - - /** - * Initializes an {@link ObjectPool} and preallocates objects in it up to capacity. - * @param capacity capacity of this pool - * @param construct the constructor of E - * @param fair whether the pool is fair (FIFO for requests) - */ - public ObjectPool(int capacity, Construct construct, boolean fair) { - objects = new ArrayBlockingQueue<>(capacity, fair); - - this.construct = construct; - - for (int i = 0; i < capacity; i++) { - objects.add(construct.instance()); - } - } - - /** - * Gets an object from the pool. If the pool is empty, it will create and return - * a new instance instead of blocking until an object is available. - * @return an object from the pool if available. Otherwise, a new instance of the object. - */ - public E get() { - if (objects.isEmpty()) { - // Rather than block, return a newly created instance instead - return construct.instance(); - } - - try { - return objects.take(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - - return construct.instance(); - } - } - - /** - * Puts an object back into the pool. - * @param element the element to put back - * @return whether the element got put back or not - */ - public boolean put(E element) { - // Use `offer` instead of `put` (which blocks until there is space), because a full - // ObjectPool is a good state to be in already. We can get into this state because the - // get() function will create a new instance instead of waiting for an object - // to be in the pool. - return objects.offer(element); - } -} diff --git a/m3/src/test/java/com/uber/m3/util/ObjectPoolTest.java b/m3/src/test/java/com/uber/m3/util/ObjectPoolTest.java deleted file mode 100644 index 5ac6f78..0000000 --- a/m3/src/test/java/com/uber/m3/util/ObjectPoolTest.java +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package com.uber.m3.util; - -import org.junit.Test; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; - -public class ObjectPoolTest { - private final Construct OBJECT_CONSTRUCT = new Construct() { - @Override - public Object instance() { - return new Object(); - } - }; - - @Test(expected = IllegalArgumentException.class) - public void negativeCapacityPool() { - ObjectPool pool = new ObjectPool<>(-3, OBJECT_CONSTRUCT); - } - - @Test(expected = IllegalArgumentException.class) - public void emptyPool() { - ObjectPool pool = new ObjectPool<>(0, OBJECT_CONSTRUCT); - } - - @Test - public void oneCapacityPool() { - ObjectPool pool = new ObjectPool<>(1, OBJECT_CONSTRUCT); - - Object obj1 = pool.get(); - // Pool should create new instances when pool is empty - Object obj2 = pool.get(); - Object obj3 = pool.get(); - - assertNotNull(obj1); - assertNotNull(obj2); - assertNotNull(obj3); - - // Should be able to put obj1 back to the pool because it's empty now - assertTrue(pool.put(obj1)); - // Should not be able to put obj2 back to the pool because it's full now - assertFalse(pool.put(obj2)); - - // Purposely don't use equals or assertEqual because we want to test that - // the object we get back is exactly the same (as opposed to sematically - // the same) as the one we put back. - assertTrue(obj1 == pool.get()); - } - - @Test - public void twoCapacityPool() { - ObjectPool pool = new ObjectPool<>(2, OBJECT_CONSTRUCT); - - // Should not be able to put a new Object in the pool because it's full now - assertFalse(pool.put(new Object())); - - Object obj1 = pool.get(); - Object obj2 = pool.get(); - // Pool should create new instances when pool is empty - Object obj3 = pool.get(); - - assertNotNull(obj1); - assertNotNull(obj2); - assertNotNull(obj3); - - // Should be able to put obj2 and obj1 back to the pool because it's empty now - assertTrue(pool.put(obj2)); - assertTrue(pool.put(obj1)); - // Should not be able to put obj3 back to the pool because it's full now - assertFalse(pool.put(obj3)); - - // Make sure we get exactly the same objects in the same order that we put them back in - assertTrue(obj2 == pool.get()); - assertTrue(obj1 == pool.get()); - } -} From c69450de4e1fc98e818cc397a33d278538027ce1 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Tue, 26 Sep 2017 12:07:36 -0400 Subject: [PATCH 32/44] Move M3Reporter constructor validation to Builder --- .../java/com/uber/m3/tally/m3/M3Reporter.java | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 0cb7924..596d4db 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -98,7 +98,7 @@ public class M3Reporter implements StatsReporter, AutoCloseable { private M3.Client client; - private final MetricBatch metricBatch; + private final MetricBatch metricBatch = new MetricBatch(); private final Object calcLock = new Object(); private TCalcTransport calc; @@ -143,32 +143,10 @@ private M3Reporter(Builder builder) { client = new M3.Client(protocolFactory.getProtocol(transport)); - Set commonTags = toMetricTagSet(builder.commonTags); - - // Set and ensure required tags - if (!builder.commonTags.containsKey(SERVICE_TAG)) { - if (builder.service == null || builder.service.isEmpty()) { - throw new IllegalArgumentException(String.format("Common tag [%s] is required", SERVICE_TAG)); - } - - commonTags.add(createMetricTag(SERVICE_TAG, builder.service)); - } - if (!builder.commonTags.containsKey(ENV_TAG)) { - if (builder.env == null || builder.env.isEmpty()) { - throw new IllegalArgumentException(String.format("Common tag [%s] is required", ENV_TAG)); - } - - commonTags.add(createMetricTag(ENV_TAG, builder.env)); - } - if (builder.includeHost && !builder.commonTags.containsKey(HOST_TAG)) { - commonTags.add(createMetricTag(HOST_TAG, getHostName())); - } - - metricBatch = new MetricBatch(); calcProtocol = new TCompactProtocol.Factory().getProtocol(new TCalcTransport()); calc = (TCalcTransport) calcProtocol.getTransport(); - freeBytes = calculateFreeBytes(builder.maxPacketSizeBytes, commonTags); + freeBytes = calculateFreeBytes(builder.maxPacketSizeBytes, builder.metricTagSet); bucketIdTagName = builder.histogramBucketIdName; bucketTagName = builder.histogramBucketName; @@ -207,7 +185,7 @@ private int calculateFreeBytes(int maxPacketSizeBytes, Set commonTags return freeBytes; } - private String getHostName() { + private static String getHostName() { try { return InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException e) { @@ -336,7 +314,7 @@ public void close() { } } - private Set toMetricTagSet(Map tags) { + private static Set toMetricTagSet(Map tags) { Set metricTagSet = new HashSet<>(); if (tags == null) { @@ -350,7 +328,7 @@ private Set toMetricTagSet(Map tags) { return metricTagSet; } - private MetricTag createMetricTag(String tagName, String tagValue) { + private static MetricTag createMetricTag(String tagName, String tagValue) { MetricTag metricTag = new MetricTag(tagName); if (tagValue != null && !tagValue.isEmpty()) { @@ -694,6 +672,7 @@ public static class Builder { // Non-generic EMPTY ImmutableMap will never contain any elements @SuppressWarnings("unchecked") private ImmutableMap commonTags = ImmutableMap.EMPTY; + private Set metricTagSet; private boolean includeHost; private int maxQueueSize = DEFAULT_MAX_QUEUE_SIZE; private int maxPacketSizeBytes = DEFAULT_MAX_PACKET_SIZE; @@ -825,6 +804,27 @@ public Builder histogramBucketTagPrecision(int histogramBucketTagPrecision) { * @return a new {@link M3Reporter} instance with the configured paramters */ public M3Reporter build() { + metricTagSet = toMetricTagSet(commonTags); + + // Set and ensure required tags + if (!commonTags.containsKey(SERVICE_TAG)) { + if (service == null || service.isEmpty()) { + throw new IllegalArgumentException(String.format("Common tag [%s] is required", SERVICE_TAG)); + } + + metricTagSet.add(createMetricTag(SERVICE_TAG, service)); + } + if (!commonTags.containsKey(ENV_TAG)) { + if (env == null || env.isEmpty()) { + throw new IllegalArgumentException(String.format("Common tag [%s] is required", ENV_TAG)); + } + + metricTagSet.add(createMetricTag(ENV_TAG, env)); + } + if (includeHost && !commonTags.containsKey(HOST_TAG)) { + metricTagSet.add(createMetricTag(HOST_TAG, getHostName())); + } + return new M3Reporter(this); } } From 3fed7820444f5c7d7ead4e3382dc3e95112fd90b Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Wed, 27 Sep 2017 18:03:31 -0400 Subject: [PATCH 33/44] Support for multi-UDP connections --- .../java/com/uber/m3/tally/m3/M3Reporter.java | 66 ++---------- .../m3/tally/m3/thrift/TMultiUdpClient.java | 100 ++++++++++++++++++ 2 files changed, 110 insertions(+), 56 deletions(-) create mode 100644 m3/src/main/java/com/uber/m3/tally/m3/thrift/TMultiUdpClient.java diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 596d4db..7a87cf2 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -27,6 +27,7 @@ import com.uber.m3.tally.CapableOf; import com.uber.m3.tally.StatsReporter; import com.uber.m3.tally.m3.thrift.TCalcTransport; +import com.uber.m3.tally.m3.thrift.TMultiUdpClient; import com.uber.m3.tally.m3.thrift.TUdpClient; import com.uber.m3.thrift.generated.CountValue; import com.uber.m3.thrift.generated.GaugeValue; @@ -132,9 +133,7 @@ private M3Reporter(Builder builder) { TProtocolFactory protocolFactory = new TCompactProtocol.Factory(); if (socketAddresses.length > 1) { - throw new UnsupportedOperationException( - "Support for a multiple SocketAddress transport not yet implemented." - ); + transport = new TMultiUdpClient(socketAddresses); } else { transport = new TUdpClient(socketAddresses[0]); } @@ -351,57 +350,6 @@ private int calculateSize(Metric metric) { } } - private void reportCopyMetric( - Metric metric, - int size, - MetricType metricType, - long iValue, - double dValue - ) { - synchronized (isOpenLock) { - if (!isOpen) { - LOG.warn("Reporter already closed; no more metrics can be flushed"); - return; - } - - Metric metricCopy = new Metric(metric.getName()); - metricCopy.setTags(metric.getTags()); - // Unfortunately, there is no finer time resolution for Java 7 - metricCopy.setTimestamp(System.currentTimeMillis() * Duration.NANOS_PER_MILLI); - metricCopy.setMetricValue(new MetricValue()); - - switch (metricType) { - case COUNTER: - CountValue countValue = new CountValue(); - countValue.setI64Value(iValue); - metricCopy.getMetricValue().setCount(countValue); - break; - case GAUGE: - GaugeValue gaugeValue = new GaugeValue(); - gaugeValue.setDValue(dValue); - metricCopy.getMetricValue().setGauge(gaugeValue); - break; - case TIMER: - TimerValue timerValue = new TimerValue(); - timerValue.setI64Value(iValue); - metricCopy.getMetricValue().setTimer(timerValue); - break; - default: - throw new IllegalArgumentException("Unsupported metric type: " + metricType); - } - - try { - metricQueue.put(new SizedMetric(metricCopy, size)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - - LOG.warn(String.format("Interrupted while putting %s: %s", - metricType, - metricCopy.getName())); - } - } - } - private String valueBucketString(double bucketBound) { if (bucketBound == Double.MAX_VALUE) { return "infinity"; @@ -496,6 +444,7 @@ public void reportHistogramValueSamples( LOG.error("Reporter already closed; no more metrics can be flushed"); return; } + // Append histogram bucket-specific tags int bucketIdLen = String.valueOf(buckets.size()).length(); bucketIdLen = Math.max(bucketIdLen, MIN_METRIC_BUCKET_ID_TAG_LENGTH); @@ -504,10 +453,14 @@ public void reportHistogramValueSamples( BucketPair[] bucketPairs = BucketPairImpl.bucketPairs(buckets); + if (tags == null) { + tags = new HashMap<>(2, 1); + } + for (int i = 0; i < bucketPairs.length; i++) { // Look for the first pair with an upper bound greater than or equal // to the given upper bound. - if (bucketPairs[i].upperBoundValue() > bucketUpperBound) { + if (bucketPairs[i].upperBoundValue() >= bucketUpperBound) { String idTagValue = String.format(bucketIdFmt, i); tags.put(bucketIdTagName, idTagValue); @@ -540,6 +493,7 @@ public void reportHistogramDurationSamples( LOG.error("Reporter already closed; no more metrics can be flushed"); return; } + // Append histogram bucket-specific tags int bucketIdLen = String.valueOf(buckets.size()).length(); bucketIdLen = Math.max(bucketIdLen, MIN_METRIC_BUCKET_ID_TAG_LENGTH); @@ -673,7 +627,7 @@ public static class Builder { @SuppressWarnings("unchecked") private ImmutableMap commonTags = ImmutableMap.EMPTY; private Set metricTagSet; - private boolean includeHost; + private boolean includeHost = true; private int maxQueueSize = DEFAULT_MAX_QUEUE_SIZE; private int maxPacketSizeBytes = DEFAULT_MAX_PACKET_SIZE; private String histogramBucketIdName = DEFAULT_HISTOGRAM_BUCKET_ID_NAME; diff --git a/m3/src/main/java/com/uber/m3/tally/m3/thrift/TMultiUdpClient.java b/m3/src/main/java/com/uber/m3/tally/m3/thrift/TMultiUdpClient.java new file mode 100644 index 0000000..e286231 --- /dev/null +++ b/m3/src/main/java/com/uber/m3/tally/m3/thrift/TMultiUdpClient.java @@ -0,0 +1,100 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.tally.m3.thrift; + +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.SocketAddress; +import java.net.SocketException; +import java.nio.ByteBuffer; + +/** + * A Thrift transport that sends to multiple connections + */ +public class TMultiUdpClient extends TTransport implements AutoCloseable { + public static final int MAX_BUFFER_SIZE = 65536; + + public final Object sendLock = new Object(); + + protected TTransport[] transports; + + protected TMultiUdpClient(SocketAddress[] socketAddresses) throws SocketException { + if (socketAddresses == null || socketAddresses.length == 0) { + throw new IllegalArgumentException("Must provide at least one SocketAddress"); + } + + transports = new TTransport[socketAddresses.length]; + + for (int i = 0; i < socketAddresses.length; i++) { + transports[0] = new TUdpClient(socketAddresses[i]); + } + } + + @Override + public boolean isOpen() { + for (TTransport transport : transports) { + if (!transport.isOpen()) { + return false; + } + } + + return true; + } + + @Override + public void open() throws TTransportException { + for (TTransport transport : transports) { + transport.open(); + } + } + + @Override + public void close() { + for (TTransport transport : transports) { + transport.close(); + } + } + + @Override + public int read(byte[] bytes, int offset, int length) throws TTransportException { + throw new UnsupportedOperationException("Reading from multiple transports is not supported"); + } + + @Override + public void write(byte[] bytes, int offset, int length) throws TTransportException { + if (!isOpen()) { + throw new TTransportException(TTransportException.NOT_OPEN); + } + + + } + + @Override + public void flush() throws TTransportException { + for (TTransport transport : transports) { + transport.flush(); + } + } +} From c3fef3b976e3087128d5d5b0bfa44fc90d579a66 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Thu, 28 Sep 2017 12:18:18 -0400 Subject: [PATCH 34/44] Remove unused MetricType; Fix multi UDP client bug --- .../java/com/uber/m3/tally/m3/MetricType.java | 30 ------------------- .../m3/tally/m3/thrift/TMultiUdpClient.java | 20 ++++--------- 2 files changed, 5 insertions(+), 45 deletions(-) delete mode 100644 m3/src/main/java/com/uber/m3/tally/m3/MetricType.java diff --git a/m3/src/main/java/com/uber/m3/tally/m3/MetricType.java b/m3/src/main/java/com/uber/m3/tally/m3/MetricType.java deleted file mode 100644 index 19f5814..0000000 --- a/m3/src/main/java/com/uber/m3/tally/m3/MetricType.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package com.uber.m3.tally.m3; - -/** - * Enum of the type of metric. - */ -public enum MetricType { - COUNTER, - GAUGE, - TIMER, -} diff --git a/m3/src/main/java/com/uber/m3/tally/m3/thrift/TMultiUdpClient.java b/m3/src/main/java/com/uber/m3/tally/m3/thrift/TMultiUdpClient.java index e286231..daa30e2 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/thrift/TMultiUdpClient.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/thrift/TMultiUdpClient.java @@ -23,24 +23,16 @@ import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; -import java.io.IOException; -import java.net.DatagramPacket; -import java.net.DatagramSocket; import java.net.SocketAddress; import java.net.SocketException; -import java.nio.ByteBuffer; /** * A Thrift transport that sends to multiple connections */ public class TMultiUdpClient extends TTransport implements AutoCloseable { - public static final int MAX_BUFFER_SIZE = 65536; + private TTransport[] transports; - public final Object sendLock = new Object(); - - protected TTransport[] transports; - - protected TMultiUdpClient(SocketAddress[] socketAddresses) throws SocketException { + public TMultiUdpClient(SocketAddress[] socketAddresses) throws SocketException { if (socketAddresses == null || socketAddresses.length == 0) { throw new IllegalArgumentException("Must provide at least one SocketAddress"); } @@ -48,7 +40,7 @@ protected TMultiUdpClient(SocketAddress[] socketAddresses) throws SocketExceptio transports = new TTransport[socketAddresses.length]; for (int i = 0; i < socketAddresses.length; i++) { - transports[0] = new TUdpClient(socketAddresses[i]); + transports[i] = new TUdpClient(socketAddresses[i]); } } @@ -84,11 +76,9 @@ public int read(byte[] bytes, int offset, int length) throws TTransportException @Override public void write(byte[] bytes, int offset, int length) throws TTransportException { - if (!isOpen()) { - throw new TTransportException(TTransportException.NOT_OPEN); + for (TTransport transport : transports) { + transport.write(bytes, offset, length); } - - } @Override From ee948041801df31df3f73120ca43c4e812082bb4 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Thu, 28 Sep 2017 13:21:11 -0400 Subject: [PATCH 35/44] Fix various edge condition bugs --- .../com/uber/m3/tally/AbstractBuckets.java | 2 +- .../java/com/uber/m3/tally/CapableOf.java | 28 +++++++++++++++++++ .../java/com/uber/m3/tally/ScopeImpl.java | 2 +- .../main/java/com/uber/m3/util/Duration.java | 8 ++++-- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/uber/m3/tally/AbstractBuckets.java b/core/src/main/java/com/uber/m3/tally/AbstractBuckets.java index 8841775..fdd60db 100644 --- a/core/src/main/java/com/uber/m3/tally/AbstractBuckets.java +++ b/core/src/main/java/com/uber/m3/tally/AbstractBuckets.java @@ -39,7 +39,7 @@ public abstract class AbstractBuckets implements Buckets { if (buckets == null) { this.buckets = new ArrayList<>(); } else { - this.buckets = Arrays.asList(buckets); + this.buckets = new ArrayList<>(Arrays.asList(buckets)); } } diff --git a/core/src/main/java/com/uber/m3/tally/CapableOf.java b/core/src/main/java/com/uber/m3/tally/CapableOf.java index 8a9f2e6..cb554c6 100644 --- a/core/src/main/java/com/uber/m3/tally/CapableOf.java +++ b/core/src/main/java/com/uber/m3/tally/CapableOf.java @@ -46,4 +46,32 @@ public boolean reporting() { public boolean tagging() { return tagging; } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other == this) { + return true; + } + if (!(other instanceof CapableOf)) { + return false; + } + + CapableOf capabilities = (CapableOf) other; + + return capabilities.reporting == reporting + && capabilities.tagging == tagging; + } + + @Override + public int hashCode() { + int code = 0; + + code = 31 * code + new Boolean(reporting).hashCode(); + code = 31 * code + new Boolean(tagging).hashCode(); + + return code; + } } diff --git a/core/src/main/java/com/uber/m3/tally/ScopeImpl.java b/core/src/main/java/com/uber/m3/tally/ScopeImpl.java index e40f01c..5f8a380 100644 --- a/core/src/main/java/com/uber/m3/tally/ScopeImpl.java +++ b/core/src/main/java/com/uber/m3/tally/ScopeImpl.java @@ -244,7 +244,7 @@ String fullyQualifiedName(String name) { * Returns a {@link Snapshot} of this {@link Scope}. * @return a {@link Snapshot} of this {@link Scope} */ - Snapshot snapshot() { + public Snapshot snapshot() { Snapshot snap = new SnapshotImpl(); for (ScopeImpl subscope : registry.subscopes.values()) { diff --git a/core/src/main/java/com/uber/m3/util/Duration.java b/core/src/main/java/com/uber/m3/util/Duration.java index fdd5e06..77d4012 100644 --- a/core/src/main/java/com/uber/m3/util/Duration.java +++ b/core/src/main/java/com/uber/m3/util/Duration.java @@ -92,8 +92,8 @@ private Duration(long nanos) { * @param nanos number of nanos in this {@link Duration} * @return a {@link Duration} of these nanos */ - public static Duration ofNanos(double nanos) { - return new Duration((long) nanos); + public static Duration ofNanos(long nanos) { + return new Duration(nanos); } /** @@ -194,6 +194,10 @@ public int compareTo(Duration other) { public String toString() { if (nanos == 0) { return "0s"; + } else if (nanos == Long.MIN_VALUE) { + // Return hard coded response as workaround because + // Math.abs(Long.MIN_VALUE) == Long.MIN_VALUE + return "-2562047h47m16.854775808s"; } boolean isNegative = nanos < 0; From e59d63d45cac959d614c35f3351640ef3cfa0183 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Tue, 3 Oct 2017 16:59:57 -0700 Subject: [PATCH 36/44] Warn instead of throw on recoverable exceptions; flush automatically after waiting a timeout --- .../main/java/com/uber/m3/util/Duration.java | 5 +-- .../java/com/uber/m3/tally/m3/M3Reporter.java | 32 +++++++++++-------- .../uber/m3/tally/m3/thrift/TUdpClient.java | 2 +- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/com/uber/m3/util/Duration.java b/core/src/main/java/com/uber/m3/util/Duration.java index 77d4012..c6ec129 100644 --- a/core/src/main/java/com/uber/m3/util/Duration.java +++ b/core/src/main/java/com/uber/m3/util/Duration.java @@ -52,7 +52,7 @@ public class Duration implements Comparable { /** * The number of milliseconds in a second. */ - public static final long MILLIS_PER_SECOND = 1000; + public static final long MILLIS_PER_SECOND = 1_000; /** * The number of seconds in a minute. @@ -209,7 +209,8 @@ public String toString() { long secondsInNanos = nanosLocal % NANOS_PER_MINUTE; int nanoOffset = (int) (nanosLocal % NANOS_PER_SECOND); - StringBuilder buf = new StringBuilder(24); + // 25 capacity from max String length produced from Long.MIN_VALUE nanos + StringBuilder buf = new StringBuilder(25); if (isNegative) { buf.append("-"); diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 7a87cf2..bac7af5 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -83,6 +83,7 @@ public class M3Reporter implements StatsReporter, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(M3Reporter.class); private static final int MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS = 1000; + private static final int MAX_PROCESSOR_WAIT_UNTIL_FLUSH_MILLIS = 1000; private static final int DEFAULT_FLUSH_TRY_COUNT = 2; private static final int FLUSH_RETRY_DELAY_MILLIS = 200; @@ -155,7 +156,7 @@ private M3Reporter(Builder builder) { addAndRunProcessor(); } catch (TTransportException | SocketException e) { - throw new IllegalArgumentException("Exception creating M3Reporter", e); + throw new RuntimeException("Exception creating M3Reporter", e); } } @@ -211,7 +212,7 @@ public void flush() { } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new RuntimeException(e); + LOG.warn("Interrupted while trying to queue flush sentinel"); } } @@ -269,10 +270,7 @@ private void flush(List metrics, int maxTries) { metrics.clear(); if (exception != null) { - throw new RuntimeException( - String.format("Failed to flush metrics after %s tries", maxTries), - exception - ); + LOG.warn("Failed to flush metrics after %s tries. Stack trace:\n%s", maxTries, exception.getStackTrace()); } } @@ -561,7 +559,7 @@ private void queueSizedMetric(SizedMetric sizedMetric) { } private class Processor implements Runnable { - List metrics = new ArrayList<>(freeBytes / 10); + List pendingMetrics = new ArrayList<>(freeBytes / 10); int metricsSize = 0; @Override @@ -572,9 +570,15 @@ public void run() { // When this reporter is closed, shutdownNow will be called on the executor, // which will interrupt this thread and proceed to the `InterruptedException` // catch block. - SizedMetric sizedMetric = metricQueue.take(); - - flushMetricIteration(sizedMetric); + SizedMetric sizedMetric = metricQueue.poll(MAX_PROCESSOR_WAIT_UNTIL_FLUSH_MILLIS, TimeUnit.MILLISECONDS); + + if (sizedMetric != null) { + flushMetricIteration(sizedMetric); + } else if (!pendingMetrics.isEmpty()) { + // If we don't get any new metrics after waiting the specified time, + // flush what we have so far. + flushMetricIteration(SizedMetric.FLUSH); + } } } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -590,7 +594,7 @@ private void flushMetricIteration(SizedMetric sizedMetric) { Metric metric = sizedMetric.getMetric(); if (sizedMetric == SizedMetric.FLUSH) { - flush(metrics); + flush(pendingMetrics); return; } @@ -598,11 +602,11 @@ private void flushMetricIteration(SizedMetric sizedMetric) { int size = sizedMetric.getSize(); if (metricsSize + size > freeBytes) { - flush(metrics); + flush(pendingMetrics); metricsSize = 0; } - metrics.add(metric); + pendingMetrics.add(metric); metricsSize += size; } @@ -612,7 +616,7 @@ private void flushRemainingMetrics() { flushMetricIteration(metricQueue.remove()); } - flush(metrics); + flush(pendingMetrics); } } diff --git a/m3/src/main/java/com/uber/m3/tally/m3/thrift/TUdpClient.java b/m3/src/main/java/com/uber/m3/tally/m3/thrift/TUdpClient.java index b9446ef..f23834b 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/thrift/TUdpClient.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/thrift/TUdpClient.java @@ -61,7 +61,7 @@ public void flush() throws TTransportException { try { socket.send(new DatagramPacket(bytes, length)); } catch (IOException e) { - throw new TTransportException("Cannot flush closed transport", e); + throw new TTransportException(e); } finally { writeBuffer.clear(); } From cb0191ed251be0c5fc5bc4719f3571db3f23d94c Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Mon, 9 Oct 2017 13:26:01 -0400 Subject: [PATCH 37/44] Don't re-interrupt thread when we want to continue business as usual --- m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index bac7af5..d56e7d0 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -210,8 +210,6 @@ public void flush() { try { metricQueue.put(SizedMetric.FLUSH); } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.warn("Interrupted while trying to queue flush sentinel"); } } @@ -245,8 +243,6 @@ private void flush(List metrics, int maxTries) { try { Thread.sleep(FLUSH_RETRY_DELAY_MILLIS); } catch (InterruptedException interruptedException) { - Thread.currentThread().interrupt(); - // If someone interrupts this thread, we should probably // not still try to flush tryFlush = false; @@ -300,8 +296,6 @@ public void close() { MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS ); } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.warn("M3Reporter closing before Processors complete due to being interrupted!"); } @@ -552,8 +546,6 @@ private void queueSizedMetric(SizedMetric sizedMetric) { try { metricQueue.put(sizedMetric); } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.warn(String.format("Interrupted queueing metric: %s", sizedMetric.getMetric().getName())); } } From 267babc2b848373dc3da9c1047d9e84c82713deb Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Mon, 16 Oct 2017 17:46:27 -0400 Subject: [PATCH 38/44] Use close sentinel metric instead of locks; code refactoring --- .../main/java/com/uber/m3/util/Duration.java | 12 +- .../java/com/uber/m3/tally/m3/M3Reporter.java | 248 ++++++++---------- .../com/uber/m3/tally/m3/SizedMetric.java | 1 + 3 files changed, 126 insertions(+), 135 deletions(-) diff --git a/core/src/main/java/com/uber/m3/util/Duration.java b/core/src/main/java/com/uber/m3/util/Duration.java index c6ec129..913a496 100644 --- a/core/src/main/java/com/uber/m3/util/Duration.java +++ b/core/src/main/java/com/uber/m3/util/Duration.java @@ -79,6 +79,9 @@ public class Duration implements Comparable { */ public static final long NANOS_PER_HOUR = NANOS_PER_MINUTE * MINUTES_PER_HOUR; + // Max string length produced from a Duration is 25 when nanos == Long.MIN_VALUE + private static final int STRING_BUILDER_INIT_CAP = 25; + // Long.MAX_VALUE nanos > 290 years, which should be good enough private final long nanos; @@ -190,6 +193,12 @@ public int compareTo(Duration other) { return Long.compare(nanos, other.nanos); } + /** + * This toString method is designed to mimic Go's time.Duration String(). In short, it is + * designed so that the least granular unit is used such that the first digit is not 0, + * e.g. 100µs instead of 0.1ms. + * @return String representation of this {@link Duration} + */ @Override public String toString() { if (nanos == 0) { @@ -209,8 +218,7 @@ public String toString() { long secondsInNanos = nanosLocal % NANOS_PER_MINUTE; int nanoOffset = (int) (nanosLocal % NANOS_PER_SECOND); - // 25 capacity from max String length produced from Long.MIN_VALUE nanos - StringBuilder buf = new StringBuilder(25); + StringBuilder buf = new StringBuilder(STRING_BUILDER_INIT_CAP); if (isNegative) { buf.append("-"); diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index d56e7d0..11a328c 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -102,6 +102,7 @@ public class M3Reporter implements StatsReporter, AutoCloseable { private final MetricBatch metricBatch = new MetricBatch(); + // calcLock protects both calc and calcProtocol private final Object calcLock = new Object(); private TCalcTransport calc; private TProtocol calcProtocol; @@ -115,14 +116,11 @@ public class M3Reporter implements StatsReporter, AutoCloseable { // Holds metrics to be flushed private final BlockingQueue metricQueue; // The service used to execute metric flushes - private ExecutorService executor = Executors.newFixedThreadPool(THREAD_POOL_SIZE); + private ExecutorService executor; // Used to keep track of how many processors are in progress in // order to wait for them to finish before closing private Phaser phaser = new Phaser(1); - private final Object isOpenLock = new Object(); - private boolean isOpen = true; - private TTransport transport; // Use inner Builder class to construct an M3Reporter @@ -154,6 +152,8 @@ private M3Reporter(Builder builder) { metricQueue = new LinkedBlockingQueue<>(builder.maxQueueSize); + executor = builder.executor; + addAndRunProcessor(); } catch (TTransportException | SocketException e) { throw new RuntimeException("Exception creating M3Reporter", e); @@ -272,37 +272,31 @@ private void flush(List metrics, int maxTries) { @Override public void close() { - synchronized (isOpenLock) { - if (!isOpen) { - LOG.warn("M3Reporter already closed"); - return; - } - - // Important to use `shutdownNow` instead of `shutdown` to interrupt processor - // thread(s) or else they will block forever - executor.shutdownNow(); - - // Wait a maximum of `MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS` for all processors - // to return from flushing - try { - phaser.awaitAdvanceInterruptibly( - phaser.arrive(), - MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS, - TimeUnit.MILLISECONDS - ); - } catch (TimeoutException e) { - LOG.warn( - "M3Reporter closing before Processors complete after waiting timeout of %dms!", - MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS - ); - } catch (InterruptedException e) { - LOG.warn("M3Reporter closing before Processors complete due to being interrupted!"); - } + // Put sentinal value in queue so that processors know to disregard anything that comes after it. + queueSizedMetric(SizedMetric.CLOSE); - transport.close(); + // Important to use `shutdownNow` instead of `shutdown` to interrupt processor + // thread(s) or else they will block forever + executor.shutdownNow(); - isOpen = false; + // Wait a maximum of `MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS` for all processors + // to return from flushing + try { + phaser.awaitAdvanceInterruptibly( + phaser.arrive(), + MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS, + TimeUnit.MILLISECONDS + ); + } catch (TimeoutException e) { + LOG.warn( + "M3Reporter closing before Processors complete after waiting timeout of %dms!", + MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS + ); + } catch (InterruptedException e) { + LOG.warn("M3Reporter closing before Processors complete due to being interrupted!"); } + + transport.close(); } private static Set toMetricTagSet(Map tags) { @@ -372,54 +366,33 @@ private String durationBucketString(Duration bucketBound) { @Override public void reportCounter(String name, Map tags, long value) { - synchronized (isOpenLock) { - if (!isOpen) { - LOG.error("Reporter already closed; no more metrics can be flushed"); - return; - } - - reportCounterInternal(name, tags, value); - } + reportCounterInternal(name, tags, value); } @Override public void reportGauge(String name, Map tags, double value) { - synchronized (isOpenLock) { - if (!isOpen) { - LOG.error("Reporter already closed; no more metrics can be flushed"); - return; - } + GaugeValue gaugeValue = new GaugeValue(); + gaugeValue.setDValue(value); - GaugeValue gaugeValue = new GaugeValue(); - gaugeValue.setDValue(value); - - MetricValue metricValue = new MetricValue(); - metricValue.setGauge(gaugeValue); + MetricValue metricValue = new MetricValue(); + metricValue.setGauge(gaugeValue); - Metric metric = newMetric(name, tags, metricValue); + Metric metric = newMetric(name, tags, metricValue); - queueSizedMetric(new SizedMetric(metric, calculateSize(metric))); - } + queueSizedMetric(new SizedMetric(metric, calculateSize(metric))); } @Override public void reportTimer(String name, Map tags, Duration interval) { - synchronized (isOpenLock) { - if (!isOpen) { - LOG.error("Reporter already closed; no more metrics can be flushed"); - return; - } - - TimerValue timerValue = new TimerValue(); - timerValue.setI64Value(interval.getNanos()); + TimerValue timerValue = new TimerValue(); + timerValue.setI64Value(interval.getNanos()); - MetricValue metricValue = new MetricValue(); - metricValue.setTimer(timerValue); + MetricValue metricValue = new MetricValue(); + metricValue.setTimer(timerValue); - Metric metric = newMetric(name, tags, metricValue); + Metric metric = newMetric(name, tags, metricValue); - queueSizedMetric(new SizedMetric(metric, calculateSize(metric))); - } + queueSizedMetric(new SizedMetric(metric, calculateSize(metric))); } @Override @@ -431,44 +404,37 @@ public void reportHistogramValueSamples( double bucketUpperBound, long samples ) { - synchronized (isOpenLock) { - if (!isOpen) { - LOG.error("Reporter already closed; no more metrics can be flushed"); - return; - } + // Append histogram bucket-specific tags + int bucketIdLen = String.valueOf(buckets.size()).length(); + bucketIdLen = Math.max(bucketIdLen, MIN_METRIC_BUCKET_ID_TAG_LENGTH); - // Append histogram bucket-specific tags - int bucketIdLen = String.valueOf(buckets.size()).length(); - bucketIdLen = Math.max(bucketIdLen, MIN_METRIC_BUCKET_ID_TAG_LENGTH); + String bucketIdFmt = String.format("%%0%sd", bucketIdLen); - String bucketIdFmt = String.format("%%0%sd", bucketIdLen); + BucketPair[] bucketPairs = BucketPairImpl.bucketPairs(buckets); - BucketPair[] bucketPairs = BucketPairImpl.bucketPairs(buckets); + if (tags == null) { + tags = new HashMap<>(); + } - if (tags == null) { - tags = new HashMap<>(2, 1); - } + for (int i = 0; i < bucketPairs.length; i++) { + // Look for the first pair with an upper bound greater than or equal + // to the given upper bound. + if (bucketPairs[i].upperBoundValue() >= bucketUpperBound) { + String idTagValue = String.format(bucketIdFmt, i); - for (int i = 0; i < bucketPairs.length; i++) { - // Look for the first pair with an upper bound greater than or equal - // to the given upper bound. - if (bucketPairs[i].upperBoundValue() >= bucketUpperBound) { - String idTagValue = String.format(bucketIdFmt, i); - - tags.put(bucketIdTagName, idTagValue); - tags.put(bucketTagName, - String.format("%s-%s", - valueBucketString(bucketPairs[i].lowerBoundValue()), - valueBucketString(bucketPairs[i].upperBoundValue()) - ) - ); - - break; - } - } + tags.put(bucketIdTagName, idTagValue); + tags.put(bucketTagName, + String.format("%s-%s", + valueBucketString(bucketPairs[i].lowerBoundValue()), + valueBucketString(bucketPairs[i].upperBoundValue()) + ) + ); - reportCounterInternal(name, tags, samples); + break; + } } + + reportCounterInternal(name, tags, samples); } @Override @@ -480,44 +446,37 @@ public void reportHistogramDurationSamples( Duration bucketUpperBound, long samples ) { - synchronized (isOpenLock) { - if (!isOpen) { - LOG.error("Reporter already closed; no more metrics can be flushed"); - return; - } + // Append histogram bucket-specific tags + int bucketIdLen = String.valueOf(buckets.size()).length(); + bucketIdLen = Math.max(bucketIdLen, MIN_METRIC_BUCKET_ID_TAG_LENGTH); - // Append histogram bucket-specific tags - int bucketIdLen = String.valueOf(buckets.size()).length(); - bucketIdLen = Math.max(bucketIdLen, MIN_METRIC_BUCKET_ID_TAG_LENGTH); + String bucketIdFmt = String.format("%%0%sd", bucketIdLen); - String bucketIdFmt = String.format("%%0%sd", bucketIdLen); + BucketPair[] bucketPairs = BucketPairImpl.bucketPairs(buckets); - BucketPair[] bucketPairs = BucketPairImpl.bucketPairs(buckets); + if (tags == null) { + tags = new HashMap<>(2, 1); + } - if (tags == null) { - tags = new HashMap<>(2, 1); - } + for (int i = 0; i < bucketPairs.length; i++) { + // Look for the first pair with an upper bound greater than or equal + // to the given upper bound. + if (bucketPairs[i].upperBoundDuration().compareTo(bucketUpperBound) >= 0) { + String idTagValue = String.format(bucketIdFmt, i); - for (int i = 0; i < bucketPairs.length; i++) { - // Look for the first pair with an upper bound greater than or equal - // to the given upper bound. - if (bucketPairs[i].upperBoundDuration().compareTo(bucketUpperBound) >= 0) { - String idTagValue = String.format(bucketIdFmt, i); - - tags.put(bucketIdTagName, idTagValue); - tags.put(bucketTagName, - String.format("%s-%s", - durationBucketString(bucketPairs[i].lowerBoundDuration()), - durationBucketString(bucketPairs[i].upperBoundDuration()) - ) - ); - - break; - } - } + tags.put(bucketIdTagName, idTagValue); + tags.put(bucketTagName, + String.format("%s-%s", + durationBucketString(bucketPairs[i].lowerBoundDuration()), + durationBucketString(bucketPairs[i].upperBoundDuration()) + ) + ); - reportCounterInternal(name, tags, samples); + break; + } } + + reportCounterInternal(name, tags, samples); } // Relies on the calling function to provide guarantees of the reporter being open @@ -558,22 +517,33 @@ private class Processor implements Runnable { public void run() { try { while (!executor.isShutdown()) { - // This `take` call will block until there is an item on the queue to take. + // This `poll` call will block for at most the specified duration to take an item + // off the queue. If we get an item, we append it to the queue to be flushed, + // otherwise we flush what we have so far. // When this reporter is closed, shutdownNow will be called on the executor, // which will interrupt this thread and proceed to the `InterruptedException` // catch block. SizedMetric sizedMetric = metricQueue.poll(MAX_PROCESSOR_WAIT_UNTIL_FLUSH_MILLIS, TimeUnit.MILLISECONDS); + // Drop metrics that came in after close + if (sizedMetric == SizedMetric.CLOSE) { + metricQueue.clear(); + break; + } + if (sizedMetric != null) { flushMetricIteration(sizedMetric); - } else if (!pendingMetrics.isEmpty()) { - // If we don't get any new metrics after waiting the specified time, - // flush what we have so far. + continue; + } + + // If we don't get any new metrics after waiting the specified time, + // flush what we have so far. + if (!pendingMetrics.isEmpty()) { flushMetricIteration(SizedMetric.FLUSH); } } } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + // Don't care if we get interrupted - the finally block will clean up } finally { flushRemainingMetrics(); @@ -619,6 +589,7 @@ public static class Builder { private SocketAddress[] socketAddresses; private String service; private String env; + private ExecutorService executor = Executors.newFixedThreadPool(THREAD_POOL_SIZE); // Non-generic EMPTY ImmutableMap will never contain any elements @SuppressWarnings("unchecked") private ImmutableMap commonTags = ImmutableMap.EMPTY; @@ -672,6 +643,17 @@ public Builder env(String env) { return this; } + /** + * Configures the executor of this {@link Builder}. + * @param executor the value to set + * @return this {@link Builder} with the new value set + */ + public Builder executor(ExecutorService executor) { + this.executor = executor; + + return this; + } + /** * Configures the common tags of this {@link Builder}. * @param commonTags the value to set diff --git a/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java b/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java index 14a4ff2..e3a1075 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/SizedMetric.java @@ -27,6 +27,7 @@ */ public class SizedMetric extends Metric { public static final SizedMetric FLUSH = new SizedMetric(null, -1); + public static final SizedMetric CLOSE = new SizedMetric(null, -2); private Metric metric; private int size; From d3ba08ffb393b8d432cb1a4be9516378884493ff Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Mon, 16 Oct 2017 18:34:38 -0400 Subject: [PATCH 39/44] New MetricBatch instantiation instead of use/clean to avoid synchronized block --- .../java/com/uber/m3/tally/m3/M3Reporter.java | 85 +++++++++---------- 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 11a328c..0737db0 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -100,8 +100,6 @@ public class M3Reporter implements StatsReporter, AutoCloseable { private M3.Client client; - private final MetricBatch metricBatch = new MetricBatch(); - // calcLock protects both calc and calcProtocol private final Object calcLock = new Object(); private TCalcTransport calc; @@ -154,13 +152,14 @@ private M3Reporter(Builder builder) { executor = builder.executor; - addAndRunProcessor(); + addAndRunProcessor(builder.metricTagSet); } catch (TTransportException | SocketException e) { throw new RuntimeException("Exception creating M3Reporter", e); } } private int calculateFreeBytes(int maxPacketSizeBytes, Set commonTags) { + MetricBatch metricBatch = new MetricBatch(); metricBatch.setCommonTags(commonTags); metricBatch.setMetrics(new ArrayList()); @@ -194,10 +193,10 @@ private static String getHostName() { } } - private void addAndRunProcessor() { + private void addAndRunProcessor(Set commonTags) { phaser.register(); - executor.execute(new Processor()); + executor.execute(new Processor(commonTags)); } @Override @@ -214,60 +213,51 @@ public void flush() { } } - private void flush(List metrics) { - flush(metrics, DEFAULT_FLUSH_TRY_COUNT); + private void flush(List metrics, Set commonTags) { + flush(metrics, commonTags, DEFAULT_FLUSH_TRY_COUNT); } - private void flush(List metrics, int maxTries) { + private void flush(List metrics, Set commonTags, int maxTries) { if (metrics.isEmpty()) { return; } - TException exception = null; - int tries = 0; boolean tryFlush = true; - synchronized (metricBatch) { - metricBatch.setMetrics(metrics); - - while (tryFlush) { - try { - client.emitMetricBatch(metricBatch); + MetricBatch metricBatch = new MetricBatch(); + metricBatch.setCommonTags(commonTags); + metricBatch.setMetrics(metrics); - tryFlush = false; - } catch (TException tException) { - tries++; - - if (tries < maxTries) { - try { - Thread.sleep(FLUSH_RETRY_DELAY_MILLIS); - } catch (InterruptedException interruptedException) { - // If someone interrupts this thread, we should probably - // not still try to flush - tryFlush = false; - - exception = new TException("Interrupted while about to retry emitting metrics", tException); - } - } else { + while (tryFlush) { + try { + client.emitMetricBatch(metricBatch); + + tryFlush = false; + } catch (TException tException) { + tries++; + + if (tries < maxTries) { + try { + Thread.sleep(FLUSH_RETRY_DELAY_MILLIS); + } catch (InterruptedException interruptedException) { + // If someone interrupts this thread, we should probably + // not still try to flush tryFlush = false; - // Store exception to throw later after clean up. - // Don't want to put everything in a finally block here, - // which will hold the lock unnecessarily long - exception = tException; + LOG.warn("Interrupted while about to retry emitting metrics"); } + } else { + tryFlush = false; + + LOG.warn("Failed to flush metrics after %s tries", maxTries); } } - - metricBatch.setMetrics(null); } - metrics.clear(); + metricBatch.setMetrics(null); - if (exception != null) { - LOG.warn("Failed to flush metrics after %s tries. Stack trace:\n%s", maxTries, exception.getStackTrace()); - } + metrics.clear(); } @Override @@ -509,10 +499,15 @@ private void queueSizedMetric(SizedMetric sizedMetric) { } } - private class Processor implements Runnable { + class Processor implements Runnable { + final Set commonTags; List pendingMetrics = new ArrayList<>(freeBytes / 10); int metricsSize = 0; + Processor(Set commonTags) { + this.commonTags = commonTags; + } + @Override public void run() { try { @@ -556,7 +551,7 @@ private void flushMetricIteration(SizedMetric sizedMetric) { Metric metric = sizedMetric.getMetric(); if (sizedMetric == SizedMetric.FLUSH) { - flush(pendingMetrics); + flush(pendingMetrics, commonTags); return; } @@ -564,7 +559,7 @@ private void flushMetricIteration(SizedMetric sizedMetric) { int size = sizedMetric.getSize(); if (metricsSize + size > freeBytes) { - flush(pendingMetrics); + flush(pendingMetrics, commonTags); metricsSize = 0; } @@ -578,7 +573,7 @@ private void flushRemainingMetrics() { flushMetricIteration(metricQueue.remove()); } - flush(pendingMetrics); + flush(pendingMetrics, commonTags); } } From ea745f563e902ed11043b727c1758bbfea77ebd8 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Tue, 17 Oct 2017 12:19:54 -0400 Subject: [PATCH 40/44] Fix flushing with close SizedMetric sentinel --- m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 0737db0..772e5de 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -570,7 +570,14 @@ private void flushMetricIteration(SizedMetric sizedMetric) { private void flushRemainingMetrics() { while (!metricQueue.isEmpty()) { - flushMetricIteration(metricQueue.remove()); + SizedMetric sizedMetric = metricQueue.remove(); + + // Don't care about metrics that came in after close + if (sizedMetric == SizedMetric.CLOSE) { + break; + } + + flushMetricIteration(sizedMetric); } flush(pendingMetrics, commonTags); From 9c3283565584e4c59f20f6c073dc02c0bcb655f2 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Tue, 17 Oct 2017 14:41:12 -0400 Subject: [PATCH 41/44] Put Processor back as private class --- m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 772e5de..573ee6d 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -499,7 +499,7 @@ private void queueSizedMetric(SizedMetric sizedMetric) { } } - class Processor implements Runnable { + private class Processor implements Runnable { final Set commonTags; List pendingMetrics = new ArrayList<>(freeBytes / 10); int metricsSize = 0; From fa7b2553695c254aa85e2d5b08c40545b44522eb Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Wed, 18 Oct 2017 13:14:38 -0400 Subject: [PATCH 42/44] Comment on HashMap sizing --- m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 573ee6d..e6dc492 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -403,7 +403,9 @@ public void reportHistogramValueSamples( BucketPair[] bucketPairs = BucketPairImpl.bucketPairs(buckets); if (tags == null) { - tags = new HashMap<>(); + // We know that the HashMap will only contain two items at this point, + // therefore initialCapacity of 2 and loadFactor of 1. + tags = new HashMap<>(2, 1); } for (int i = 0; i < bucketPairs.length; i++) { @@ -445,6 +447,8 @@ public void reportHistogramDurationSamples( BucketPair[] bucketPairs = BucketPairImpl.bucketPairs(buckets); if (tags == null) { + // We know that the HashMap will only contain two items at this point, + // therefore initialCapacity of 2 and loadFactor of 1. tags = new HashMap<>(2, 1); } From cccaeafea11b5fb1da0aa35d3bfec4229f66fd08 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Mon, 23 Oct 2017 12:09:29 -0400 Subject: [PATCH 43/44] Remove flush retry logic --- .../java/com/uber/m3/tally/m3/M3Reporter.java | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index e6dc492..43f238e 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -85,9 +85,6 @@ public class M3Reporter implements StatsReporter, AutoCloseable { private static final int MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS = 1000; private static final int MAX_PROCESSOR_WAIT_UNTIL_FLUSH_MILLIS = 1000; - private static final int DEFAULT_FLUSH_TRY_COUNT = 2; - private static final int FLUSH_RETRY_DELAY_MILLIS = 200; - private static final int DEFAULT_METRIC_SIZE = 100; private static final int DEFAULT_MAX_QUEUE_SIZE = 4096; @@ -214,15 +211,10 @@ public void flush() { } private void flush(List metrics, Set commonTags) { - flush(metrics, commonTags, DEFAULT_FLUSH_TRY_COUNT); - } - - private void flush(List metrics, Set commonTags, int maxTries) { if (metrics.isEmpty()) { return; } - int tries = 0; boolean tryFlush = true; MetricBatch metricBatch = new MetricBatch(); @@ -235,23 +227,7 @@ private void flush(List metrics, Set commonTags, int maxTries tryFlush = false; } catch (TException tException) { - tries++; - - if (tries < maxTries) { - try { - Thread.sleep(FLUSH_RETRY_DELAY_MILLIS); - } catch (InterruptedException interruptedException) { - // If someone interrupts this thread, we should probably - // not still try to flush - tryFlush = false; - - LOG.warn("Interrupted while about to retry emitting metrics"); - } - } else { - tryFlush = false; - - LOG.warn("Failed to flush metrics after %s tries", maxTries); - } + LOG.warn("Failed to flush metrics: " + tException.getMessage()); } } From c9177dc6965c196e097a09610cfce071479903a1 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Tue, 24 Oct 2017 13:28:22 -0400 Subject: [PATCH 44/44] Configurable max processor wait time --- .../java/com/uber/m3/tally/m3/M3Reporter.java | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java index 43f238e..9c5c30a 100644 --- a/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java +++ b/m3/src/main/java/com/uber/m3/tally/m3/M3Reporter.java @@ -83,10 +83,8 @@ public class M3Reporter implements StatsReporter, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(M3Reporter.class); private static final int MAX_PROCESSOR_WAIT_ON_CLOSE_MILLIS = 1000; - private static final int MAX_PROCESSOR_WAIT_UNTIL_FLUSH_MILLIS = 1000; private static final int DEFAULT_METRIC_SIZE = 100; - private static final int DEFAULT_MAX_QUEUE_SIZE = 4096; private static final int DEFAULT_MAX_PACKET_SIZE = 1440; @@ -102,6 +100,8 @@ public class M3Reporter implements StatsReporter, AutoCloseable { private TCalcTransport calc; private TProtocol calcProtocol; + private int maxProcessorWaitUntilFlushMillis; + private int freeBytes; private String bucketIdTagName; @@ -141,6 +141,8 @@ private M3Reporter(Builder builder) { freeBytes = calculateFreeBytes(builder.maxPacketSizeBytes, builder.metricTagSet); + maxProcessorWaitUntilFlushMillis = builder.maxProcessorWaitUntilFlushMillis; + bucketIdTagName = builder.histogramBucketIdName; bucketTagName = builder.histogramBucketName; bucketValFmt = String.format("%%.%df", builder.histogramBucketTagPrecision); @@ -215,20 +217,15 @@ private void flush(List metrics, Set commonTags) { return; } - boolean tryFlush = true; - MetricBatch metricBatch = new MetricBatch(); metricBatch.setCommonTags(commonTags); metricBatch.setMetrics(metrics); - while (tryFlush) { - try { - client.emitMetricBatch(metricBatch); + try { + client.emitMetricBatch(metricBatch); - tryFlush = false; - } catch (TException tException) { - LOG.warn("Failed to flush metrics: " + tException.getMessage()); - } + } catch (TException tException) { + LOG.warn("Failed to flush metrics: " + tException.getMessage()); } metricBatch.setMetrics(null); @@ -498,7 +495,7 @@ public void run() { // When this reporter is closed, shutdownNow will be called on the executor, // which will interrupt this thread and proceed to the `InterruptedException` // catch block. - SizedMetric sizedMetric = metricQueue.poll(MAX_PROCESSOR_WAIT_UNTIL_FLUSH_MILLIS, TimeUnit.MILLISECONDS); + SizedMetric sizedMetric = metricQueue.poll(maxProcessorWaitUntilFlushMillis, TimeUnit.MILLISECONDS); // Drop metrics that came in after close if (sizedMetric == SizedMetric.CLOSE) { @@ -532,7 +529,6 @@ private void flushMetricIteration(SizedMetric sizedMetric) { if (sizedMetric == SizedMetric.FLUSH) { flush(pendingMetrics, commonTags); - return; } @@ -579,6 +575,7 @@ public static class Builder { private boolean includeHost = true; private int maxQueueSize = DEFAULT_MAX_QUEUE_SIZE; private int maxPacketSizeBytes = DEFAULT_MAX_PACKET_SIZE; + private int maxProcessorWaitUntilFlushMillis = 10_000; private String histogramBucketIdName = DEFAULT_HISTOGRAM_BUCKET_ID_NAME; private String histogramBucketName = DEFAULT_HISTOGRAM_BUCKET_NAME; private int histogramBucketTagPrecision = DEFAULT_HISTOGRAM_BUCKET_TAG_PRECISION; @@ -680,6 +677,17 @@ public Builder maxPacketSizeBytes(int maxPacketSizeBytes) { return this; } + /** + * Configures the maximum wait time in milliseconds size in bytes of this {@link Builder}. + * @param maxProcessorWaitUntilFlushMillis the value to set + * @return this {@link Builder} with the new value set + */ + public Builder maxProcessorWaitUntilFlushMillis(int maxProcessorWaitUntilFlushMillis) { + this.maxProcessorWaitUntilFlushMillis = maxProcessorWaitUntilFlushMillis; + + return this; + } + /** * Configures the histogram bucket ID name of this {@link Builder}. * @param histogramBucketIdName the value to set