From 0500b9ca10053ad2253b14da2708e7b3bf06dddf Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sun, 17 Dec 2017 17:21:36 -0500 Subject: [PATCH 01/45] Added new Datasource install file --- NEW_DATASOURCE.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 NEW_DATASOURCE.md diff --git a/NEW_DATASOURCE.md b/NEW_DATASOURCE.md new file mode 100644 index 000000000..d24560c9a --- /dev/null +++ b/NEW_DATASOURCE.md @@ -0,0 +1,15 @@ +# Add New Data Source + +The following are the instructions for adding a new data source + +## Add the following files for Creating a new Datasource + +backend/persistent/datastores/Datastores.js +backend/persistent/datastores/NameOfDatastoreToConnec.js +app/constants/constants.js +app/images +app/actions/sessions.js +app/components/Settings/Tabs/Tab.react.js +app/components/Settings/Settings.react.js + +## Datastore file should have the following functions From a62260aae4c8af637704f10396a8bf490b75ec5d Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sun, 17 Dec 2017 19:16:53 -0500 Subject: [PATCH 02/45] Check in latest instructions --- NEW_DATASOURCE.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/NEW_DATASOURCE.md b/NEW_DATASOURCE.md index d24560c9a..afdb2249e 100644 --- a/NEW_DATASOURCE.md +++ b/NEW_DATASOURCE.md @@ -13,3 +13,16 @@ app/components/Settings/Tabs/Tab.react.js app/components/Settings/Settings.react.js ## Datastore file should have the following functions + +function query(queryObject, connection); + +function connect(connection); + +function schemas(connection); + +function tables(connection); + +//Question are these required? +function files(connection); + +function storage(connection); \ No newline at end of file From f6aa99c26e664f5c992bb44f68585ff467ec4f53 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Tue, 19 Dec 2017 20:24:39 -0500 Subject: [PATCH 03/45] Check in new file based on comments --- NEW_DATASOURCE.md => NEW_CONNECTION.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename NEW_DATASOURCE.md => NEW_CONNECTION.md (100%) diff --git a/NEW_DATASOURCE.md b/NEW_CONNECTION.md similarity index 100% rename from NEW_DATASOURCE.md rename to NEW_CONNECTION.md From 605e306f4d49c96e14c368cea2ba1460ec374f84 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Tue, 19 Dec 2017 20:30:11 -0500 Subject: [PATCH 04/45] Add icons for Athena --- app/images/athena-logo-small.png | Bin 0 -> 3827 bytes app/images/athena-logo.png | Bin 0 -> 30978 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 app/images/athena-logo-small.png create mode 100644 app/images/athena-logo.png diff --git a/app/images/athena-logo-small.png b/app/images/athena-logo-small.png new file mode 100644 index 0000000000000000000000000000000000000000..a731a2bd55ac23a041435d66e719efa2e6d33e86 GIT binary patch literal 3827 zcmb_fc{tQ-`yV6QAd$gXnhvQDUdfszB8R~!Oq0~u*RjTsH6bBeqO#{4`(TW%vJ7L3 zb1YfL&aod^r))*M-fig`Z&H;CrwW}7(0=@!QR#8zD^!VUq*eciPx1`j*t5IPuIi;!SKR;oX zg>h|KWR?`{r!FS8%Z(jS1t=$4#{A`nn;=pqDT$QZ+N$*vOSKUSbfW$7aV6cVtVkd8 zQzs~vU@80+n6~N|PyhRBjYWv{9aF1F==l$@<(d4%<4a!^#yDK{!X~Zw!A#E#*O0Ap zqj80})6x4cKhNzaU*|_j-eW$Q-Pclc%$^n7jmvG46Y1_jpCyi;$o{=xu6tD3xWjq$ zNolJpbIqyj1cVGX+BVNg&M5zZY?OG&K%mp~&npGUsLQSxQVep{`F6Q-W$gUy{t_n8 z!-X>;DGP@Cv~Rf}xMHx*jwcY;S*-cq!yiE@1F5X8m80_)6yvAGkstStWmo$Ks#X?8 z1r8^~JH9vAX)aQ}duMGmH^RC>mXHcQ-^#1W5;rXhud(Y(zNIVn>$ZJ}z}bzX`xl84 z_byf{53wMQQP8)JBto7WI?U;)6-vYKZqq}zwa1)2pDJ{o%*<*_jRxAA9-I>Bz2?1W z*O}Asl|q$D8+&of?C=5#E%Aj{(`Tls#?JG|&_->3FXpUcMaX^K==?pYgO8G2DVU4z z^0&KxTI({j;Uozd)E)_KSQ>$Ew`1<+9dVJnEV%mR8@>dn1BYU~e`vqL(fGh)d?H?C-J@l6TP3OrXe0>Z`ELOp){fug_yQ<%sOgPQPiR!?edW z%n%*BSplzC94p8Wt z~{_RQ|N<7+}M<7zkB<)9r~ ze|aWOG*~x!4S2TIc@fZe@-?y=b<)3w zc*$FS{pj#o%}=a>DFxS!N2#Dk{~$kG*1R1SYd@Jd6kTZY1B zm((Aw6}o7K3rzUhb_564I;_|OUMx;hF_JNQ2~+cSZ#M;>peTw7u3P^3)e;x#Q`DQO z*{Erkt$T@Gl9qY7@OHU2vGgWfxj@IU+EWST}V7T|eCFb#=ayoQP8uVTni) z?=fKzv~*|sJ{f-v*N=nqpGWC`P9q+*Ha|Zy4T5n!Iy^{VH^@NL1y@P}1IlUA?+XC^&OuzctmiLaehY zNICGPdR_3_BK+uyATOrb$A{_kOp;B^|V3T0rrOB;D;@8igklCe6-0#aG zd?QD|y#1fb8-}Z|L6^NtUjE^7EC>9x5V{}~?f9l(wp)U9nsMy0Q<-aS3g-&XZ>)aE zfi^y!szPpQT{I0Ca!b3R+4;s75gk3u&mUhm>XeE;OWa4?$vu;6&f&^$dpWaBev}H2 z3XGkt^y)y?hIj`BS?75ZpTDiu7#0c>>Ap7irLSF%@|?Rg2z~j9+uYZb=}|(vt#iPe z1SOH1ua-wX{5DriDL^Kcx7D0f>N)V<8ZJL1bL%af>0OIs2~ixF@q%h+V><4B;cKN-bJPVb4dU^zCby3gQC*c``Ww-d0Ff@Qtk_s?F3 zF{Z`jL}+RV^YQbMhme}=*_QPu%7VT)2Vk%g2ptw? zy;=WD*pV7B4mCBN5`wb=SCX)v?@Xx)I*_{uhaIa|SskgC7vZ!)Fo}&tH*$EqYO#3c z86zz!S)a$|I^h||Y4Mg8TKu{BNMNeScl?|>#BmOnyWp0ZcoSR8HAHS!zxr9E{x)+< z6QyHLt4@c%n|bB4FDoMxf#8?P-#=Xr56Ckb5k1rhf^I- z*EBW#q%A+M4s0F9MYe5rMzZ(l#FkW>>ZSMi*YQlHdG1(a=bGRAq+$gRxfJ2gmzELy zs5`v$`RFDnL1PE4nib@*8@z$62^O4qEn9|3N9^havHmsBvqqCAo9^ z1^HH~eZ=^uo5>_bpQTlKwxv}6P4~&9@nM7f4pQi+ZUY%6?8JWfM6LPTXD7nbTX*)S zLv(bcUJrT(&IG)sKA+Aq?m;6D7Pn2eJN(ul$n~krL*5=ygl-S^R_L+@+m1f8?FGf- zjtblzqk`$9S8LDes=ZJ~Gu{nP6kGN9#q>yPuiilK51PcVac7T>(hyc$PwDhVbQ+kdHhl1oM@of<3o1$w5sYw@%RckGv!Tf+3aT z0nwO>fKyOq7eS^6#PbXAlT~)0{5`TuC#bNAm*hbMhSbQRrwlJw!cg!2*yDvTl3IkiW-TBF<W(a4t?>*92^h}lh7c?h%=f7Mf@f4ud)P!O=k`d zf*EKB_%Nh$qb|VkM}OY|Apo6_3>+C5!X+K!zjzJ09jE+eL93NWEK-i=yfnpwj>*lW z$;Su`Hj6Fg_sdf#36KlA{q=U6r5^4V+Y0)$8tQmAb9(vEF^heT+(Kd6EWT|7$OST_ zgxzJ_8j6m9jLuEqlwua&Nit2&ob$umD03@rR59NmIf4#&2FZ~$ z&~<8%{D=+&2g&rBF22EjdQBJ4U_ZS^OFA};=?F+r37v}6Sgl;i-<6)5sm>{ zP+Gvj=}{grOxhXEWRB)WTSLtvQX1K?`ZP_c1vYGR3mvdywY~vPZ1cAi7gmc7hS*kB zqs&c!m%u;bOZQ-=L{L9YGh?PnP^U9nNUJ(wTK8pA6HWpTJb`{vShordh!RvHX}gap z7SN6s)3nG-E94%@psRz^O=*53ag24^bTGuc!$30$RprDY=u9=UB6~>=cpf?e+`v|V fd)H@+ zW$Z>lLDzWj=TL$W9S87_?C$DD?)uJl?p~G;Y$=p&oULqmwC`KOZEx6G+W5G(+A2^` zct6lqQ@-hKwpe2scE-?i+7Dd<;kd_h;_@{rk(oPZu7xw`TySua-hS+tLfz9$KU{x8 z5+|Y%z4dvmLv-Qx#eSj928DqW0WU5rc<+sW_(fp*y=gJ&Na5`##Ya zJU(*nnkxVMv{N^~9AoK_-*=m|(XfN74TqH*$Ho%&ZM~2RvFy^i!%wTXz)b>A1?lXH1l;vev=>b-N1C9Q*M*bX1UoPc3vB&l%v8W>$tN z_}~-575XUV;4@?L8tBxIK{)?^If(i0+ZYAtfjB;|M`q1`*3ESC72StbEN|v>%9I7p z440;~@1_|AE=ePw4iC7@G}A9t7VCvNtrN017B+@E8%(EDt7eB!lnKyk zPyhJ+B7}d?zP_}N)D4LD=4{qW|GhL0WNP_rLt3>=!u|_B_vj|>1sSjT_xbbXSKKUR zMmWmN^b#41@INFvaGYoFUA*KD%rjb(Pnx`4E6<@qlUg}?)`yGtdM+!1_vpll=qB*O z#z?1XI}Lw+RXYZOPjthMx-7LZNQT(t{5Y|0Id|IEG?lI1Gw&v2IiBf`tjDKugj2jy zIvw8svOHB`MahN|9NAL!%5sEN(d&!XQtt_pjbmCaG@APe&ykF3x832Zee%`XmRhAO zKhAgYQse1H?&iI#Fpi2=Xy%fm)XhlR*S337i~`9sl12U~NF$p{HUB<@SA9K8Mv9R6 zy~quFhP?rvH*cjv+g)u^5z9;|NW?li+4+6Xf0wpJ3yi)oop(?+~u?1 z`1<7cM2@Q#U06{H4vzVt=`6Ck@)aJSC9iz+ni3iv-JBPSWuUw+8RX_l-EAfIH7U+6 zzQqV+YQ-oTjj17KyDjgfxPixdJdW8*b^w}Mdg_kF!^mSAb>;IKnOh^{ ztBqVr(k?wUs(pUDFWe02-fm5QOTVuy8MK-*N+Jt6@uFr^w(VUF(MRDlN%zGV!sV%* zDg&o{HS6%h26nXC@&{seJlS$Ym-|)ugkJPWO4y}gXD7Lo8Hc|7&0&j&3sEvPrn8Oe zhwd;~wg_p7F>x(V^XC~V%UlIN8yVT(%jQ6FI)r>tLtg(T#;IyFb=c;-Yt@UK2f{Tg zZ_0v8XaI?6$!m*pCclav8F)-TvNFmgy)*OmDpnJNOmEoCPcPOyIy1;T9d(qKG9GgW z{pjjj{pIj8AlLUP4Gk}yeDOYnEmB*Cgka*73j%wI@Mgu(J#W+m)eLXeIE%uH+>aDt>s^mz3GWorCoUJ@B zZT@A&I+r2Yu*O#?LXu=~NVt4@iCAIgvTAb5G1_$l&Nq9o${ajH_tAqTy*w$Q$}8Yf z6wqks|dZ%srhNs-!CaS&@RV%a&JrP9H{HnA9|?G?-Mmi5kkSC2X~=nX#$x zyh+{gpjRq;#-VCtvp*GZN)hss9)L_6zR!+~Eqh3kirP3fdSv^ioT1H&n>H;RQpM9v z!IsEs@%eQ;qAzcAq^rj$V#8MP98{(!-Lua;?9od`>r@k)(ip`B${S$cw-=jjlwRCD zy0S2-FU=ltp4Gfi$D!?omh_6x!sVJ#U*$9UVwttq%djIZbjE{Zi(6QkW97vC>LMR& z8cBaU8ts<{P~>=IMh6*o0_9q{90IG!F+677d~ER!K~BzjzVS-`IIU&kRj*YXU(_sl zZMV3HT&O+(C-}T3JApTBIr?wwA|jin-&xC< zm%rup_qFfiisX6qYWU=+S(t4|3<%KMmZKC5GQP|ArT3Z;3_;Ltq;DHW#!_M83bgJ4 zEkx~Ee-XLk_C=k>G!D)kA7==i3GJMDem8mQ>oYx=yCTEX?pj6lJ5=aW@j~De>|~+B zSi**gD%;I5Z}6uNrki;cJ@ivH6E}|ts>JjWirr^6pF#c4LBF%rr(kIz_6*hH7a$qN zYBsNs5i~y29?X0*Hz5k=HE27U)I7{3uWS1Z8e{h|)#~mRem6?9Xy0c8Ih~)49^P%6 zSi_kkO6%HCXvXR7WMXg$^?~6Q^GvSi)MBLz59RiRd|19x(b5ly@v7AZyyiu1Vq|%& z;t-|iq@aU8ZN-LhXPIx#jWNMeQ1UjhO3@4P#n94wGNqOyOlEkob#uyXWk`erT>AL! zz)(Jpv77%u-Je;dFw+RmX9Rily81s~UYyg>MVM2iMMQW?wwHuIGu(F@>q{!Q8Q8v& zGJnOiVY6=7EtDEF!nQsf8>V_k#-{FkQY08}W#7ChxEow@>|n|1MjB0x&po;e3isTb z?F{&#AutZT{s-%~#EW2~w{n(+!71|Q&hR z+O&LV7OapkH%7eWY%nL?qNkQZ=B(*)t6F7hk3=(BA%sl@^WW)d{jdb7wrWO{SNA8m z`GX-mJ@_+g>qOY=Z#{+3u8^khqYbb78_{1WZqNBs!tBh9`n}o>&8t1^FAY*&v5`w{ zXiP4IoJViHaSZUsH^JwFa#yo=ey#FM3Ol2|8(~~hSlnNXuDVVzp0NQ~r6mU%l(!1O z=wz_|>o@+xzASUlpm+XZl8m#Qs#;p&0Pl*?DD1eb#h+Ojy;@_6Y|UDBz7pc%&m~YWq?@PGqt}@PA<|teK8S*>CgRhs&A!bgU}4cgF#-LZS6=- zei2EHvJ9W_a~&uBlDKThdCX>wo|fMMo=HVU2+S%p-G-mc?TMO9_jhiPkjInmw}Kwf zPR#vP0(NK5#Ose}?u_PProL|d+*r>4?r>#83I;cC{WjtMaG(%0sCl4v>zF17FEh0D`)CE>J*IaLk_Z zrx84j2tPNUVQIZH`zMwF)akxo&TozV3*^+YQjeM^w1!Xo)YwVKvEN7gA1v?RBydYb zLiQWt4}3j9E>6|*%4$I+$o)qLRwR&kU0mE2Ddi{(LlzqUIaSVm=a0WM_DWoexcCKD zdZmU_4O_TR#2g7(@x31!Yx!!?@7K|?FkkQXiTH>)ZIsD{2Glmh*0b%OB=B!A@NcFl z$p0=kKI9obQ}W`Eso2=-0m{t}MpaJ0hFvq4t7n%GKybJhNUG%s+&?=Vum{JG7OqY! z$+|B2zsh9*{|1VQD|Jk9)4`wz1g%WPy;d%-Rg_sToxdnI&Ui~5Yg&aH?s^TwPPOp8 zYl3ZU7$W_a-i_s--Sf00XaeFu4wfZSZaBuB=;?Mz+JFNdtoxGG5=9>#Y^nQ zo&9zJ^-?eG&1@*KEB?ch;_^G{!+k?A_q;$Ju`R+oHAjQVQS;h`kma#s*?mPU1alTY z;$3sAj(lt-3qBg=wX}5D_=d}!iEWvf?5YiwsmV#bCF13?8B{+Y3BFD5Toqpz(1cVz zq7A=ilzIQ=24QXbV-CV=T`I9{9;Yg0KTS_yY8YeKmo|5;nRc>&N3R2gI|)8oc0AH0 zO?ac%IJUm0(;M3tzco=ou0E&tR_dG~IV`%|>+N28j-s?ova{FbTFKX>s*-sWUn;UT ztib^juI26r3irPHTzA!%W~+8Tf}?h;G|w8w!6B|hzkvO~swNTs`q}M$p*_OD*AbG9 zAevbsXnq8-t7)D1fp^E{y1MkD%6L>|JQ|Q3PBP4gF0xk< zvaoc(a&L`>df(hkxeJ1cIr;5xz;~Tio+~7fjQX8!`-erZ>O6nnH?+Y^hzM)U znuM>6$}4Lj077kcXam*rVumzN9ahWUHr_Dq<5iW}8Lp70=jYwt^DM?$0X!}W0^se_ zh|{wQ7b4FJM}!{u%^|{24@PdlNf?Z?+v~CJ%zLJ{{5_E>a_exo7{g;8G4c~} zuif45O-6AVhV*>Zju~Q;Rl;n_yy9uOtzzj{ep?r$Nkh&RqhkT+tAXo9Zp(E`!_?05 zTPt5>tEmSr8=L2ghBR$<@)I7)DWO59iYk7%2P#bZ_ihF%Ew-9?5vYd}Rxvgtbjx^z2;+WH9SCj}xtSS}@2dLrN{2dOz?Ubd z`rdDV)joO^oqP5-!2a;a>HapKc#dBRP_V4MkJjf?3S-&^oeR%*eG#pLYan7q{2I76 zHI}{?D|*cr^(R zwV{oa!zg3GdgaUQ)UK1J1ubWAl+4B6({Hml6uuoWJ=4I_Wv4K$>O(=^>Wvf31Li(8 zz2w|5=)3%lv%FZMI>Ay=xbyE|e;XEZEjKFGq>z*yFu_)TVRu&NO#~Ogb^hgN?%-I|%`3hK<`n_$vi~8{wit&&}rF$mSk$-fk~IepiPaiY{gW8_wT5%HTSY zA6>jhKH`)glE3jJo;JK#K*@yOCc&`tr-vvyx>C$x_lj+Qp}&H96ScR<=z|Ia=gCR zLHb2W&JE5>eg}v9@>vaE*}ltT%4nAuLzfx)k|rJXu+DAVH_6g#!vkrrl3ImCn7e3R z;AY@x~D+aaz`jKU&2h@gu$1X(VW?#Mtzx&D zt+v1;C?w5Z=L|BQ4Hs^3k~o_}qxI3x>NATikK?@x>LcAD8J+o9!h% znxyb=PYr47`WR!STvc@>Uxl4Xzqi-oxY$N`BblJ^v83jWckhhe)ZE7Wbi6aw0E zQ-$6ZERNZ(OijofGAXJ2I7Skh&f{!*v{}UL#-1q=9N-xzL%@EQ2oQWwePItzo5oQ2 z$7HMtF_<6?zr8iTrK!*{H|SX4w~?Kmq+w(r$z*A}YU}|tkf^vdQRQ5DEZ}&vI?t+^ z4do%ZlQiNT{pZ)ZQnWJP)Ov`1Z=WwFzBYzK=rP>nYJn5QX{#`W2Fz zltjLcJwWuX#_Qg8tPE%m;Pz2!a~?v_wjKp`j>p!(wCOnzz(aai-MCz1P7l=E(#8AtQ^9^5dXcYZb588Pj^S$|0}Cn zst7g`+k4m;Z?N`*%VxN3qx{}>>0-ufwoOWU?ca-)#zn7wvCsav=wl+iMqw@w-us3Z zIf|m*RJ;Fd7en|kdg&ITccX{BKAAvVoRQoCb7L$0Q3I-XRy9+Zfw?lS5mIp<%dK$t6qaX2f2Q5@vn ze$?>lJn^>qCW&!%@^7##hgMLXDbzi5F*}+z$WFkbOLoG;dESn@dgfsh*}?}S# zunpAehtY@MQ1_*Q*ay@cfwBhF;C;Svh}mBCwxeQGoCN%EDl6$LqCW6e-it5c9pSnuiYqmwbj-Pj#&BO(&LREUkVWJafziM5>+q?e=h#Wezv6TUGsx+&U$JXx4rKq1}G1bdkAO> z&A3EdRUY=9-}&EIh%B)DXpiFs*AX@}QmTg{F~{=#0OcH4rxxC%ZXb$VOu{m0gu=YD!3p}eZ)1*`Y=br35L}+|g zmYolkE;@sR9Ld39M^Ou_);sUTP-$pagTKf9-GFH7eKIL>-NL@-&@^r~YQhU+WZy44 zWInE&D@z(6ANeEx%-zkM&y)=2fu*{PCDG%WO|aJE*buOM z-(3BN&*u14U$Y-Ka@X}983GhjOwOVZW;AN%Sm*!lycH?aKS$bP13q*RdIT5ZrrGM} zl;#`~(^z3*|0az$Jv3Oy=bN#ROPN`>@BP_3uZc1}B>f&i!ZIb6(7{rC%` z5FLYWf|B-P;pzl2;_L&}Qp0b3yWPuUcaA(V_!~`gkMarm;&pt6^P$J7UrKo9rxNmd<%j*h!V4f8j69VEQdP=aou?G4Bp3OlLm6Q%&jkQjO~)pb=Vune?i<_` z1{~oIA(&X}50W1BRZ^1P`%nn0!1rx^tX&{Nr_iAr1yWVNV#^=Pw?GSFdtCPrA{};M0=JXwDU<1Y-`++Yvb^ZeE8!0-?P7>Ue1nRBf zZ8&;qy{2qrHacJ&DAxEb9YkVFd!fta6M+E(72p7L1-J(H4i-1PO!{{ZfK=40#!I(|ET@FA*bEe zEctj14&-A?5v6<`xZph9qi+`wO} z-C(zYtN4oQIcqil8@KYy<`K_>`iO^p9_Mc56wzYwhz8WC+g8AONW_fX1`?k>>|jV- znz0v7=xf??P0HE7W4<}L?T!n%flRvK#{wmrp4*?fhdr4Z!RdT5kQd!L68+9eD%P9e zt1?3#_EnBLEU$FyIk+8+(td?%aTOD7^u6UTqjVIA4jIuG&9DObxy$64yfF|&pnX$(Izy%6Z=$_g#wg z@h`?Wo7MZ1p5VEek*i-e420#RITbz8;HL(6Yz?X9Er#^X24o2wV z)s{zcnt#Z>K=v0zrm$XLiB0i&+C6S9i(A%)@<*rcdZR4xXSYAbZ>96;nm}vh3)uN=LT5oXFJo+PhTBE?!(Z-tOsqXR)dwlT=hhIJ%QXn-+e+uG!qmggmU+ z>)#W4SAa^C?G{=BK)BY`Y4-X@+f$^+H34yur{4bORkdy(L4)Ejmb@U|;~nxjEw!F> zMLgfMFOlxTWBY}{%b{%5V#omkn$c_ZOGOpEx}(A=f>dgGYlU07+w2SvC00F1D^$A4 zNO8gB2Aza&zoC(^q0rqs_8jG~+FJiiL#Dt9GNkA2Ef(eBdg1%|0;v*!ldI46w%UHr z8U>*pL9HwrY<#SZi%)&?FXUoR(E4B9{zEZV8@hFsxRY~d zt*7T|0j*8I!Nf_YSknm4*H|Y-8O`5Eulj)NPZ8ry0ubi&?X`g_0e?%`YfrDFP)H4M z&of!1@3w&6zkjSg9o$zVf)PG*tJj=6J)xrJ0&!*ml!OI|G#jJ0Hu&_CN)4Ix*vCpw zMBuXKtYN~cY(fDddS!<#8YYc1>ATtMFB9YS18Bh;RI%{(HC*w=rCUDVfQ&6r-I(QE zqfLW7MtR+e1*izXk`r`!6J!N`Ot!^dPXh=LfzXDQ@s7&X=A$+OxJym1G3gnAiJ!%T z(};|0Y@&d8ZXn8ayDRMvv{C%e15YYNq29uz3zJ&j|{sFqMTYI0CsY(AEI>3e2a)uo2 zoS&V)@pi1Pf(D?G|E9;wOke&MhyAw6aPxA&Xa-wb`8@3a%$p}Eu>4_)Kv~8;u&6y6 zpI$4qChQ87bsX9 z@zERa?A?C)D2Q+&Ocsw`xo%!eTeI!+@4OK884G~(UnoD*KWQzn!TPfQQ-1k3)0&!M zkT14ouLsQk&ponL{*kwO;gcvM8xj*^b>P~P;WYmqrMBqd)ZmHRPlim2^N~mesdVT? z7};e^BDi0r!<&`#3yptl_&+j9I@GaC@r%C#)d;zt{2kY!Vr2_t@^Y7lJe9927$6p? z(V)bB)T>qN?f(>>&S1x<59lL1rTdF}Z{xLW=_39~Yk>_;4CTx@AZE9mMJYdL|BELA zqGow2ce+n<;lE7l)W4e6gViwPKq2jfFD&}cJp$hCXYkXhaOKq>mC+1004>*a4;f8^ zIt>}Z7Q>@4RuH#qQ&878l2`WX7e5U2VRX4)bTPgJ2bMdtt2jdJsw2r(!?w5+8`pvC zGO*$Q7~Y3skGyW(G|Hsl_@)_hNu5hr4qCl*-_;IMSOk2tck6!DYk^cC9r@Ur=E@&Q zl|Erru>dpwQtlSP{1-n9KnFmw4qsWr2o^=t9+JEIv0FPsg8Cn|_P6_9CrYyhSs48L zI7mu4Y73;=uUI1-h&BFZTCzTkm<6K{ml_oM&pn1L{o#}CQqSrFsoM;;<0}6gEdI^M zz@hY=GvS@lu;DUVNGV8iR6S69NrLcXR{%(102_AN*+6^{Ba)Bp{5_Y9Y`_eY#@=)& zp9BwGzfS!#LjOlN2VD6N=X#<2OC*XF^?|ymE$91>%A&Tl4i@Spm+(&VQ37-oYm(HT zP)k!hyOS~&@ZhJb5CH#(VBGyi4;;=*8=CY+{hp*3jCyhv%!FP7g48lyc#sKg+x{Q< z-Ji{XQ!29Ly1H~0q<{pqhB5d~V#xkXgjB-}MTU5)N9yXONk=7NDIB57Ys5yP3qyf? z%rNvT_^?S27x>z7NG?*rJlS8#15JO%Y|&e>a8hUzmMD{2ha292+Xh|>_aP(^{1m14 zZf_!9sE`+?|b?Rexn*duOcuylTn(JILXi$7HHT8s~mKDoXLweX+X71 z2X3Wjaa=uPR-Fo}gq4}aIKTaBc#zg5q%`idehrV6DVl5`i1*J|ybSdgsovuqBXyb9 zK~!#2P(kdUmhJDX-J5>Xso%7GW%|LIvGHIM{_a!#{7EiJFxSS;*AZEymtCn$Gc@iD-l$d;KE18`WKq*M5a=bs zvn6HXrtfD)so#mmBCGh&->kEw9i@dtzs?7}njZFrtP%*&t-|==5=yd-&gsL*2)nur z@sR%alk;nV&o#h<^hy#&-dFFpvASMhHjmZi5ca&qfOsB#KRW@VTsHkdEp{H8n~P%6 z=5YhOe^)C+v@`gVUeiBvKli7@5sQW!9~DOE9-fSbBb5NY0s5JlJAL?Bk+DOD&<<}Y z1!kdhj=ges6yq<|yFW8FEg8;Gio(xt+63J4(z^;qa^h~do`5mK zxyJhFD{pKQa94@Hlrac(wh$~n`Lkf9km|8RUeo*^Wlh<71sZ_FvZQOf(G;@60lt|- zCMl&eS`v+wN}bt{JF?a)bQ5fdt>o9g@K0BlyHHa8Hus#py`TDTF1;KLtpfuopz~v; zli?iB053fk*pviY6&52e{Dnq_W6%Bz>l4ov5BY$H!f}a(E%I)C7~urS(;|{EcqmEN zuGDmURSO-z2qixLi?@G08Zk+~^|AVSInA@oS9NS&DcdTDK!qJ&%0IIq0T28m%$fF! zoafVPaZPI(|42B+zp%ah*jy2ZdKEszzLPDvkrJ&Tp8z@X1aG+E;aBvjG-t>p0@{Ew zHU{I>!ZY7RW9VFWXuGUldH9>dR^5MWpYT6J?Poz61GyKv6ZWQbhA}OM$Sf>y-T=vH z)r5wYTfsg3Nw0;TfW<~j{zBINKi%HxhSRqFw*wyVb4wi6l9vSh?$jSFUW9(5lo*58TlzY^7Fme0f;MPRaw^z4T-knBLi$m+-9cjj z>3%hF$oO2nMzan?Z1Q~O9`BUax{QG2_}PQ#FaoFqZ9Gn=2&ne%`QN5%(`tM5ciV>&)9HSwU# zA^S)AhU~pcJoJ~2jzZL`q5lpacJ7U#TFEoaWFn<>g#J5DD+1o z`svBReL_Ul4t0KIgR}q8$=c1h`8t<|4V3JI9{Z2GUnaJuHzPw&ojhmcnx|_{0gBsQJw-<(h@UoU3)-q7dBvR>2^@6!SNyrzv}67@hiuTWB49wct^ z{m*2PUiu=nTDxswAKnJOa&y=>{~n>{Q9-3!gth*#+Fs%SYDkk0K+l>Q7_h;Xem&}SZ8&=1HFA=`}4 z+V&aBa_pje?H^V(MpDjuZ^Dug?9Oa8TYQ*(9n)ohgS59nyw^~wY zBAwHQ=PN%lN;nkhahsjo(Y9+Jdqe`L=zx(TxOy^ElX>$$lvEe~-h?<#g={jm1d3W> zT=W6tgm}@gO9<1Z5nI=l9E#DTXJVQH-eAZOaQA$c8&KvQG|WYsOqyTX-QUUP@F$%O z^pKQe+R2CU*~3G0sj$;X?qn|~s#?#>Ku@H?UFg|%ID0l{pq(w^xxPIy7U5ohTTBS| zVUoD3EYc9o>OAl7)RH6I>egQD1saS#+B&5YF|OAfOjCZ3g==KyGeY{R=q*{|`q!&azUrBG z(=$)^{NPB-%em~mJM6%9nDHy204k>%1SFwBgIgN0lk{Ux%;-~K#=Y_`PR?T$kEcm` z4dwagMje)+!Osu6a)mSW>zjlcsJ%wq1tOO6yg=bic*Pc0DG_oFJ`Ykh@_uaR11ky@Jl&)sg{uvuUPaC< zYO|9_I0+gED<#le{_=?2=&Tw_mwVU?y}z8ZL@SvcrggsX%X2(iy?<2{EhLu{#38L8 ze!M=Pl;+0PPHnE|vT_=d@d3Cz@gSC`IE3yd`u15Rwb!D43#c5iM=mn%XqbNk($XJy zHV`IlEy|+6I)PPV2Cjf@>a*7yrC^?MstV}feU@{?cEtlI@+V+<6wLwSZZ~6-?=U>I zy$9U5J`Sua_KPl~9?<>pg|Q3%NK9mN1#H3og?vK}BX-I>kh?b5Pzps}$m2lBJz#v= zDhG?#W#DB?NINJ=m+C zlTohgKcS{PXRR!i{UmyVF{_UT@(j3IsKuV4zn-E9W6drr(rMyy>}GuNL_n>q^020y z3NylvDtEqWg!vX(i6t^nU#9~&vWZ6tS!U%DK2kn6m|i3#bmQvZPXc@_NQ~IkCLUa= zNN9S&CN;L$QaCu2v~RU|Q4AD3E-^}dV;N-($$CsxNE2!DZ5#o5{7E8IQmEZSKG9OA z)B(?+(tB`7S9H{;HojPqjBa1bu^ms&XKS0i9xv%?Y2%`J1VF7u^!g8Z!Tir zHEV@%PwdYVUX$($Q3UQMoN~Rew(f}2n6w$MbuZ#C1!_*&5|rCAzwmFJJr#nG&g-XZO97^B^%MzH^YSZ)>?G2{BG^iq4m#s`0k zF%=WJ)ot|aCxE}<)|rXARK62@L3zl_}Ocu^fU z>O||u09>47Wt1MNo6sl{p}tNYFq-!-tx-!B`|71~i~ssPt<+E(nxr|CIoGt&jb;q* zE)RKOQ`4IYUBj(;n{rgP>}5bT7RT%|JQyBbE9PT5M$JQuuLxHo?U=Bbu()s52HT=y z=zxozgFzl|?7Ft$w%Y}q*dWh)cZZwu?oAbnCOTBK5=)x~13EYLF2T>uJS$Nb$kE~q zZ>phtl5+QR2jk3nDn=m?-wA6PTY3J=d!BJ0OcYJNA61gGreRP(V-W@0?57C+#z!Vv zO}+NgV}Lj0iw>tvC zw54j1l_6SRC(T9!W(%u!I`Z~g*&MDQFcP_fw{2wSh8A-3g6{W$0`(;W=Sw5LXZX(%;#C zE#~>UzM$-cYphiDMgA6=&Ldwm3btu)AR$VO{@mp!>Eu*8>Z)%9(BlW$e zHJu$bLsrf)1Q!W=?{^Aj`ASFTI9>1C4j)mX_XE?2pe{xk+(1PWd>%+d#@`-JqhKEm z+}LW340cpHkke>2sTwSQ)}s1mVBNCk7TqxUbI{P@RufE1PT5Dbdus>U%iitu$p;!8hXtdL`2y*#cI zKI85Bpy2Xwkf2hfL+8}7VqFu2v(U~hQ8GWTzOn{K@DdD||GxPYEJWMicW!tk)eQ`m z-wH2LMh$x`bzKAAR#3Jtez%%J%&E~whjNPn4-EkCdRQyOLOO~P8JN;sD#qMM$l9UB zA*O6Sr$j&#TVv_Plv{6VxE?N)pBm4ETl1U zAI@;xpoLU=w>)|-hE5wD)3ciCbXtBe{)A^~3;u}LecxXC69#5aI>+WE_31-0#`Ua> zooHOTR$^9v4;D7>ZCum=iemvR{In_>bU8g-cSvBp2j@b~RYapU8|+2D>*S)|nD$J* zk(kl7$;&X`<5YT;RwB4tIi0Z?t{{sx!0?{ZTtfen#eJ`#T*5B zj_*w|xg!a-qBk^{e}=@fsa~W>xtbff4`TAxyMc)(YF2giHXqiFJE0OsOxb1knG&(s z@jkW!bWJnCmcU@_sAup{Vc2=jaILQ?mrfD38(ksETK*EMGV-TcK6uQvrV*$I!7DB>PDjl`gFwyT&ssv%l;0$%ZWdXOCO^5vlpOx`9Q^>g~!*9i6U5;4N#BQG}3R_iR zh~^)}Od-nYC+bW5Zl!kKh^?(d**%HtTZCE&PZGK%gvQ+(2$T%l6`ZW+818NaPZHr%c~~az72GIkF^*ba5|YQ6G{@n+V8|f$QB}so6olbDI1$%uWD-))a?e`ikQVw-Y}g5Xx;)xpa!>hm8f$0`kfh*#xb^ro zE!ASPk-C!3&M>M(<*sbWOIv3DsAJ{QJhkb9l$uV0yam!v)=f5MBk#tepBVoV7lQ|_ zrFx*Ubv(QU}F{}h1|u@dYOHFM9w*PU*G@_ z;9`nYtQ#x&?u+MV!7FG*Cr+}K8qd-RZzne@Rkc>$9Z_YS4;|$nJAaD?c)~?^K4*q8 zLto#ue|`s*h#K{xHl*7tSFK?$t;R|muT&CQWHA{AgJ6fO^DOQ#y@6g|UDQ^49f!%C z*brF(mUQr%gH&^`4NbAI92JFY*h`Mf=;`U(qaMWabDIf4$2HC(m2JMqls>zq%~f}i z`G{A$@R1hy6e`aafK)!szO>j0X8Yf7_YS}eVI zsR`1S_SUpm{hd(7-OKqjN9C>#_rob;sdS<=8C6ToT=?%_IY5eo2XdZu4bWP~e#58< z3d{zJMapnqV&k5=nh6cI^Sx`omdk6^{KX!}LqWoR4hR8h)ft?+p%$xfm+|+;}_vKA`*I^t3X z^V6+IY70{~%`?yg^ZtRF7QjgAnTH8b(n*OBJ)KN+MWVf`@HGa$iw-Yjd`@aIG+uS$ zf9m-Wcx>vY_<;1vXK7dO)<{(wT6#53t9vyhlJdPg+r?f&aOu3K@sT;b&t|lx_W_Y@ z{W|TVlEJYea%sSC%PAs7@L2$5V2lm(iNo@SLtuafyw>aRK$ z4=lrkO_GJ!c&{aXYYpzg9%$STi~!caYSX11B<9le06PZa7--1cnsyTrvxz&^&aOXb zneZ0d`Tck^6=c=lGS32$6vpE7C2!!D;?qV!96zzTdb8gmPe#~=HE@5wJvu-C^xP|l zinqPrkKbYtzFywx#GP0pimbm(Aloe!16-ekeXU=`oI4Z)c&RufyH(Uane>v@N zlmT~b#Z3grByWCMT$_Izjl0PKbOQ9B8GcImE}~_uf!QtmEUW;ohU; zBpxaK(nXf{cRw*^y&RBo%@Pl~0oV6IS|2iT`hA#akW(xwObO}Wv;PTz)p@;kVDYwa zN@H?hstwsO&!PilRc}0+Z^fHnAAziR6R7+bjd$S?X-wPu95_o)zqxm(opgA3(5}wJ zP2@21F)O$mmENnDud?uWK64fPGI^luSbkh}T^VG088~-s>_=hTmu9YPnMPj|3kYS9 zF5`65nEwkI4D;|=6uZj7qhZ?C=T-Z~BXuPhLX#bK!e`eVhasa${8D^?SgqIWXuZGURKOG<`fEm|cb6Ne;1arB8PZ(ECf= zB_-{>rg3~z(aDfA2ajk;HNPCIPISkub;MY{`Se;3T+0pRr4#o;=K;3ZIMH+1%f-Vy z(5^REC#xcnil%M-&gV_H^S$46EVAR#zD3U}Q}}j=7KM^T<+q&4qcG^f%QbP$G96Y=WP$tgIrRQlgkOc_$)O-_NgLMU0ENBl$+Z#YpH-^bQ6IBFOdM) zynEiTz>X4YBNQ3eWiq>tSHAMn_5$B~+_k7Al7o}4cf$sGRvJG8$>0L!(fIsuZ^F;( zl%@Av=0g36?)Q|8hmIn4bM{i67%>gaLxEAZ6dFnNe&&fAF~@>wz7Ewt9d?&sVJMd} zbP?>)m)2LS<5SYrnQC$KJ3MVxjvsbg1zTJEjE!Btynhq)C7Vf3>^~?BaYDC!>W2-cwr_nXfVo>f@G$#YX*JI}!htuxD$v$G+ z{o~5rw*_cgHN+KBVFJ~YAuw6b$@r>bx*-WzhzA*1un#%KeNY-Gnsr@DM8vw!K~liG z>xSFKj8Qp#ulzROxnWxZ?~JQyMeZato+@Hk&$!OD_g#AAS>VPC`Q4=aa`Fl35{A(P z4K}1g0VY+6bj@7eOi&g0dl^@N_THZBP6#4SlC5@GpL>iSs63GSv z89F2xY?D5_yjb;ktQ_yOhsv+y+o{_hK4~1$Ecb1ofpw+_yBV)*CM>rDm4L0!itWgl z0glN0$oTqvzD(%8IfJs6xK=3$K+v3IrPEbikBLj`7!IdZLb%jm(KYninP=XQ!Vf(T zKqF+8oPJ(tlvvy|w0$(w6PhroITTNtMiM_d`(9ztD&;&FWoB`)>T~|s1L^&3zop%} zfNvWp`8bU`=CS*C8>g)(5|t3nh)nvDHPlWx+v_nOml(e*`)hq6MA@x zvI(Vqnsk&o1^A--QAk=_?60Odqv_yQ!%pqp<0RWL52OYr7MADMsA+BBztc`5E!-JB7LnvoXNSCde;3T!EN27d#wldI!q z4Eq%fx)_kLFf1AMJ&CN_aHVaXp8QVVEAJ25Hfs~jZ`1C$*&b>p&3$^OYS36G)GU&@ zLPpkjM`^foDFzAt(HYE)nN#hj_B*2;xL#_(GhYBB`jz~|C@6Jjj05vL!t;`EcgjGD zEvRm8U{n5U4UUZs2V4(6y#f%Tz9kxyssRouU4CI_$LC@hY&zH|wuz0hO?+mZPkYTf zt!ey8|Gq`}{3&64pNQl|?&HeAd$_fGpfhK{KpsvWT|7^A1=zF(mh@SLZ2xt^1Gn-l zHlc5950;ZfB)ZaWbKhqN*0BDAAC)PuuZ6lF05r9f8D%b>av_@*9Tt@HbvZ1-ahLpb z_V+4>QZO-jjVL~Qj$I4>3E2A7W6zK{oCCm%sV9n^Hlo1K#m%?pSlhH_HxV0y&;Z+- zdSgc6w?)Cm^U3h|_{TstqufBY!+oc@Ots-pa#~P2NZaey5L@b4HY&~@S4tguqOT(g zfhnukZUmURxkrn_UIB~SixDQCm)4TH0sTL>1GnRRj^s{BUXsJR7hC^l=bj}y5kBJX zM~a0uk^!mDa~pjgL7-wPPD&%P+G8380$PzeLA}xypbpSBZU|e*0T5>DOQ%U3HdIaS z5I;JAzgur<;GmnmORX~4jV-<g`T_1qXJl6t@yw7yb){ivCG2Ju=&`pa^rG1sb_2eG_ z{vb9wFtY%N(z9he#_jd&V1+-|eMgQfLcG&eX=VM5%p|xMOk6ZS9Qt>;RT`XeZ+;wl zHWVul0umHlL_~LH%%Ar(fP2uB5?+Hlt;@WNyg?0lCf~sZ(p6P?=?o z_2l3#zXzyS?l5>;CF4*BOQZ+2a%ffq3s`f`GMn6RA5^kH3%)8KDD85aaZ7b@9}i95 zperA)rPsw0nw9X+>4cgDnM=Q}*)B7{(2$Q35&sZnrp4OZ)t ziC*w~@adH;Gut?W^mW{Q@QSfWj92k|1#;4R?)2tU@o&-A)d1{sb@xusLIj$dy+;E# ztNgT|!Ydif?vLB%iGwjlv45$fsRs{hU5I$URC(LAAT9;qn`u*`i~alsKfmFcf>ElU z7QAZ&=m0$e|q__5+7rLOQC(cNz;u#a%D^P2quchzi zcfGc_YxVOEn&MEgDibWj9>@Hdk->yT@e_TSHXa6yTR6jZL(X?KhKROPsGImDW?w%Mz)cLk`%0n669yr^M{`P(%fNsapzKvwDP+^FGt5hn zbO0@IC&vMYs@4MoBl;WMhigh+^3QX;XM;%qILXsC5VutdB9V49aEWooPe=(pxD658 z%jD5;8EnNu7pUisD`tjt;edxLv=4L!{+#iS^F>>*F@}>RKIHrLPgKe{`7OW9@XId ze@eshm;oev2}QCa++}Nh3tQZwayW}L7~+ldK|W*fhhGK207@k;={J5b6*JZx&#g(& zl}8Qp!&%I=fH+U~G-cp8Pnf~XsjG8LvOg*4d~b`aN7}P0F>NQCy8wM+`1CBI6&vpa z_!TZ2BHf1Q7AMsc)79M85Zy&0+8}pbWe48FFDc5S(y)sv3_%D{eV1ap$zJ!DH3^2F zZ@Q!|PgoH;z&StaOB~Jm6u>Ybd>Go>aWG_teg`5ocB(s0+H^aW51%r*INf=7))~-I ziq)6ez;uO~4aNuDeuL!+AtMntj18ldgg@HipiOm+|*t?;R>btgmZBa0p4!|4q zltGoPhmh{LizTiEb|bn)tr!2UzVutvKXK#e1pv2b0pIVxtbzb_q2o*FbZS#5YhT|_bp_|lF@5vEFJtt;sL~N zKwAY{5ZhwAwb?EA@s2LYvs-sd1d~5dLqskd{In>Kh69|fXvQm0e18(y`oL13e>Ob9 z7VOOMq=FIoWbu9rzMp~qN-)C_x89{T|M{pIywA}PM_iuU_Pk`;4S9U)?vh|*C>dnw ztKO&3mxW`4SuiHoJcYg`{XMy?i{(Xr(K+?PvN-ojAT!yxUGEu|mhSupQohSDZh{3} z+m7faH7)cTwJhp$_Cxv|cx#CHVHZZp>wscwuF!OIpiYGe!#0aBV3f#I4fcInY{wP6B7;@ghNL#r zZa$pFGqf1;XFNmze71_{255dVl)UxpJCk8Sl_a@PRVF3s7rs=ki`aN(h66@q{gu7m z?USk#MVg4Qd-rng2c|z(7IKtxo=z(q3`6V$AN~G(u35j~fn3Wx(-o zNfGp=vcHu>^4>A*XOtcJ?^!VfF!jF{KnYq&fp4b+zl|%?#_8Jw366&pDxE@}7l7qm zW7M4ZFSfR3MawBu4}JtMoo-C@Yp47~(vwt!7Nxg9FgtaxGSn<^nygU70VG#VrcKS8 z^V?4wgJ33Od6jJN>+_m0Mk*K4JRdasi{j=cX36iTIdTCT@7-J(1u2)j3^H{;YJR$1 zC}c7U{0bRU1dgPzun^fde--R;-}$) z%PpHN0Onf}bU@}xXR%i{={xA{LgBI=px?hT6mwO6aJ$7$)voPBVS4` zqSKPs_Q_GB*Q!sB|4z(NRaf`kH{PsE%sRZ*{9RN1`Dko$O>3*R5+kLV@oX!1(?}$} zmiR$?{|JLtH&>Cu{NDL`PQ#b~i$1ll(nF^J?Cvbn}of!t;4yb{Wri)LgR`g3E>kl zpbxDb9VN}=4?YYory>DnaJGH)^MQtK67qKa&zWOQ)ViuE68#rR{7$+5Ze2<3S`%7* z>(hfOm>QZiMjgi-RJ8RjeQEOhb}o4L^481dvyXj$)o=$bTxg81YV;|%?p8Tp%D-=8f~Mx`)@4An zsVs`B@5X5Hms|X<`UEeo&VZ4ndc#gOo91;HdpGG!)-rX>L6E)HaOKhDk39?_Thzes z&=@JH)>R1O@RyR6D4`xYN10}PLL!2!k#wn&6x!^J%Z>%TPBXFRgk!iHb-AdA<8e}r ziBzr}{KL_ZlI7K(U6wisb326*b1Vp7p8cxnFinZOM9n15O(@LwS3^boD@mrttP;wW z1T=L6;qYK0AFn)85xas?Hu_ZA?f;NU`%MZo(_aR#5}1 ziZ(C;mqFq2WKZIHbN@5e0bC5RdYOzS-eHw7-QPO{3G79(%RjqJUeH9}FuxxsLfKvX zb%Lqosf8em-KD{<_RYwlY!}MIC{$76Sf}W0zv|&QbLD$ftsVnTX_VEhrpI@5c*;W}-!6;2C{S1cR=`o1Z9k`eBpA zF?e|<+Vep(tx6q#A_Q01rW=APx^uR8j-R=U-DTCI`Z{V@xBPZz)Jv_WiQh+xpxohQ zE3^BYF4|;dc>gIl0>*QqI6*&5YZr!1M)KJ?*(j-Xl0-@rT6a1ym<#@usrA&>szk>P zCXPehms(7Owo2QI{%hir^%FY(v0*_uWQ>z1ta^-d?vux^Qpfr@U5T~^rNNspf<8GS zl{{p#&Ds5aCr>gmNy?j=n?Ry2?C-c-iZkJRJhLvf^9X`DaHC0Y^>}iHJx!Z;qxDWR zPu+qQ2oJuQ^#PwG7r4QWd2AoxJ6$R6h?Snl@;&}x(P>9PrPI@tDL1iSiC6@}D!a9L ztir_fdM-^vje7TFLdq4Xo!oH48;y94C9$!cN;WJS-ODHb`aP^EA&SV$uvnyQY8x7 z)yz>Ljl4`bapm-d=^?arOta)lh zcPbPu58hOZDX&62Rzf+Y-CSN8#frMDDzgcE3K{FML_yEXYIXURpr-3kZgiz^Bf4EQ zSs=WWk{Hc(G;FZ!vw9-ug`8j{OOW}HW0aF$DOljRJion`;&Dx9(LZ5QUEAi|L0$&- zy`91D;Wuru9ZZf*5NQ)ZrWeY{!(r3U^^kSkhqb(D{_s0f`q*ZW%#YqabLhR3zyZ3sXl=dH{!S0!JWeec%z@NuMV6a1#4 zrdZnV2EyVVCUN<*DEM$z^il@zqo?k+zaTakEz=N#;rC3V2LMt|Bw`O!inFLGXWEQQnyU{dHqC|LbB7>*?UFPoZQ|8peKB`J~sCj_}jYgb?tM@dkYo+Uw_EOpS_pA@q|?~ z3HQQW-KMl?y3X(^T`6Xhy=A!cR_XHB=7(x9tSc$^Yxcl3*S4^ajNSD@o_JyzWDAu@&sM2O?bne+r#2Ag0dTD3iGl?jO%@#$CTA~U2e+Q^HP;TvORXC4rg8*r zc@!k=q?}WF*j^+eJ#~imF61-aFw{)DU9@=lx*X1ta@j@nu=$tPcA?6D8Vpcvm^WAb zKh{XxFN*ne)Wm`Kv0seZpyv&+ zpjCzRv%&jnA@JpkTerI22t#Uv%LVOZ2SN1|H>B%2`!jmB&{mO$R1>Dq&T-!U>Nm|qada%6^a78(k8y;O#rzQQ{99Nw8@eWYPf@%dTtg4kuxxu*vE536!Q|DBXql*u;? zLp`W+I&#=R<$l}1Kv$wgCqI*2wlcBpuU9V97n>-C?+>*`Sy(kwYPP(hS=T4-jrEetC3SbBz1pj&_C;No6Wk=HGV>qKy6Eqb~H0#_BAEGb#PN%jsM zP(0`ue*Vb9q3@9}Ka?9D&3WZ>G*vzGBVRKh1Vp99h-{D8!ajeYy2)igux~V+hh5oT z_`S^>aUN#Ni1SQb{MPnl>&}#)%B=Z@6=dXctVR*0>GPnhP2bD%Z>6XSVRs@N_)V}|GMghluKU6opDtksXs?9;XbwFEwtSMlBEt}UsaJsnp6$86sbENKHZVT(a= z6T~2bn#D(j!yZ)&%FPYiaLG{6p=)8H;*X_T+xkcqCn#``1+31kEq-n-StMd0-1oEE z0~gKeXTMKks9|lrj#EmRjUMVQn-i*}hXZFC|1K{Mu^mGuwLcWS7vL`kpR#xiRmPUR ztsXli9*%uzy~aYE+HdV}Mv%YwnQG7O!IYW6FnNsx!{`Esf)g?at4Qp$U%*sFyb$_Q zVQa|*%sL|;d=V7$zQ9GBaQ5-Y{5>VtFZUNKTFmCmav`XhG|iv~ZZkqh)JfjAF7&FH zB3^8@i~7!pNST!R>2I;38o&0P6>_RPE~}oD&=K0?;%9pOV6kI(jWzXRv@!Vjxgi1x zYfnC=QXW6zzy}9UfcV!Gam$^ zsca!tFf+r{*i#%A`d@#SzBa&oOoZdO;A9FNHI_|+DAwFN+!Mi`V*7DY&;)V0{F=%u z|F60r`_;m}%DNt)eX?nb)Y)bk9BFww>YpQW&2_8AobsFexv_)E#K0=blc23Ve65hL z?ANHt%8Hc@x?=g4P)Ll?Z?g}2O01S*UTSQ!h@^}amE~$UM)8>PgIleo=}}>EIMHCW z*Eho4E>usQD!$n0RBHI^nOGL;S=w*&h}a9v2eFb_euYQq%5#W&Q*HZhX;x&+mP?T+ zI3us2&o8bVT#WOLPJf72Saz|&_`f<=6S);+$Ry+OD&u~J#kf}#$~n!I>QgPce&#@O z`Sr82B1zU{Oc_If;m6aNPmqdCOwrnyw<|5i$4qXXL$B@6K1n67`XYZy1bgg_vq*qJ-L>w7%@?+mFPa$Cbpi3$QRRnn9{hv}#o#}_O*P4gk?6T;UjMdQh%l)*McW|ooe!8`$8t=-BBAMeKMy$tFEhb~c4J8w%8`UOA05dU z884evmGUPud#76CB2*4xI%y{aiw#EJ+NqnDZ-c3}l1_vWwTj*hMrCyLIK z6JE-UuXSoea25GSv0`80gZs>`ak7GOJE|%H%8&hHmj0 e!^-<~ru@ literal 0 HcmV?d00001 From 9324b38960df066f639ba169561d47ac2ab99a10 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Tue, 19 Dec 2017 21:10:15 -0500 Subject: [PATCH 05/45] Updated Instructions --- NEW_CONNECTION.md | 15 ++++++++++++++- app/constants/constants.js | 30 +++++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/NEW_CONNECTION.md b/NEW_CONNECTION.md index afdb2249e..555424f5e 100644 --- a/NEW_CONNECTION.md +++ b/NEW_CONNECTION.md @@ -25,4 +25,17 @@ function tables(connection); //Question are these required? function files(connection); -function storage(connection); \ No newline at end of file +function storage(connection); + +## Images +It is recommended to add the images as png and a good idea to include a regular and small icon + +## Updating the constants.js file +The following are the instructions for adding the Constants File: +1. Add an entry in DIALECTS +2. Add an entry in SQL_DIALECTS_USING_EDITOR +3. Add an entry under CONNECTION_CONFIG + + +## Updating the Tab.react.js +The following are the instructions for adding the Tab React JS file \ No newline at end of file diff --git a/app/constants/constants.js b/app/constants/constants.js index 2d8d3ec46..7f7f31ba7 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -14,7 +14,8 @@ export const DIALECTS = { IBM_DB2: 'ibm db2', APACHE_SPARK: 'apache spark', APACHE_IMPALA: 'apache impala', - APACHE_DRILL: 'apache drill' + APACHE_DRILL: 'apache drill', + ATHENA: 'athena' }; export const SQL_DIALECTS_USING_EDITOR = [ @@ -26,7 +27,8 @@ export const SQL_DIALECTS_USING_EDITOR = [ 'sqlite', 'ibm db2', 'apache spark', - 'apache impala' + 'apache impala', + 'athena' ]; const commonSqlOptions = [ @@ -178,7 +180,29 @@ export const CONNECTION_CONFIG = { 'value': 'secretAccessKey', 'type': 'password' } - ] // TODO - password options for apache drill + ], // TODO - password options for apache drill + + /** + * accessKeyId: 'xxxx', + secretAccessKey: 'xxxx', + region: 'xxxx', + */ + [DIALECTS.ATHENA]: [ + { + 'label': 'S3 Bucket', + 'value': 'bucket', + 'type': 'text', + 'description': ` + The S3 connection will import CSV files from any + directory in your S3 bucket.` + }, + {'label': 'S3 Access Key ID', 'value': 'accessKeyId', 'type': 'text'}, + { + 'label': 'S3 Secret Access Key', + 'value': 'secretAccessKey', + 'type': 'password' + } + ] }; From 4d9ee13150be3c3495df2314edccb7292f82bb38 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Tue, 19 Dec 2017 21:17:14 -0500 Subject: [PATCH 06/45] Check in constants for Athena --- NEW_CONNECTION.md | 3 +++ app/constants/constants.js | 37 +++++++++++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/NEW_CONNECTION.md b/NEW_CONNECTION.md index 555424f5e..fec9f7928 100644 --- a/NEW_CONNECTION.md +++ b/NEW_CONNECTION.md @@ -35,6 +35,9 @@ The following are the instructions for adding the Constants File: 1. Add an entry in DIALECTS 2. Add an entry in SQL_DIALECTS_USING_EDITOR 3. Add an entry under CONNECTION_CONFIG +4. Add the location of the logo in +5. Add the preview query in the PREVIEW_QUERY section +6. Add an entry in the SAMPLE_DBS ## Updating the Tab.react.js diff --git a/app/constants/constants.js b/app/constants/constants.js index 7f7f31ba7..5032195f2 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -188,19 +188,31 @@ export const CONNECTION_CONFIG = { region: 'xxxx', */ [DIALECTS.ATHENA]: [ + { + 'label': 'S3 Access Key','value': 'accessKey','type': 'password' + }, + { + 'label': 'S3 Secret Access Key','value': 'secretKey','type': 'password' + }, + { + 'label': 'AWS Region','value': 'region','type': 'text' + }, { 'label': 'S3 Bucket', 'value': 'bucket', 'type': 'text', 'description': ` - The S3 connection will import CSV files from any - directory in your S3 bucket.` + The Athena will put the output results of the query in this location.` }, - {'label': 'S3 Access Key ID', 'value': 'accessKeyId', 'type': 'text'}, { - 'label': 'S3 Secret Access Key', - 'value': 'secretAccessKey', - 'type': 'password' + 'label': 'Database','value': 'database','type': 'text' + }, + { + 'label': 'Athena SQL Query', + 'value': 'query', + 'type': 'text', + 'description': ` + The SQL Query that will be executed against Athena.` } ] }; @@ -218,7 +230,8 @@ export const LOGOS = { [DIALECTS.MSSQL]: 'images/mssql-logo.png', [DIALECTS.SQLITE]: 'images/sqlite-logo.png', [DIALECTS.S3]: 'images/s3-logo.png', - [DIALECTS.APACHE_DRILL]: 'images/apache_drill-logo.png' + [DIALECTS.APACHE_DRILL]: 'images/apache_drill-logo.png', + [DIALECTS.ATHENA]: 'images/athena-logo.png' }; export function PREVIEW_QUERY (dialect, table, database = '') { @@ -232,6 +245,7 @@ export function PREVIEW_QUERY (dialect, table, database = '') { case DIALECTS.MARIADB: case DIALECTS.POSTGRES: case DIALECTS.REDSHIFT: + case DIALECTS.ATHENA: return `SELECT * FROM ${table} LIMIT 1000`; case DIALECTS.MSSQL: return 'SELECT TOP 1000 * FROM ' + @@ -402,5 +416,12 @@ export const SAMPLE_DBS = { sqlite: { dialect: 'sqlite', storage: `${__dirname}/plotly_datasets.db` - } + }, + [DIALECTS.ATHENA]: { + bucket: 'plotly-s3-connector-test', + accessKeyId: 'AKIAIMHMSHTGARJYSKMQ', + secretAccessKey: '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', + region:'us-west-2', + database:'default' + }, }; From 273e522822062f20cc0d33aa1822f8a0c4a50ea4 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Tue, 19 Dec 2017 21:22:04 -0500 Subject: [PATCH 07/45] Got UI to detect Athena --- NEW_CONNECTION.md | 3 ++- app/components/Settings/Tabs/Tab.react.js | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEW_CONNECTION.md b/NEW_CONNECTION.md index fec9f7928..9dae65f9e 100644 --- a/NEW_CONNECTION.md +++ b/NEW_CONNECTION.md @@ -41,4 +41,5 @@ The following are the instructions for adding the Constants File: ## Updating the Tab.react.js -The following are the instructions for adding the Tab React JS file \ No newline at end of file +The following are the instructions for adding the Tab React JS file +1. Add an entry for the connection label \ No newline at end of file diff --git a/app/components/Settings/Tabs/Tab.react.js b/app/components/Settings/Tabs/Tab.react.js index 70d3b6b94..94ab4533d 100644 --- a/app/components/Settings/Tabs/Tab.react.js +++ b/app/components/Settings/Tabs/Tab.react.js @@ -33,6 +33,8 @@ export default class ConnectionTab extends Component { label = `Apache Spark (${connectionObject.host}:${connectionObject.port})`; } else if (connectionObject.dialect === DIALECTS.ELASTICSEARCH) { label = `Elasticsearch (${connectionObject.host})`; + } else if (connectionObject.dialect === DIALECTS.ATHENA) { + label = `Athena (${connectionObject.database})`; } else if (connectionObject.dialect === DIALECTS.SQLITE) { label = connectionObject.storage; } else { From 632f9da20762d88d878b0daf2897a9db282d0184 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sat, 23 Dec 2017 14:05:11 -0500 Subject: [PATCH 08/45] Check in latest version --- backend/persistent/datastores/Athena.js | 45 +++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 backend/persistent/datastores/Athena.js diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js new file mode 100644 index 000000000..edb9f73b8 --- /dev/null +++ b/backend/persistent/datastores/Athena.js @@ -0,0 +1,45 @@ + +//TODO +//1. What is expected object for connection +//2. What is expect results for the schems. Is this a list of databases or +// table definitions. Must check against Postgres +//3. What is the expected results from the list of tables? +//4. What is the expected results for the query +//5. Hard code the format of the data to test + + +/* + * The connection object will open a connection to the Athena database + * @param {object} connection + */ +export function connect(connection) { + let conn = {}; + return new Promise(function(resolve, reject) { + resolve( conn ); + }); +} + +/** + * The following method will execute a query against the specified connection + * TODO. What is expected Format + * @param {object} queryObject + * @param {object} connection + */ +export function query(queryObject, connection){ + +} + + +/** + * Should return a list of databases that are defined within the database. + * @param {object} connection + */ +export function schemas(connection){ + +} + +/** + * Should return a list of tables that are in the database + * @param {*} connection + */ +function tables(connection); \ No newline at end of file From c1f37ba4ceef996c87607576f437c3ce14443de6 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Thu, 28 Dec 2017 16:15:52 -0500 Subject: [PATCH 09/45] Check in first version of schemas --- backend/persistent/datastores/Athena.js | 54 +++++++++++++++++++-- backend/persistent/datastores/Datastores.js | 5 ++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js index edb9f73b8..ad9c2dbeb 100644 --- a/backend/persistent/datastores/Athena.js +++ b/backend/persistent/datastores/Athena.js @@ -13,9 +13,24 @@ * @param {object} connection */ export function connect(connection) { - let conn = {}; + console.log( 'Athena Connection', connection); + const { + username, password, database, port, dialect, accessKey, secretKey + } = connection; + + let con = { + username, + password, + database, + port, + dialect, + accessKey, + secretKey + }; + + console.log( 'Connection object', con); return new Promise(function(resolve, reject) { - resolve( conn ); + resolve( con ); }); } @@ -26,7 +41,14 @@ export function connect(connection) { * @param {object} connection */ export function query(queryObject, connection){ + console.log( 'Athena Query Conn', connection); + console.log( 'Athena Query Obj', queryObject); + return []; + + return new Promise(function(resolve, reject) { + resolve( [] ); + }); } @@ -35,11 +57,35 @@ export function query(queryObject, connection){ * @param {object} connection */ export function schemas(connection){ - + console.log( 'Schemas', connection); + + let columnnames = ['FirstName', 'LastName', 'Date']; + let rows = []; + + for( let i=0; i< 4; i++){ + let r = []; + r.push( `John-${i}`); + r.push( `Smith-${i}`); + r.push( `0${i}/12/2017`); + + rows.push(r); + } + + + return new Promise(function(resolve, reject) { + resolve( {columnnames, rows} ); + }); } /** * Should return a list of tables that are in the database * @param {*} connection */ -function tables(connection); \ No newline at end of file +export function tables(connection){ + console.log( 'Tables', connection); + return []; + + return new Promise(function(resolve, reject) { + resolve( [] ); + }); +} \ No newline at end of file diff --git a/backend/persistent/datastores/Datastores.js b/backend/persistent/datastores/Datastores.js index 88a5336a8..995c8c7c6 100644 --- a/backend/persistent/datastores/Datastores.js +++ b/backend/persistent/datastores/Datastores.js @@ -5,6 +5,7 @@ import * as ApacheDrill from './ApacheDrill'; import * as IbmDb2 from './ibmdb2'; import * as ApacheLivy from './livy'; import * as ApacheImpala from './impala'; +import * as Athena from './Athena'; /* * Switchboard to all of the different types of connections @@ -27,6 +28,8 @@ import * as ApacheImpala from './impala'; function getDatastoreClient(connection) { const {dialect} = connection; + + console.log( 'Selection dialect', dialect); if (dialect === 'elasticsearch') { return Elasticsearch; } else if (dialect === 's3') { @@ -39,6 +42,8 @@ function getDatastoreClient(connection) { return ApacheImpala; } else if (dialect === 'ibm db2') { return IbmDb2; + }else if (dialect === 'athena') { + return Athena; } return Sql; } From ca49c638651948a9de744b6c71a6bfbc9f8c2c8c Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Thu, 28 Dec 2017 20:46:33 -0500 Subject: [PATCH 10/45] Check in schema definition --- NEW_CONNECTION.md | 35 ++++++++++++++----------- backend/persistent/datastores/Athena.js | 26 +++++++++++++----- backend/persistent/datastores/Sql.js | 2 ++ 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/NEW_CONNECTION.md b/NEW_CONNECTION.md index 9dae65f9e..e202b7ffa 100644 --- a/NEW_CONNECTION.md +++ b/NEW_CONNECTION.md @@ -12,20 +12,6 @@ app/actions/sessions.js app/components/Settings/Tabs/Tab.react.js app/components/Settings/Settings.react.js -## Datastore file should have the following functions - -function query(queryObject, connection); - -function connect(connection); - -function schemas(connection); - -function tables(connection); - -//Question are these required? -function files(connection); - -function storage(connection); ## Images It is recommended to add the images as png and a good idea to include a regular and small icon @@ -42,4 +28,23 @@ The following are the instructions for adding the Constants File: ## Updating the Tab.react.js The following are the instructions for adding the Tab React JS file -1. Add an entry for the connection label \ No newline at end of file +1. Add an entry for the connection label + +## Datastore file should have the following functions +```javascript +function query(queryObject, connection); + +function connect(connection); + +/** + * The following method will return a list of schemas that are + */ +function schemas(connection); + +function tables(connection); + +//Question are these required? +function files(connection); + +function storage(connection); +``` \ No newline at end of file diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js index ad9c2dbeb..a73f68b85 100644 --- a/backend/persistent/datastores/Athena.js +++ b/backend/persistent/datastores/Athena.js @@ -59,14 +59,27 @@ export function query(queryObject, connection){ export function schemas(connection){ console.log( 'Schemas', connection); - let columnnames = ['FirstName', 'LastName', 'Date']; + let columnnames = ['Table', 'column_name', 'data_type']; let rows = []; for( let i=0; i< 4; i++){ + + + let r = []; + r.push('Test'); + r.push(`first_name_${i}`); + r.push(`varchar`); + + rows.push(r); + } + + for( let i=0; i< 2; i++){ + + let r = []; - r.push( `John-${i}`); - r.push( `Smith-${i}`); - r.push( `0${i}/12/2017`); + r.push('Sample'); + r.push(`last_name_${i}`); + r.push(`varchar`); rows.push(r); } @@ -83,9 +96,8 @@ export function schemas(connection){ */ export function tables(connection){ console.log( 'Tables', connection); - return []; - + return new Promise(function(resolve, reject) { - resolve( [] ); + resolve( ['Sample', 'Test'] ); }); } \ No newline at end of file diff --git a/backend/persistent/datastores/Sql.js b/backend/persistent/datastores/Sql.js index 35f6638c3..4cd37b050 100644 --- a/backend/persistent/datastores/Sql.js +++ b/backend/persistent/datastores/Sql.js @@ -127,6 +127,7 @@ export function query(queryString, connection) { queryString, {type: Sequelize.QueryTypes.SELECT} ).then(results => { + console.log( 'SQL Results', results); return parseSQL(results); }); } @@ -137,6 +138,7 @@ export function tables(connection) { {type: Sequelize.QueryTypes.SELECT} ).then(tableList => { + console.log( 'List of Tables', tableList); let tableNames; if (connection.dialect === 'postgres' || connection.dialect === 'redshift') { From 0d389f746ec94326b0ced7d66a20c7a9a74cf039 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Fri, 29 Dec 2017 16:00:14 -0500 Subject: [PATCH 11/45] Check in more documentation for this --- NEW_CONNECTION.md | 38 ++++++++++++++++++++--- backend/parse.js | 2 ++ backend/persistent/datastores/Athena.js | 41 +++++++++++++++++++++++-- backend/persistent/datastores/Sql.js | 1 + 4 files changed, 75 insertions(+), 7 deletions(-) diff --git a/NEW_CONNECTION.md b/NEW_CONNECTION.md index e202b7ffa..34df35655 100644 --- a/NEW_CONNECTION.md +++ b/NEW_CONNECTION.md @@ -5,7 +5,7 @@ The following are the instructions for adding a new data source ## Add the following files for Creating a new Datasource backend/persistent/datastores/Datastores.js -backend/persistent/datastores/NameOfDatastoreToConnec.js +backend/persistent/datastores/NameOfDatastoreToConnect.js app/constants/constants.js app/images app/actions/sessions.js @@ -32,19 +32,49 @@ The following are the instructions for adding the Tab React JS file ## Datastore file should have the following functions ```javascript -function query(queryObject, connection); +/** + * The following function will connect to the datasource + * @param {Object} connection + */ function connect(connection); /** - * The following method will return a list of schemas that are + * The following function will return a list of schemas. The return + * array should return a list of array with each array containing the + * following information: + * -row[0] schema name + * -row[1] column name + * -row[2] data type + * @param {Object} connection + * @return {Array} */ function schemas(connection); +/** + * The following function will return a list of tables. + * The return result should an array of strings + * @param connection {object} + * @return {Array} + */ function tables(connection); +/** + * The following function will execute a query against the + * data source. The function should return a tuple which contains + * two elements. The first is an array of strings which is the column names + * The second is a two dimensional array which represents the rows to be displayed + * @param {string|object} queryObject + * @param {object} connection + * @return {object} -- tuple {columnames, rows} + */ +function query(queryObject, connection); + //Question are these required? function files(connection); function storage(connection); -``` \ No newline at end of file +``` + +## Steps to run application in development mode + diff --git a/backend/parse.js b/backend/parse.js index f0c0c5535..08c8cccf4 100644 --- a/backend/parse.js +++ b/backend/parse.js @@ -53,6 +53,8 @@ export function parseSQL(data) { // plotly workspace requires keys (columnnames, ncols, nrows, rows) // and they should be in this format + console.log( 'Returning column names', columnnames); + console.log( 'Returning rowss', rows); return {columnnames, rows}; } diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js index a73f68b85..feca9400c 100644 --- a/backend/persistent/datastores/Athena.js +++ b/backend/persistent/datastores/Athena.js @@ -1,4 +1,4 @@ - +import {parseSQL} from '../../parse'; //TODO //1. What is expected object for connection //2. What is expect results for the schems. Is this a list of databases or @@ -44,10 +44,29 @@ export function query(queryObject, connection){ console.log( 'Athena Query Conn', connection); console.log( 'Athena Query Obj', queryObject); - return []; + let results = []; + + for( let i=0; i< 20; i++){ + + let r = {}; + if( i % 2 === 0){ + r.first_name_0 = 'John'; + r.value = i; + results.push( r ); + }else{ + r.first_name_0 = 'Joan'; + r.value = i; + results.push( r ); + } + } + + console.log( 'Returning results to query'); + let columnnames = ['first_name_0', 'value']; return new Promise(function(resolve, reject) { - resolve( [] ); + + + resolve( parseSQL( results ) ); }); } @@ -73,6 +92,14 @@ export function schemas(connection){ rows.push(r); } + let rowData = []; + rowData.push('Test'); + rowData.push( 'value'); + rowData.push('integer'); + + rows.push( rowData ); + + for( let i=0; i< 2; i++){ @@ -84,6 +111,14 @@ export function schemas(connection){ rows.push(r); } + rowData = []; + rowData.push('Sample'); + rowData.push( 'value'); + rowData.push('integer'); + + rows.push( rowData ); + + return new Promise(function(resolve, reject) { resolve( {columnnames, rows} ); diff --git a/backend/persistent/datastores/Sql.js b/backend/persistent/datastores/Sql.js index 4cd37b050..585480b36 100644 --- a/backend/persistent/datastores/Sql.js +++ b/backend/persistent/datastores/Sql.js @@ -164,6 +164,7 @@ export function tables(connection) { tableNames = tableList.map(object => values(object)[0]); } + console.log( 'SQL Table names', tableNames); return uniq(sort((a, b) => gt(a, b) ? 1 : -1, tableNames)); }); From 2c943a8c703c5d46db1e94db665b41d275647aee Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Fri, 29 Dec 2017 16:00:44 -0500 Subject: [PATCH 12/45] Check in updated new connection --- NEW_CONNECTION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEW_CONNECTION.md b/NEW_CONNECTION.md index 34df35655..72ef9effc 100644 --- a/NEW_CONNECTION.md +++ b/NEW_CONNECTION.md @@ -77,4 +77,4 @@ function storage(connection); ``` ## Steps to run application in development mode - +TBD From 3577b06c55471e35b29492f52a08b9aae4797a33 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Fri, 29 Dec 2017 16:02:17 -0500 Subject: [PATCH 13/45] Check in draft of new connection: --- NEW_CONNECTION.md | 80 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 NEW_CONNECTION.md diff --git a/NEW_CONNECTION.md b/NEW_CONNECTION.md new file mode 100644 index 000000000..72ef9effc --- /dev/null +++ b/NEW_CONNECTION.md @@ -0,0 +1,80 @@ +# Add New Data Source + +The following are the instructions for adding a new data source + +## Add the following files for Creating a new Datasource + +backend/persistent/datastores/Datastores.js +backend/persistent/datastores/NameOfDatastoreToConnect.js +app/constants/constants.js +app/images +app/actions/sessions.js +app/components/Settings/Tabs/Tab.react.js +app/components/Settings/Settings.react.js + + +## Images +It is recommended to add the images as png and a good idea to include a regular and small icon + +## Updating the constants.js file +The following are the instructions for adding the Constants File: +1. Add an entry in DIALECTS +2. Add an entry in SQL_DIALECTS_USING_EDITOR +3. Add an entry under CONNECTION_CONFIG +4. Add the location of the logo in +5. Add the preview query in the PREVIEW_QUERY section +6. Add an entry in the SAMPLE_DBS + + +## Updating the Tab.react.js +The following are the instructions for adding the Tab React JS file +1. Add an entry for the connection label + +## Datastore file should have the following functions +```javascript + +/** + * The following function will connect to the datasource + * @param {Object} connection + */ +function connect(connection); + +/** + * The following function will return a list of schemas. The return + * array should return a list of array with each array containing the + * following information: + * -row[0] schema name + * -row[1] column name + * -row[2] data type + * @param {Object} connection + * @return {Array} + */ +function schemas(connection); + +/** + * The following function will return a list of tables. + * The return result should an array of strings + * @param connection {object} + * @return {Array} + */ +function tables(connection); + +/** + * The following function will execute a query against the + * data source. The function should return a tuple which contains + * two elements. The first is an array of strings which is the column names + * The second is a two dimensional array which represents the rows to be displayed + * @param {string|object} queryObject + * @param {object} connection + * @return {object} -- tuple {columnames, rows} + */ +function query(queryObject, connection); + +//Question are these required? +function files(connection); + +function storage(connection); +``` + +## Steps to run application in development mode +TBD From d72d5b7bf7839f204739d6efcb79cbe761ac2a0c Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sat, 30 Dec 2017 17:39:25 -0500 Subject: [PATCH 14/45] Check in fixes based on Nicolas comment --- NEW_CONNECTION.md | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/NEW_CONNECTION.md b/NEW_CONNECTION.md index 72ef9effc..d8ca2a8c5 100644 --- a/NEW_CONNECTION.md +++ b/NEW_CONNECTION.md @@ -2,19 +2,25 @@ The following are the instructions for adding a new data source -## Add the following files for Creating a new Datasource +## The following are the files that need to be updated -backend/persistent/datastores/Datastores.js -backend/persistent/datastores/NameOfDatastoreToConnect.js -app/constants/constants.js -app/images -app/actions/sessions.js -app/components/Settings/Tabs/Tab.react.js -app/components/Settings/Settings.react.js +1. app/constants/constants.js +2. app/actions/sessions.js +3. app/components/Settings/Tabs/Tab.react.js +4. app/components/Settings/Settings.react.js +5. backend/persistent/datastores/Datastores.js +## The following are the new files that need to be added + +1. app/images +2. backend/persistent/datastores/NameOfDatastoreToConnect.js + +# Detailed Implementation Instructions ## Images It is recommended to add the images as png and a good idea to include a regular and small icon +You can add the new icon in the directory app/images. It recommended to use png format +and name the file in lower case with dashes (ex: mysql-logo.png ) ## Updating the constants.js file The following are the instructions for adding the Constants File: @@ -25,12 +31,14 @@ The following are the instructions for adding the Constants File: 5. Add the preview query in the PREVIEW_QUERY section 6. Add an entry in the SAMPLE_DBS - ## Updating the Tab.react.js The following are the instructions for adding the Tab React JS file 1. Add an entry for the connection label -## Datastore file should have the following functions +## New Datastore file +This file should be added in the following location +backend/persistent/datastores + ```javascript /** @@ -70,11 +78,17 @@ function tables(connection); */ function query(queryObject, connection); -//Question are these required? -function files(connection); - -function storage(connection); ``` -## Steps to run application in development mode +## Updating the Datastores.js +The following are the instructions for updating the Datastores.js + +1. Import the new Data source file +2. Update the function getDatastoreClient +``` javascript + } else if (dialect === 'new dialect') { + return MyNewDialect; + } +``` +# Development and Debugging Setup TBD From 837b95a00ad9bd335b481cb81852b81369e4ca4d Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sat, 30 Dec 2017 17:41:00 -0500 Subject: [PATCH 15/45] Updated Datastores.js instructions --- NEW_CONNECTION.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEW_CONNECTION.md b/NEW_CONNECTION.md index d8ca2a8c5..ad7ac78e9 100644 --- a/NEW_CONNECTION.md +++ b/NEW_CONNECTION.md @@ -86,6 +86,8 @@ The following are the instructions for updating the Datastores.js 1. Import the new Data source file 2. Update the function getDatastoreClient ``` javascript + //The dialect name must match what was specified under + //app/constants/constants.js for the Dialect name } else if (dialect === 'new dialect') { return MyNewDialect; } From f879c180830a0d3641428da9df3bfd03e9e4ff5b Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sat, 30 Dec 2017 17:45:05 -0500 Subject: [PATCH 16/45] Check in small fix --- NEW_CONNECTION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEW_CONNECTION.md b/NEW_CONNECTION.md index ad7ac78e9..c08a18153 100644 --- a/NEW_CONNECTION.md +++ b/NEW_CONNECTION.md @@ -18,7 +18,7 @@ The following are the instructions for adding a new data source # Detailed Implementation Instructions ## Images -It is recommended to add the images as png and a good idea to include a regular and small icon +It is recommended to add the images as png and a good idea to include a regular and small icon. You can add the new icon in the directory app/images. It recommended to use png format and name the file in lower case with dashes (ex: mysql-logo.png ) From d2d7975625136e7c4f91cecb6625004792b6af59 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Tue, 2 Jan 2018 22:14:12 -0500 Subject: [PATCH 17/45] Check in settings to get SQL --- app/components/Settings/Settings.react.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/components/Settings/Settings.react.js b/app/components/Settings/Settings.react.js index 582c803f7..5ccee7493 100644 --- a/app/components/Settings/Settings.react.js +++ b/app/components/Settings/Settings.react.js @@ -199,7 +199,8 @@ class Settings extends Component { DIALECTS.APACHE_SPARK, DIALECTS.IBM_DB2, DIALECTS.MYSQL, DIALECTS.MARIADB, DIALECTS.POSTGRES, - DIALECTS.REDSHIFT, DIALECTS.MSSQL, DIALECTS.SQLITE + DIALECTS.REDSHIFT, DIALECTS.MSSQL, DIALECTS.SQLITE, + DIALECTS.ATHENA ])) { if (connectRequest.status === 200 && !tablesRequest.status) { this.setState({editMode: false}); From 06c74fcefa7d56b7f676b092cea20c8e4696e6d1 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Tue, 2 Jan 2018 22:18:42 -0500 Subject: [PATCH 18/45] Check in small update for Settings.js --- NEW_CONNECTION.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NEW_CONNECTION.md b/NEW_CONNECTION.md index c08a18153..4e37107fc 100644 --- a/NEW_CONNECTION.md +++ b/NEW_CONNECTION.md @@ -35,6 +35,12 @@ The following are the instructions for adding the Constants File: The following are the instructions for adding the Tab React JS file 1. Add an entry for the connection label +## Updating the Settings.react.js +Update the fetchData function to make the calls to appropriate sessions.js backend. +Currently there is Dialect types for SQL, ElasticSearch, S3 and Apache Drill. If +your new data source is an SQL Type you will need to add the new Dialect in the +fetchData section for SQL types. + ## New Datastore file This file should be added in the following location backend/persistent/datastores From 6b443bc0ba69b451c00ec4e967505df8524dc3cc Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Wed, 3 Jan 2018 22:30:21 -0500 Subject: [PATCH 19/45] Check in first draft of unit tests --- backend/persistent/datastores/Athena.js | 11 +++++ package.json | 1 + test/backend/datastores.athena.spec.js | 58 +++++++++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 test/backend/datastores.athena.spec.js diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js index feca9400c..286dc8db5 100644 --- a/backend/persistent/datastores/Athena.js +++ b/backend/persistent/datastores/Athena.js @@ -11,6 +11,12 @@ import {parseSQL} from '../../parse'; /* * The connection object will open a connection to the Athena database * @param {object} connection + * @param {string} connection.username - username for connection + * @param {string} connection.password - password for connection + * @param {string} connection.database - database for connection + * @param {string} connection.port - port for connection + * @param {string} connection.accessKey - AWS Access Key + * @param {string} connection.secretKey - AWS Secret Key */ export function connect(connection) { console.log( 'Athena Connection', connection); @@ -18,6 +24,11 @@ export function connect(connection) { username, password, database, port, dialect, accessKey, secretKey } = connection; + //Validate that the connection has a + + if( !username ){ + throw new Error(`The username was not defined`); + } let con = { username, password, diff --git a/package.json b/package.json index 6ddc567e0..6fd3a5b93 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "test-unit-ibmdb": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.ibmdb.spec.js", "test-unit-impala": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.impala.spec.js", "test-unit-livy": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.livy.spec.js", + "test-unit-athena": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.athena.spec.js", "test-unit-oauth2": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/routes.oauth2.spec.js", "test-unit-scheduler": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/QueryScheduler.spec.js", "test-unit-routes": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/routes.spec.js", diff --git a/test/backend/datastores.athena.spec.js b/test/backend/datastores.athena.spec.js new file mode 100644 index 000000000..8a19c3be7 --- /dev/null +++ b/test/backend/datastores.athena.spec.js @@ -0,0 +1,58 @@ +import {assert} from 'chai'; + +import {connect} from '../../backend/persistent/datastores/Athena'; + + +// Suppressing ESLint cause Mocha ensures `this` is bound in test functions +/* eslint-disable no-invalid-this */ +describe('Athena Unit Tests:', function () { + + it('connect succeeds', function( done ) { + const conn = { + username: 'user' + }; + connect( conn ).then( connection =>{ + assert.isDefined( connection, 'Connection is defined'); + done(); + }).catch( err =>{ + console.log( 'Error', err); + done('Should have obtained a valid connection'); + }) + }); + + it('connect fails missing username', function( done ) { + const conn = { + }; + try{ + connect( conn ).then( connection =>{ + assert.isUnDefined( connection, 'Connection should not have been defined'); + done('Should have not have occured do to missing username'); + }).catch( err =>{ + done(); + }) + }catch( error ){ + done(); + } + }); + + it('connect fails username is empty', function( done ) { + const conn = { + username: '' + }; + try{ + connect( conn ).then( connection =>{ + assert.isUnDefined( connection, 'Connection should not have been defined'); + done('Should have not have occured do to missing username'); + }).catch( err =>{ + done(); + }) + }catch( error ){ + done(); + } + }); + + + + +}); +/* eslint-enable no-invalid-this */ From 3ffd87ce9c64ea6b8df603558ca533917ccf1e39 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Wed, 3 Jan 2018 22:33:12 -0500 Subject: [PATCH 20/45] Updating unit tests --- backend/persistent/datastores/Athena.js | 9 ++++++++- test/backend/datastores.athena.spec.js | 10 ++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js index 286dc8db5..85143850e 100644 --- a/backend/persistent/datastores/Athena.js +++ b/backend/persistent/datastores/Athena.js @@ -25,10 +25,17 @@ export function connect(connection) { } = connection; //Validate that the connection has a - if( !username ){ throw new Error(`The username was not defined`); } + + if( !password ){ + throw new Error(`The password was not defined`); + } + + if( !database ){ + throw new Error(`The database was not defined`); + } let con = { username, password, diff --git a/test/backend/datastores.athena.spec.js b/test/backend/datastores.athena.spec.js index 8a19c3be7..f427f0f51 100644 --- a/test/backend/datastores.athena.spec.js +++ b/test/backend/datastores.athena.spec.js @@ -9,7 +9,9 @@ describe('Athena Unit Tests:', function () { it('connect succeeds', function( done ) { const conn = { - username: 'user' + username: 'user', + password: 'password', + database: 'database', }; connect( conn ).then( connection =>{ assert.isDefined( connection, 'Connection is defined'); @@ -22,6 +24,8 @@ describe('Athena Unit Tests:', function () { it('connect fails missing username', function( done ) { const conn = { + password: 'password', + database: 'database', }; try{ connect( conn ).then( connection =>{ @@ -37,7 +41,9 @@ describe('Athena Unit Tests:', function () { it('connect fails username is empty', function( done ) { const conn = { - username: '' + username: '', + password: 'password', + database: 'database', }; try{ connect( conn ).then( connection =>{ From 463b4c65a862fc5d426a659e413ccbb78004e52f Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Wed, 3 Jan 2018 22:34:40 -0500 Subject: [PATCH 21/45] Check in unit tests for connection --- test/backend/datastores.athena.spec.js | 35 ++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/backend/datastores.athena.spec.js b/test/backend/datastores.athena.spec.js index f427f0f51..98214e65d 100644 --- a/test/backend/datastores.athena.spec.js +++ b/test/backend/datastores.athena.spec.js @@ -57,6 +57,41 @@ describe('Athena Unit Tests:', function () { } }); + it('connect fails missing password', function( done ) { + const conn = { + username: 'user', + database: 'database', + }; + try{ + connect( conn ).then( connection =>{ + assert.isUnDefined( connection, 'Connection should not have been defined'); + done('Should have not have occured do to missing password'); + }).catch( err =>{ + done(); + }) + }catch( error ){ + done(); + } + }); + + it('connect fails username is empty', function( done ) { + const conn = { + username: 'user', + password: '', + database: 'database', + }; + try{ + connect( conn ).then( connection =>{ + assert.isUnDefined( connection, 'Connection should not have been defined'); + done('Should have not have occured do to missing password'); + }).catch( err =>{ + done(); + }) + }catch( error ){ + done(); + } + }); + From aaef90f39dc01ae58f5cd3776f451c0e1f7c16de Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Wed, 3 Jan 2018 22:35:58 -0500 Subject: [PATCH 22/45] Check in latest tets for database --- test/backend/datastores.athena.spec.js | 34 ++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/backend/datastores.athena.spec.js b/test/backend/datastores.athena.spec.js index 98214e65d..826847876 100644 --- a/test/backend/datastores.athena.spec.js +++ b/test/backend/datastores.athena.spec.js @@ -92,7 +92,41 @@ describe('Athena Unit Tests:', function () { } }); + it('connect fails missing database', function( done ) { + const conn = { + username: 'user', + password: '', + database: 'database', + }; + try{ + connect( conn ).then( connection =>{ + assert.isUnDefined( connection, 'Connection should not have been defined'); + done('Should have not have occured do to missing database'); + }).catch( err =>{ + done(); + }) + }catch( error ){ + done(); + } + }); + it('connect fails database is empty', function( done ) { + const conn = { + username: 'user', + password: 'password', + database: '', + }; + try{ + connect( conn ).then( connection =>{ + assert.isUnDefined( connection, 'Connection should not have been defined'); + done('Should have not have occured do to missing database'); + }).catch( err =>{ + done(); + }) + }catch( error ){ + done(); + } + }); }); From 154a6b7f905d37ca7d14581d13ea370a58ca1206 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Wed, 3 Jan 2018 22:44:16 -0500 Subject: [PATCH 23/45] Update to do for test cases --- backend/persistent/datastores/Athena.js | 6 ++++++ test/backend/datastores.athena.spec.js | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js index 85143850e..6edc3e7f8 100644 --- a/backend/persistent/datastores/Athena.js +++ b/backend/persistent/datastores/Athena.js @@ -18,6 +18,12 @@ import {parseSQL} from '../../parse'; * @param {string} connection.accessKey - AWS Access Key * @param {string} connection.secretKey - AWS Secret Key */ +export function validateConnection( connection ){ + + //TODO Move validation of connection into common function +} + + export function connect(connection) { console.log( 'Athena Connection', connection); const { diff --git a/test/backend/datastores.athena.spec.js b/test/backend/datastores.athena.spec.js index 826847876..94ce68cce 100644 --- a/test/backend/datastores.athena.spec.js +++ b/test/backend/datastores.athena.spec.js @@ -2,6 +2,18 @@ import {assert} from 'chai'; import {connect} from '../../backend/persistent/datastores/Athena'; +//TODO +//Query tests case. +//1. Validate connection and query object +//2. Validate empty results with column names and rows empty +//3. Validate one row and column +//4. Validate multiple rows and columns + +//Schema +//1. Validate connection object + +//Tables +//1. Validate connection object // Suppressing ESLint cause Mocha ensures `this` is bound in test functions /* eslint-disable no-invalid-this */ From 8f14f9286318e31cac9955552ed52304adab4b98 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Thu, 4 Jan 2018 21:30:51 -0500 Subject: [PATCH 24/45] Check in git ignore update --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index f39b2a6a8..5902686be 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,6 @@ backend/headlessBundle.js backend/headlessBundle.js.map npm-shrinkwrap.unsafe.json + +#AWS Credentials +aws.json From cc09ab52aed576928edef41f9b9aa537645f740d Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Thu, 4 Jan 2018 22:16:52 -0500 Subject: [PATCH 25/45] Check in first function for testing --- .../datastores/drivers/AWSAthenaDriver.js | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 backend/persistent/datastores/drivers/AWSAthenaDriver.js diff --git a/backend/persistent/datastores/drivers/AWSAthenaDriver.js b/backend/persistent/datastores/drivers/AWSAthenaDriver.js new file mode 100644 index 000000000..e5d92d810 --- /dev/null +++ b/backend/persistent/datastores/drivers/AWSAthenaDriver.js @@ -0,0 +1,83 @@ +'use strict'; + +const AWS = require( 'aws-sdk' ); +const credentials = require('../../../../aws.json'); + +/** + * The following function will create an AWS Athena Client + * @param {object} connection + * @param {string} connection.accessKey - AWS Access Key + * @param {string} connection.secretAccessKey - AWS Secret Key + * @param {string} connection.region - AWS Region + */ +export function createAthenaClient( connection ){ + /*let connectionParams = { + accessKeyId: connection.accessKey, + secretAccessKey: connection.secretAccessKey, + region: connection.region, + maxRetries: 5 + }*/ + + let connectionParams = { + apiVersion: '2017-05-18', + accessKeyId: credentials.accessKey, + secretAccessKey: credentials.secretAccessKey, + region: credentials.region, + maxRetries: 5 + } + let athenaClient = new AWS.Athena( connectionParams ); + + return athenaClient; +} + +/** + * The following method will execute the sql statement to query the + * athena database + * @param {object} athenaClient - Object created using create athena client + * @param {object} params + * @param {string} params.dbName - Database name to connect to + * @param {string} params.sqlStatement - SQL statement to execute + * @param {string} params.s3Outputlocation - Location will Athena will output resutls of query + * @return {string} requestId + */ +export function startQuery( athenaClient, params ){ + let client = athenaClient; + let dbName = params.dbName; + let sqlStatement = params.sqlStatement; + + let queryParams = { + QueryString: params.sqlStatement, + ResultConfiguration: { + OutputLocation: s3Outputlocation, + EncryptionConfiguration: { + EncryptionOption: 'SSE_S3' + } + }, + QueryExecutionContext: { + Database: params.dbName + } + }; + return new Promise(function(resolve, reject) { + + client.startQueryExecution( queryParams, (err, data) =>{ + if (err) { + reject( err ); + }else{ + let queryId = data; + resolve( queryId ); + } + }); + }); +} + +export function queryResultsCompleted(){ + +} + +export function cancelQuery(){ + +} + +export function getQueryResults(){ + +} \ No newline at end of file From 938cece584e88b79c463c145081737f3ff42d062 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Thu, 4 Jan 2018 22:30:11 -0500 Subject: [PATCH 26/45] Check in basic test to start query --- .../datastores/drivers/AWSAthenaDriver.js | 2 +- test/manual/AthenaClient.js | 17 +++++++++++++++++ test/manual/REMOVE_DIRECTORY.md | 0 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 test/manual/AthenaClient.js create mode 100644 test/manual/REMOVE_DIRECTORY.md diff --git a/backend/persistent/datastores/drivers/AWSAthenaDriver.js b/backend/persistent/datastores/drivers/AWSAthenaDriver.js index e5d92d810..e975e606d 100644 --- a/backend/persistent/datastores/drivers/AWSAthenaDriver.js +++ b/backend/persistent/datastores/drivers/AWSAthenaDriver.js @@ -48,7 +48,7 @@ export function startQuery( athenaClient, params ){ let queryParams = { QueryString: params.sqlStatement, ResultConfiguration: { - OutputLocation: s3Outputlocation, + OutputLocation: params.s3Outputlocation, EncryptionConfiguration: { EncryptionOption: 'SSE_S3' } diff --git a/test/manual/AthenaClient.js b/test/manual/AthenaClient.js new file mode 100644 index 000000000..d1c4f5d61 --- /dev/null +++ b/test/manual/AthenaClient.js @@ -0,0 +1,17 @@ + + +import {createAthenaClient, startQuery} from '../../backend/persistent/datastores/drivers/AWSAthenaDriver'; + + +const client = createAthenaClient(); + +let params = { + dbName :'shopscreen_logs', + sqlStatement: 'select * from clean_logs', + s3Outputlocation: 's3://aws-athena-query-results-575576301786-us-east-1/' +}; +startQuery( client, params ).then( rst =>{ + console.log( 'Got the query id back', rst); +}).catch( err =>{ + console.log( 'err', err); +}); \ No newline at end of file diff --git a/test/manual/REMOVE_DIRECTORY.md b/test/manual/REMOVE_DIRECTORY.md new file mode 100644 index 000000000..e69de29bb From 765474c4a181261971bd83ebaa201eb8e3844836 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Thu, 4 Jan 2018 22:59:01 -0500 Subject: [PATCH 27/45] Check in version for getting connection status --- .../datastores/drivers/AWSAthenaDriver.js | 76 +++++++++++++++++-- test/manual/AthenaClient.js | 20 ++++- 2 files changed, 88 insertions(+), 8 deletions(-) diff --git a/backend/persistent/datastores/drivers/AWSAthenaDriver.js b/backend/persistent/datastores/drivers/AWSAthenaDriver.js index e975e606d..5abbf231c 100644 --- a/backend/persistent/datastores/drivers/AWSAthenaDriver.js +++ b/backend/persistent/datastores/drivers/AWSAthenaDriver.js @@ -42,8 +42,6 @@ export function createAthenaClient( connection ){ */ export function startQuery( athenaClient, params ){ let client = athenaClient; - let dbName = params.dbName; - let sqlStatement = params.sqlStatement; let queryParams = { QueryString: params.sqlStatement, @@ -56,26 +54,92 @@ export function startQuery( athenaClient, params ){ QueryExecutionContext: { Database: params.dbName } - }; + }; return new Promise(function(resolve, reject) { client.startQueryExecution( queryParams, (err, data) =>{ if (err) { reject( err ); }else{ - let queryId = data; + let queryId = data.QueryExecutionId; resolve( queryId ); } }); }); } -export function queryResultsCompleted(){ +/** + * The following method will check to see if the query results + * have completed. It will return -1 if the query errored, 0 + * if it is still executing or 1 if it has completed + * @param {object} athenaClient - Object created using create athena client + * @param {string} queryExecutionId + * @param {int} -1 : Error, 0 : Still running, 1 : Completed + */ +export function queryResultsCompleted( athenaClient, queryExecutionId ){ + let client = athenaClient; + let queryParams = { + QueryExecutionId: queryExecutionId + }; + + return new Promise(function(resolve, reject) { + + client.getQueryExecution( queryParams, (err, data) =>{ + if (err) { + reject( -1 ); + }else{ + + let state = data.QueryExecution.Status.State; + let queryState = 0; + switch (state) { + case 'QUEUED': + queryState = 0; + break; + case 'RUNNING': + queryState = 0; + break; + case 'SUCCEEDED': + queryState = 1; + break; + case 'FAILED': + queryState = -1; + break; + case 'CANCELLED': + queryState = -1; + break; + default: + queryState = -1; + break; + } + resolve( queryState ); + } + }); + }); } -export function cancelQuery(){ +/** + * The following method will stop the query execution based on the query id + * @param {object} athenaClient - Object created using create athena client + * @param {string} queryExecutionId + */ +export function stopQuery( athenaClient, queryExecutionId ){ + let client = athenaClient; + let queryParams = { + QueryExecutionId: queryExecutionId + }; + + return new Promise(function(resolve, reject) { + + client.stopQueryExecution( queryParams, (err, data) =>{ + if (err) { + reject( err ); + }else{ + resolve( data); + } + }); + }); } export function getQueryResults(){ diff --git a/test/manual/AthenaClient.js b/test/manual/AthenaClient.js index d1c4f5d61..fc41f6aee 100644 --- a/test/manual/AthenaClient.js +++ b/test/manual/AthenaClient.js @@ -1,17 +1,33 @@ -import {createAthenaClient, startQuery} from '../../backend/persistent/datastores/drivers/AWSAthenaDriver'; +import {createAthenaClient, startQuery, stopQuery, queryResultsCompleted} from '../../backend/persistent/datastores/drivers/AWSAthenaDriver'; const client = createAthenaClient(); let params = { dbName :'shopscreen_logs', - sqlStatement: 'select * from clean_logs', + sqlStatement: 'select * from clean_logs limit 1000', s3Outputlocation: 's3://aws-athena-query-results-575576301786-us-east-1/' }; startQuery( client, params ).then( rst =>{ console.log( 'Got the query id back', rst); + + queryResultsCompleted( client, rst ).then( queryState =>{ + if( queryState === -1 ){ + console.log( 'Error Querying '); + } else if( queryState === 0 ){ + stopQuery( client, rst ).then( stopRst =>{ + console.log( 'Query Stopped', stopRst); + }); + }else{ + console.log( 'Query has completed and is done'); + } + }); + //Stop the query + //stopQuery( client, rst ).then( stopRst =>{ + //console.log( 'Query Stopped', stopRst); + //})// }).catch( err =>{ console.log( 'err', err); }); \ No newline at end of file From 6094c2109f856cf65a31728b4df9166d1249a9b4 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sat, 6 Jan 2018 14:06:38 -0500 Subject: [PATCH 28/45] First stab at execution method --- .../datastores/drivers/AWSAthenaDriver.js | 87 ++++++++++++++++++- test/manual/AthenaClient.js | 9 +- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/backend/persistent/datastores/drivers/AWSAthenaDriver.js b/backend/persistent/datastores/drivers/AWSAthenaDriver.js index 5abbf231c..de2e57257 100644 --- a/backend/persistent/datastores/drivers/AWSAthenaDriver.js +++ b/backend/persistent/datastores/drivers/AWSAthenaDriver.js @@ -2,6 +2,7 @@ const AWS = require( 'aws-sdk' ); const credentials = require('../../../../aws.json'); +const NUMBER_OF_RETRIES = 5; /** * The following function will create an AWS Athena Client @@ -142,6 +143,90 @@ export function stopQuery( athenaClient, queryExecutionId ){ }); } -export function getQueryResults(){ +/** + * The following method will get the query results based on the query id + * @param {object} athenaClient - Object created using create athena client + * @param {string} queryExecutionId + */ +export function getQueryResults( athenaClient, queryExecutionId ){ + let client = athenaClient; + + let queryParams = { + QueryExecutionId: queryExecutionId + }; + + return new Promise(function(resolve, reject) { + + client.getQueryResults(queryParams, (err, data) => { + if ( err ) { + return reject(err); + } else { + return resolve(data); + } + }); + }); +} +/** + * The following function will create an AWS Athena Client + * @param {object} queryParams + * @param {string} queryParams.accessKey - AWS Access Key + * @param {string} queryParams.secretAccessKey - AWS Secret Key + * @param {string} queryParams.region - AWS Region + * @param {string} queryParams.region - AWS Region + * @param {string} params.dbName - Database name to connect to + * @param {string} params.sqlStatement - SQL statement to execute + * @param {string} params.s3Outputlocation - Location will Athena will output resutls of query + * @param {number} params.timeout - The timeout interval when the query should stop + */ +export function executeQuery( queryParams ){ + let client = createAthenaClient( queryParams ); + + + return new Promise(function(resolve, reject) { + + startQuery(client, queryParams, (err, data) => { + let queryExecutionId; + if ( err ) { + return reject(err); + } else { + + queryExecutionId = data; + //Define a function for interval + //1. Set number of timeout before starting + let retryInterval = params.timeout/ NUMBER_OF_RETRIES; + + let retryCount = 0; + //If retry interval is not defined or less 0 set retry to 1000 + if( ( !retryInterval ) || (retryInterval < 1 ) ){ + retryInterval = 1000; + } + + let checkQueryStatus = ()=>{ + console.log( `Checking Query Status`); + retryCount++; + queryResultsCompleted( client, queryParams).then( queryStatus =>{ + console.log( `Checking Query Status: ${queryStatus}`); + + if( queryStatus < 0 ){ + return reject( `There was an error completing the query`); + }else if( queryStatus === 1){ + console.log( 'Have query results'); + return getQueryResults( client, queryExecutionId ); + }else{ + //Loop again + console.log( 'Starting time out'); + return setTimeout( checkQueryStatus, retryInterval ); + } + }).catch( err =>{ + console.log( `Unexpected error checking query results ${err}`); + return reject( err ); + }); + }; + + checkQueryStatus(); + + } + }); + }); } \ No newline at end of file diff --git a/test/manual/AthenaClient.js b/test/manual/AthenaClient.js index fc41f6aee..36e6b4d0d 100644 --- a/test/manual/AthenaClient.js +++ b/test/manual/AthenaClient.js @@ -10,7 +10,7 @@ let params = { sqlStatement: 'select * from clean_logs limit 1000', s3Outputlocation: 's3://aws-athena-query-results-575576301786-us-east-1/' }; -startQuery( client, params ).then( rst =>{ +/*startQuery( client, params ).then( rst =>{ console.log( 'Got the query id back', rst); queryResultsCompleted( client, rst ).then( queryState =>{ @@ -30,4 +30,11 @@ startQuery( client, params ).then( rst =>{ //})// }).catch( err =>{ console.log( 'err', err); +});*/ + +executeQuery( queryParams ).then( queryResult =>{ + console.log( `Received response `); + console.log( queryResult ); +}).catch( err =>{ + console.log( `Unexpected error returning the query result ${err}`); }); \ No newline at end of file From 1db266744bd84f0d68e5d9913eea866e189240c8 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sat, 6 Jan 2018 14:24:26 -0500 Subject: [PATCH 29/45] Fixing Query Results --- .../datastores/drivers/AWSAthenaDriver.js | 102 +++++++++--------- test/manual/AthenaClient.js | 9 +- 2 files changed, 56 insertions(+), 55 deletions(-) diff --git a/backend/persistent/datastores/drivers/AWSAthenaDriver.js b/backend/persistent/datastores/drivers/AWSAthenaDriver.js index de2e57257..89d68de38 100644 --- a/backend/persistent/datastores/drivers/AWSAthenaDriver.js +++ b/backend/persistent/datastores/drivers/AWSAthenaDriver.js @@ -57,13 +57,12 @@ export function startQuery( athenaClient, params ){ } }; return new Promise(function(resolve, reject) { - - client.startQueryExecution( queryParams, (err, data) =>{ + return client.startQueryExecution( queryParams, (err, data) =>{ if (err) { - reject( err ); + return reject( err ); }else{ let queryId = data.QueryExecutionId; - resolve( queryId ); + return resolve( queryId ); } }); }); @@ -85,13 +84,16 @@ export function queryResultsCompleted( athenaClient, queryExecutionId ){ }; return new Promise(function(resolve, reject) { - - client.getQueryExecution( queryParams, (err, data) =>{ + console.log( 'Start checking query execution'); + return client.getQueryExecution( queryParams, (err, data) =>{ + console.log( 'Start returning checking query execution'); if (err) { - reject( -1 ); + console.log( `Error getting query execution ${err}`); + return reject( -1 ); }else{ - + console.log( `Got Response ${data}`); let state = data.QueryExecution.Status.State; + console.log( `The query Result state ${state}`); let queryState = 0; switch (state) { case 'QUEUED': @@ -113,7 +115,7 @@ export function queryResultsCompleted( athenaClient, queryExecutionId ){ queryState = -1; break; } - resolve( queryState ); + return resolve( queryState ); } }); }); @@ -177,56 +179,54 @@ export function getQueryResults( athenaClient, queryExecutionId ){ * @param {string} params.dbName - Database name to connect to * @param {string} params.sqlStatement - SQL statement to execute * @param {string} params.s3Outputlocation - Location will Athena will output resutls of query - * @param {number} params.timeout - The timeout interval when the query should stop + * @param {number} params.queryTimeout - The timeout interval when the query should stop */ export function executeQuery( queryParams ){ let client = createAthenaClient( queryParams ); + //let query = startQuery; return new Promise(function(resolve, reject) { + //console.log( 'Starting query', query); + return startQuery(client, queryParams).then( queryExecutionId => { + + console.log( `Got Query Id ${queryExecutionId}`); + //Define a function for interval + //1. Set number of timeout before starting + let retryInterval = queryParams.queryTimeout/ NUMBER_OF_RETRIES; + + let retryCount = 0; + //If retry interval is not defined or less 0 set retry to 1000 + if( ( !retryInterval ) || (retryInterval < 1 ) ){ + retryInterval = 1000; + } - startQuery(client, queryParams, (err, data) => { - let queryExecutionId; - if ( err ) { - return reject(err); - } else { - - queryExecutionId = data; - //Define a function for interval - //1. Set number of timeout before starting - let retryInterval = params.timeout/ NUMBER_OF_RETRIES; - - let retryCount = 0; - //If retry interval is not defined or less 0 set retry to 1000 - if( ( !retryInterval ) || (retryInterval < 1 ) ){ - retryInterval = 1000; - } - - let checkQueryStatus = ()=>{ - console.log( `Checking Query Status`); - retryCount++; - queryResultsCompleted( client, queryParams).then( queryStatus =>{ - console.log( `Checking Query Status: ${queryStatus}`); - - if( queryStatus < 0 ){ - return reject( `There was an error completing the query`); - }else if( queryStatus === 1){ - console.log( 'Have query results'); - return getQueryResults( client, queryExecutionId ); - }else{ - //Loop again - console.log( 'Starting time out'); - return setTimeout( checkQueryStatus, retryInterval ); - } - }).catch( err =>{ - console.log( `Unexpected error checking query results ${err}`); - return reject( err ); - }); - }; - - checkQueryStatus(); + let checkQueryStatus = ()=>{ + console.log( `Checking Query Status`); + retryCount++; + queryResultsCompleted( client, queryExecutionId).then( queryStatus =>{ + console.log( `Checking Query Status: ${queryStatus}`); + + if( queryStatus < 0 ){ + return reject( `There was an error completing the query`); + }else if( queryStatus === 1){ + console.log( 'Have query results'); + return getQueryResults( client, queryExecutionId ); + }else{ + //Loop again + console.log( 'Starting time out'); + return setTimeout( checkQueryStatus, retryInterval ); + } + }).catch( err =>{ + console.log( `Unexpected error checking query results ${err}`); + return reject( err ); + }); + }; + + checkQueryStatus(); - } + }).catch( err =>{ + return reject( err ); }); }); } \ No newline at end of file diff --git a/test/manual/AthenaClient.js b/test/manual/AthenaClient.js index 36e6b4d0d..510f4ab63 100644 --- a/test/manual/AthenaClient.js +++ b/test/manual/AthenaClient.js @@ -1,14 +1,15 @@ -import {createAthenaClient, startQuery, stopQuery, queryResultsCompleted} from '../../backend/persistent/datastores/drivers/AWSAthenaDriver'; +import {createAthenaClient, startQuery, stopQuery, queryResultsCompleted, executeQuery} from '../../backend/persistent/datastores/drivers/AWSAthenaDriver'; const client = createAthenaClient(); let params = { dbName :'shopscreen_logs', - sqlStatement: 'select * from clean_logs limit 1000', - s3Outputlocation: 's3://aws-athena-query-results-575576301786-us-east-1/' + sqlStatement: 'select * from clean_logs limit 100000', + s3Outputlocation: 's3://aws-athena-query-results-575576301786-us-east-1/', + queryTimeout: 20000 }; /*startQuery( client, params ).then( rst =>{ console.log( 'Got the query id back', rst); @@ -32,7 +33,7 @@ let params = { console.log( 'err', err); });*/ -executeQuery( queryParams ).then( queryResult =>{ +executeQuery( params ).then( queryResult =>{ console.log( `Received response `); console.log( queryResult ); }).catch( err =>{ From 1a1fcf762688b03fc1954fc4954ffc144ba3c2d3 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sat, 6 Jan 2018 14:38:19 -0500 Subject: [PATCH 30/45] Check in latest version --- .../datastores/drivers/AWSAthenaDriver.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/persistent/datastores/drivers/AWSAthenaDriver.js b/backend/persistent/datastores/drivers/AWSAthenaDriver.js index 89d68de38..4c6d3567f 100644 --- a/backend/persistent/datastores/drivers/AWSAthenaDriver.js +++ b/backend/persistent/datastores/drivers/AWSAthenaDriver.js @@ -135,11 +135,11 @@ export function stopQuery( athenaClient, queryExecutionId ){ return new Promise(function(resolve, reject) { - client.stopQueryExecution( queryParams, (err, data) =>{ + return client.stopQueryExecution( queryParams, (err, data) =>{ if (err) { - reject( err ); + return reject( err ); }else{ - resolve( data); + return resolve( data); } }); }); @@ -158,8 +158,7 @@ export function getQueryResults( athenaClient, queryExecutionId ){ }; return new Promise(function(resolve, reject) { - - client.getQueryResults(queryParams, (err, data) => { + return client.getQueryResults(queryParams, (err, data) => { if ( err ) { return reject(err); } else { @@ -201,8 +200,9 @@ export function executeQuery( queryParams ){ retryInterval = 1000; } + console.log( `The Interval ${retryInterval} The retry count ${retryCount}`); let checkQueryStatus = ()=>{ - console.log( `Checking Query Status`); + console.log( `Checking Query Status ${retryCount}`); retryCount++; queryResultsCompleted( client, queryExecutionId).then( queryStatus =>{ console.log( `Checking Query Status: ${queryStatus}`); @@ -214,7 +214,7 @@ export function executeQuery( queryParams ){ return getQueryResults( client, queryExecutionId ); }else{ //Loop again - console.log( 'Starting time out'); + console.log( 'Starting time out'); return setTimeout( checkQueryStatus, retryInterval ); } }).catch( err =>{ From 9fa4f7c18b4c73e1ab1773c5063a752deff9a2f8 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sat, 6 Jan 2018 14:43:18 -0500 Subject: [PATCH 31/45] Check in test information --- backend/persistent/datastores/Athena.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js index 6edc3e7f8..6bf75d7d4 100644 --- a/backend/persistent/datastores/Athena.js +++ b/backend/persistent/datastores/Athena.js @@ -1,11 +1,9 @@ import {parseSQL} from '../../parse'; -//TODO -//1. What is expected object for connection -//2. What is expect results for the schems. Is this a list of databases or -// table definitions. Must check against Postgres -//3. What is the expected results from the list of tables? -//4. What is the expected results for the query -//5. Hard code the format of the data to test + +const SHOW_TABLES_QUERY = `SHOW TABLES`; +const SHOW_SCHEMA_QUERY = `DESCRIBE ${tableName}`; + + /* From 400bb7b11389aafc1baa7801d8f88e544d8ebedb Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sat, 6 Jan 2018 17:03:51 -0500 Subject: [PATCH 32/45] Check in latest parsing --- .../datastores/drivers/AWSAthenaDriver.js | 25 ++++++++---------- test/manual/AthenaClient.js | 26 ++++++++++++++++--- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/backend/persistent/datastores/drivers/AWSAthenaDriver.js b/backend/persistent/datastores/drivers/AWSAthenaDriver.js index 4c6d3567f..84f181f6e 100644 --- a/backend/persistent/datastores/drivers/AWSAthenaDriver.js +++ b/backend/persistent/datastores/drivers/AWSAthenaDriver.js @@ -158,10 +158,11 @@ export function getQueryResults( athenaClient, queryExecutionId ){ }; return new Promise(function(resolve, reject) { - return client.getQueryResults(queryParams, (err, data) => { + client.getQueryResults(queryParams, (err, data) => { if ( err ) { return reject(err); } else { + console.log( `Got response for query results`); return resolve(data); } }); @@ -183,15 +184,10 @@ export function getQueryResults( athenaClient, queryExecutionId ){ export function executeQuery( queryParams ){ let client = createAthenaClient( queryParams ); - //let query = startQuery; - return new Promise(function(resolve, reject) { - //console.log( 'Starting query', query); return startQuery(client, queryParams).then( queryExecutionId => { - console.log( `Got Query Id ${queryExecutionId}`); - //Define a function for interval - //1. Set number of timeout before starting + //Define the wait interval let retryInterval = queryParams.queryTimeout/ NUMBER_OF_RETRIES; let retryCount = 0; @@ -200,21 +196,22 @@ export function executeQuery( queryParams ){ retryInterval = 1000; } - console.log( `The Interval ${retryInterval} The retry count ${retryCount}`); let checkQueryStatus = ()=>{ - console.log( `Checking Query Status ${retryCount}`); retryCount++; queryResultsCompleted( client, queryExecutionId).then( queryStatus =>{ - console.log( `Checking Query Status: ${queryStatus}`); - if( queryStatus < 0 ){ return reject( `There was an error completing the query`); }else if( queryStatus === 1){ - console.log( 'Have query results'); - return getQueryResults( client, queryExecutionId ); + return getQueryResults( client, queryExecutionId ).then( rst =>{ + + if( rst && rst.ResultSet && rst.ResultSet.Rows ){ + return resolve( rst.ResultSet.Rows ); + }else{ + return resolve( [] ); + } + }); }else{ //Loop again - console.log( 'Starting time out'); return setTimeout( checkQueryStatus, retryInterval ); } }).catch( err =>{ diff --git a/test/manual/AthenaClient.js b/test/manual/AthenaClient.js index 510f4ab63..f4c0b53b3 100644 --- a/test/manual/AthenaClient.js +++ b/test/manual/AthenaClient.js @@ -5,11 +5,13 @@ import {createAthenaClient, startQuery, stopQuery, queryResultsCompleted, execut const client = createAthenaClient(); +const SQL_STATEMENT = `select * from clean_logs limit 1000`; +//const SQL_STATEMENT = `SHOW TABLES`; let params = { dbName :'shopscreen_logs', - sqlStatement: 'select * from clean_logs limit 100000', + sqlStatement: SQL_STATEMENT, s3Outputlocation: 's3://aws-athena-query-results-575576301786-us-east-1/', - queryTimeout: 20000 + queryTimeout: 5000 }; /*startQuery( client, params ).then( rst =>{ console.log( 'Got the query id back', rst); @@ -33,9 +35,25 @@ let params = { console.log( 'err', err); });*/ +console.log( `Starting of executing query`); executeQuery( params ).then( queryResult =>{ console.log( `Received response `); - console.log( queryResult ); + console.log( queryResult ); + console.log( 'Geting resposne'); + for( let i=0; i< queryResult.length; i++ ){ + console.log('Have row'); + let row = queryResult[i].Data; + //console.log(row); + let rowData = ''; + for( let j=0; j< row.length; j++ ){ + rowData = `${rowData}, ${row[j]}`; + //console.log( row[j] ); + } + console.log( rowData ); + } + }).catch( err =>{ console.log( `Unexpected error returning the query result ${err}`); -}); \ No newline at end of file +}); + +console.log( `Completing Executin of query`); \ No newline at end of file From 775692d3d5112bb568c88bba0cf9ca37a7eb8b87 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sat, 6 Jan 2018 17:14:09 -0500 Subject: [PATCH 33/45] Check in latest connection properties --- backend/persistent/datastores/Athena.js | 56 +++++++++++++++---------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js index 6bf75d7d4..c399ccd77 100644 --- a/backend/persistent/datastores/Athena.js +++ b/backend/persistent/datastores/Athena.js @@ -3,18 +3,23 @@ import {parseSQL} from '../../parse'; const SHOW_TABLES_QUERY = `SHOW TABLES`; const SHOW_SCHEMA_QUERY = `DESCRIBE ${tableName}`; - +//TODO +// 0. Get the connection parameters working and validation +// 1. Get the list of tables +// 2. Get the list of schemas +// 3. Get the query working correctly /* * The connection object will open a connection to the Athena database * @param {object} connection - * @param {string} connection.username - username for connection - * @param {string} connection.password - password for connection - * @param {string} connection.database - database for connection - * @param {string} connection.port - port for connection * @param {string} connection.accessKey - AWS Access Key - * @param {string} connection.secretKey - AWS Secret Key + * @param {string} connection.secretAccessKey - AWS Secret Key + * @param {string} connection.region - AWS Region + * @param {string} connection.dbName - Database name to connect to + * @param {string} connection.sqlStatement - SQL statement to execute + * @param {string} connection.s3Outputlocation - Location will Athena will output resutls of query + * @param {number} params.queryTimeout - The timeout interval when the query should stop */ export function validateConnection( connection ){ @@ -25,32 +30,41 @@ export function validateConnection( connection ){ export function connect(connection) { console.log( 'Athena Connection', connection); const { - username, password, database, port, dialect, accessKey, secretKey + region, accessKey, secretKey, dbName, sqlStatement, s3Outputlocation, queryTimeout } = connection; //Validate that the connection has a - if( !username ){ - throw new Error(`The username was not defined`); + if( !region ){ + throw new Error(`The AWS Region was not defined`); + } + + if( !accessKey ){ + throw new Error(`The AWS access key was not defined`); + } + + if( !secretKey ){ + throw new Error(`The AWS secret key was not defined`); + } + + if( !dbName ){ + throw new Error(`The Database Name was not defined`); + } + + if( !sqlStatement ){ + throw new Error(`The SQL Statement was not defined`); } - if( !password ){ - throw new Error(`The password was not defined`); + if( !s3Outputlocation ){ + throw new Error(`The Athena S3 Results Output location was not defined`); } - if( !database ){ - throw new Error(`The database was not defined`); + if( !queryTimeout ){ + throw new Error(`The Athena S3 Results Output location was not defined`); } let con = { - username, - password, - database, - port, - dialect, - accessKey, - secretKey + region, accessKey, secretKey, dbName, sqlStatement, s3Outputlocation, queryTimeout }; - console.log( 'Connection object', con); return new Promise(function(resolve, reject) { resolve( con ); }); From b033703b69e9638ad87489e24b546e324d5363a2 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sat, 6 Jan 2018 17:28:36 -0500 Subject: [PATCH 34/45] Updated the test cases --- backend/persistent/datastores/Athena.js | 6 +- test/backend/datastores.athena.spec.js | 259 +++++++++++++++++++++--- 2 files changed, 232 insertions(+), 33 deletions(-) diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js index c399ccd77..d3b8fcaa1 100644 --- a/backend/persistent/datastores/Athena.js +++ b/backend/persistent/datastores/Athena.js @@ -1,7 +1,7 @@ import {parseSQL} from '../../parse'; const SHOW_TABLES_QUERY = `SHOW TABLES`; -const SHOW_SCHEMA_QUERY = `DESCRIBE ${tableName}`; +const SHOW_SCHEMA_QUERY = `DESCRIBE `; //TODO // 0. Get the connection parameters working and validation @@ -58,8 +58,8 @@ export function connect(connection) { throw new Error(`The Athena S3 Results Output location was not defined`); } - if( !queryTimeout ){ - throw new Error(`The Athena S3 Results Output location was not defined`); + if( !queryTimeout && queryTimeout < 0){ + throw new Error(`The Athena Query Timeout was not defined`); } let con = { region, accessKey, secretKey, dbName, sqlStatement, s3Outputlocation, queryTimeout diff --git a/test/backend/datastores.athena.spec.js b/test/backend/datastores.athena.spec.js index 94ce68cce..53d83557d 100644 --- a/test/backend/datastores.athena.spec.js +++ b/test/backend/datastores.athena.spec.js @@ -21,9 +21,13 @@ describe('Athena Unit Tests:', function () { it('connect succeeds', function( done ) { const conn = { - username: 'user', - password: 'password', - database: 'database', + region: 'us-east-1', + accessKey: 'XXXXXXXX', + secretKey: 'XXXXXAAAA', + dbName: 'PLOT.LY-TEST', + sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', + s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', + queryTimeout: 10000 }; connect( conn ).then( connection =>{ assert.isDefined( connection, 'Connection is defined'); @@ -34,15 +38,19 @@ describe('Athena Unit Tests:', function () { }) }); - it('connect fails missing username', function( done ) { + it('connect fails missing region', function( done ) { const conn = { - password: 'password', - database: 'database', + accessKey: 'XXXXXXXX', + secretKey: 'XXXXXAAAA', + dbName: 'PLOT.LY-TEST', + sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', + s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', + queryTimeout: 10000 }; try{ connect( conn ).then( connection =>{ assert.isUnDefined( connection, 'Connection should not have been defined'); - done('Should have not have occured do to missing username'); + done('Should have not have occured do to missing region'); }).catch( err =>{ done(); }) @@ -53,14 +61,18 @@ describe('Athena Unit Tests:', function () { it('connect fails username is empty', function( done ) { const conn = { - username: '', - password: 'password', - database: 'database', + region: '', + accessKey: 'XXXXXXXX', + secretKey: 'XXXXXAAAA', + dbName: 'PLOT.LY-TEST', + sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', + s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', + queryTimeout: 10000 }; try{ connect( conn ).then( connection =>{ assert.isUnDefined( connection, 'Connection should not have been defined'); - done('Should have not have occured do to missing username'); + done('Should have not have occured do to missing region'); }).catch( err =>{ done(); }) @@ -69,15 +81,19 @@ describe('Athena Unit Tests:', function () { } }); - it('connect fails missing password', function( done ) { + it('connect fails missing AWS access key', function( done ) { const conn = { - username: 'user', - database: 'database', + region: 'us-east-1', + secretKey: 'XXXXXAAAA', + dbName: 'PLOT.LY-TEST', + sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', + s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', + queryTimeout: 10000 }; try{ connect( conn ).then( connection =>{ assert.isUnDefined( connection, 'Connection should not have been defined'); - done('Should have not have occured do to missing password'); + done('Should have not have occured do to missing AWS access key'); }).catch( err =>{ done(); }) @@ -86,16 +102,63 @@ describe('Athena Unit Tests:', function () { } }); - it('connect fails username is empty', function( done ) { + it('connect fails AWS Access key is empty', function( done ) { + const conn = { + region: 'us-east-1', + accessKey: '', + secretKey: 'XXXXXAAAA', + dbName: 'PLOT.LY-TEST', + sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', + s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', + queryTimeout: 10000 + }; + try{ + connect( conn ).then( connection =>{ + assert.isUnDefined( connection, 'Connection should not have been defined'); + done('Should have not have occured do to missing aws access key'); + }).catch( err =>{ + done(); + }) + }catch( error ){ + done(); + } + }); + + it('connect fails missing secret key', function( done ) { + const conn = { + region: 'us-east-1', + accessKey: 'XXXXXXXX', + dbName: 'PLOT.LY-TEST', + sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', + s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', + queryTimeout: 10000 + }; + try{ + connect( conn ).then( connection =>{ + assert.isUnDefined( connection, 'Connection should not have been defined'); + done('Should have not have occured do to missing secret key'); + }).catch( err =>{ + done(); + }) + }catch( error ){ + done(); + } + }); + + it('connect fails secret key is empty', function( done ) { const conn = { - username: 'user', - password: '', - database: 'database', + region: 'us-east-1', + accessKey: 'AAAAAAA', + secretKey: '', + dbName: 'PLOT.LY-TEST', + sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', + s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', + queryTimeout: 10000 }; try{ connect( conn ).then( connection =>{ assert.isUnDefined( connection, 'Connection should not have been defined'); - done('Should have not have occured do to missing password'); + done('Should have not have occured do to missing secret key'); }).catch( err =>{ done(); }) @@ -104,16 +167,19 @@ describe('Athena Unit Tests:', function () { } }); - it('connect fails missing database', function( done ) { + it('connect fails missing db name', function( done ) { const conn = { - username: 'user', - password: '', - database: 'database', + region: 'us-east-1', + accessKey: 'XXXXXXXX', + secretKey: 'XXXXXX', + sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', + s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', + queryTimeout: 10000 }; try{ connect( conn ).then( connection =>{ assert.isUnDefined( connection, 'Connection should not have been defined'); - done('Should have not have occured do to missing database'); + done('Should have not have occured do to missing db name'); }).catch( err =>{ done(); }) @@ -122,16 +188,41 @@ describe('Athena Unit Tests:', function () { } }); - it('connect fails database is empty', function( done ) { + it('connect fails db name is empty', function( done ) { + const conn = { + region: 'us-east-1', + accessKey: 'AAAAAAA', + secretKey: 'XXXXXX', + dbName: '', + sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', + s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', + queryTimeout: 10000 + }; + try{ + connect( conn ).then( connection =>{ + assert.isUnDefined( connection, 'Connection should not have been defined'); + done('Should have not have occured do to missing db name'); + }).catch( err =>{ + done(); + }) + }catch( error ){ + done(); + } + }); + + it('connect fails missing sql statement', function( done ) { const conn = { - username: 'user', - password: 'password', - database: '', + region: 'us-east-1', + accessKey: 'XXXXXXXX', + secretKey: 'XXXXXX', + dbName: 'PLOT.LY-TEST', + s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', + queryTimeout: 10000 }; try{ connect( conn ).then( connection =>{ assert.isUnDefined( connection, 'Connection should not have been defined'); - done('Should have not have occured do to missing database'); + done('Should have not have occured do to missing sql statement'); }).catch( err =>{ done(); }) @@ -140,6 +231,114 @@ describe('Athena Unit Tests:', function () { } }); + it('connect fails db name is empty', function( done ) { + const conn = { + region: 'us-east-1', + accessKey: 'AAAAAAA', + secretKey: 'XXXXXX', + dbName: 'PLOT.LY-TEST', + sqlStatement: '', + s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', + queryTimeout: 10000 + }; + try{ + connect( conn ).then( connection =>{ + assert.isUnDefined( connection, 'Connection should not have been defined'); + done('Should have not have occured do to missing sql statement'); + }).catch( err =>{ + done(); + }) + }catch( error ){ + done(); + } + }); + + + it('connect fails missing S3 Output locaiont', function( done ) { + const conn = { + region: 'us-east-1', + accessKey: 'XXXXXXXX', + secretKey: 'XXXXXX', + dbName: 'PLOT.LY-TEST', + sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', + queryTimeout: 10000 + }; + try{ + connect( conn ).then( connection =>{ + assert.isUnDefined( connection, 'Connection should not have been defined'); + done('Should have not have occured do to missing S3 Location'); + }).catch( err =>{ + done(); + }) + }catch( error ){ + done(); + } + }); + + it('connect fails db name is empty', function( done ) { + const conn = { + region: 'us-east-1', + accessKey: 'AAAAAAA', + secretKey: 'XXXXXX', + dbName: 'PLOT.LY-TEST', + sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', + s3Outputlocation: '', + queryTimeout: 10000 + }; + try{ + connect( conn ).then( connection =>{ + assert.isUnDefined( connection, 'Connection should not have been defined'); + done('Should have not have occured do to missing S3 Location'); + }).catch( err =>{ + done(); + }) + }catch( error ){ + done(); + } + }); + + it('connect fails missing S3 Output locaiont', function( done ) { + const conn = { + region: 'us-east-1', + accessKey: 'XXXXXXXX', + secretKey: 'XXXXXX', + dbName: 'PLOT.LY-TEST', + sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', + s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', + }; + try{ + connect( conn ).then( connection =>{ + assert.isUnDefined( connection, 'Connection should not have been defined'); + done('Should have not have occured do to missing S3 Location'); + }).catch( err =>{ + done(); + }) + }catch( error ){ + done(); + } + }); + + it('connect fails db name is empty', function( done ) { + const conn = { + region: 'us-east-1', + accessKey: 'AAAAAAA', + secretKey: 'XXXXXX', + dbName: 'PLOT.LY-TEST', + sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', + s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', + queryTimeout: -1 + }; + try{ + connect( conn ).then( connection =>{ + assert.isUnDefined( connection, 'Connection should not have been defined'); + done('Should have not have occured do to missing S3 Location'); + }).catch( err =>{ + done(); + }) + }catch( error ){ + done(); + } + }); }); /* eslint-enable no-invalid-this */ From 7d596b699a326c8cf46506aba193ade1a4ce7edf Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sun, 7 Jan 2018 14:13:44 -0500 Subject: [PATCH 35/45] Check in latest vesrion with getting table names --- backend/persistent/datastores/Athena.js | 35 +++++++++++++++++-- .../datastores/drivers/AWSAthenaDriver.js | 8 ++--- test/manual/AthenaClient.js | 12 +++++-- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js index d3b8fcaa1..d8fb3c398 100644 --- a/backend/persistent/datastores/Athena.js +++ b/backend/persistent/datastores/Athena.js @@ -1,4 +1,5 @@ import {parseSQL} from '../../parse'; +import {executeQuery} from './drivers/AWSAthenaDriver'; const SHOW_TABLES_QUERY = `SHOW TABLES`; const SHOW_SCHEMA_QUERY = `DESCRIBE `; @@ -161,14 +162,42 @@ export function schemas(connection){ }); } + /** * Should return a list of tables that are in the database - * @param {*} connection + * @param {object} connection + * @param {string} connection.accessKey - AWS Access Key + * @param {string} connection.secretAccessKey - AWS Secret Key + * @param {string} connection.region - AWS Region + * @param {string} connection.region - AWS Region + * @param {string} connection.dbName - Database name to connect to + * @param {string} connection.s3Outputlocation - Location will Athena will output resutls of query */ export function tables(connection){ console.log( 'Tables', connection); - + + connection.sqlStatement = SHOW_TABLES_QUERY; + connection.queryTimeout = 5000; return new Promise(function(resolve, reject) { - resolve( ['Sample', 'Test'] ); + executeQuery( connection ).then( dataSet =>{ + + let rst = []; + console.log( 'data set', dataSet ); + if( dataSet && dataSet.length > 0){ + rst = dataSet.map( data =>{ + if( data && data.Data && data.Data.length > 0 ){ + return data.Data[0].VarCharValue; + } + }); + } + resolve( rst ); + }).catch( err =>{ + reject( err ); + }); }); + + + + + } \ No newline at end of file diff --git a/backend/persistent/datastores/drivers/AWSAthenaDriver.js b/backend/persistent/datastores/drivers/AWSAthenaDriver.js index 84f181f6e..f99828c8f 100644 --- a/backend/persistent/datastores/drivers/AWSAthenaDriver.js +++ b/backend/persistent/datastores/drivers/AWSAthenaDriver.js @@ -176,10 +176,10 @@ export function getQueryResults( athenaClient, queryExecutionId ){ * @param {string} queryParams.secretAccessKey - AWS Secret Key * @param {string} queryParams.region - AWS Region * @param {string} queryParams.region - AWS Region - * @param {string} params.dbName - Database name to connect to - * @param {string} params.sqlStatement - SQL statement to execute - * @param {string} params.s3Outputlocation - Location will Athena will output resutls of query - * @param {number} params.queryTimeout - The timeout interval when the query should stop + * @param {string} queryParams.dbName - Database name to connect to + * @param {string} queryParams.sqlStatement - SQL statement to execute + * @param {string} queryParams.s3Outputlocation - Location will Athena will output resutls of query + * @param {number} queryParams.queryTimeout - The timeout interval when the query should stop */ export function executeQuery( queryParams ){ let client = createAthenaClient( queryParams ); diff --git a/test/manual/AthenaClient.js b/test/manual/AthenaClient.js index f4c0b53b3..741dfe6f9 100644 --- a/test/manual/AthenaClient.js +++ b/test/manual/AthenaClient.js @@ -2,6 +2,8 @@ import {createAthenaClient, startQuery, stopQuery, queryResultsCompleted, executeQuery} from '../../backend/persistent/datastores/drivers/AWSAthenaDriver'; +import {tables} from '../../backend/persistent/datastores/Athena'; + const client = createAthenaClient(); @@ -36,7 +38,7 @@ let params = { });*/ console.log( `Starting of executing query`); -executeQuery( params ).then( queryResult =>{ +/*executeQuery( params ).then( queryResult =>{ console.log( `Received response `); console.log( queryResult ); console.log( 'Geting resposne'); @@ -56,4 +58,10 @@ executeQuery( params ).then( queryResult =>{ console.log( `Unexpected error returning the query result ${err}`); }); -console.log( `Completing Executin of query`); \ No newline at end of file +console.log( `Completing Executin of query`);*/ + +tables( params ).then( queryResult =>{ + console.log( 'Retrieved the results', queryResult); +}).catch( err =>{ + console.log( 'Error getting tables', err); +}); \ No newline at end of file From 699644409922e54e382547ed6b076913206427cc Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sun, 7 Jan 2018 14:18:38 -0500 Subject: [PATCH 36/45] Checkn in cleanup for get tables --- backend/persistent/datastores/Athena.js | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js index d8fb3c398..0600cacac 100644 --- a/backend/persistent/datastores/Athena.js +++ b/backend/persistent/datastores/Athena.js @@ -2,15 +2,15 @@ import {parseSQL} from '../../parse'; import {executeQuery} from './drivers/AWSAthenaDriver'; const SHOW_TABLES_QUERY = `SHOW TABLES`; -const SHOW_SCHEMA_QUERY = `DESCRIBE `; +const SHOW_SCHEMA_QUERY = `SELECT column_name, data_type FROM information_schema.columns WHERE table_schema = `; +const DEFAULT_QUERY_TIMEOUT = 5000; //TODO -// 0. Get the connection parameters working and validation -// 1. Get the list of tables // 2. Get the list of schemas // 3. Get the query working correctly + /* * The connection object will open a connection to the Athena database * @param {object} connection @@ -22,12 +22,6 @@ const SHOW_SCHEMA_QUERY = `DESCRIBE `; * @param {string} connection.s3Outputlocation - Location will Athena will output resutls of query * @param {number} params.queryTimeout - The timeout interval when the query should stop */ -export function validateConnection( connection ){ - - //TODO Move validation of connection into common function -} - - export function connect(connection) { console.log( 'Athena Connection', connection); const { @@ -174,10 +168,9 @@ export function schemas(connection){ * @param {string} connection.s3Outputlocation - Location will Athena will output resutls of query */ export function tables(connection){ - console.log( 'Tables', connection); connection.sqlStatement = SHOW_TABLES_QUERY; - connection.queryTimeout = 5000; + connection.queryTimeout = DEFAULT_QUERY_TIMEOUT; return new Promise(function(resolve, reject) { executeQuery( connection ).then( dataSet =>{ @@ -195,9 +188,4 @@ export function tables(connection){ reject( err ); }); }); - - - - - } \ No newline at end of file From a6c84348792e78365bfdd82c647f5531f44a5bb0 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sun, 7 Jan 2018 14:37:09 -0500 Subject: [PATCH 37/45] Check in get list of Schemas --- backend/persistent/datastores/Athena.js | 46 ++++++++++++++++--- .../datastores/drivers/AWSAthenaDriver.js | 5 -- test/manual/AthenaClient.js | 4 +- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js index 0600cacac..46d6a0a50 100644 --- a/backend/persistent/datastores/Athena.js +++ b/backend/persistent/datastores/Athena.js @@ -2,7 +2,7 @@ import {parseSQL} from '../../parse'; import {executeQuery} from './drivers/AWSAthenaDriver'; const SHOW_TABLES_QUERY = `SHOW TABLES`; -const SHOW_SCHEMA_QUERY = `SELECT column_name, data_type FROM information_schema.columns WHERE table_schema = `; +const SHOW_SCHEMA_QUERY = `SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema `; const DEFAULT_QUERY_TIMEOUT = 5000; //TODO @@ -106,13 +106,47 @@ export function query(queryObject, connection){ * Should return a list of databases that are defined within the database. * @param {object} connection */ +/** + * Should return a list of tables and their that are defined within the database. + * @param {object} connection + * @param {string} connection.accessKey - AWS Access Key + * @param {string} connection.secretAccessKey - AWS Secret Key + * @param {string} connection.region - AWS Region + * @param {string} connection.region - AWS Region + * @param {string} connection.dbName - Database name to connect to + * @param {string} connection.s3Outputlocation - Location will Athena will output resutls of query + */ export function schemas(connection){ - console.log( 'Schemas', connection); - let columnnames = ['Table', 'column_name', 'data_type']; let rows = []; - for( let i=0; i< 4; i++){ + connection.sqlStatement = `${SHOW_SCHEMA_QUERY} = '${connection.dbName}'` ; + connection.queryTimeout = DEFAULT_QUERY_TIMEOUT; + return new Promise(function(resolve, reject) { + executeQuery( connection ).then( dataSet =>{ + + let rows = []; + if( dataSet && dataSet.length > 0){ + for( let i=0; i< dataSet.length; i++){ + let data = dataSet[i]; + + if( data && data.Data && data.Data.length > 0 ){ + if( i != 0){ + let row = []; + row.push( data.Data[0].VarCharValue ); //Table Name + row.push( data.Data[1].VarCharValue ); //Column Name + row.push( data.Data[2].VarCharValue ); //DataType + rows.push( row ); + } + } + } + } + resolve( {columnnames, rows} ); + }).catch( err =>{ + reject( err ); + }); + }); + /*for( let i=0; i< 4; i++){ let r = []; @@ -153,7 +187,7 @@ export function schemas(connection){ return new Promise(function(resolve, reject) { resolve( {columnnames, rows} ); - }); + });*/ } @@ -175,7 +209,7 @@ export function tables(connection){ executeQuery( connection ).then( dataSet =>{ let rst = []; - console.log( 'data set', dataSet ); + //console.log( 'data set', dataSet ); if( dataSet && dataSet.length > 0){ rst = dataSet.map( data =>{ if( data && data.Data && data.Data.length > 0 ){ diff --git a/backend/persistent/datastores/drivers/AWSAthenaDriver.js b/backend/persistent/datastores/drivers/AWSAthenaDriver.js index f99828c8f..8961621a7 100644 --- a/backend/persistent/datastores/drivers/AWSAthenaDriver.js +++ b/backend/persistent/datastores/drivers/AWSAthenaDriver.js @@ -84,16 +84,12 @@ export function queryResultsCompleted( athenaClient, queryExecutionId ){ }; return new Promise(function(resolve, reject) { - console.log( 'Start checking query execution'); return client.getQueryExecution( queryParams, (err, data) =>{ - console.log( 'Start returning checking query execution'); if (err) { console.log( `Error getting query execution ${err}`); return reject( -1 ); }else{ - console.log( `Got Response ${data}`); let state = data.QueryExecution.Status.State; - console.log( `The query Result state ${state}`); let queryState = 0; switch (state) { case 'QUEUED': @@ -162,7 +158,6 @@ export function getQueryResults( athenaClient, queryExecutionId ){ if ( err ) { return reject(err); } else { - console.log( `Got response for query results`); return resolve(data); } }); diff --git a/test/manual/AthenaClient.js b/test/manual/AthenaClient.js index 741dfe6f9..5dad76da5 100644 --- a/test/manual/AthenaClient.js +++ b/test/manual/AthenaClient.js @@ -2,7 +2,7 @@ import {createAthenaClient, startQuery, stopQuery, queryResultsCompleted, executeQuery} from '../../backend/persistent/datastores/drivers/AWSAthenaDriver'; -import {tables} from '../../backend/persistent/datastores/Athena'; +import {tables, schemas} from '../../backend/persistent/datastores/Athena'; const client = createAthenaClient(); @@ -60,7 +60,7 @@ console.log( `Starting of executing query`); console.log( `Completing Executin of query`);*/ -tables( params ).then( queryResult =>{ +schemas( params ).then( queryResult =>{ console.log( 'Retrieved the results', queryResult); }).catch( err =>{ console.log( 'Error getting tables', err); From 6d9e904af68ca0387eed9eff673a6679ce6afc15 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sun, 7 Jan 2018 14:47:48 -0500 Subject: [PATCH 38/45] Code Clean up --- backend/persistent/datastores/Athena.js | 47 +------------------------ 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js index 46d6a0a50..902d073ff 100644 --- a/backend/persistent/datastores/Athena.js +++ b/backend/persistent/datastores/Athena.js @@ -6,10 +6,7 @@ const SHOW_SCHEMA_QUERY = `SELECT table_name, column_name, data_type FROM inform const DEFAULT_QUERY_TIMEOUT = 5000; //TODO -// 2. Get the list of schemas -// 3. Get the query working correctly - - +// 1. Get the query working correctly /* * The connection object will open a connection to the Athena database @@ -146,48 +143,6 @@ export function schemas(connection){ reject( err ); }); }); - /*for( let i=0; i< 4; i++){ - - - let r = []; - r.push('Test'); - r.push(`first_name_${i}`); - r.push(`varchar`); - - rows.push(r); - } - - let rowData = []; - rowData.push('Test'); - rowData.push( 'value'); - rowData.push('integer'); - - rows.push( rowData ); - - - for( let i=0; i< 2; i++){ - - - let r = []; - r.push('Sample'); - r.push(`last_name_${i}`); - r.push(`varchar`); - - rows.push(r); - } - - rowData = []; - rowData.push('Sample'); - rowData.push( 'value'); - rowData.push('integer'); - - rows.push( rowData ); - - - - return new Promise(function(resolve, reject) { - resolve( {columnnames, rows} ); - });*/ } From f5c86fe3a9eaac8aa69422f79c4a480a8ef527af Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sun, 7 Jan 2018 15:14:16 -0500 Subject: [PATCH 39/45] Check in fix for unit tests --- app/constants/constants.js | 19 ++++--- backend/persistent/datastores/Athena.js | 9 +--- test/backend/datastores.athena.spec.js | 67 ++----------------------- 3 files changed, 20 insertions(+), 75 deletions(-) diff --git a/app/constants/constants.js b/app/constants/constants.js index 5032195f2..57b5be71e 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -198,11 +198,15 @@ export const CONNECTION_CONFIG = { 'label': 'AWS Region','value': 'region','type': 'text' }, { - 'label': 'S3 Bucket', - 'value': 'bucket', - 'type': 'text', - 'description': ` - The Athena will put the output results of the query in this location.` + 'label': 'Database','value': 'dbName','type': 'text' + }, + { + 'label': 'S3 Results Bucket','value': 'bucket','type': 'text', + 'description': + `Athena will put the results of the query to the following bucket.` + }, + { + 'label': 'Query Timeout','value': 'queryTimeout','type': 'number' }, { 'label': 'Database','value': 'database','type': 'text' @@ -422,6 +426,9 @@ export const SAMPLE_DBS = { accessKeyId: 'AKIAIMHMSHTGARJYSKMQ', secretAccessKey: 'Urvus4R7MnJOAqT4U3eovlCBimQ4Zg2Y9sV5LWow', region:'us-west-2', - database:'default' + database:'ploytly', + sqlStatement: 'SHOW TABLES', + s3Outputlocation:'s3://plotly-athena-results-us-west-2/', + queryTimeout:5000 }, }; diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js index 902d073ff..0ac400365 100644 --- a/backend/persistent/datastores/Athena.js +++ b/backend/persistent/datastores/Athena.js @@ -15,14 +15,13 @@ const DEFAULT_QUERY_TIMEOUT = 5000; * @param {string} connection.secretAccessKey - AWS Secret Key * @param {string} connection.region - AWS Region * @param {string} connection.dbName - Database name to connect to - * @param {string} connection.sqlStatement - SQL statement to execute * @param {string} connection.s3Outputlocation - Location will Athena will output resutls of query * @param {number} params.queryTimeout - The timeout interval when the query should stop */ export function connect(connection) { console.log( 'Athena Connection', connection); const { - region, accessKey, secretKey, dbName, sqlStatement, s3Outputlocation, queryTimeout + region, accessKey, secretKey, dbName, s3Outputlocation, queryTimeout } = connection; //Validate that the connection has a @@ -42,10 +41,6 @@ export function connect(connection) { throw new Error(`The Database Name was not defined`); } - if( !sqlStatement ){ - throw new Error(`The SQL Statement was not defined`); - } - if( !s3Outputlocation ){ throw new Error(`The Athena S3 Results Output location was not defined`); } @@ -54,7 +49,7 @@ export function connect(connection) { throw new Error(`The Athena Query Timeout was not defined`); } let con = { - region, accessKey, secretKey, dbName, sqlStatement, s3Outputlocation, queryTimeout + region, accessKey, secretKey, dbName, s3Outputlocation, queryTimeout }; return new Promise(function(resolve, reject) { diff --git a/test/backend/datastores.athena.spec.js b/test/backend/datastores.athena.spec.js index 53d83557d..0fe6f6b4a 100644 --- a/test/backend/datastores.athena.spec.js +++ b/test/backend/datastores.athena.spec.js @@ -25,7 +25,6 @@ describe('Athena Unit Tests:', function () { accessKey: 'XXXXXXXX', secretKey: 'XXXXXAAAA', dbName: 'PLOT.LY-TEST', - sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', queryTimeout: 10000 }; @@ -43,7 +42,6 @@ describe('Athena Unit Tests:', function () { accessKey: 'XXXXXXXX', secretKey: 'XXXXXAAAA', dbName: 'PLOT.LY-TEST', - sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', queryTimeout: 10000 }; @@ -65,7 +63,6 @@ describe('Athena Unit Tests:', function () { accessKey: 'XXXXXXXX', secretKey: 'XXXXXAAAA', dbName: 'PLOT.LY-TEST', - sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', queryTimeout: 10000 }; @@ -86,7 +83,6 @@ describe('Athena Unit Tests:', function () { region: 'us-east-1', secretKey: 'XXXXXAAAA', dbName: 'PLOT.LY-TEST', - sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', queryTimeout: 10000 }; @@ -108,7 +104,6 @@ describe('Athena Unit Tests:', function () { accessKey: '', secretKey: 'XXXXXAAAA', dbName: 'PLOT.LY-TEST', - sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', queryTimeout: 10000 }; @@ -129,7 +124,6 @@ describe('Athena Unit Tests:', function () { region: 'us-east-1', accessKey: 'XXXXXXXX', dbName: 'PLOT.LY-TEST', - sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', queryTimeout: 10000 }; @@ -151,7 +145,6 @@ describe('Athena Unit Tests:', function () { accessKey: 'AAAAAAA', secretKey: '', dbName: 'PLOT.LY-TEST', - sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', queryTimeout: 10000 }; @@ -172,7 +165,6 @@ describe('Athena Unit Tests:', function () { region: 'us-east-1', accessKey: 'XXXXXXXX', secretKey: 'XXXXXX', - sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', queryTimeout: 10000 }; @@ -194,7 +186,6 @@ describe('Athena Unit Tests:', function () { accessKey: 'AAAAAAA', secretKey: 'XXXXXX', dbName: '', - sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', queryTimeout: 10000 }; @@ -210,57 +201,12 @@ describe('Athena Unit Tests:', function () { } }); - it('connect fails missing sql statement', function( done ) { - const conn = { - region: 'us-east-1', - accessKey: 'XXXXXXXX', - secretKey: 'XXXXXX', - dbName: 'PLOT.LY-TEST', - s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', - queryTimeout: 10000 - }; - try{ - connect( conn ).then( connection =>{ - assert.isUnDefined( connection, 'Connection should not have been defined'); - done('Should have not have occured do to missing sql statement'); - }).catch( err =>{ - done(); - }) - }catch( error ){ - done(); - } - }); - - it('connect fails db name is empty', function( done ) { - const conn = { - region: 'us-east-1', - accessKey: 'AAAAAAA', - secretKey: 'XXXXXX', - dbName: 'PLOT.LY-TEST', - sqlStatement: '', - s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', - queryTimeout: 10000 - }; - try{ - connect( conn ).then( connection =>{ - assert.isUnDefined( connection, 'Connection should not have been defined'); - done('Should have not have occured do to missing sql statement'); - }).catch( err =>{ - done(); - }) - }catch( error ){ - done(); - } - }); - - it('connect fails missing S3 Output locaiont', function( done ) { const conn = { region: 'us-east-1', accessKey: 'XXXXXXXX', secretKey: 'XXXXXX', dbName: 'PLOT.LY-TEST', - sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', queryTimeout: 10000 }; try{ @@ -275,13 +221,12 @@ describe('Athena Unit Tests:', function () { } }); - it('connect fails db name is empty', function( done ) { + it('connect fails S3 Output location is empty', function( done ) { const conn = { region: 'us-east-1', accessKey: 'AAAAAAA', secretKey: 'XXXXXX', dbName: 'PLOT.LY-TEST', - sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', s3Outputlocation: '', queryTimeout: 10000 }; @@ -297,19 +242,18 @@ describe('Athena Unit Tests:', function () { } }); - it('connect fails missing S3 Output locaiont', function( done ) { + it('connect fails missing Query Timeout', function( done ) { const conn = { region: 'us-east-1', accessKey: 'XXXXXXXX', secretKey: 'XXXXXX', dbName: 'PLOT.LY-TEST', - sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', }; try{ connect( conn ).then( connection =>{ assert.isUnDefined( connection, 'Connection should not have been defined'); - done('Should have not have occured do to missing S3 Location'); + done('Should have not have occured do to missing Query Timeout'); }).catch( err =>{ done(); }) @@ -318,20 +262,19 @@ describe('Athena Unit Tests:', function () { } }); - it('connect fails db name is empty', function( done ) { + it('connect fails query timeout is less then zero', function( done ) { const conn = { region: 'us-east-1', accessKey: 'AAAAAAA', secretKey: 'XXXXXX', dbName: 'PLOT.LY-TEST', - sqlStatement: 'SELECT * FROM TEST_TABLE LIMIT 100', s3Outputlocation: 's3://aws-athena-query-results-11111111-us-east-1/', queryTimeout: -1 }; try{ connect( conn ).then( connection =>{ assert.isUnDefined( connection, 'Connection should not have been defined'); - done('Should have not have occured do to missing S3 Location'); + done('Should have not have occured do to missing Query Timeout'); }).catch( err =>{ done(); }) From 6621ab64ac26ffcf8a51372172efa3563a0b2461 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sun, 7 Jan 2018 15:36:38 -0500 Subject: [PATCH 40/45] Check in latest version --- app/constants/constants.js | 12 ------------ backend/persistent/datastores/Athena.js | 1 + .../persistent/datastores/drivers/AWSAthenaDriver.js | 8 ++++---- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/app/constants/constants.js b/app/constants/constants.js index 57b5be71e..988779ca4 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -182,11 +182,6 @@ export const CONNECTION_CONFIG = { } ], // TODO - password options for apache drill - /** - * accessKeyId: 'xxxx', - secretAccessKey: 'xxxx', - region: 'xxxx', - */ [DIALECTS.ATHENA]: [ { 'label': 'S3 Access Key','value': 'accessKey','type': 'password' @@ -210,13 +205,6 @@ export const CONNECTION_CONFIG = { }, { 'label': 'Database','value': 'database','type': 'text' - }, - { - 'label': 'Athena SQL Query', - 'value': 'query', - 'type': 'text', - 'description': ` - The SQL Query that will be executed against Athena.` } ] }; diff --git a/backend/persistent/datastores/Athena.js b/backend/persistent/datastores/Athena.js index 0ac400365..3126c815e 100644 --- a/backend/persistent/datastores/Athena.js +++ b/backend/persistent/datastores/Athena.js @@ -109,6 +109,7 @@ export function query(queryObject, connection){ * @param {string} connection.s3Outputlocation - Location will Athena will output resutls of query */ export function schemas(connection){ + console.log( 'Getting Athena Schemas'); let columnnames = ['Table', 'column_name', 'data_type']; let rows = []; diff --git a/backend/persistent/datastores/drivers/AWSAthenaDriver.js b/backend/persistent/datastores/drivers/AWSAthenaDriver.js index 8961621a7..1329fc5dc 100644 --- a/backend/persistent/datastores/drivers/AWSAthenaDriver.js +++ b/backend/persistent/datastores/drivers/AWSAthenaDriver.js @@ -12,20 +12,20 @@ const NUMBER_OF_RETRIES = 5; * @param {string} connection.region - AWS Region */ export function createAthenaClient( connection ){ - /*let connectionParams = { + let connectionParams = { accessKeyId: connection.accessKey, secretAccessKey: connection.secretAccessKey, region: connection.region, maxRetries: 5 - }*/ + } - let connectionParams = { + /*let connectionParams = { apiVersion: '2017-05-18', accessKeyId: credentials.accessKey, secretAccessKey: credentials.secretAccessKey, region: credentials.region, maxRetries: 5 - } + }*/ let athenaClient = new AWS.Athena( connectionParams ); return athenaClient; From e7aaa8ea84431b3c299d102a575ce7ee2672d907 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sat, 13 Jan 2018 14:16:38 -0500 Subject: [PATCH 41/45] Check in latest changes based on @n-riesco comments --- NEW_CONNECTION.md | 60 +++++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/NEW_CONNECTION.md b/NEW_CONNECTION.md index 4e37107fc..cd518d4c4 100644 --- a/NEW_CONNECTION.md +++ b/NEW_CONNECTION.md @@ -1,6 +1,5 @@ # Add New Data Source -The following are the instructions for adding a new data source ## The following are the files that need to be updated @@ -15,31 +14,54 @@ The following are the instructions for adding a new data source 1. app/images 2. backend/persistent/datastores/NameOfDatastoreToConnect.js -# Detailed Implementation Instructions +# Add New Data Source ## Images -It is recommended to add the images as png and a good idea to include a regular and small icon. -You can add the new icon in the directory app/images. It recommended to use png format -and name the file in lower case with dashes (ex: mysql-logo.png ) +It is recommended to add the images as png and a good idea to include. +Please verify that the icon looks good with the following CSS specs: +```css + max-width: 100px; + max-height: 70px; +``` ## Updating the constants.js file The following are the instructions for adding the Constants File: 1. Add an entry in DIALECTS 2. Add an entry in SQL_DIALECTS_USING_EDITOR 3. Add an entry under CONNECTION_CONFIG -4. Add the location of the logo in +4. Add the location logo into the LOGOS Constants 5. Add the preview query in the PREVIEW_QUERY section -6. Add an entry in the SAMPLE_DBS +6. Add sample credentials for your connector in the SAMPLE_DBS ## Updating the Tab.react.js The following are the instructions for adding the Tab React JS file -1. Add an entry for the connection label +1. Customise the label for your connector in the render function: + +```javascript + let label; + if (dialect === DIALECTS.S3) { + label = `S3 - (${connectionObject.bucket})`; + } else if (dialect === DIALECTS.NEW_CONNECTION) { + //Specify the label + label = `NEW CONNECTION (${connectionObject.host})`; + } + .... +``` ## Updating the Settings.react.js Update the fetchData function to make the calls to appropriate sessions.js backend. -Currently there is Dialect types for SQL, ElasticSearch, S3 and Apache Drill. If -your new data source is an SQL Type you will need to add the new Dialect in the -fetchData section for SQL types. +Add your new connectino to the following list: + +```javascript + if (contains(connectionObject.dialect, [ + DIALECTS.APACHE_IMPALA, + DIALECTS.APACHE_SPARK, + DIALECTS.IBM_DB2, + DIALECTS.MYSQL, DIALECTS.MARIADB, DIALECTS.POSTGRES, + DIALECTS.REDSHIFT, DIALECTS.MSSQL, DIALECTS.SQLITE, + DIALECTS.DATA_WORLD + ])) { +``` ## New Datastore file This file should be added in the following location @@ -50,18 +72,16 @@ backend/persistent/datastores /** * The following function will connect to the datasource * @param {Object} connection + * @returns {Promise} that resolves when the connection succeeds */ function connect(connection); /** - * The following function will return a list of schemas. The return - * array should return a list of array with each array containing the - * following information: - * -row[0] schema name - * -row[1] column name - * -row[2] data type + * The following function will return a list of schemas. * @param {Object} connection - * @return {Array} + * @returns {Promise} that resolves to { + * columnnames: [ 'tablename', 'column_name','data_type' ], + * rows: [[tablename1, columnname1, datatype1], ...]] } */ function schemas(connection); @@ -69,7 +89,7 @@ function schemas(connection); * The following function will return a list of tables. * The return result should an array of strings * @param connection {object} - * @return {Array} + * @returns {Promise} that resolves to an array of table names */ function tables(connection); @@ -80,7 +100,7 @@ function tables(connection); * The second is a two dimensional array which represents the rows to be displayed * @param {string|object} queryObject * @param {object} connection - * @return {object} -- tuple {columnames, rows} + * @returns {Promise} that resolves to { columnnames, rows } */ function query(queryObject, connection); From 77b7710aaf123b64f74ff951fcacc7cbf1840cdd Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Sat, 13 Jan 2018 14:50:26 -0500 Subject: [PATCH 42/45] Check in updated new Connection --- NEW_CONNECTION.md | 73 ++++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/NEW_CONNECTION.md b/NEW_CONNECTION.md index c08a18153..463479079 100644 --- a/NEW_CONNECTION.md +++ b/NEW_CONNECTION.md @@ -1,39 +1,50 @@ # Add New Data Source -The following are the instructions for adding a new data source - -## The following are the files that need to be updated - -1. app/constants/constants.js -2. app/actions/sessions.js -3. app/components/Settings/Tabs/Tab.react.js -4. app/components/Settings/Settings.react.js -5. backend/persistent/datastores/Datastores.js - -## The following are the new files that need to be added - -1. app/images -2. backend/persistent/datastores/NameOfDatastoreToConnect.js - -# Detailed Implementation Instructions - ## Images -It is recommended to add the images as png and a good idea to include a regular and small icon. -You can add the new icon in the directory app/images. It recommended to use png format -and name the file in lower case with dashes (ex: mysql-logo.png ) +It is recommended to add the images as png and a good idea to include. +Please verify that the icon looks good with the following CSS specs: +```css + max-width: 100px; + max-height: 70px; +``` ## Updating the constants.js file The following are the instructions for adding the Constants File: 1. Add an entry in DIALECTS 2. Add an entry in SQL_DIALECTS_USING_EDITOR 3. Add an entry under CONNECTION_CONFIG -4. Add the location of the logo in +4. Add the location logo into the LOGOS Constants 5. Add the preview query in the PREVIEW_QUERY section -6. Add an entry in the SAMPLE_DBS +6. Add sample credentials for your connector in the SAMPLE_DBS ## Updating the Tab.react.js The following are the instructions for adding the Tab React JS file -1. Add an entry for the connection label +1. Customise the label for your connector in the render function: + +```javascript + let label; + if (dialect === DIALECTS.S3) { + label = `S3 - (${connectionObject.bucket})`; + } else if (dialect === DIALECTS.NEW_CONNECTION) { + //Specify the label + label = `NEW CONNECTION (${connectionObject.host})`; + } + .... +``` + +## Updating the Settings.react.js +Update the fetchData function to make the calls to appropriate sessions.js backend. +Add your new connectino to the following list: + +```javascript + if (contains(connectionObject.dialect, [ + DIALECTS.APACHE_IMPALA, + DIALECTS.APACHE_SPARK, + DIALECTS.IBM_DB2, + DIALECTS.MYSQL, DIALECTS.MARIADB, DIALECTS.POSTGRES, + DIALECTS.REDSHIFT, DIALECTS.MSSQL, DIALECTS.SQLITE + ])) { +``` ## New Datastore file This file should be added in the following location @@ -44,18 +55,16 @@ backend/persistent/datastores /** * The following function will connect to the datasource * @param {Object} connection + * @returns {Promise} that resolves when the connection succeeds */ function connect(connection); /** - * The following function will return a list of schemas. The return - * array should return a list of array with each array containing the - * following information: - * -row[0] schema name - * -row[1] column name - * -row[2] data type + * The following function will return a list of schemas. * @param {Object} connection - * @return {Array} + * @returns {Promise} that resolves to { + * columnnames: [ 'tablename', 'column_name','data_type' ], + * rows: [[tablename1, columnname1, datatype1], ...]] } */ function schemas(connection); @@ -63,7 +72,7 @@ function schemas(connection); * The following function will return a list of tables. * The return result should an array of strings * @param connection {object} - * @return {Array} + * @returns {Promise} that resolves to an array of table names */ function tables(connection); @@ -74,7 +83,7 @@ function tables(connection); * The second is a two dimensional array which represents the rows to be displayed * @param {string|object} queryObject * @param {object} connection - * @return {object} -- tuple {columnames, rows} + * @returns {Promise} that resolves to { columnnames, rows } */ function query(queryObject, connection); From 812dde5a6c865f889433a280e25bd6da9d93dcbc Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Mon, 15 Jan 2018 19:52:21 -0500 Subject: [PATCH 43/45] Checking in updates for environment and troubleshooting --- CONTRIBUTING.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 438dbd4ee..bcbe7499f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,9 @@ Note that this section targets contributors and those who wish to set up and run If you're interested in using the distributed App, [download the latest release.](https://github.com/plotly/falcon-sql-client/releases) +## Prerequisites +It is recommended to use node v6.12 with the latest electron-builder + ## Install Start by cloning the repo via git: @@ -125,3 +128,14 @@ $ yarn run test-e2e - Mac builds are created in TravisCI tests and automatically hosted on [amazon](https://s3.console.aws.amazon.com/s3/buckets/falcon-travis-artifacts/plotly/falcon-sql-client/?region=us-east-1&tab=overview). Select the latest build (largest number *note:* the folders are not necessarily sequential) -> release.zip Builds are uploaded to https://github.com/plotly/falcon-sql-client/releases. + +## Troubleshooting +The Falcon Configuration information is installed in the users home directory. +For example Unix and Mac (~/.plotly/connector). If you have tried the install +process and the app is still not running, this may be related to some corrupted +configuration files. You can try removing the existing configuration files and then +restarting the build process + +```bash +rm -rf ~/.plotly/connector/ +``` \ No newline at end of file From 0b79f128ec4771e02d41d6b6cb9c04b96e80293c Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Tue, 16 Jan 2018 19:12:02 -0500 Subject: [PATCH 44/45] Check in fixes based on @n-riesco --- NEW_CONNECTION.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/NEW_CONNECTION.md b/NEW_CONNECTION.md index 463479079..deeb93706 100644 --- a/NEW_CONNECTION.md +++ b/NEW_CONNECTION.md @@ -9,7 +9,7 @@ Please verify that the icon looks good with the following CSS specs: ``` ## Updating the constants.js file -The following are the instructions for adding the Constants File: +The following are the instructions for updating the Constants File: 1. Add an entry in DIALECTS 2. Add an entry in SQL_DIALECTS_USING_EDITOR 3. Add an entry under CONNECTION_CONFIG @@ -18,7 +18,7 @@ The following are the instructions for adding the Constants File: 6. Add sample credentials for your connector in the SAMPLE_DBS ## Updating the Tab.react.js -The following are the instructions for adding the Tab React JS file +The following are the instructions for updating the Tab React JS file 1. Customise the label for your connector in the render function: ```javascript @@ -34,7 +34,7 @@ The following are the instructions for adding the Tab React JS file ## Updating the Settings.react.js Update the fetchData function to make the calls to appropriate sessions.js backend. -Add your new connectino to the following list: +Add your new connection to the following list: ```javascript if (contains(connectionObject.dialect, [ From 4b74e8bc12139d69104c760e192935855753bc09 Mon Sep 17 00:00:00 2001 From: Shannon Lal Date: Tue, 16 Jan 2018 19:17:08 -0500 Subject: [PATCH 45/45] Check in fixes based on @n-riesco code review --- CONTRIBUTING.md | 4 ++-- NEW_CONNECTION.md | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bcbe7499f..5c4697dd3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -130,8 +130,8 @@ $ yarn run test-e2e Builds are uploaded to https://github.com/plotly/falcon-sql-client/releases. ## Troubleshooting -The Falcon Configuration information is installed in the users home directory. -For example Unix and Mac (~/.plotly/connector). If you have tried the install +The Falcon Configuration information is installed in the user's home directory. +For example Unix and Mac (~/.plotly/connector) and for Windows (%userprofile%\.plotly\connector\). If you have tried the install process and the app is still not running, this may be related to some corrupted configuration files. You can try removing the existing configuration files and then restarting the build process diff --git a/NEW_CONNECTION.md b/NEW_CONNECTION.md index deeb93706..1562599b6 100644 --- a/NEW_CONNECTION.md +++ b/NEW_CONNECTION.md @@ -1,7 +1,8 @@ # Add New Data Source ## Images -It is recommended to add the images as png and a good idea to include. +The connector must provide an icon to be displayed on the selection tab and +it is recommended to add the images as png format. Please verify that the icon looks good with the following CSS specs: ```css max-width: 100px;