From 8eb630a66b9655bdd25e77cfcbd4961e56a302bd Mon Sep 17 00:00:00 2001 From: i344511 Date: Tue, 9 Jun 2020 22:58:02 +0800 Subject: [PATCH] Add python3 support for HiveOperator --- HiveOperator/README.md | 26 +++-- .../{ => py2}/HiveOperator-1.0.1.tar.gz | Bin .../solution/{ => py2}/HiveOperator-1.0.tgz | Bin .../solution/py3/HiveOperator-1.1.0.tar.gz | Bin 0 -> 10115 bytes .../examples/HiveOperator/Dockerfile | 0 .../examples/HiveOperator/Tags.json | 0 .../vflow/graphs/HiveOperator_test/graph.json | 0 .../HiveOperator/Apache_Hive_logo.svg | 0 .../examples/HiveOperator/configSchema.json | 0 .../examples/HiveOperator/operator.json | 0 .../operators/examples/HiveOperator/script.py | 0 HiveOperator/src/{ => py2}/vsolution.json | 0 .../examples/HiveOperator/Dockerfile | 23 +++++ .../examples/HiveOperator/Tags.json | 5 + .../vflow/graphs/HiveOperator_test/graph.json | 64 ++++++++++++ .../HiveOperator/Apache_Hive_logo.svg | 51 +++++++++ .../examples/HiveOperator/configSchema.json | 97 ++++++++++++++++++ .../examples/HiveOperator/operator.json | 37 +++++++ .../operators/examples/HiveOperator/script.py | 59 +++++++++++ HiveOperator/src/py3/vsolution.json | 1 + 20 files changed, 354 insertions(+), 9 deletions(-) rename HiveOperator/solution/{ => py2}/HiveOperator-1.0.1.tar.gz (100%) rename HiveOperator/solution/{ => py2}/HiveOperator-1.0.tgz (100%) create mode 100644 HiveOperator/solution/py3/HiveOperator-1.1.0.tar.gz rename HiveOperator/src/{ => py2}/vrep/vflow/dockerfiles/examples/HiveOperator/Dockerfile (100%) rename HiveOperator/src/{ => py2}/vrep/vflow/dockerfiles/examples/HiveOperator/Tags.json (100%) rename HiveOperator/src/{ => py2}/vrep/vflow/graphs/HiveOperator_test/graph.json (100%) rename HiveOperator/src/{ => py2}/vrep/vflow/subengines/com/sap/python27/operators/examples/HiveOperator/Apache_Hive_logo.svg (100%) rename HiveOperator/src/{ => py2}/vrep/vflow/subengines/com/sap/python27/operators/examples/HiveOperator/configSchema.json (100%) rename HiveOperator/src/{ => py2}/vrep/vflow/subengines/com/sap/python27/operators/examples/HiveOperator/operator.json (100%) rename HiveOperator/src/{ => py2}/vrep/vflow/subengines/com/sap/python27/operators/examples/HiveOperator/script.py (100%) rename HiveOperator/src/{ => py2}/vsolution.json (100%) mode change 100755 => 100644 create mode 100644 HiveOperator/src/py3/vrep/vflow/dockerfiles/examples/HiveOperator/Dockerfile create mode 100644 HiveOperator/src/py3/vrep/vflow/dockerfiles/examples/HiveOperator/Tags.json create mode 100644 HiveOperator/src/py3/vrep/vflow/graphs/HiveOperator_test/graph.json create mode 100644 HiveOperator/src/py3/vrep/vflow/subengines/com/sap/python36/operators/examples/HiveOperator/Apache_Hive_logo.svg create mode 100644 HiveOperator/src/py3/vrep/vflow/subengines/com/sap/python36/operators/examples/HiveOperator/configSchema.json create mode 100644 HiveOperator/src/py3/vrep/vflow/subengines/com/sap/python36/operators/examples/HiveOperator/operator.json create mode 100644 HiveOperator/src/py3/vrep/vflow/subengines/com/sap/python36/operators/examples/HiveOperator/script.py create mode 100644 HiveOperator/src/py3/vsolution.json diff --git a/HiveOperator/README.md b/HiveOperator/README.md index 0f22141..d2d8d83 100644 --- a/HiveOperator/README.md +++ b/HiveOperator/README.md @@ -2,7 +2,12 @@ HiveOperator [(Source code)](https://github.com/SAP/datahub-integration-examples ------------ This operator provides functionality to query a Hive Metastore server using a HiveQL string and returns a response in the format of a delimited string. -The operator runs on a custom Docker images that extends the SAP-deliver docker image `com.sap.python2.7` and uses the Kerberos client binary `krb5-user` as well as `libsasl2` for Ubuntu. The PyHive python module is developed and maintained by Dropbox: https://github.com/dropbox/PyHive +Two implementations are provided for this operator based on different versions of python. +- For python2, it runs on a custom Docker images that extends the SAP-deliver docker image `com.sap.python2.7`. +- For python3, it runs on a custom Docker images that extends the python3.6 docker image. +> Since SAP Data Intelligence version3.0, it only supports python3. + +Both use the Kerberos client binary `krb5-user` as well as `libsasl2` for Ubuntu. The PyHive python module is developed and maintained by Dropbox: https://github.com/dropbox/PyHive ![alt text](./graph.jpg "Graph") @@ -25,7 +30,7 @@ Before you start using the example, please make sure that: - Install Kerberos client libraries **2. Custom operator 'HiveOperator'** - - Derived from Pythin20Operator + - Derived from Python20Operator - Uses image tags `python27:""` and `pyhive:pip2` - **input port `inSql` of type string:** expects a single HiveQL-compliant string without a semicolon - **output port `output` of type string:** outputs the response from the Hive Metastore server, columns are delimited by a comma (default) but can be overriden using the `delimiter` configuration parameter (See description below) @@ -33,24 +38,27 @@ Before you start using the example, please make sure that: **3. Sample graph HiveOperator_test** - Provides an interactive terminal to query a Hive Metastore server and display the results. Note, the HiveOperator can only process one HiveQL statement at a time. +> Python3 implementation uses the same steps only diff in the base image and operator tags. + ## How to run - - Import [solution/HiveOperator-1.0.tgz](solution/HiveOperator-1.0.tgz) via `SAP Data Hub System Management` -> `Files` -> `Import Solution` + - For python2, import [solution/py2/HiveOperator-1.0.1.tar.gz](solution/py2/HiveOperator-1.0.1.tar.gz) via `SAP Data Hub System Management` -> `Files` -> `Import Solution` + - For python3, import [solution/py3/HiveOperator-1.1.0.tar.gz](solution/py3/HiveOperator-1.1.0.tar.gz) via `SAP Data Hub System Management` -> `Files` -> `Import Solution` - Run the `Graph` -> `examples.HiveOperator_test` **Operator configuration parameters** - database: Specify which database in Hive metastore to connect to - delimiter: Used to separate columns in HiveOperator output e.g. 1.34;Hello;World; + database: Specify which database in Hive metastore to connect to + delimiter: Used to separate columns in HiveOperator output e.g. 1.34;Hello;World; hive_hostname: Hostname or IP address to Hive Metastore server - hive_port: Port used by Hive Metastore server - http_mode: If hive.server2.transport.mode is set to http, set this parameter to true + hive_port: Port used by Hive Metastore server + http_mode: If hive.server2.transport.mode is set to http, set this parameter to true kerberos_enabled: If Hive cluster is kerberized set to true and read additional notes below kerberos_keytab_filename: The file name of the uploaded keytab file (case sensetive) kerberos_principal: Kerberos principal used with uploaded keytab file kerberos_realm: Kerberos realm used with principal and keytab file - username: Username for plain authentication - password: Password for plain authentication + username: Username for plain authentication + password: Password for plain authentication **Kerberos configuration** (Optional) Upload .keytab and krb5.conf file via the HiveOperator designer. These will be copied into the docker container at runtime. Remember to specify the kerberos realm and principal name in the operator's configuration section when designing your graph. diff --git a/HiveOperator/solution/HiveOperator-1.0.1.tar.gz b/HiveOperator/solution/py2/HiveOperator-1.0.1.tar.gz similarity index 100% rename from HiveOperator/solution/HiveOperator-1.0.1.tar.gz rename to HiveOperator/solution/py2/HiveOperator-1.0.1.tar.gz diff --git a/HiveOperator/solution/HiveOperator-1.0.tgz b/HiveOperator/solution/py2/HiveOperator-1.0.tgz similarity index 100% rename from HiveOperator/solution/HiveOperator-1.0.tgz rename to HiveOperator/solution/py2/HiveOperator-1.0.tgz diff --git a/HiveOperator/solution/py3/HiveOperator-1.1.0.tar.gz b/HiveOperator/solution/py3/HiveOperator-1.1.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..efa92729d0b1dc4fc2564b0bc413e4d064403367 GIT binary patch literal 10115 zcmV-}Cw$l+iwFP!000001MFOFZyV>8^;ZkXe|W$E!7h*)J?}57)ozNoNU*5`xXJE^ zZV}MK5ykb0!;+M2Hwdsleb2c=%Cb$_lI$>xtp&J@?#m@38%5cF}&nXqV%4 zzpPAcYJK?NQy&^f3gN;3@Z^g}t=HP%KXQF@JTNXs7oAPUKG4QQogS$0$yc)DIA5*D zOQjxs>%0%f>|@IROUF0w`u}3tE3`#5gy|MunhY;h+b+`|7+ z+xS3Rt=&EU@8!6I|F6DsBQV^W|E=}j-1~nY$H)5r>&qvupMwsL-e2c`1FvuWpQ3T_ zzxNLMU+=>J|K$Gv)6?(&tQN1U>&kDvU9`ug!7zy0dT zPk8jn509UGeLU~yv-uhTXx4y&IyiXx;}bPrtgC6eR_BY!c-^WmzEJ;lz>8`&hlRUN zRj)SlxN0$5ynaC5+VRk&y}16N=gViII$yQR>mTW6&sO7AXWu{2 zF{{pI>&;7N&nL5`I(*)~T92Qt4h}w7-)>$XR>~okm?B+XD{$)nuY1^ptFFJl~#9m;LzyZ>Ytxy=doa)h?I)QsI@RpRdQW zIi5ZpuhjB<{?+P9KY!ZyYk7TkHlDWX#SHV|p5i08C5w?U$C$C2DpQi(OoeT8@~bzh z^UdG0;hBRU|MK{$Tsq=E2mkr~(|`Z^+o$UAv_ESH?hxdB^ZlP6A78onTciE{c>DNR z{{LY-UDf}+>gVs7;{*JUld$1`3kIhC^H2cy{J)Rm=g$rvjs}_<9jOudY4qi12MWh% z!^Dv{E-!9Ki`Rs_4>{OxI3Geg&9*!aZFMD`* zjS>cF{pVl4Ibde+aG>j?ohV1QS?+`QJbtJUxpE<7BeC_J0Z$DP?;c{pnN zvqe8g#llf=Qlq@8&)2iA-aagY;mNahJx1*~zI^xLs2e|PJHGhCo6kNRy~I`POBVfaKE)MdFE5?XIPc`ud@;tY?bX%! zvvxj3wZ%REb-0L?qy8EVCx6>MiL&fuHWhoy75P_8ixt3_{U)BSGu2jZ*H?q@CZ;-l z_xZtG$^LrQYG1C~`D*s8ql#QF&)fH#0sjmrlM+;HDTd?Y^R^+xE)D$iwOd~Fv-z6O zjZBGe?=8K(A!Xf%9~!*)%f`&BW%I^#OYuy9zFwTKFHO6Ng01Oa`}K{X*3JY+`j%d#@zq@Z!gE){l8UFAC?9;_rKLq ze*fW&yU+i7IX;E|KlIC>Tlqi8_x}{`^Z#CsPwM|`ei?Lo{!f6yz5n-d)Q9!qzkS!w zr{B%y&wr&kw*B9&uiyHsO-OHG0cc4nK2YENO6z&!*!usU^!WAS|EtBgIc2JN;`{V6nAO3&;_aEnMB>#JBV$y;9?~G0Q{`=29jz@oZdDf|mcDb7M^Wzcx zKT_?y=_j-K^mz2+4}YriNWnz&$++w1?eS>dkN)ucgGUHvg*WG`qvhn}cyzj6FOCim zU%Yrxzi@TGoF1A|%AvN0)>e3;TD_XD$1kh->hsa?-mZ5n;BbhMM{4-$a6Bp+>yhsb zSX=8xS66B_BONu(OIc&T3tevk{hZyt_ zbIjTof9zi#kF?SXNY#qV`2B%;G=(Xa{d1V%^OH&2S~J>Qs5Wa1wXr#l)OtCduTJ{q z+3`sJYd2oE|5DWfSkEfPI{)iPoy@wfTAp_pjX&w@Clk)09`S>dnjDYLLP@oAD!Nps za$$d@O8<)8!o6)x{(sBY+SM_t zU||;9R9L;?I*bXG<%c_qU1NTWvsgP*85p6^ib>(W#S$ubAXr5|*jO128(DzY*A7cg z_){G0X`^W>XX})V^0qeKR=(7Q9sqPpsa%HdGseW#z(+KO39&9ZA_@Q;j2VbZ!_@_4 zu$*j^9Xy3S6wK-a76HEnK8FWFtl&f!eI;LPaSJvKrAkNx@2DTg-?2I ztnvUC<~9yC!63v`g4w;Xjvqr|?Fu93=u|efi*y4lip|08IP(sR^c|xESit_VywFrw zu8UYFR+*DH34wr+(E(=|!DU>mBcPXS2cK$P0}#$vz=ZY<0#%2oJbj43hhvjCq2wsO zguz2Xq~s2s&pA|xDr-Vx#I7o1Q80ojV1Zl(*xbULuo5N@hOVKt32Ps>(7xyCD81dt3cctO4=%CDaNV<$<VSHHzH`h zc$MP#`55-h#9IRf(ZvFaMof-H2j~}N;p>d6btv8_SPk(DZZ47TBp9)II36#;(>Z5w zafm)B#;$=aVO|GN;uTYx051jtmi;jN$sE(b+!##41GvKr#VgDc;Yf?X)|u!7;FN`Z z0XsO-21E<^le_^pSyCMs0-mru4Lr42PKBs<0T6}DJWyyPvaKdD;RR3t0fvb}9XvM_ zy~Vq@6!?TSBv0JZLP`#FkI0LR2e>_F?9wEYF>rAF9FK@IxK?6e2!M=;M9kTd(Xcj# zt!rSggn%b?Sdt=M^Px&usV;Gzm$@h; z!YX*x&mIu z)|2(jQW`#y$UufC781~itrs6Vpr&ZSax%LH25D&oBhr_~3gI{GlfteBK7el|al`0x zOqfJIHy=e8q-bGNt^`S#m=1=_D9(UQy@QBgx+;#oNez~enJPSB?^Ga05LDnYm@?BO zu>uwm@puiii~s~J2H4wDnD~mZjrqwY%#TLmx*C?uaa_zh_J>?z=@w$Ak;gJohumc) zPcbDRmQrBWvG60ag=BL65y&7Cms<5*)cP(I-$`1Wo_|op@J4I)QBzBGnrKRpCgP zf>iVuIG@Z#5k#_%R7L>6yQm0YND33|pdg$m$w2=RXUw@+8`+tv346{|7!@=!u*l2; zc#~v2!=q5RRkRRb1T-a0B0)Se83gDPFbJOXA^{C04H6TW91#G^Nif6GP?Ma8#w0fY zXRspli%FVN02N0f?}I&xQCm@?jhw;NU=L^&6J$Nwkr-J9i4%(1#U)u72Of*iyBvaA z*$F38m5>rBT8wCEFyaOWl&J74LXo-&+kyFFM2t92gcpDWQXYz}3R)93X$(B)Mba{@ zgD+4B0jwE`7!JiFh$ReF=*7%zlqmE;q8`@@0xLU_;>pQ&@{T+fOC%@_hBI+RI~F8s zycD3=xmD*F+Bssgb`y^93fD~m%F;kXy`?A-j-*(usJ=`ojyN}iSioSUP^<@AOOV@^ zXtABA4GIY?Ilm*zg7rkA!(9=1%q8eJiX}>sp*AV38d=B%)=dZ&L=K~mx-vp(!%tWy z@FeNvs+O;&-U& z4qOzassi>AzzwG_qUV!%flhFue2rk53YzZFA|4Xef+a*kAsrDOmSajNfFs~Cb^stRm82jlP$+MPVJza-tXbir9)dBL zP_!+Q`Z8q>;n9wCY)7qz1@^85Qex2hxK>L-HhK7bKmb zkljcZ#o`u83I>GWpk#siR-QV7LWt@lQnJELTz_U6Z{tw309c6g;Do9Q6cbn`Ysqcw z6ChSWYOKwHb(+RebxYo2rsTc!NHPR3adgAa#V8AzF1-rYXDNT5966-k`uo~ zgflBVFa0Pc9rhOx)e^DfDv)Us;|1x=X;6)nkckC2^F1iII7uUHl3)ewyury-h%k00 zhyua@uQ};0!4j-M*{$e zkwq^Xmf&C$l42mBNrO?k5@uJd4a!r>69gMQKz_z7aEoJvQ!*KVXgFcp!$`f&cw(6@ zEg-;0OfK@?QdM!k*czc#U`WG9rG#`w&M5i3+eSx7ySP#ZX(K_;WwU~foXB&qGn+v$ zIGv^=8;h7L3EnhekLBbffo;VQMIuM+-w7j9crn+J!8r=^pHR1fkRZ1b@Km6ZlED~S zX@NUJ6}WW(4}DxHwhc8St4t(9w!J98z;f6ZDX(* zJ)~b!@EK~|Z$qCvI zAzWLjot)gIOpAUHUQl|PDijqd9E&YkFD6XIGbHlRR<((3u@XFmHY7rjIvi<%wYClH z1SM%MFA|Pu*?2>_${G0U zVk1hj?2@x=%K(3r>kZ@EP~0FNAdK1F&CKiGvx5cp)e_>#5zB%7FQh#!im9fDsR}lc z{`w5>8EMl6CkhwZ0+j$2LzH%m7u}22wP-G7pQw1a;%n)Asa@*R!JIO z&>pC{+3kd|x6;zgdeeBN=ygHr`a-S6!V}C%Md*bynJS&IHEhU!oAiPCF^A|0(#c@K z267cS!c@S8Gh0+FFj?-fKVVpTIWi7ne`v#N!kKcQu(m?7e|L+37rO;cvQNV)q8Jn` z#IHY&8OckMLB%TUMj{j5?hgZqYR~w(ej6lWNB?jX(5pE0X5GrS0Gb5m-Na0O_Y|W9+_rIT@$dXAFibO*#*z$Ru~@*V zI!%*yGSk-=Pm_M?%V;19vQbkMOHyg#jQ{s{e`f)Tq$u0t#%f-l@IfL$V0W>5_wKpp zTmYXfUpDi0w-!6GyVxoFe0n{3bv$2OzgV9w);=kgv|ER|=b4%~juQ_SGKjM&z%B%Mz_yx2C5PD*4a?YK z^tD*D%)l3j6Ss+9^2BDus|Bip1=wa^=xDIe1uoeFT$WN>MDP}n&Ss?M%K>s(F3!0y zQvouum^d$^ETKi=g9(Ak6w)?Js|cEisj^CpEujEJKtZ5WRVob_kr9`QWrdEk#@l*Q zpfm(ULZQksP2d_67yV!>M7D< zRiO_=R4Y;tqbkIq1jtAQ4`Mo5bqq6u#3}-HLRuPWSjYOo%BV9WporcbYh%hfOdt#g z3jD5^=XADuC{vUZsfx6$<=!#@fh;KIteDgy)r8oBBlh!kF4oUN4)Y8dHOZG1psqeL z5N~aL;JhfSiovqgD~ECL5RVy7TC{!z1T9&L7^z_X42|6pd9g48G|cJPd3BJJs>Ph< z4n@V;_`*D(LzIfV>0FP|rIG!utmne*WlK}gk}XyYAF9^Xz=%@OwUQD% ze$t#KJkDef<;iz;BYHo za!!Lbzb%g|FS869JY^hI3~X*AJ3*?zNs74igm|e!!y0}o2lha0rNBN~!CYWbV2afl zzL?2d3Br4HPeTouiK}fsr9!Z%e@H!I6N$ZH!-@)|pNn~Sph3;cQkPfY{c*M><*`24 zB^Lo~ml`l(XCC<%FY)<=&_hJKID;$`BVx!7nu(<@ml&4dBBaKo5=U)$7R4f&-{EWu zxfg@uhOv){m_x2`!P7ASI;|dcAp$K#Qm-bxz&S#5$wHYZHE@jA;DCOu`ymAg7VNkZ z{TG01V*_ncnk`UXOWX;hy^5foI&~2_E%7`(w8SGt}zo!2^|LpuY(c~tUTsk zT7*J08DTAQsh9F3bwxP|^qW5xrmLlo1sGGh+)cPL3lRZzHh&=RJ)`Vk7oijk+7J{m z3sG*6tw}lrd{srIRFqouN0lOeTSXO2O_9cETw$&u1=0AZG0aJujOaC`@?QuHfki>u zkl8h6ZA(Rkl;))P83ZmOHAWYek^8QOOF}sab)X`rPZ)JLNlU}Wy%O}pqya7k%T3X7^_TB zS^`p2jTQ6NA~r`q}JR`7-)$a`+!3M@5BpIPoa3;eO&noyU{Ul2l(A*P z1-PPu@Go~N2QCGrnfSR>cL)vS(uGPvBD86Mxst4ucwpW=4-@eSB{E1I%vA7;Ub=@@ zzoC#81V_b*G!L>JGi7$wLPuU$&b}x@DPq7u7)!=Mt6i#-C=#^VVTu+W_!n#BP$E(fWQ`RT6&D*NV05T9#M{*&X#5lFY7SYOuTVf zlGNpGaw53c443^E418e|5Od6`Se<5baJjN4bEOE_`U2CI#yRRUsgbr&kW3KpYZio} zF_PtgVq(i2U1Xt?rg#u*UKb%I$N4DJFe3=6>=&xeB^p8J2H|^W9qPqd%L|s~0yu;) z9$nfggQO^=auF31y}I7nJAwdDauZWh!DUGbjdh0wo&_cw8E5OqkZNXepl6Lii)MjC zIRXTeGh(AuC`lk7wWU5rAk+#wj^nfWRudw)6k?R1vld+#m;SU6m+vKr6J9g)C8-%H z8sBjM_{hg)OgyAM0gkMMnUqt`C9b5+}WZiLjN_Jkm^_|*`*gV?Yi+P zPMSi4@SfAZHcaVkK^Vk;Ku*Xs*QGaT7kK+-B0!)IE})Ewo@pl_}%{L??!xsnD6nae&K7#36dxw+rO zqZ-JkDZABdvsP?n>wQwfnLCkh=JsmzyNq1GIyx>FM9_Oug9<2|?Qcsi8Vmu%Qs)L+ z2C+xOWD|6I;qW|litVc+)4U8K(ky;YyuxVd42X6SNP{sHQm_;B%EImCfCvN-`yw3_ zVtSU%-Wt{(_>q$u2XMJ0L|X$($kH2TV)y&IM1?F@Zgk&95T@YAyrCH$P|;3*@qBLZc}{{wewVgw4I&c zb0t`$rQvtFgv=wQFR`7tWoHT}=sei%-v zg((D)&*np7&{c^ut>kSDOO=J^9a&w;;5>MLYeb)TF? z004Nvdy7MgK-mdb<*5|6I-)D3s8mt4jDQQN#LGPeX(|Tca>ZA|4;hzTgg7N~*eiSm zp{tpkY?fp;`-OFt07hnI?a~v*6174|qE?0+S>>co#Z+b==@zZ?p74QOG$jR)E<*;c z4$g#)oOD+?+=%_MI7Q9umHrgebO?Wnq!)r%pNcf1jIf-Q0Oh7+363@AudfOu3p4Hl zK&ON%b$qFKHJ5%6Vv>*NZy{mJ$JCp^>7KN6jnFwWlwOh&t8vj@1jb%ziayzDx~E)Y|2^s`x|Zy`|ar8eZSm)+V;!1 z|GT(*|F?U*|N9}fk9YrXeDdQrlb7vP^S)1tecT>;dx-IW`~Bakq2^-!{NJ4C|C8+U zc>ni1vtWEbYRQCRxtgN!Lu1$;*aU z1>G%drz`fhmglvey}VwoX3f>UVIEqn7JM}v_aPOT0iTgS@wQ67{r0(XERuHCoKM?} z`zkuWzMi(tY=2GtGiu*@CC@qLp2BB0SLfE5d&}mbpgGZ-R;n$`_K8lncl3Q5Ptt*BnIs(`jqp=d(V4wXYY6K z;S1LlEZSyz^?=U$|95Z4cd4E6Ua5PW)nmW^tNpG|eeaJK?LTh1g`C}O4{RWTdl%fj z3*_!vw$FpV&Et+@!~e4$r2c<&`(^My=^g&343GTJLu?-}{vUpSy)7w5A z-iCAwkA5}O{xeW$+Zc@ZSGAYT&2$^0ZSX(i&+dnQ_Uq+8E@mg}r9u0#V9z_fJ;1*A zkh_7v`1|Wca}xLO1(BO!v<}{T)DX{&KmYRNtIxmu`s?R+Q0p^2+A-Zn*^OxsL!ljJ z-J2(GP%P@P@fOlvH7|B}%H5Z%ZvL|Qe%0IkcV4j?YxpRD`Q!Qjb?3e7{eMN44e$TU zvpn^B{;PZB{~u(#TwGuEuPINuD<($5=aKf?@b|CI8h)EJ)9I-HjE^acBp$kYx#%ue zC%VInUr)aM%6!BZ(~ftQ_a|)b8jS`=lbj~ayz5=s$(9dFMmlfn0QQXZa`dg;!>k+) z6y?rY{oc{w9JhWQc8+#^*LEShzm9hNm>oSH-V^_HoIKfk?vBeF-F0mDRC@dSlVtq6 z`-+$28_7TKy@s{t9*w%ogWW4Sd?r`JC?EW7JeiM=#y^)k7(aQB-Z)zK-}t-3kJgk| zi*v7*P-eTD?9}!2X1REJ-p!uA>SmoSo#W$I_IIbrN6FoRv0_HH;3n6zGebT%s{?_W zzE5B3sl>zE4f~(s@y*4>*?L^gmN#>3{9v%?Q$8Feg9V=sA8eN$cFLclgwvllq|ZE^ zOglT_`wnoDFq2M)>2#E=r{|xoruZo2ao@gXj`@DNeD>*boP3(BtG}%|cGB6p3_E^f zMjuei(QT8s;q`QA*6yh%%XamJdEV=*uiECKT`UiN8h?Ic!?);OHv*fu=Te;aVt{?>rdvwcVG^Hn=7`Slln{pQ=R|NQHBwnMcI zSGeg4Hm;5?u|s=sr|vn3KK!;5Pxl+<;bF4pS~d&(;A8z+;71Dor!iR$&54Ros8nq{%e3Go? z*<^K{EZccweyxon?BsfSb2VF1PmFa&_t!K2pV@fSPyWuPL9U#Sn>^euI_~Yt~ zVLQV|o;Q + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/HiveOperator/src/py3/vrep/vflow/subengines/com/sap/python36/operators/examples/HiveOperator/configSchema.json b/HiveOperator/src/py3/vrep/vflow/subengines/com/sap/python36/operators/examples/HiveOperator/configSchema.json new file mode 100644 index 0000000..0aec337 --- /dev/null +++ b/HiveOperator/src/py3/vrep/vflow/subengines/com/sap/python36/operators/examples/HiveOperator/configSchema.json @@ -0,0 +1,97 @@ +{ + "$schema": "http://json-schema.org/draft-06/schema#", + "$id": "http://sap.com/vflow/examples.HiveOperator.configSchema.json", + "type": "object", + "properties": { + "codelanguage": { + "type": "string", + "sap_vflow_constraints": { + "ui_visibility": false + } + }, + "script": { + "type": "string", + "sap_vflow_constraints": { + "ui_visibility": false + } + }, + "delimiter": { + "title": "Delimiter", + "description": "Delimit Hive results using a special character", + "type": "string" + }, + "hive_hostname": { + "title": "Hive Hostname", + "type": "string" + }, + "http_mode": { + "title": "HTTP-mode enabled", + "type": "boolean" + }, + "port": { + "title": "Hive Port", + "type": "number" + }, + "username": { + "title": "Username", + "type": "string" + }, + "password": { + "title": "Password", + "type": "string" + }, + "database": { + "title": "Hive Database", + "type": "string" + }, + "kerberos_enabled": { + "title": "Kerberos enabled", + "type": "boolean" + }, + "kerberos_principal": { + "title": "Kerberos Principal", + "type": "string", + "sap_vflow_constraints": { + "ui_visibility": [ + { + "name": "kerberos_enabled", + "value": true, + "text": "kerberos_enabled=true" + } + ] + } + }, + "kerberos_realm": { + "type": "string", + "sap_vflow_constraints": { + "ui_visibility": [ + { + "name": "kerberos_enabled", + "value": true, + "text": "kerberos_enabled=true" + } + ] + } + }, + "kerberos_keytab_filename": { + "type": "string", + "sap_vflow_constraints": { + "ui_visibility": [ + { + "name": "kerberos_enabled", + "value": true, + "text": "kerberos_enabled=true" + } + ] + } + } + }, + "required": [ + "delimiter", + "hive_hostname", + "http_mode", + "port", + "database", + "kerberos_enabled" + ] +} \ No newline at end of file diff --git a/HiveOperator/src/py3/vrep/vflow/subengines/com/sap/python36/operators/examples/HiveOperator/operator.json b/HiveOperator/src/py3/vrep/vflow/subengines/com/sap/python36/operators/examples/HiveOperator/operator.json new file mode 100644 index 0000000..60b1bc3 --- /dev/null +++ b/HiveOperator/src/py3/vrep/vflow/subengines/com/sap/python36/operators/examples/HiveOperator/operator.json @@ -0,0 +1,37 @@ +{ + "description": "hiveOperator", + "component": "com.sap.system.python3Operator", + "inports": [ + { + "name": "inSql", + "type": "string" + } + ], + "outports": [ + { + "name": "output", + "type": "string" + } + ], + "icon": "puzzle-piece", + "iconsrc": "Apache_Hive_logo.svg", + "config": { + "$type": "http://sap.com/vflow/examples.HiveOperator.configSchema.json", + "database": "default", + "delimiter": ";", + "hive_hostname": "fqdn.hive.com", + "http_mode": false, + "kerberos_enabled": false, + "kerberos_keytab_filename": "vora.keytab", + "kerberos_principal": "vora", + "kerberos_realm": "AD.HADOOP", + "port": 10000, + "script": "file://script.py", + "username": "hive" + }, + "tags": { + "pyhive": "pip3", + "python36": "", + "tornado": "5.0.2" + } +} \ No newline at end of file diff --git a/HiveOperator/src/py3/vrep/vflow/subengines/com/sap/python36/operators/examples/HiveOperator/script.py b/HiveOperator/src/py3/vrep/vflow/subengines/com/sap/python36/operators/examples/HiveOperator/script.py new file mode 100644 index 0000000..70833f9 --- /dev/null +++ b/HiveOperator/src/py3/vrep/vflow/subengines/com/sap/python36/operators/examples/HiveOperator/script.py @@ -0,0 +1,59 @@ +from pyhive import hive +from subprocess import call +import base64 +from thrift.transport.THttpClient import THttpClient + +hostname = api.config.hive_hostname +port = api.config.port +user = api.config.username +password = api.config.password +database= api.config.database +kerberos_enabled = api.config.kerberos_enabled +kerberos_keytab = "/keytabs/" + api.config.kerberos_keytab_filename +kerberos_principal = api.config.kerberos_principal + "@" + api.config.kerberos_realm +http_enabled = api.config.http_mode + +if(kerberos_enabled): + call(["cp","/vrep/vflow/subengines/com/sap/python36/operators/examples/HiveOperator/"+api.config.kerberos_keytab_filename,"/keytabs"]) + call(["cp","/vrep/vflow/subengines/com/sap/python36/operators/examples/HiveOperator/krb5.conf","/etc"]) + call(["/usr/bin/kinit","-kt",kerberos_keytab, kerberos_principal]) + +def on_input(inSql): + hiveconnection(inSql) + +def add_http_mode_support(username=user, password=password, port=port, httpPath="/cliservice", host=hostname, transportMode="http"): + ap = "%s:%s" % (username, password) + _transport = THttpClient(host,port=port,path=httpPath) + _transport.setCustomHeaders({"Authorization": "Basic "+base64.b64encode(ap).strip()}) + return _transport + +def hiveconnection(inSql): + if(kerberos_enabled): + auth = "KERBEROS" + kerberos_service_name = "hive" + password = None + else: + password = api.config.password + auth = 'CUSTOM' + kerberos_service_name = None + + if(http_enabled): + conn = hive.connect(thrift_transport=add_http_mode_support()) + else: + conn = hive.Connection(host=hostname, port=port, username=user,password=password,database=database, auth=auth,kerberos_service_name=kerberos_service_name) + + cur = conn.cursor() + cur.execute(inSql) + resultList = cur.fetchall() + + string = "" + for x in resultList: + for y in x: + string = string + str(y) + api.config.delimiter ## Delimiter to separate Hive columns in output + string = string + "\n" + + api.send("output",string) + + + +api.set_port_callback("inSql", on_input) \ No newline at end of file diff --git a/HiveOperator/src/py3/vsolution.json b/HiveOperator/src/py3/vsolution.json new file mode 100644 index 0000000..f83b34a --- /dev/null +++ b/HiveOperator/src/py3/vsolution.json @@ -0,0 +1 @@ +{"name":"HiveOperator","version":"1.1","AdditionalFields":{"manifest":{"name":"HiveOperator","version":"1.1"},"paths":["/vflow"]}} \ No newline at end of file