From 67518a0564d1f550c73908fd031952936847c539 Mon Sep 17 00:00:00 2001 From: Kristian Nordman Date: Thu, 23 Mar 2023 19:19:23 +0100 Subject: [PATCH 1/6] Initial Zendesk integration --- app/data_sources/zendesk.py | 192 +++++++++++++++++++++++ app/static/data_source_icons/zendesk.png | Bin 0 -> 33154 bytes 2 files changed, 192 insertions(+) create mode 100644 app/data_sources/zendesk.py create mode 100644 app/static/data_source_icons/zendesk.png diff --git a/app/data_sources/zendesk.py b/app/data_sources/zendesk.py new file mode 100644 index 0000000..8c6df2b --- /dev/null +++ b/app/data_sources/zendesk.py @@ -0,0 +1,192 @@ +import logging +from abc import abstractmethod, ABC +from datetime import datetime +from enum import Enum +from typing import Dict, List, Optional +from data_source_api.exception import InvalidDataSourceConfig +from data_source_api.basic_document import BasicDocument, DocumentType +from indexing_queue import IndexingQueue +import re +import json +import base64 +import requests +from parsers.html import html_to_text +from pydantic import BaseModel + +from db_engine import Session +from schemas import DataSource + +#import http.client as http_client +#http_client.HTTPConnection.debuglevel = 1 + +class HTMLInputType(Enum): + TEXT = "text" + TEXTAREA = "textarea" + PASSWORD = "password" + + +class ConfigField(BaseModel): + name: str + input_type: HTMLInputType = HTMLInputType.TEXT + label: Optional[str] = None + placeholder: Optional[str] = None + + def __init__(self, **data): + name = data.get("name") + label = data.get("label") or name.title() + data["label"] = label + data["placeholder"] = data.get("placeholder") or label + super().__init__(**data) + + class Config: + use_enum_values = True + + +class ZendeskDataSource(ABC): + + cached_users={} + cached_sections={} + cached_categories={} + + @staticmethod + def get_config_fields() -> List[ConfigField]: + """ + Returns a list of fields that are required to configure the data source for UI. + for example: + """ + return [ + ConfigField(label="BaseUrl", name="baseurl", type="text", placeholder="https://example.zendesk.com"), + ConfigField(label="Email", name="email", type="text", placeholder="example@example.com"), + ConfigField(label="Token", name="token", type="password", placeholder="paste-your-token-here") + ] + + + @staticmethod + def validate_config(config: Dict) -> None: + """ + Validates the config and raises an exception if it's invalid. + """ + print("Validating zendesk config") + print(json.dumps(config)) + auth=base64.b64encode(bytes('{}/token:{}'.format(config['email'], config['token']), 'utf-8')) # bytes + headers = {'Authorization': 'Basic {}'.format(auth.decode("utf-8"))} + response=requests.get('{}/api/v2/help_center/en-us/articles'.format(config['baseurl']), headers=headers) + if response.status_code != 200: + raise InvalidDataSourceConfig + + @staticmethod + def _is_valid_message(message: Dict) -> bool: + return 'client_msg_id' in message + + @classmethod + def get_display_name(cls) -> str: + """ + Returns the display name of the data source, change GoogleDriveDataSource to Google Drive. + """ + pascal_case_source = cls.__name__.replace("DataSource", "") + words = re.findall('[A-Z][^A-Z]*', pascal_case_source) + return " ".join(words) + + def auth_headers(self): + auth=base64.b64encode(bytes('{}/token:{}'.format(self._config['email'], self._config['token']), 'utf-8')) # bytes + return {'Authorization': 'Basic {}'.format(auth.decode("utf-8"))} + + def zendesk_get(self, endpoint): + return requests.get('{}{}'.format(self._config['baseurl'], endpoint), headers=self.auth_headers()) + + def zendesk_get_user(self, userid): + if not userid in self.cached_users: + user=requests.get(self._config['baseurl']+'/api/v2/users/{}'.format(userid), headers=self.auth_headers()) + parsedUser=json.loads(user.content) + self.cached_users[userid] = parsedUser['user'] + + return self.cached_users[userid] + + def zendesk_get_section(self, sectionid): + if not sectionid in self.cached_sections: + url=self._config['baseurl']+'/api/v2/help_center/sections/{}'.format(sectionid) + section=requests.get(url, headers=self.auth_headers()) + parsedSection=json.loads(section.content) + self.cached_sections[sectionid] = parsedSection['section'] + + return self.cached_sections[sectionid] + + def zendesk_get_category(self, categoryid): + if not categoryid in self.cached_categories: + category=requests.get(self._config['baseurl']+'/api/v2/help_center/categories/{}'.format(categoryid), headers=self.auth_headers()) + parsedCategory=json.loads(category.content) + self.cached_categories[categoryid] = parsedCategory['category'] + + return self.cached_categories[categoryid] + + def _feed_new_documents(self) -> None: + """ + Feeds the indexing queue with new documents. + """ + total_fed = 0 + current_page=1 + while True: + response=self.zendesk_get('/api/v2/help_center/en-us/articles?page={}'.format(current_page)) + articles=json.loads(response.content) + + parsed_docs = [] + for article in articles['articles']: + print(article['title']) + last_modified = datetime.strptime(article['updated_at'], "%Y-%m-%dT%H:%M:%SZ") + if last_modified < self._last_index_time: + continue + + doc_id = article['id'] + plain_text = html_to_text(article['body']) + author=self.zendesk_get_user(article['author_id']) + author_image_url = '' + if 'photo' in author and author['photo'] != None: + if 'content_url' in author['photo'] and author['photo']['content_url'] != None: + author_image_url = author['photo']['content_url'] + + section=self.zendesk_get_section(article['section_id']) + category=self.zendesk_get_category(section['category_id']) + + parsed_docs.append(BasicDocument(title=article['title'], + content=plain_text, + author=author['name'], + author_image_url=author_image_url, + timestamp=last_modified, + id=doc_id, + data_source_id=self._data_source_id, + location=category['name'], + url=article['html_url'], + type=DocumentType.DOCUMENT)) + + total_fed += len(parsed_docs) + IndexingQueue.get().feed(docs=parsed_docs) + parsed_docs = [] + + print("page: {}, pagecount: {}, current_page: {}".format(articles['page'], articles['page_count'], current_page)) + if articles['page'] == articles['page_count']: + break + + current_page+=1 + + + + def __init__(self, config: Dict, data_source_id: int, last_index_time: datetime = None) -> None: + self._config = config + self._data_source_id = data_source_id + + if last_index_time is None: + last_index_time = datetime(2012, 1, 1) + self._last_index_time = last_index_time + + def _set_last_index_time(self) -> None: + with Session() as session: + data_source: DataSource = session.query(DataSource).filter_by(id=self._data_source_id).first() + data_source.last_indexed_at = datetime.now() + session.commit() + + def index(self) -> None: + try: + self._set_last_index_time() + self._feed_new_documents() + except Exception as e: + logging.exception("Error while indexing data source") diff --git a/app/static/data_source_icons/zendesk.png b/app/static/data_source_icons/zendesk.png new file mode 100644 index 0000000000000000000000000000000000000000..c725585d567dff27d27f98e3f3d8eb5de5c3f0d8 GIT binary patch literal 33154 zcmeEtRZ|>ov^5UFoxulp85mpx1PSi$9$W*#9fC`62@(kI!5s#dAcMOGcbAX%RGr`O z-JFZAuIlcKNBUWN?Y-8HQc;q@KqWzifq}u0la*3~fq_lN0?U0S@1)&6nDe|DNMqUz6+`Vyts9ATWt_k!ba)X4XH9X2!Y zdjaE!#P7iOhm2_XaR0qz4gJ5D|F;PL-~WVH4dj(P5G^7BF!P$3{-mae{A92pVA%;L zWhM%zKrYH;z@)$7pXJhZikw8oZDr!MUs~?JBSgIEbUDI&P>6V7D$BA;9X z<8J#&gAqST?LCk?lYnB>>Sh0}pVi&!Q=}gokg!HkGIbx_Owt{xJ)#GttwBMg?3`Vs zApJ{lXyD0V@k_Rk68B#eGj#&-n#xF>0y((EPP7g3;t!#J(<=^W;6Mllsz60$`ldw<0C?jG#l=_t&0~igizxfkS2w zL-qJ@R&xGsF9Wj}p?`~a@K5^U=;(tq2&Vrwmpo^Q3x@6V7Bs@k%|0_6(Q%O3<1EBYhE;%30B)NgjO zbcq*Uku1XWqfFlr=b9l>AyFn$oV;eVa8$aTpedwFPCM-P#iCw~x&B%Tk4#bZf8}<#{A2oGHvCmz{I;bxnnOIil;_pOgQNeO4 z%)sHAS{IWW!kFlyj+YhQ$>I#)N?PV7k+=E@mH?Lqb4hbg{p*Hdr+0!YYM=eK&^rs-IAft^N44i7-sY0y`)RC(ttrd!J&BMbfP$aZi(=kvG!RP^eVL)J!`ru_g z1OlLIlJj>q4jjE+D;m81;xKJr-)nC6=`Y)9>$(#18ri0tO#DIcZH)?rCy^c%>Sdsi zbD$;IWLo#dAOp$ZQb!T_m7zi_)-*bm*e;NgviV4qde(`*5I(wj)7@mRW62}Pj4B-tf ze^H8l`$v^DjR>^Pq{v@gzi|ct!)5o>oIEhvuSCvzsL`54e_|8^T`fGqD4+5vfraBN zR9hY5;t)oRwave^oV1zHMBnv1r%46;Td7A&sT6;2+aQdB@&_j${IHIfs;}{AR zN#*{!OlhJ>#PtZs<1@nZW9>SN?~#49>UBJ&N1}3za|L(9g2X33ZrZcpG64(m5+>%? zR`c=+u*bzpu<&5z*y4!QRb(IbB`IG~&U^=jbVXc4H=E5Krl^oYU=`8%ySi4jYbFt; z0s+g@N5bAl8=0uYKQwq)ebm^$u+W(oPhdtDq1rhHM!FiC;k3hYHseddyTf8SpcR!4 z`^F(2(oava(f}cmjjJD{X+Q}=5s0X1p#`HHZV<@Wi!Ly$S>i8a%#1Wb6!w=I%zZ9q zoS53#?sfmiJg--@TxF-y$>xe~l0*80T>)Hk(&w5ct>eXscd)bJmP*8v81Mp9CwHsu z{j+o!Ef^)I2GHQ~g?_Zst9lYZ%Q2z{X_O+n2lL25x$}PImf1Kbq58~qkIOh&+3|PZ zrhUzIhehYE{0m@Un{T0rYkA5VHDD}(51I6J*r06+V zXiZ}@k}lrCNeHEnnPf{I;-YJKC=%ye>woQ25Xkcq5-e9GUnt2lJT&HBL@-0B)!9cW z8b*rC;SUJ!3dgp|WI_V}lAP`=q6{nzyRj>EYziqXU!$$kbS?r)bqf>OMTD z4ZbmJ;s?upBu-w$Xf^(l3HxLqE{N7xO42W5Uk&8rY~IFnY<(d7Mr0%#05uiKvRi8X znC@A@a>rImS;jWKoMLilfE~$II@tE;V=#Yj6wl0Rgokp4krx#>=9|mUKXx266QZ_% zvUT^9HO}Emy0-muH|K9V`_v0NiOM+E?4h*6)A>U`++$ z*pXp#Ld`n%i}fA%^0B^kkL7dY?T=^YmnWn=whYscC$~fMu5-73(qkAf zoP~E01aXIbyJh2cqVt4&Tw)@u>Npj&PgNzYx5l!-jazjKv|lcUO8T-0u>&=Bb7cJs*!t9Af40sCkOr`J1exp}^tNSq3vAL`Aw8xSh;L zKCcgpSbCz$?pJXZU|lqr(gzghS?d9jYM>G})~pR1tOhr8cFv!aoH0v@n%=_xn3@+; zC#L7B%KG0pD(~Z*#CgDB# zJY{r{m{)%h1gm_^_F0Sm6oP!_JI+Mtl2b-U(f!r{HFBm69X> zM}MPj7_hC_<+7-^iiFpoJ%r)q*AKUEKD)hRs#yYI1EM7g_1{uc9EK==7UW_zWeFbQ zBiLj@JJ$dbR&|-V?aPKAgfgUyb_} zk4W*6FnEjodX zKNbx}Eg7l{2{lBNfeM; z+@LM$Yeps7-!+$fzmZNNY#b^`dS*GV0BKU>tbxb)oEuf^bVFXnr-F@>&H!I)G9l(Y z&j`S8>J{H>Oh++Kdn?*n*MMGyKN@ZFGezW7fnlbazv8aiHwdZ&L@4h!B7P5=XS;zF zLvAQ=di0{R;*d0zdONFO&D(w_I1F?H-iatel<#fcYldKP<9ALZb$s`H@akC#Hvk@2 zZC!>YFl902@aXkG8QYvO zHI;sMQ9#CU9$-%I5$x)E{UMPnwvo_`HZ<@mmtIQLd(o0#UPuPf7R4T|oF4Pq% zNv-56`fEACTVIJNB6Q0y{5OuXXZ?5>m>`~C(c3N(gpDyvKOX%2)P<=6+-Wx`aUi8-4pR5m6N`#n7FGE0Lr#TEH77ypQQz$$7DENbz@^bmHMQ5{ zqvrV5EWA%c`G$ZIXv+0l#f+90_UQIyTd0pyLFSfDs3AK1OW>rT3u0d#?b!|EH<8S` z_Ny5tYvi2BA>SizxRBQV$6&=M;k!>9ThYcZ%pqk~I(&dUJhP%Q<`G_wii#?;%3pfM z>o0mEWKXR~`0!Ln2Dw(2iaFN9Vn%a3u=h{O0ZAwtfgZyJidUAlp^%L3Y*GUgC@aC9 zcB1dq@*3na2wuv>%@3n(a=yM(3|j~dc1e`2&&yyMh`<`_2JU7tzKbw;-0Twfm9DLW zgQ_>>KNea$6M`Ap!Xv%N#?Xy^GS=Flk$<2fF; zy-HW<=|*AZjiV-t?S&{jG9eeZZL)#O&qolo2`hWj4K9iw0P_!uKM!t#V#Zs07lWGu z&ehgjhUvgiH5&Uhe<9v0*%&?FyU5g^Cnyel*;FK>H5JV})AB9==4pQGSS&Ga8=994 zh2>mar!R?IJRjI!Hjs%^=y*qKCT76-Ib~ZKO76P~#buHt+cUE!J>Qgx!;psRDBjLF z^7b18o=fag=z0XCEr$=`DaFY!boGqP1S@~lp)dns0O(c|h)D?jfo{HAly5<7AHY&@ z3QwX1R-N1-0y0{?3ZzjO=RsHj>4MU}szq@(P{Y#fRf~+mvM*X#6;q)W8@hJHpY9=1 zJh?Z|M6NRqgErYRTv!uoT68Q+eh*w3yZ_v&(yfsj&0kRuUplxs1aFO z>K|>=xHXh$+GGb;UR49T-i~+QF@@1`Mn#K96(NnN?fJ97GB&k$?mMlnW%tOAr$Ijq zFL}AGZ+Uzq{lH#GMH~Y0DcdBZM~dkOi$v9NG6UTd^eIablOO>WpL&+pZ>pl?i>lxO z$$LoJN;C|~@&B5_Limi`aWwHbClc>`LV0hEt?~Gm8$f1CtWg9`e&3;HaU)Oer!DEmZ zgR$Oj{}Ph_F3b%CX}b8U+ZI2vRvIx@Qv;So{5z!ltBVf;mzPOZ^vgjmKlMN%^z@T0 z_tWx=_L<vj_#h!NIVfHLn>*L%0y}TuMJ8*h~jtb1{bwPBEchIOCNl@htMgn+U zcki|?yV5njMjpg($&_&n);WsOk*1o0p?rww+P^cpaFxha{oK#S_KTP%RU4%c)Hz?s z?;E-tw>ENld3npt@-_Lt97!tKKoh%Gs=a7j&eZ~gmshuQ1CIAFyzB8PU}L1(sPEe< zwu9Y+YnPN)9?IL*nugb>b@_h4s z2&EYSu2JqI4n0Q=gj$gn`jIrLB-t><>`$naEK4ZYIECV3_!KgVreyyVOrx8D|A~k2 z83Bj+M&J($O?BK>E<$R6Oa*l<&bJP_!`4NAnUPhCcdo}71%>sTc!@WC7{WnvK%*6$ z95s4bAL5#{(hX5|vmW1bHt+U{aJ!{-vm7TxCWv39?9q_&thBWHbKxar<@ep1Gu3ns zTj6uWeEUVbO$C*XJm*X&8zhHj} z=k&oEQ3~_#*^P}nb&?Uq4EW;3rd$e_!F5C$;tL%@h2=J7Wk^Q{z2C$<7On5Il#QfS zEi_;atgN|8i#+I;-1Pc(UhE+`uj1V2p=}q8#mbW^JD)N|=Dr=aQca3z1|G#xZ2M6* zyZd*z!e7BvW#fzyM%`9A@WJz&M^SB9TVXoCWV;Y7e4$<=@&8vFZ|=rf@FCcPxO2W_ zZN)4JHt=-30kQn}haU&fN_&)H?ip@YAO8CfO*8zFGE>_dm}yFCiYyX@%+^%vxgia`zW{4zjUk!5iC}%0To9nWq;Ly$8C#J zZe?xFiez-7^X)ASHE$ijhK=^HUAooVFwbj`>3Uv}xp<-InYwlHutu8EqSy1uraGpk zZWtXTF^^L0_E`@%BmEpyDOd!I5^D>M7CZllq_{DJNL1%TLmrJfT;VM3p8zCO37)k~--R>I_|G%kou;bDmK1 z0+WL|MQRrAtD8)&;*Cs$#Q z89TgE>xG?6g_0ZRqQXz;hiew$-zP`2Z=q=$e+o0Xim{EK^|V-XRXKH2%S21&XW`C5 zTbfTuRRql2I$R4;NnflbpSoMYT5V$0-#65- zEIdt2IAXML2M0H%sbN^0d(U6dLnvZqh=<6F1mExd4H@%1SDs_HZj{5ioOy;!SGTMN zAsNA99=)u^Q^*G=9>@e-+E1M{@J9@Igf=BrvvNCT&dM~JPnC?dx3LhZp89!4H{0Qj z3vq66PjX6;{FL&Vb`pura%}tI>mBw{)3h#TYl>>5mA-+xuJ(^81BFm^*)ahDcKSCy zD^e7PTP_H;q+aP5Ao_+FVvSyos=gzKlUVy(A2XA!5^>LN zUp(&@XMp#DgH%I5hL(suEvDd>N;!Y}NDlX}n?0y)LGWWtLt7(>K#n_AYrX7h1Daov zd$~T%$LwFQuUCbX<7{77dUQfRBV^N%$j>2UE9V@b*y`%F(m7gBOe&;`FQPpeiMI+` zm_3b!S<02UTR`UKq;C++-{HrKINoQgnx>nR9*qg6u&)0hd92ZmtNMFgNHOd1HxWpo zmXw~IMmjs8C9Cku;^cO*zLO{l*|e$aWh?EnbW~LGT7Z7Eg#9zOIgFD0)fn$Uf*`c*Ze;7&4ik(b9WFPBaIh+@JD6^u!@rNz7|4o1Dt__!xx zI1&j;AKSh}PZJ7PB(Z$W;4m6MGkP}j4aoSY8{!g~qr0G3Y)Oa!a+$y`>X4XDe5f(9 zbh&K_OwX8Kyn~rR7g&b-#|yF*S7*Ym58iOgeLdcHzTu1KqSu8a2Zg)scho}~>&T-d zoJza-`z{(WQd6H$RHW~FdFC%y)}KU>@=aNLtu$4`QASy2JwgyNzRQ4@B?K%Ft2C5I zap69#!JB|dLgeherQm#4JK7$dUmdCzc^JOd#^Nf2CuM-@iro2T)_qZ)4zH56(`(qm zi9MFxSCSeVyT|^YVRrXl!g5-5s-0D2Wfynw5e60w|*y0tv)F5VOQ$iji+jXdXz z{oUNj#0^4xS7XrA6IpLXL!CPWW+1obyp*u#PC0KxQ&u{31SG*GfdZSnn?hA5Kyo!( z@0css+3MPXqwdX%`A(Vzw_JMM@+&+>nkj3f-7Qdxw>=+QR5x zKO7SnZ&>?Zr0JBCFc+UzEWBEDn){E~<3w^VR~5L?GyM&|kZ+#y2;BRWD>zXp z_t!!7e$m($U3>jpk#{6~`$S>5FIKVcmn8rlbi$mJpM%j-F1)4Cn}K+yD+No+)-R#(!<$3skvHK)g9WL`7k?tvJZ{^7~1T! zzb9f`rKm+&#atpm;;69@k~qiMhp<7BAeOFI3>}g9ApG=-c${cA0T+$!D@$MUd`D-b zev}?*k~E_M+UPl%b5N|B)~z_7CKi)o+w=17>d8aJlkoTqQ^Z&GdhL_j&3M0Tv8V0` z0I57HYVe%~H3Bsiuf{Gm3(nr>lV|9SoBJl7;qWOnlLy(<+TuLiv=tqFBFXaJMn-4O zKheCXH!z?AMJL21HIfP-PreyfNj1Oj$|ClbZfwO`Dp>=ERH%xo`01ydX(z5abRv5Y zCUMAFIJ=|%*71z`>qECsy1K#FrfQ6-GrF%*70Wg7$+-_d$#6JlyT8qxr!xo#aTq>% zKu5mgSt_npNzV`5=!2Tm1;>)xm_>TPjxs38pF{pQS)`Gw-GQy99j&wHn`u6MD+ynaG#5AUe$2D`{|g77^Dkv`ZB zGi@J*lPtD*uiSWhQ|w~K;3-xgX6Q-Y9$m(S&BA+RgGvHlK>@MPTwqEcV!9`x98?-) z8OBW?Y5_f8er;J*(#i-e%jn-eRofY*Q-&y`!`oyXR$je%4<+n?X93fjwNs@O4tobisd%t|KjGs& zDoPo)-pwuc*-G~HNn&LdkIjbwC)~_5F8fP_f9Rj_D}hrhWZ@C5SJmsc@=L!ys=~G) zJ3||&aZx-JS7*UgxCJ%e4mf4v$!T;3%_-iz*3O|{i!eRXRLuafgW~$k6$7Mb9xyoV zpyc95gGlN`N0U`dCAG5C%HBVQxE9L+t7{Kmgv{tWgs{s5HD8f@6Wx{{33<*tyB<;E zeWsn9eZf}cw!c3)zfg=rkrcVZSBkGj$9^#2@XeGLe`jbRvwqKv8(p15GF}EeW78LK zSBbb3OKg>3eIb#vM>qPD-t2FeyVWWc6tX=y>{&(Yw5rS5D~KPnnPSghzKhMljtqm~ z6lYA@2X)b)ih{_6w5tCBprCvD&ENFhm8&wo_8_~ej#MJyGXb(O?#CAjxRmWA>QT^| zqS(_MzT^>hbv1+|Y>Gk%%8 zaLFy(x$i9I>O$E+6XIQltt;xBP-k1*`ZsqD$V_65y#%*Nvw9dZ&ED}Zr%x~1g)nbd zBcDo7-pTUENAlk}`|trTIIr82!oeJd0eE(bs5gxr!D?GY-Tm06VR}@&cOFoB(qKv0)H#3^K#@EwR_ILN-Kcr&5(EGsBW7%+j!Lx z*~^@!w2TwPM}D5`s0t_!KB+5N?HD??vO3+%^@4lKHsn$pDeScmg|>SYQGQa(?0Pc~ z81)f7|M=x-fh-Ndywt3SUEb`DRI!X3mmoo8`s~ITg3I6s48mSgq}HvLSp=hJg>MM= z33A%Ny(3i0+jXSOdFowz#Im#`vv_+U5=qt`e6IjKojY$PLWn~Xcl-%1kFl-$99q$D zo-Cr2ZkXl+zqg#Xqrre|j(``z;7O+13I|LbL1&aM*FY8INKOj$pIxC~jCwW{P(yEG zKg~(X@OSr_MydwU)}{qpu=_HY62s3#r& zR6h&1HA015f$tbQ(>X8B8ETiIqPbZ4m;Wiq zoMS~W>+hF6CCe|}K$gK$AJpB6o>9RAQXxMr%rwhS(&F6&40n?*vfV{EsV*kWr#g|}z44SJNt4poJ> z={Po_fB73FY>+jHv5_q(juoQ&1+haKhb`%;XFU`(HNhtmTF|wpVHT=D5MckHc#9%l zJ~IJHM(>PRiw7dmulHvJ)dh3?$ZZ))*!8qDe;(cSSnhjJNV$9dUftU!vtx+ce)PNa z7xVCJBKXATY0f3m=3MgX49`k{sD!YLfND9Qm@%!>b%Z6eXT#q59`>>ID&no}bEN*A)kVn;U#>D1c=h_y9L=R#d~ z)S`C-9O~h+1j&C*`fE^A z{&M~%;e`-Xf9oTC=m~hEY2Pbo!ZTp@ij_O9#D_ztcdKNOM*(5~%IVcZFh_?jt?C2o z?_8B5BSE3`Dtxe6zZe@n>T*}Wdw2ajAhHjq4dm~861{Zsza*jm~FZrnpwcHrl%#E^dq9V2Z z7ZZf`U6HeWMIys~zIcj(hZuT}p?Wf8vbF~;=eXx?s?4%t;DP=N|r8*ueQl3oz*V1Z655pw{TjJWP0=r1lYcHMq zJ@TWcg~LqQ_zscT!aam-XP=A;(mg!_TUNGnza%I>DsPVw0X{lvKa5`+T7Eq?e14*Q zr1C#43HVOwwa};Y2gwOT6UEEokULxs%_7bxKl=mr5(7}27;%0g)5pxfWmR!~L($2H zG~i;M($vqc+Z)RZhLw#QGx7BjmsB7Aag+tU7XMQaDEz_%Z~f@+(anOMP>`JrjdG}q zt3ybLC-d3r*#ywOs?w#oslSloM%Mpu@C`nW_DjeO-p&M9RpH^FQO5WqH#K$O1tAqN zk@mV*%6|Js7{dM{zkl$W_r9vOW@fTNYu*Hj>8fbGTeTdWJ;X{y37#2vhi@k9a8gC< zP`zL^K=YmgeV3GB^o@^g+y+-ch=}jA4}|?~qav|sU6&z9l&=di4+jjQ6Av<&SA}Ai zG|ysK0sQN4C+nRwI9XE(sqP(LMN%Yi%hjkSbSij!ACBf-E}l4cX#$=;IJUe18f#~v zb?!STgEEqbb-@L|{zwy>;0*_90kZA(!UHzdd4j%gAwT9cA)0a|rjEtRjRYgWjcttv zYdQFdP|5Q0=c&HVJZq8dCY^~Suj{}*WbMTjoxW@Bug_X7(PJ!++1Qi*ji9pkf~bt} zWupBL0!f_9tBJz6%C;Y~TbzdCA!FHPLUK)#W3(KmDHfStq^+DM9 zX2i}6VtCMH;0{j?AB&Js3Q-jy!Or~AQ&`}J3HKL4vMz}XBw zfnqnIs7Fei%tAX*%sH^EEV2-EjapX8_Q+!E7g0vmB%f^s&Nt;GG0DlIgyVLjkyao* zv`>Vp=C0@k$Kx<023%FXWoma^{vtail?cqWlM$Q8ke8Q7zw5YZeSuxcA*&PWbuY+7 zeP4+@-+mfh%2*Upc=OVcopbk8a;gJ}(%DC|uMzc0RNR(w0y*!oI%qaDw<)kBt~>|P zMGv9%Ya)Hhy(Ejq4FF~zy+5H?`XkJ{;(I{}Az}(QN(wXlVDK=$+;Rztzkt47s6KX3 ztY3B`UpjRi2|c`?y%@cgL7@g%Pqdn1Z^sFF(_%7BQ|JzW)pjo_dT<&Shofzv-GK%Tp zk7i>f>;ocoy7%$jZ_QWlZ(+R(tVyVAX1m>EWBCn9dTqS_H`Q-*A;TRd^BgE1eQ&QJ zVv%3t=nRyZf(GhYyxu{%9s!`Hwm5mm$SUj;n_`y z<7t9*>|xB+ul2x4rz0Kf)fwXvXrgH5^pD$Ill)Q@^3nsNYvc1C~gOH{ZQmG{WjNFaT%M2?a`E>6-qy}-p%#u0HB-Bzzi|5)%Rct$Y+kMdsa7hGFo$AMl?Z==fd=&of?p)7U;o^e6NJ3puN(`-jxJ;zPy7QJ{_6py}T(um)W+ntUlI2OjIE)konf z^A%)*3XzGeA7&leJ3TPBF({^s;_K2ro4hvJT6mYuP9q$js9G@t&Cq*bASzgY4l1iu zwCT1h6)^#$=9E`D6jyFy^5kDo_k(0@;MNF|S&*o8W**nilsvox3LONGy(TAGJ7!3V z@vincFfl0!f8JSfO2bq#TKDMNj0E{vr3;%FlI{^Njtr z&XauMx28O4q#bs6govKDs(FSmVm{o+)KV-5<2sSuQz>EkC&7z^TW9$G;gshPdM@WN2Dqc!G@$e@b11nzJ5yMBv{9X zHkQokSCL&<`t_R0-il88>_M$`LUDm}I48vg8bI}kiBeG}hRbE}+iYFe>2JNUGERk{TxRK*^x4U^D$&JROI#e68R=M8 ztpYgLR6J>>OvHT`QFyfhN=ab9I3O*-5`7rJ+glZzZ1&e7dc{M73bt>%FiCIYscS@VmREf$~Z&r<$|_E4P7 zRVP0*dEVHob8*8TfII_w+fcXfDBPIN!?AVi|8l!`nDQ!iYbC9x;}-r%NP~hK+s`cn zy7%8b3J{&TU4qQw9uhR9idxwUoG34A6veRGMMnnh30w{F2{ZH}S;A!IRmVr%gGVOc zy4lKKQMMU|ws5g{*dev4UknnxBSyK*V!$4fDJ1)3^4>~@bGCun=}La5F+KVbF1gHF zKEK18kew%5RhCjy6b$vJA6GF-2h4D^d=Q(~+~MC|UeYcdh;k!BJ&Auv57lSOW;#;Z zEeE(g`X#&{V{-@P^6I$cs6ke%OJvxBcT08HsJa%+L52#-|rUyc3C1n3NC44*y*i{qQ&)lT3@c3XM7Dhfp4oKHx^ zgtkgcbnCzSz#~J8dr5kx#^vU!EGgh|NhQ>C4sQvimt<=IG!p{fg+q!>OBoY-2(=J6 zdY2qzyhrn%L2su%T|`*d7SAr3kOp-O zo!qsI*p02xMdrNPhsdj#!*6v8wJ0X!kk%9|^CH&K2k)hn`uD`WKHV_`+Cp&m$7#%7 zq71(^b_e^jd^alBRpy$E#WJBH?NVeqC-F-MQR|ZeWvjj&ys+XICLOzJ=FkvWFa|Ov zx%5bq0^Fqk&YC@;Uhu1P??_-ZCGEf z6axL+sT_|4Oq=$5pN|g-H#s>FDK#|>;8>A-=R+oZ351^VL!3D`p4}!l^RfJ~-_G8Z ztrq#wWc_pcNBi5ARBOWWKF3qq|*ek;b9)1EV$fI~PT^V!cle5D+OM6%q3m8MZkl z_c;IK6kDXS+)>lvFqOkjf=z=yALq8#BpkWy)Ed?JB_?g?@Q}G&bbHmQCGzS~fY2t= z>$u^ECG1%w{RCfq82%68_R#G1L+zqZq)@n*~isd_mW~AZ14!=nOTdI>!8JDP?e~sHiIF> zs3Je<%=55dKe4#9-W$D+srZ`mv1UT$vnyWb#=BU){dUh}mRe#gFd4PYpd+wPHd6Mb zggeTegsUUY7SUq)CZ!(p1pYk?yAkn zu9m)8D!&Ck5j&WTxhs5_YLKT9N<%Uggg<+kGIo9E@xOG8EWykm(UPdZyIGGJvr{h_0#i`e^<3O|{(Al?`MTWI zuAiRqT}nadlaEiArFt^Cfn{?iWb+kQ*=wcx-2ClzJrUkWxITCR4131KxoUG`0b8;d zu^bTSj*)^-U2!vio8tJXnP5d#fS%)9jHFb>0O0<5ziLNdGC=g@hh|)?yw;>Z-uO61 z1rI9+hdH`Ec}Ljp)nrLk8|PUH1Zk<^)PCO9tJ(E7pDn>JC_^WVnAj%s zHRm&*XST(%beUG`XVymHrDRW!MH>aER2*yNc|F#fOYT*QvGRN@P!$LU4u}1)loO^c zPQ1IyKD4@HaAOM`)~6a+L;aMvq7J3IXx=KHzuK`Dj;>_1Pa%K>{Gh2p+Gn%WtZnR^ zc);>T2^k+JpU;w+*|C?=2U-g-R1Y;}LBn&BDM$(w@j$00M5DfUTmFZiH!NfT@ZPDl zjkObpiEazRE^q-GKqPq)vKhDB-kp2ZzPsm1tlh5xSO4%63W8&3uLk)Ai4PqYbq*Q{ zj4%H*t^d*TC?m0C)_TH+m7rC0)S0)?&E!xxL{0wXJxLp^Bad+U8*cJ>b$xqpoNc5% zewZr~ClP5M__-UVbAJ0iipRHl@;0N&;+thkJ>~m($Y?Ws8!wm8PCTV2yhDUTkC1#E z9kq9KOscF>b8-wEeNP_^0{^DNcTiAv?o{^U+5qo{c^Y3}$4I8qX(`lgGsOPj#kakn zoXG&DS@|t>XIQun%lb)^I03)QNwV%Diw<9jD4#}sH|>l|T!>-iV}f~1EST|1{Bui0 zB?Np_h*Djv-CX^+bka->r2}j;zJUw)ofF>E#*rQ*FoC?tWZT+>mYH5V_E~-zDT3re zOoev8FC);T<_CwgF5*3?kdQzE=m8fTL;QFG$(-#~$?H-D7Ee z;7Eszc2oN55=<29+56|>>b0(?E$?DAXZW0d9zV%NQd(E`+3T%S7`^vKOc3i87_?!} z_o=Op@l$S)l-bO?DcB2Ke86EEq#6S_YT1FO+PxEv%nvxKRl;Q%M;kP1>zw`) zM!M$J)?=+=Q{Nj=);07yvdTYBsABLw#2P zrCoku0?Gt83@3s@BHoMawQARq+c;tul>RB!FOPU?C9F(^P|a8rR*pMjbQz12>zfgZ zgavl_?7gbGH8)>7*}p8!kEAl*D|mVRwKs~-(_Fd%QL37W1Omt${CTb?2upA)>+j8$ zcY+LdGyrrdQ(-`z%Q62xxhT5kz*o`h{PtS{jy`xqarEJN(HY&g_F1uE-5+gZMObo6 zZ)~K6NxMJjzHD@f-q@0f)IvC=d;_9P4H@p{$Om=B{E;7m=7faW>abLhi|hE8f0z=P z&)WN}D(%g1wX=Icvh>M}^tMi%zTvTC48q4n)&@#lQs-5e8fjs4thimS5$vV_5?UP# zt5~;Ni|09h2M)eII=S9>%;OnOTz<~O)7QtQ(L}lLi?~=*{Xqx$jnSI-cUJq92R*@( zcpOS7FF$NdkrgfKYjp?Z zS3I12+sxqtbEejwpjz;|F9YK|So@hz^oc`TMH&SPsvtGcGVu|~ET)3U8fqptsN8Ze z14nXg+PH9M)-L3BMJZx+MD7fr@suUn*9SM>G&BcH?4l(jFcXP;2D^j&xz)F0vk zuGrqK$=35+X)O-ZIMo>O>eSSso`xxJc^!9g^*hXG?OhBSbBg5jvfOZ%`3SObI1V_e zF|jy0K-Z+2VR{+jC|W77NfMC{EX{JDqK-Mo!rG1XOVzq%8lriY$4-cGyH~7g_S7n$ z#+<;u_Zz<^pU0#PSY&$a_mNOurcc}HiGEYLl1SH;@T9<`>Sxo+U#z2FiaRA@=u7}% zI+wdF0nf(wTiRVCiavL^6+K5fdi1{^$CPb$3iCgh@{r+}nJ{s^q!%+y2qb}NC}SC= zX;S|UdoS#B1 zhbv`;VVxg>1&Ou^mYXPWVO0L4*An%Pnu({DzI3b({A{|aU z=|+np*SPn%b;FZ+2UG&w`yf6y;}04 z6d+bw9G{?aTW>D}2J?P1w&^lvUnCC>;2r-NH`}YX*}zqJ5|jj}9;CMe@vR`*AQWw4 z4jl4#nQ#J3DhGs+<$9|O+ZKHKFbv$QukOdH{mAHQ(!YFf#6ggCuW-@i&>)?P84Z|R zY6uTeiKvbNH7qA5+D!<2rbK33;XiNRx^dcJ@VTuM9FbdM3xCnv8-1IvpD@3#X~m3< zw6`UJwQZLC=h=qeMErDXuD4$f{#|TjT(Ei?7wb>Q$B$Vwuy^2#vyhGEv+(h+7OL!}j;==6< zWs#}f42o3 zFYk8$dl$YuYB5WYVkwtd3{!#W>c5+uw`*toMx)p9>2=pQK*7o7!!L{%k)|a2KS?KY zn&~W?y$3?J;9V? zm;ld1C7{%k(X|S*cHLtX<%!poO5CXBrs@k|Otb;C%w~K?6bY4|6^P{k#E4;BB)g|Y zsFD{1FCeu<>^Nu}mrjp?5$Mna37=?CQ%Po(jU{>G>o7-CUv}mJ9d2{oQe)$fAWwm) zf8&m#p1+Vf?KGbG6TREbazKnSCg6{OKmZ=gb=}8YbKIpb3y1HstzmkfM@p|vNDrhD zLhuvchM{ z^)LoTgw%Er*dEo~+*LUA3#2}fmwsX+ZzuPEy#PE3ffo|NHEu46SnXGOo*cL5M(2S% zv7|2jI{SeBdt>OyW~7^lE8=^55c&Gka#tTc6BWvrFP#CBeJ8&z#7Qto=}_sgeH*J< z{~nB)sYQcO%Yg*>bm#eB4F~-ugfS4f3a6w&BQi}Mk>ShY#li{yR=jgWn1kRJC_|6B zuIW1O=c>1UK$h^&_Ow}@Y?}T4XL-Sb6Id#!4j49JSBRD%>J-Es0qu@Q9^uy{kEeE2qun}}?H2h>;>ikM7SLk@!DPIs zHJ`Pw;1GU=Vw0)u*uZsYLF#iK@r|CdM{B9R%aoZoScVCbFAoiF;!Wm_<;rEC1Xtg_ zj^^rnZ*sigiYhXyn&8DKF1j6cVu&{8Z%)}F6q%|sB3w}THXh)?qO)?i3!FH%J#;Ux z^*T2K5g9{WIn7*5VJq=}<&7mc@{?IOW<|=VfX-a(Hoc%fI4mkzYjY@!?4U^SDbuWM zY);R$?;9mc1H{M@uYo4%NjlRBA}m48=xW?-gQ|TuuXIeFBuGxORB0;-fsFW3w$f$} zsncmckP8@PXwKnKZz~A-H<%`U37d|M2bjjE1cIS9St|VTcF+{31={xou>bz@bVr4I z;31ff?Ht1MO|Exe?8?@&$VKy`2wq8<&xo733MpoEKP>`|mX@}hAe$@!;*^oBcc0K& zU;OI{(rx8!`N(~(APnSZOnxaFv_?16v>hYM>?zI?dV4l#L!}ywPnY*PTf*=0go*D0 zx0!xHO_9vy8~Z`XcXqaZZHxL7%%`&p-esL6l_mK`W|%*S7ZzZ@FTTP0F9Xu$%% z!xjI&Ll!-@*1VRj<%caYuffKyXD#-YvjKOLc7A+WI*Qmq7Yo%LZ>7ve6mi|0C9#LS zug#}4l;NzBc8X?rQg*)Q2$U*;jGqibYVtSF|7VK;I}@&|YsUWX|thubc5+Cv(~$HI;&Mxc}GOH8ob&1l!oQZ6_1kwv)-k z$;7rbu_m@{J14enJDJ$N=Xh6sPMP2zaZ@(^AjzXyR!X17#b93c09jzA$#PWj;(?qLm)-?>{*Lc!% zRaeE+|KSP^s38#kvjk<0piAp>y3+2-_h;>##w@4sl`wsS zW@d*s{(FZCk4tx932Kr8D?!SOTR;Bx9oETg%n?=Ge)%*eGI78<;y#7V9f*HMzy1-Te3C{{H!aSub!dL-Ma#o zFX5vNQkv5uCmeJ6n0GhWl49F=_R8(I-#Z_I95Qc24^Hf{?pyGT#vU3NH;KPKLbM3EAZHOY5 zgRW2%l^!M4+TYD$ai4AZ$wmQwQ7mpsd@cO+bOzk1X6cOY#rU@*}VtKkaranoMKR6Ai2hyHJMbBn+@F0s^fT|R{ z2SY6i0=WP5KaNc8&Pwi!AIhdy#)wrc4SNY7ee)A7=+15WDrIAB!gDP-^c+Yvi94K z)1pW=Ym(;hSb8qH%)2fB7b*S6^Od|9qjy;9I`Cm4P43E@h2>i0FYkl~6<=USsZ^Oioc$pPAcWv4>Vj?j%G zK3*(HOn{B0u_k{g=eYp7nr2we`WEE)Uicr)VVRBA+goHn?k~b%2^xL$lkR;OBG+A#(&By?nw*BJT-P-Rf;KzC8T2#|cpcP~4 zq_8SEo8iR*EuW3izLi_zVTI^$CBnHCyqOFuPCV|E_s%2ImFN0cxVvcp5)mZ#3C#pI ztpe?zWxVT8ZFM}PK+CB#Ns%kUoVJEt|CfFLL-(Giu&Q?9`Lp`ECUHT$`i2HkccZ|n zr2+=0s%8ucR!XrS#A~w-;ugn&Hjc-(%i7`~=nzNz)OH)g3%L*CId?B3d4b~A>_oRn zO;ieM;t(eUCA!GnKKsIWN`ZArm>vqf zeL}WS%gtUqzgfq*dw0Sg25>%Q z5Q~PauCK=WLk}yhr;3Av0JW-bugQ;xcLjn6y0*VcVYGOC@O(wd0cDLivBmW1>pctD z-Ta|=;+*@!f`xKUSktiQjRnE1d+f>QK68UT zOJ4J=2{(>1LRcG}4Gq4HL%dw$TqZsqQm~|O3-{|do!o|d{N-j}U5_NS%uOLFVc}Z0 zZMwnxXqa@s*LuK^TPT_OZ5;37yaz$1NI>(K!}x6)!&?%1YO(JU{`TY!upBSGXtTqe z8J@2t(UmU`6FLKD^;C2qTLgIe@Fc7~@!##cBG{Ps#tldNVfmE%8e4$egBG(U7*D~B z3J?B#*&(H-Zk6OvzHs2Kump*UW_jnB;r69&+O93N5=?}Gvs%$vhweh>s%9ZX2~It^?!J5OYp5>!(zO4mJo++rGR&; zKWiGv5*os+c#iur)Q$^EenGaavv9H}brBNdL4(O<4ZB6z)Hi{xJu!skxW74@$}_|A z5@Z16GB%r~_Hs2>>ooiA%WVEc=a5ategzByG8|ImF9CCrr}MC~E_ztnL#mYa-#5^& zmBgRuo*OJQQKzvMaw@z@FeO`N+>>MLgpy`54@OAXe`@#$8@|-cpGLiOU}SH2tFYs} z1Sb%zwn!JDa@QPu<|yW_ohG*vw*Elcz3m~NF*5$8v}ACh5#^g;={3%nUx7H0ae+1OVfM?Dj=w|}5EW9Lo^M73G*juPkHb7WOO-SDKUIH^l?s01`cz8%Hj3!` z($M8|W4VKru97PmZXUAQF?U`et>K0(D7IraDe>pMQQ3c1ajLs!^|bv}RJqqyiW+~A z=K2?WXti$fyR|FpObL^?{gDDovGGfOG z;$&SUZ+2M;n*PU%9IOi>0eg=wihR_Fsj_M=Zv3SXcC&@F$A~yWqX2Wk{d<`>Jd~lV zt=~`?1%?LPx^=t1<#J+|aGcKuhX0Nfd5=)b08&_PX#QWi3dBiy2@VT>rSd^A>{N_F zuC)QYv@ag|<6Wf9rcvR@Ru#oYN^Fsvl|b+3-7fd`{2?j7j|6c|*s_sN5K{-Uk&;65 zX6>`bB{Z>>%G)+SC&SR^&1Qmw80xZaJDcIdLSpScoV&v^L6OgY6ew-GzN9`)hP0$X z2bIU~Fp!h^`tRE3Z_j6`zws1oRHxgX=$2}$OaS@A2s;5L9%%an8luikjyQgj`>t#k zgx>*r?ru}IoOD16$5KTQQ87ohTjnkSnkYF@*m#qNy@k05?yn;uLj|y%k0*nhzBoUO z&P#E1bM3X()afU}*~@t=cNiwaB>mKP94(*Pcf?F5F?GzCNV0~x9agNG+k0B2GPJ?7 zr~0dl$>!{{l@Vs!b9)Q!40h701A=P-os|4$Dv!7jkR=v9g-Pos^r*X7b^7g>^Oc)4 zIU;GWCiWuPeAEUC+!K}N=Cq;=v+eMaK1aLH@u;N;gys>FAX&pxP?k@Hwh`TBna>_SQ4Ow=G^?t|d-qWdUd z>kobOas^&iUtwIU*{|m`gEeUiQL2pv6+v}_Sr2>>N4R33ow^D!2Yf=1K}`^4t@l^+ z+6ZjT5v|{!=FvQ+8*>Mua^4XrgbM;eWTliKwTbc9PND4W-f(Ry|CCptg@NdXa4*a5 zL1xiQ&m$nyz<;|8Om#gZ*P5@Cm)6_c3QOxI=J*Uz#7VgDEVtv{l8M{z-Y|#f;b1|^ zT1K;4sQK2wkU|6r>tJYG19N4>{GK{N5KXK_)ky^|c@WkZeRx%K+^}s80Syo)!q!a@ zw&=Zb0lR(BgA|!B@OZJD|LIDa)unYchb(Q7yQ7SbFtiBYj8o}A_jCuez5o41z4&eT z0P}_TtBYa5KIdRz`&r>HE}B89;9AUp5~A(OCXvz%vR-LJ2>$n1t-4wv5Zm68MaRNh z_Dmhxa{jaz{&Vl<1DL7-GPndgeBwL@Gxith53y(wYAl~T3T3Z2)rtz~JXB+e;Kg(8 z;~oYSt$j?3LSUV$W#oMUxo+?ubzWm46zobp$* z_n{!B!N#gxyK(uw|Hu`|GPDt}E8!G{#mVHZ%nUhE>;b_cQP=ti|1=s-2k^6YC#^j1de<3Xz~0ItOK@CLV}PswoB&Rhhh+E-0R*BEroFa}N=|_ytPtCx5+jp@&&@yQp!Gmgm;rI&{c97^t^oh5P zY36doh{#TznQ{P9yy1Qfg=4LA4jZcn$uniSjK_($(BO{G_xppGa4W*)9|@>R>mtig zX<_ydFLrfT#1O01+C`~Mv@~*uY0P+a%w&_*@qCbC44D+eCUe^zefflVDy%=dl49Z^ zi@NiRs;SWrHG?!U?l~zO5Dau22%%!Lcq**aL8@i4Ax$c*zuh4Q>-J7(fg(hsbR(c* zAB+p;CHEge9ytt7Uv79#THl_pC>NbNFN5Y$Q-w>{rGjA zEx7V{Kfnc$h2X+5j3H&kD=v^a>K>8lC1+^exRus;kqICPjE}n^CvBE_AnO{@RzT&a zvIt^EwO?(i3x^jy6Cfg0S7Cq^@1tGM5Ah*{~!^5co?@ zK1J@ozo>J1@8-#{x8Jv=VJITZ-doJvay`g^oJwsIhwXczua)n9ij+B)QioXL&x!A& z9bgF*@yAUVFcKgxL~#J+hcjc~R8U?=36Jpl>yWks*wfBKjV~O3`&t>sv@7;=Jl0Sl zVX9HFYbK@nFmG=lt=lA|0jey8Xm4Zb9kf>$x*6im5AnK~W(24+DT41yTI- z>s1Pw|0c*gHW!}+D=iKj3gagA`+=hx%N_prn22p!}wO2uAR z5MJx`*!?TH=w*h&)1;gE@RS!h#IaQMl#;d0f7w>2#d1brfzsBT_Mf;Q9PJWTss1*K zNs%&|who?(P(^Ule2=DrE|(lQB*o*6;GFeK%$aH2*^P3}N?eA)_-~wk2Q|@X`DUPS zvcd?_K9f>$PJ5U=4_drNDg<-@_O#P9}N@Ak`g-qo8|ws z_D^3|j|;@`E0X9Gj=JjES9i&T12Q?xz=7!ds{h_&F3}$gRP&7-aF6ZZm-{i6dn45I zj{ULI?TsUoq<{@U9i;GM5J%DzbQ}cE1l{NCw24m|C6sGN)OrLdi@7XJVdZ%;DM_AH zhxzv%wv1~l2bojbf2)+vHO$tmMJ2|4vV%7gU4R~?kughbm}FeOwT}$=@cgJFu`bn+ zanxRk60qTXe|M)0>C20&hmy*oNW)p>d+54CwBO71e=zA{qLJg=)zqxhkzKYc98aKtK(lV!n7-sc z-Vq4;&40vp<#~*2v2RhPe&P7uRw#WTRGop~Y1}%5PV|M6b8{1>47T$Y=D#cCcYyzK zmivjzb-5;YB>;4^7GiwXsNwmwbA}-bKcGGZUwZdt+@q}f;g?>cvTMaqgV!7RR14Qy z*nrMW0VBX>HT2@wfzEU?Xe3j_AP~}%ZZ>vVm||dJURcOr_T7W05N`dIjPe^CoV9)D zO8deQTPn%e(K4c&zA`$z`x+4Z?0XaJbN_S4c*h}J|CFd#T;i35?1csO*$qDYf}2Rrm5?Y_U z&L#ry$KDmdmz3W3E3+Y=EmlgZ&b`^=bUAuWMM(P~f$XO&BdKclFJ;qiiDk_d;gF&Q zQt7S2&}{;;~hU;-90JFs3`cZD_i@6bR}=NUxWklk9f~e1s?K%QNnG6@dj~g zft=F}3Rsy05Z0T+V8do(8G53}fcN`ZVCP!@_C0gJU9-?Gyulk<>%R>pR=j#&&xq!| z_&(j(F}BD8!`s8e)7l!cgG|$IkKRG9^sn-}Pb88n#ISLnm&QFsv-vW}bagk4dUtx+ zz-1Z}VaufXr75&u(ByTC7b;c%4q|FQD06O3!w$v+AVpOYCc{Uv!r5)`L|B~yMfcZ< zcD~+;Z$c}#WxXw7SyPt!0$7-w1z)S*{8i?_SxMEC9ZS38DyWhNJHPt7Hn8ApNm7(k>>!Z9)g1q(=@~SyQaPAx+aYm7#3;GH zy4vX#-9MH-_Yo;v{s?&Qy3c%SR<4}2}aOEJUM<`}^DotDzD zAzHP3%_1Vwvw9u8TtWYzxOgxZYk+dqlT2z0X&>wPvkV`hXnN!Gu^=CICxem4rFG4| zWvYSV2AwQ(k>p=o93*GRPT;ETj+XuN^ z>GqDGi&~L%Ez(|DC$sUDOw}mI%-lIyqr~PLd=N9Dm)Twuwr?!*&by#eKg2K!e<~^urMG9Pf}j2oO{1t zc6JWHiN8>GKDT#JmxWHqQpm=Yza8_4^7f3pL^Nf{IgS$qp zE@dhd7D&_^_TD$g5R~R8KOU#=UA!l~f;cKumaT?2nlh|#249l@dOK;$vac}Ko~U@?68h+m zYP(778O8?OX>31N5>r2eGK*$YFD|QgBor<6ziW4?fD~#RjX}#N&(2Jg;N6&8h-#1E zIS;^ZIfSdgWfh)2>)5nPi@)?XuQ=VVm5%s8H}jOm{9(EML6y4_K1$d}!9Wvv+b`4j zV{mEpJ`K=OfHAI%gJj{_nb$`F)!Nc;Xhu8%|BDT57<2P}^2bZ9!8fm#;N!^|8v1np zsELM|X!F?O@aBwT=8iLa#14PxwQxzszeTPGB3eRD7}&flYDh&J_qKi9nOz~@Gr26k zdAp9~Ss8?3KHlX79XTF6ql#ulU<~?8$f^q>v~Vu4M{j_14rY`!8Z3cP@jGv z^5jejkTgMb&%*EsnZfYp6SH?t z=b>bbFu|9i0x)!)nQ~$0znsrKi1WK{UF}{Z2dxS~oK;li1cWz6EYl7BFBc#?bgbl< zD9#VPtY@H&3K;WiJX&?*V5(@b6~h*0VQ9CccxU^;`9CZ%-yV(#qzv+PL>M)-L?Q() zYU}Bc2%?Dgdd4Ax7Q0rBxHf&ibOJ#o0yFc^m`KMk`O&K4o`&2>3zr-I{e;}(ZhPMa z59#B#om{lIF)2dy;r*IXmq-Fm?itbOPWT@iroWJf99$7YdAk2dNWMSUcbs6QtkeG9 zw#6p)SlRIWUa~j0{l+64P?z~~Do`glA6`7D56pP;JLr;Fx)m9MoL3fx+2xm|*VPGy z3&M{b4Is4a58HWk(|^88H3|M*zGYlz&kTR*Hpx|PMUe9RvTJf=Qpg%%dyH`F1``F7#U+|a$Xzd-y=qvZQxA`^g~>ebeTBjqli zp-!BqbA2G+^7ze}l3Xa_>k}iY@vRghPAa>1o<=8HFJ7jt1ZL0XcK@ zw)280fAep8-uARHKHvIb{(VwoI^rk~KDx?QeLQ~N!V z+gO*<5L3giTVdUYwgzP&BC@wm5Dp}@2bYhW*{*}9-{OcHH6ZOn&}Ynz<862gFmXQ< z0B+w6j{HutrN9v0-(dlH$vW{Yc68fIu)AWZA!7a~*?q%znynikY3$kjN(`=BmC7lD z(QTz>7-s&-5N(G5>^q<#O()z)v805}XP~_UJ0Z{=@qy7ruPE{!IJTJHWff_9dSt|S z7C%U&O4-Ys84-L#y#n~6LN-NHVGLN6yo%G5Fb>#84cqn%$5o^_44--gO@N6S)-ROcAc1gS945XD6MqSwWcecVE(K`= zXl|A|u0Uus5NQS9*u3}`U`DCRV)|Vy4A;YJ;eC{BD>ATI2oHq|R*WbNDGVv|evL^9 zz4#G%?ZBf}-2H>$lrfR_IDfY>MPA%ob})sFbR_sClOpJU_9pf z4DtGn5eUI%iYl<@TS+IPNaD<}Hh zI&}+rhAaS6MLMe>(6m49|DkjomHUO!>CFhEN;$z{#jn?owlWNYqFWt}*okoNW+#;h z?vvZGwQReh`C9}$qW-p8H<&Yz(IU#$HP6ccj)9(guAd8V^E0d4o7#=tv`<$wC)gRt z4+Aj3@}eX`W0ulc@rNo$Oi_UF`<^HQ_6AEgD}Zqugz`e1+Vg|9{IPfuoi?g}0!SgL zjPaP1STxuXp%MEbY@9FqwbzBl1A7SFb$(z2O~Eafo6lwp-d(-Lk1MCL2xu_$nL;NP z)@9|`I~=bwHyUVP-Y9r!l{#O?z@bT}D@*qi`W|NiTSCMYnx2SbT!(9_A8SHjBw*rK#d!%AJ(3diZ~Oo;P2)zY?T9gUV82Qxv4sJK>&Nw92R zC*dNy2V2hawSX{P7=76ZaddH!6{jMX$0%TTh2<6^n6+*-V(&lVVYEIqr@i)j-{A2a zVMFm&+hULa9oTc22R!PL+qbDjtauo!@5{TC-}S4A2f-A<){M@P=KW+u3+w~+>3AApDL)W`{3<6GNxu z*y60kTuef9&T_5IZ%U>(l?({(n8++Ts*2=a3amG4 zQ_05olBT2rYlanlXGH8QXRP$tVar+K247Fq+kT7q5pl->P!;Em5ShAiq~FZ}&ewoU zWc_EmX!=<*#gcGRsdR2iiz2hAFU%f4w6*T^nKEZU?hFcr+gK? zbkNA6Sk1+|+LZ010ADPHJT!7^iwBT^l4(xRUm;F$RVDd*5q4JB%oHF+OJg9alh=Kq zqtuy2E%b(3)awwxFu_ng7{3Z(ix3jvPN@%;t=MYWFO~i}w?+fgfmieMmzz&STY@6J zSY%d;s@_}M$Zf-k064sOmaLsapxcgWV%NQ80@x zjUMoWM8$`nY@cLJz1MynwGmWfr|{ycYMS8(NsXgL4{Jx&^czfb>5tBtpO3Q<9rKvY zC28&wCQ;g&aGR|#33_aG7O3?XB{a})t-ds`Gmg)}WOC?_k~+rcB)L;wGTs2Y4Sv@C51W1B)J@-vxl``_U}~ndL`)_ zC`fl&k!j^REM?~m4wBH_Z*L7=QE=&=;-hmOB(rOU{(9P%uo8|ySsnAr3t%gwQA4y; z6>DGI7hrbzYVt%0cz&f{hmV|Opltn?@04`^P3p zPfv5L{m*R<&Lsim4z+&U+l8YfSXj@Owpt54=Wr&O>S|J51Q~YwY%uguG^Ds;RT+y zUC8C&;k&*tYj-V$-no6w#+rtE)g#=}c|j(V2vLWh6$a+aoM0D|BvM!FVdAY~Fd*$c zKQB(rWd2b=NLNuoaMjB~ubY>}EssixFWWCX%OwxYu9GBlRCL|BSmLpI&0Vkz}vHoNj z|5SGXcu*Ut?{6!{^q05z)P=V2Qq4^yXc~LY z`d&}G*8<>9z!JRG!=Xmb={oZ<1R-X>u$BO{$j)C3H!>c4r}-_1)Zu~hklAo6cB!VC=7-@U!U6{IhD2MZ?ESA6+DIO% zN|NzD>Tvv1=;fMNJz!y?u@}Oe?-gZWgHxhH*RH6YyvvX@~^5lT;5S z6a_xs-F{7PJRW!QvORLsw~9^%ugoMbM~n4=0T7Cy?z#?DQu)(z{ceJ1mmtK0s1?wu zRGV&9T6uv?wFEEo^VJe1DJ+O_-*;_&eaXA5V$I=e{cj%Cyh=>Gn__(G3q2L%#5VPE zjuaB3W9%L82Q_we<>|R6 z7V991iFLO;4a>W}y*xs+4L`e=a1a%>N72UTZ;w|1P?vCLv_bBD{r{>4_YGCs52{p_h z9)SPr`uL6r4nMb9Owb<=1qEBX4p{u@hO3J4fQ3C=YziTvoRRn7k>>YV`7;}6Sp)Md zT)nEQ9_u&UK=LT!8JKpLHCJ1@#96fu8mJA%;dAd@S{PkmNI8277+5>fW&FCx$vezs)&&A@_+^CI(CEmUBBuVt|Y@z*9rak>yktBo*+hBif_l>H%@A};d6MHtN9=X&>VhsOx8e)}wxt+Xz@T#QPjcW^UgyvPQ*Oz0 z*{OA_({mM^xx}`l<|<2*ecmxTPhCTeN*_7y1UAl}pZaPwP((7dma+XvVXP3Zen+3 zd3A=(l)UCw2Wiq8IaK(~PmnkWY=OzjE{s1?9_wbQb7QJa^2<~wnm>qvC;DIPInPD8BVj>*D8LCk1$$Su6HAw@#H#SN9J0vKaGqA;ch~A|@u_V# zhSNLU63~A0&|4tP)JGcsqgu}96~p)oZQx+<#>A{eTIEG|U3D_2w=5}ahi-!4XNzdf zcrn%AhxQ>SZf-y)E$@`Q99ui?X`-5b+pY1`+Nx2 zIe&ziHnJs9#SRjqt&51^G>QU7`L?8@Iw>J2bvmKTcv>l}o)sKX)zLoRM1I$AJBJ@? zZuz1NKl6k0mP|!x(u?--CLFN~7~tk9@;9y=a-*1{xzwAtJsmWP1M5%b24~_ps$z<_ zQE|2On>4aJT0*_E{fQ(;T*c)z{0u1UT#I+A<;h7}MO2Rjl2q#_JDzc@>zB#;SXbTY zb6*cfxWA)26G0*d`+={{Sjor>V8yv=xZ=lpCoOYp1wjzqEeI(X?)wfpjN0tGhL*4r zXq?j@bk{PaL1{5kUmg$)$Ps-QR?T^w7!QF~!;-S^s{r$hwkV>);?fDGn-)#Whryt- zAzFY33H~ossfk*Tb@NSY6OOK<;6UXaPjO^yy3NwYq2YqGc;j>shO8Zxf`+M0hts7&SPek=@6e|Ip7!^p4W67lFiq?aYb0 zFnJ&f#^`0% zoL?ACO=O5U_liO>?)r~dAXxJV1}Uss72$CKZFRRsB|Ty$7MwBiLetrX0-uFoDobW) zm|UDA7VHHJXAW<))C(o!zDYTb&>|vR5oJAk&6P4C1ZGwWAULtcN5AGVFp@lB!8Z6{ zFWDpLG3kt@Rjrda6CWj7SQL0xDerxtw8(kyACJ#_4Rz0 zcYg({Z?2TLZpeqz`sg*#r-Cn-&4MXgA%LuO8K|tGyM++MidT(Un(H*~fdnc!qEkR7 zAI3Byux;tx*v>p#r&EEH!(vJNsC%NDXSBF$f8f@`!9ZZ z1(o}q=othxdLDxCcNKkghN`%YD=2C>WM(CldIXeus7y$K5QSR5YU$Jr|FH`Hb9t%} z=xyxasw?3*FbKolx%t7VT|!ifAo5AXalgN(=C4R2H_dw=?dq#iU3!iJbAeoL9Cq|= ze~siuAalnX>rc+Mz|>h%`rFIvH+#1z$t?s4V-PqrQ`W2%v7sl%V7fpwgdPfk2hx=u zxAR_5NIXg>{b^{$Rm(y;Y9Hx5lhzbMXC$6${mc8`O%1cu4NWs%zrWv;;SNTN)SBm@ z2e%l51i6YMe`9JyBW1fD8`VWlDsa2zN!avqD`cMZpm z=AHU;Q|LXw+PxvAuLmXRg+X(i(~j6ID$x~2>z@~5m3t~L!%QE+N{_FW|2lJt*IcmA zQLT3FUrT3j3~^obttJ*$z)s3lc#-3eXeiJ|v9}-uvcrrVdvbbQke&9HDzy&T)ttyM z&gT(GnY8SFJ+*Y6f|O6!4)QROs0|d-r{|2?e+}2(Yx%!oKlHL702}DL+w0i3f=s(V zCVj04`EoB&0lutH!f0(-(P-ok0^?FGUnkCptx})#O>bkaIqr7(GjfKzw|4C^5q)=o z0fTAdS{?p#5y9#5PtE)jme@zpba?Xs5R7SAZ4|lupLUfvFB?6t*2})FdI}oS3!`40 z<%OD^fy+?RJAwA^3&hvVMzomOGt!Vd60qT8DafC#(PuQhbc?YEf|TKZ6VOQAcAk==j01$X0%PA%j~+NwU+!xG``*ksjZnu4 zxS}M3S0WyIZp?>g42;)nD#JQ|PXBejtKltg+%{agfxcLHPgOD-u(~kRqK1>k!$hC; z&Q{Yf5H}rU!YQXb`+U_3HZ6qZ90-CfT1H*@1^)`Q%)qT#e)x>sdjHEqjPn-TuIC1N zR{&m=LI78P!Ts+^&z@^CY;b8Qa$}|T^_<#&nD}EG`CP_158e^iLU3{J7=k8rBoP+* zq~p^opvWf!CjfU<<=`v+cpcTFDCERaejkB zUkgHFvM!Y9V}q*Ul<2e}sx+i5VIFi+r{MI>lQ;pYWPf5!K<_+~F)2|^NQQ;s0)1t> z0KOCc+nWjf2P1~2VD@%)ubn;L(E9xE;Tk(CqDBQQ%xZ7$JS;`Rt&@|P#{08oP666_ z_@Ct=s;4O`#g^$Er`)|w6LM#b*X`-ZJ|AbL(f0Of^9cXdKuqBT~DL}l{YcV40>m`d4IP8nCbc0=s`2!FTH>+ zdFHD*Jrf-XbmHyr62~Sc8Bj*sfI}0e5WP|1=7!`7NaQ_$3LNb~5P3F+24mbE(%zFrMb0 zQ$Yw&A5NVNlfmj2gHAUohXwzFhs>k?3*?Ugd6287V{{$#dH@|@>mHORt>v{F{w2W5 z_7HD`&-*r8FLwqsef@IU4%`Hz_V%7nL)Z|8&oxX$&3w^=IAgDV{EN9M(zJ_9W6|-{ zj{HDS=TH|wHximB*s^AmUi#ND#|p6%qhw&xp_{BmQ<<%Lp#lr+vyLpR*N>kib<(bf zyasyQD$k&L?V8%ShWg%wHYq;c*?heCN^*kEgppP`PS$X`c|IOI`XJE5 zRpQFzS2~N?usffdS^-UjS34bzj1uF{k@Bt~dYRUBZ7y>w=W;1>%3Lhh+*pXE-b6NH_A#D_HBK1B z_>g7Br4sYvGPF(~H8khh(|rBXsI8(}8Ah3nQqn1!WijHVe{;^0awm~?Q77Xoq+k0NsgKv<-Dr4bOhd$=P#BVH-OgH!aehP`wPV| zJ?8!DmOwUpF>llVTtbw>HyV}g6;H#ywm@#{7r|A+L>yQSr* ze>&qAd^Y6Fz)D%nogy2^K1;-Cz z3K<0i6d6@?aPfr2L6E-`9IXEje*spk*g!x)ab?6s)c?QE@&9mMU(n0td#P6*Qmr7s OPewvfyhhY8=zjqE8dpvL literal 0 HcmV?d00001 From 0df4925cecbf8c87d8c48d163ff7b24282df8027 Mon Sep 17 00:00:00 2001 From: Kristian Nordman Date: Thu, 23 Mar 2023 19:29:30 +0100 Subject: [PATCH 2/6] Write setup instructions --- ui/src/components/data-source-panel.tsx | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ui/src/components/data-source-panel.tsx b/ui/src/components/data-source-panel.tsx index ddafaa5..c4c5593 100644 --- a/ui/src/components/data-source-panel.tsx +++ b/ui/src/components/data-source-panel.tsx @@ -260,6 +260,17 @@ export default class DataSourcePanel extends React.Component + + Follow these instructions to create a token. + +
+ + Email should be of a user with access to all relevant articles + + + )} {this.state.selectedDataSource.value === 'google_drive' && ( Follow these instructions From 2abaf2adeb669dafa8edef99535fba6b9817e848 Mon Sep 17 00:00:00 2001 From: Kristian Nordman Date: Fri, 24 Mar 2023 15:01:23 +0100 Subject: [PATCH 3/6] bugfixing and trimming --- app/data_sources/zendesk.py | 37 ++----------------------------------- 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/app/data_sources/zendesk.py b/app/data_sources/zendesk.py index 8c6df2b..442fbc7 100644 --- a/app/data_sources/zendesk.py +++ b/app/data_sources/zendesk.py @@ -5,6 +5,7 @@ from typing import Dict, List, Optional from data_source_api.exception import InvalidDataSourceConfig from data_source_api.basic_document import BasicDocument, DocumentType +from data_source_api.base_data_source import BaseDataSource, ConfigField, HTMLInputType from indexing_queue import IndexingQueue import re import json @@ -19,30 +20,8 @@ #import http.client as http_client #http_client.HTTPConnection.debuglevel = 1 -class HTMLInputType(Enum): - TEXT = "text" - TEXTAREA = "textarea" - PASSWORD = "password" - -class ConfigField(BaseModel): - name: str - input_type: HTMLInputType = HTMLInputType.TEXT - label: Optional[str] = None - placeholder: Optional[str] = None - - def __init__(self, **data): - name = data.get("name") - label = data.get("label") or name.title() - data["label"] = label - data["placeholder"] = data.get("placeholder") or label - super().__init__(**data) - - class Config: - use_enum_values = True - - -class ZendeskDataSource(ABC): +class ZendeskDataSource(BaseDataSource): cached_users={} cached_sections={} @@ -178,15 +157,3 @@ def __init__(self, config: Dict, data_source_id: int, last_index_time: datetime last_index_time = datetime(2012, 1, 1) self._last_index_time = last_index_time - def _set_last_index_time(self) -> None: - with Session() as session: - data_source: DataSource = session.query(DataSource).filter_by(id=self._data_source_id).first() - data_source.last_indexed_at = datetime.now() - session.commit() - - def index(self) -> None: - try: - self._set_last_index_time() - self._feed_new_documents() - except Exception as e: - logging.exception("Error while indexing data source") From d9566a6ef659ee58c8497fdf7293a5f58bf52041 Mon Sep 17 00:00:00 2001 From: Kristian Nordman Date: Fri, 24 Mar 2023 21:38:38 +0100 Subject: [PATCH 4/6] more cleanup refactor api-calls --- app/data_sources/zendesk.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/app/data_sources/zendesk.py b/app/data_sources/zendesk.py index 442fbc7..e30af8a 100644 --- a/app/data_sources/zendesk.py +++ b/app/data_sources/zendesk.py @@ -7,18 +7,17 @@ from data_source_api.basic_document import BasicDocument, DocumentType from data_source_api.base_data_source import BaseDataSource, ConfigField, HTMLInputType from indexing_queue import IndexingQueue + import re import json import base64 import requests from parsers.html import html_to_text -from pydantic import BaseModel from db_engine import Session from schemas import DataSource -#import http.client as http_client -#http_client.HTTPConnection.debuglevel = 1 +logger = logging.getLogger(__name__) class ZendeskDataSource(BaseDataSource): @@ -36,7 +35,7 @@ def get_config_fields() -> List[ConfigField]: return [ ConfigField(label="BaseUrl", name="baseurl", type="text", placeholder="https://example.zendesk.com"), ConfigField(label="Email", name="email", type="text", placeholder="example@example.com"), - ConfigField(label="Token", name="token", type="password", placeholder="paste-your-token-here") + ConfigField(label="Token", name="token", type="password", placeholder="paste-your-token-here",input_type=HTMLInputType.PASSWORD) ] @@ -45,8 +44,6 @@ def validate_config(config: Dict) -> None: """ Validates the config and raises an exception if it's invalid. """ - print("Validating zendesk config") - print(json.dumps(config)) auth=base64.b64encode(bytes('{}/token:{}'.format(config['email'], config['token']), 'utf-8')) # bytes headers = {'Authorization': 'Basic {}'.format(auth.decode("utf-8"))} response=requests.get('{}/api/v2/help_center/en-us/articles'.format(config['baseurl']), headers=headers) @@ -75,7 +72,7 @@ def zendesk_get(self, endpoint): def zendesk_get_user(self, userid): if not userid in self.cached_users: - user=requests.get(self._config['baseurl']+'/api/v2/users/{}'.format(userid), headers=self.auth_headers()) + user=self.zendesk_get('/api/v2/users/{}'.format(userid)) parsedUser=json.loads(user.content) self.cached_users[userid] = parsedUser['user'] @@ -83,8 +80,7 @@ def zendesk_get_user(self, userid): def zendesk_get_section(self, sectionid): if not sectionid in self.cached_sections: - url=self._config['baseurl']+'/api/v2/help_center/sections/{}'.format(sectionid) - section=requests.get(url, headers=self.auth_headers()) + section=self.zendesk_get('/api/v2/help_center/sections/{}'.format(sectionid)) parsedSection=json.loads(section.content) self.cached_sections[sectionid] = parsedSection['section'] @@ -92,7 +88,7 @@ def zendesk_get_section(self, sectionid): def zendesk_get_category(self, categoryid): if not categoryid in self.cached_categories: - category=requests.get(self._config['baseurl']+'/api/v2/help_center/categories/{}'.format(categoryid), headers=self.auth_headers()) + category=self.zendesk_get('/api/v2/help_center/categories/{}'.format(categoryid)) parsedCategory=json.loads(category.content) self.cached_categories[categoryid] = parsedCategory['category'] From 4e803948ee6b0787fb2858802a89a91df68061b2 Mon Sep 17 00:00:00 2001 From: Kristian Nordman Date: Fri, 24 Mar 2023 21:43:18 +0100 Subject: [PATCH 5/6] cleanup imports actual logging --- app/data_sources/zendesk.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/app/data_sources/zendesk.py b/app/data_sources/zendesk.py index e30af8a..d69043b 100644 --- a/app/data_sources/zendesk.py +++ b/app/data_sources/zendesk.py @@ -1,8 +1,6 @@ import logging -from abc import abstractmethod, ABC from datetime import datetime -from enum import Enum -from typing import Dict, List, Optional +from typing import Dict, List from data_source_api.exception import InvalidDataSourceConfig from data_source_api.basic_document import BasicDocument, DocumentType from data_source_api.base_data_source import BaseDataSource, ConfigField, HTMLInputType @@ -14,9 +12,6 @@ import requests from parsers.html import html_to_text -from db_engine import Session -from schemas import DataSource - logger = logging.getLogger(__name__) @@ -48,6 +43,8 @@ def validate_config(config: Dict) -> None: headers = {'Authorization': 'Basic {}'.format(auth.decode("utf-8"))} response=requests.get('{}/api/v2/help_center/en-us/articles'.format(config['baseurl']), headers=headers) if response.status_code != 200: + logger.error("Failed to call Zendesk API. Responsecode: {}".format(response.status_code)) + logger.error(response.content) raise InvalidDataSourceConfig @staticmethod @@ -104,9 +101,11 @@ def _feed_new_documents(self) -> None: response=self.zendesk_get('/api/v2/help_center/en-us/articles?page={}'.format(current_page)) articles=json.loads(response.content) + logger.info("Found {} articles.".format(articles['count'])) + parsed_docs = [] for article in articles['articles']: - print(article['title']) + logger.info("Indexing {}".format(article['title'])) last_modified = datetime.strptime(article['updated_at'], "%Y-%m-%dT%H:%M:%SZ") if last_modified < self._last_index_time: continue @@ -137,7 +136,7 @@ def _feed_new_documents(self) -> None: IndexingQueue.get().feed(docs=parsed_docs) parsed_docs = [] - print("page: {}, pagecount: {}, current_page: {}".format(articles['page'], articles['page_count'], current_page)) + logger.debug("page: {}, pagecount: {}, current_page: {}".format(articles['page'], articles['page_count'], current_page)) if articles['page'] == articles['page_count']: break From 7e4ca97a48a6f6dcf86e3e59809deaba0b8d4b99 Mon Sep 17 00:00:00 2001 From: Kristian Nordman Date: Fri, 24 Mar 2023 21:55:40 +0100 Subject: [PATCH 6/6] even more logging --- app/data_sources/zendesk.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/data_sources/zendesk.py b/app/data_sources/zendesk.py index d69043b..878d6b3 100644 --- a/app/data_sources/zendesk.py +++ b/app/data_sources/zendesk.py @@ -101,11 +101,11 @@ def _feed_new_documents(self) -> None: response=self.zendesk_get('/api/v2/help_center/en-us/articles?page={}'.format(current_page)) articles=json.loads(response.content) - logger.info("Found {} articles.".format(articles['count'])) + if current_page == 1: + logger.info("Found {} articles.".format(articles['count'])) parsed_docs = [] for article in articles['articles']: - logger.info("Indexing {}".format(article['title'])) last_modified = datetime.strptime(article['updated_at'], "%Y-%m-%dT%H:%M:%SZ") if last_modified < self._last_index_time: continue @@ -134,12 +134,12 @@ def _feed_new_documents(self) -> None: total_fed += len(parsed_docs) IndexingQueue.get().feed(docs=parsed_docs) + logger.info("Processed page {}/{}. Sent {} for indexing".format(articles['page'],articles['page_count'],len(parsed_docs))) parsed_docs = [] - logger.debug("page: {}, pagecount: {}, current_page: {}".format(articles['page'], articles['page_count'], current_page)) if articles['page'] == articles['page_count']: break - + current_page+=1