Merge branch 'develop' – Fix ingest_ns_document

Author: szabozoltan69

api/management/commands/ingest_ns_document.py

@@ -76,7 +76,8 @@ def fetch_country_documents(self, api_key, country_ns_code):
         if not data:
             return None
         documents = []
-        for item in data["documents"]:
+        items = data[0]["documents"] if len(data) == 1 else data["documents"]
+        for item in items:
             document_info = {
                 "name": item["name"],
                 "url": item["url"],

lang/translation.py

@@ -1,4 +1,5 @@
 import logging
+import threading
 
 import boto3
 import requests
@@ -12,6 +13,9 @@
 # Array of language : ['en', 'es', 'fr', ....]
 AVAILABLE_LANGUAGES = [lang for lang, _ in settings.LANGUAGES]
 
+IFRC_TRANSLATION_CALL_COUNT = 0
+IFRC_TRANSLATION_CALL_LOCK = threading.Lock()
+
 
 class BaseTranslator:
     def _fake_translation(self, text, dest_language, source_language):
@@ -99,6 +103,11 @@ def translate_text(self, text, dest_language, source_language=None):
             # NOTE: Mocking for test purpose
             return self._fake_translation(text, dest_language, source_language)
 
+        global IFRC_TRANSLATION_CALL_COUNT
+        with IFRC_TRANSLATION_CALL_LOCK:
+            IFRC_TRANSLATION_CALL_COUNT += 1
+            logger.info(f"IFRC translation API call count: {IFRC_TRANSLATION_CALL_COUNT}")
+
         # A dirty workaround to handle oversized HTML+CSS texts, usually tables:
         textTail = ""
         if len(text) > settings.AZURE_TRANSL_LIMIT:
@@ -124,6 +133,7 @@ def translate_text(self, text, dest_language, source_language=None):
             # NOTE: Sending 'text' throws 500 from IFRC translation endpoint
             # So only sending if html
             payload["textType"] = "html"
+        logger.info(f"IFRC translation API call content: {text[:30]}... to {dest_language} from {source_language}")
         response = requests.post(
             self.url,
             headers=self.headers,