Stabilize high level scan zip upload and page iterator (#9)

* bump requests

* better page load fail handling

* handle upload failures better

Author: cx-nathan-leach

.vscode/launch.json

@@ -10,7 +10,8 @@
             "request": "launch",
             "program": "${file}",
             "console": "integratedTerminal",
-            "env": { "PYTHONPATH" : "${workspaceFolder}:${workspaceFolder}/tests"}
+            "env": { "PYTHONPATH" : "${workspaceFolder}:${workspaceFolder}/tests"},
+            "envFile": "${workspaceFolder}/.env"
         }
     ]
 }

cxone_api/high/scans.py

@@ -10,6 +10,7 @@
 from ..low.scan_configuration import retrieve_tenant_configuration
 from ..low.uploads import generate_upload_link, upload_to_link
 from .projects import ProjectRepoConfig
+import asyncio, logging
 
 
 class ScanInvoker:
@@ -96,12 +97,31 @@ async def scan_get_scanid(cxone_client : CxOneClient, project_repo : ProjectRepo
 
 
     @staticmethod
-    async def __upload_zip(cxone_client : CxOneClient, zip_path : str) -> str:
-        upload_url = json_on_ok(await generate_upload_link(cxone_client))['url']
+    async def __upload_zip(cxone_client : CxOneClient, zip_path : str, max_retries : int = 5, retry_delay_s : int = 3) -> str:
+        _log = logging.getLogger("ScanInvoker.__upload_zip")
+
+        upload_url = None
+        retries = 0
+        while retries < max_retries:
+            if not retries == 0:
+                await asyncio.sleep(retry_delay_s)
+            retries += 1
+
+            link_response = await generate_upload_link(cxone_client)
+            if link_response.ok:
+                upload_url = link_response.json()['url']
+            else:
+                _log.debug(f"Failed to generate link: {link_response.status_code} {link_response.text}")
+                continue
+
+            upload_response = await upload_to_link(cxone_client, upload_url, zip_path)
+            if not upload_response.ok:
+                _log.debug(f"Failed to upload zip: {link_response.status_code} {link_response.text}")
+                upload_url = None
+                continue
 
-        upload_response = await upload_to_link(cxone_client, upload_url, zip_path)
-        if not upload_response.ok:
-            return None
+        if upload_url is None:
+            raise ScanException("Failed to upload the zip for scan.")
 
         return upload_url
 

cxone_api/util.py

@@ -1,4 +1,4 @@
-import re, urllib, requests
+import re, urllib, requests, logging, functools, inspect, asyncio
 from datetime import datetime
 from requests import Response
 from requests.compat import urljoin
@@ -83,6 +83,8 @@ def normalized_string(s):
 
 
     def decorator(wrapped):
+
+        @functools.wraps(wrapped)
         async def wrapper(*inner_args, **inner_kwargs):
 
             normalized = {}
@@ -113,7 +115,8 @@ async def wrapper(*inner_args, **inner_kwargs):
 
 
 
-async def page_generator(coro, array_element=None, offset_param='offset', offset_init_value=0, offset_is_by_count=True, **kwargs):
+async def page_generator(coro, array_element=None, offset_param='offset', offset_init_value=0, offset_is_by_count=True, 
+                         page_retries_max=5, page_retry_delay_s=3, **kwargs):
     """
     An async generator function that is used to automatically fetch the next page
     of results from the API when the API supports result paging.
@@ -128,24 +131,42 @@ async def page_generator(coro, array_element=None, offset_param='offset', offset
         offset_is_by_count - Set to true (default) if the API next offset is indicated by count of elements retrieved.
                              If set to false, the offset is incremented by one to indicate a page offset where the count
                              of results per page is set by other parameters.
+        page_retries_max - The number of retries to fetch a page in the event of an error.  Defaults to 5.
+        page_retry_delay_s - The number of seconds to delay the next page fetch retry.  Defaults to 3 seconds.
+        
         kwargs - Keyword args passed to the coroutine at the time the coroutine is executed.
     """
+    _log = logging.getLogger(f"page_generator:{inspect.unwrap(coro).__name__}")
+
     offset = offset_init_value
     buf = []
+    retries = 0
+
 
     while True:
         if len(buf) == 0:
-            kwargs[offset_param] = offset
-            json = (await coro(**kwargs)).json()
-            buf = json[array_element] if array_element is not None else json
-
-            if buf is None or len(buf) == 0:
-                return
-
-            if offset_is_by_count:
-                offset = offset + len(buf)
-            else:
-                offset += 1
+            try:
+                kwargs[offset_param] = offset
+                json = (await coro(**kwargs)).json()
+                buf = json[array_element] if array_element is not None else json
+                retries = 0
+
+                if buf is None or len(buf) == 0:
+                    return
+
+                if offset_is_by_count:
+                    offset = offset + len(buf)
+                else:
+                    offset += 1
+            except BaseException as ex:
+                if retries < page_retries_max:
+                    _log.debug(f"Exception fetching next page, will retry: {ex}")
+                    await asyncio.sleep(page_retry_delay_s)
+                    retries += 1
+                    continue
+                else:
+                    _log.debug(f"Abort after {retries} retries", ex)
+                    raise
 
         yield buf.pop()
 

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "cxone_api"
 dynamic = ["version"]
 dependencies = [
-    "requests==2.32.3",
+    "requests==2.32.4",
     "jsonpath_ng==1.7.0",
     "validators==0.34.0",
     "dataclasses-json==0.6.7",