retry reading from job_metadata.json on NFS (#1300)

* accommodate for apparent NFS latency #1255

* avoid delay in non-result get job metadata #1255

Author: bossie

openeogeotrellis/backend.py

@@ -32,6 +32,7 @@
 import pkg_resources
 import pystac
 import requests
+import reretry
 import shapely.geometry.base
 from deprecated import deprecated
 from geopyspark import LayerType, Pyramid, TiledRasterLayer
@@ -2830,6 +2831,12 @@ def load_results_metadata(self, job_id: str, user_id: str) -> dict:
 
         return {}
 
+    @staticmethod
+    @reretry.retry(exceptions=FileNotFoundError, tries=5, delay=1, backoff=2, logger=logger)
+    def _load_results_metadata_from_file(metadata_file: Path):
+        with open(metadata_file) as f:
+            return json.load(f)
+
     @staticmethod
     def _load_results_metadata_from_uri(results_metadata_uri: Optional[str], job_id: str) -> Optional[dict]:
         # TODO: reduce code duplication with load_results_metadata
@@ -2843,10 +2850,9 @@ def _load_results_metadata_from_uri(results_metadata_uri: Optional[str], job_id:
         uri_parts = urlparse(results_metadata_uri)
 
         if uri_parts.scheme == "file":
-            file_path = uri_parts.path
+            file_path = Path(uri_parts.path)
             try:
-                with open(file_path) as f:
-                    return json.load(f)
+                return GpsBatchJobs._load_results_metadata_from_file(file_path)
             except FileNotFoundError:
                 logger.debug(
                     f"File with results metadata {file_path} does not exist; this is expected and not "

openeogeotrellis/job_registry.py

@@ -6,6 +6,7 @@
 import random
 from datetime import datetime, timedelta
 from decimal import Decimal
+from pathlib import Path
 from typing import Any, List, Dict, Callable, Union, Optional, Iterator, Tuple
 
 import kazoo
@@ -900,7 +901,7 @@ def _load_results_metadata_from_uri(results_metadata_uri: Optional[str], job_id:
         uri_parts = urlparse(results_metadata_uri)
 
         if uri_parts.scheme == "file":
-            file_path = uri_parts.path
+            file_path = Path(uri_parts.path)
             try:
                 with open(file_path) as f:
                     return json.load(f)