Merge pull request #49 from nextmv-io/merschformann/fix-large-input-upload

Fixes making large input runs

Author: sebastian-quintero

nextmv/cloud/application.py

@@ -875,11 +875,9 @@ def upload_large_input(
         if isinstance(input, Dict):
             input = json.dumps(input)
 
-        _ = self.client.request(
-            method="PUT",
-            endpoint=upload_url.upload_url,
+        _ = self.client.upload_to_presigned_url(
+            url=upload_url.upload_url,
             data=input,
-            headers={"Content-Type": "application/json"},
         )
 
     def upload_url(self) -> UploadURL:

nextmv/cloud/client.py

@@ -180,6 +180,53 @@ def request(
 
         return response
 
+    def upload_to_presigned_url(
+        self,
+        data: Union[Dict[str, Any], str],
+        url: str,
+    ) -> None:
+        """
+        Method to upload data to a presigned URL of the Nextmv Cloud API.
+        Args:
+            data: data to upload.
+            url: URL to upload the data to.
+        """
+
+        upload_data = None
+        if isinstance(data, Dict):
+            upload_data = json.dumps(data, separators=(",", ":"))
+        elif isinstance(data, str):
+            upload_data = data
+        else:
+            raise ValueError("data must be a dictionary or a string")
+
+        session = requests.Session()
+        retries = Retry(
+            total=self.max_retries,
+            backoff_factor=self.backoff_factor,
+            backoff_jitter=self.backoff_jitter,
+            backoff_max=self.backoff_max,
+            status_forcelist=self.status_forcelist,
+            allowed_methods=self.allowed_methods,
+        )
+        adapter = HTTPAdapter(max_retries=retries)
+        session.mount("https://", adapter)
+        kwargs = {
+            "url": url,
+            "timeout": self.timeout,
+            "data": upload_data,
+        }
+
+        response = session.put(**kwargs)
+
+        try:
+            response.raise_for_status()
+        except requests.HTTPError as e:
+            raise requests.HTTPError(
+                f"upload to presigned URL {url} failed with "
+                + f"status code {response.status_code} and message: {response.text}"
+            ) from e
+
     def _set_headers_api_key(self, api_key: str) -> None:
         """Sets the API key to use for requests to the Nextmv Cloud API."""
 
@@ -202,5 +249,8 @@ def get_size(obj: Union[Dict[str, Any], IO[bytes]]) -> int:
         obj.seek(0)  # Reset the cursor to the beginning of the file
         return size
 
+    elif isinstance(obj, str):
+        return len(obj.encode("utf-8"))
+
     else:
         raise TypeError("Unsupported type. Only dictionaries and file objects are supported.")