Use requester-pays for mirroring in AnVIL deployments (#7794)

Author: nadove-ucsc

src/azul/drs.py

@@ -359,15 +359,21 @@ class DRSObject:
     _http_client: HttpClient
     _url: furl
 
-    def get(self, access_method: AccessMethod = AccessMethod.https) -> Access:
+    def get(self,
+            access_method: AccessMethod = AccessMethod.https,
+            access_headers: Mapping[str, str] | None = None
+            ) -> Access:
         """
         Returns access to the content of the data object identified by the
         given URI. The scheme of the URL in the returned access object depends
         on the access method specified.
         """
-        return self._get(access_method)
+        return self._get(access_method, access_headers)
 
-    def _get(self, access_method: AccessMethod) -> Access:
+    def _get(self,
+             access_method: AccessMethod,
+             access_headers: Mapping[str, str] | None
+             ) -> Access:
         url = self._url
         while True:
             response = self._request(url)
@@ -387,9 +393,9 @@ def _get(self, access_method: AccessMethod) -> Access:
                     # https://github.com/ga4gh/data-repository-service-schemas/issues/361
                     assert access_method is AccessMethod.gs, R(
                         'Unexpected access method', access_method)
-                    return self._get_access(access_id, AccessMethod.https)
+                    return self._get_access(access_id, AccessMethod.https, access_headers)
                 elif access_id is not None:
-                    return self._get_access(access_id, access_method)
+                    return self._get_access(access_id, access_method, access_headers)
                 elif access_url is not None:
                     scheme = furl(access_url['url']).scheme
                     assert scheme == access_method.scheme, R(
@@ -406,11 +412,15 @@ def _get(self, access_method: AccessMethod) -> Access:
             else:
                 raise DRSStatusException(url, response)
 
-    def _get_access(self, access_id: str, access_method: AccessMethod) -> Access:
+    def _get_access(self,
+                    access_id: str,
+                    access_method: AccessMethod,
+                    access_headers: Mapping[str, str] | None
+                    ) -> Access:
         url = self._url.copy()
         url.path.add(['access', access_id])
         while True:
-            response = self._request(url)
+            response = self._request(url, headers=access_headers)
             if response.status == 200:
                 response_data = json_dict(json.loads(response.data))
                 scheme = furl(response_data['url']).scheme
@@ -429,8 +439,8 @@ def _get_access(self, access_id: str, access_method: AccessMethod) -> Access:
             else:
                 raise DRSStatusException(url, response)
 
-    def _request(self, url: furl) -> urllib3.BaseHTTPResponse:
-        return self._http_client.request('GET', str(url), redirect=False)
+    def _request(self, url: furl, **kwargs) -> urllib3.BaseHTTPResponse:
+        return self._http_client.request('GET', str(url), **kwargs, redirect=False)
 
 
 class DRSStatusException(Exception):

src/azul/indexer/mirror_service.py

@@ -748,7 +748,13 @@ def _repository_url(self, file: File) -> furl:
         assert file.drs_uri is not None, R(
             'File cannot be downloaded', file)
         object = self.repository_plugin.drs_object(file.drs_uri)
-        access = object.get(AccessMethod.gs)
+        if config.is_anvil_enabled(self.catalog):
+            project = config.google_project()
+            assert project is not None, R('No Google project configured')
+            access_headers = {'x-user-project': project}
+        else:
+            access_headers = None
+        access = object.get(AccessMethod.gs, access_headers)
         assert access.method is AccessMethod.https, access
         return furl(access.url)