Add Google Drive and AWS S3 as a Remote Storage option

datashuttle/configs/aws_regions.py

@@ -0,0 +1,39 @@
+from typing import List, Literal, get_args
+
+# -----------------------------------------------------------------------------
+# AWS regions
+# -----------------------------------------------------------------------------
+
+AwsRegion = Literal[
+    "us-east-1",
+    "us-east-2",
+    "us-west-1",
+    "us-west-2",
+    "ca-central-1",
+    "eu-west-1",
+    "eu-west-2",
+    "eu-west-3",
+    "eu-north-1",
+    "eu-south-1",
+    "eu-central-1",
+    "ap-southeast-1",
+    "ap-southeast-2",
+    "ap-northeast-1",
+    "ap-northeast-2",
+    "ap-northeast-3",
+    "ap-south-1",
+    "ap-east-1",
+    "sa-east-1",
+    "il-central-1",
+    "me-south-1",
+    "af-south-1",
+    "cn-north-1",
+    "cn-northwest-1",
+    "us-gov-east-1",
+    "us-gov-west-1",
+]
+
+
+def get_aws_regions_list() -> List[str]:
+    """Return AWS S3 bucket regions as a list."""
+    return list(get_args(AwsRegion))

datashuttle/configs/canonical_configs.py

@@ -17,6 +17,7 @@
     Literal,
     Optional,
     Union,
+    get_args,
 )
 
 if TYPE_CHECKING:
@@ -25,18 +26,30 @@
 
 import typeguard
 
+from datashuttle.configs.aws_regions import AwsRegion
 from datashuttle.utils import folders, utils
 from datashuttle.utils.custom_exceptions import ConfigError
 
+connection_methods = Literal["ssh", "local_filesystem", "gdrive", "aws"]
+
+
+def get_connection_methods_list() -> List[str]:
+    """Return the canonical connection methods."""
+    return list(get_args(connection_methods))
+
 
 def get_canonical_configs() -> dict:
     """Return the only permitted types for DataShuttle config values."""
     canonical_configs = {
         "local_path": Union[str, Path],
         "central_path": Optional[Union[str, Path]],
-        "connection_method": Optional[Literal["ssh", "local_filesystem"]],
+        "connection_method": Optional[connection_methods],
         "central_host_id": Optional[str],
         "central_host_username": Optional[str],
+        "gdrive_client_id": Optional[str],
+        "gdrive_root_folder_id": Optional[str],
+        "aws_access_key_id": Optional[str],
+        "aws_region": Optional[AwsRegion],
     }
 
     return canonical_configs
@@ -101,7 +114,8 @@ def check_dict_values_raise_on_fail(config_dict: Configs) -> None:
 
     check_config_types(config_dict)
 
-    raise_on_bad_local_only_project_configs(config_dict)
+    if config_dict["connection_method"] not in ["aws", "gdrive"]:
+        raise_on_bad_local_only_project_configs(config_dict)
 
     if list(config_dict.keys()) != list(canonical_dict.keys()):
         utils.log_and_raise_error(
@@ -130,6 +144,29 @@ def check_dict_values_raise_on_fail(config_dict: Configs) -> None:
             ConfigError,
         )
 
+    # Check gdrive settings
+    elif config_dict["connection_method"] == "gdrive":
+        if not config_dict["gdrive_root_folder_id"]:
+            utils.log_and_raise_error(
+                "'gdrive_root_folder_id' is required if 'connection_method' "
+                "is 'gdrive'.",
+                ConfigError,
+            )
+
+        if not config_dict["gdrive_client_id"]:
+            utils.log_and_message(
+                "`gdrive_client_id` not found in config. default rlcone client will be used (slower)."
+            )
+
+    # Check AWS settings
+    elif config_dict["connection_method"] == "aws" and (
+        not config_dict["aws_access_key_id"] or not config_dict["aws_region"]
+    ):
+        utils.log_and_raise_error(
+            "Both aws_access_key_id and aws_region must be present for AWS connection.",
+            ConfigError,
+        )
+
     # Initialise the local project folder
     utils.print_message_to_user(
         f"Making project folder at: {config_dict['local_path']}"

datashuttle/configs/config_class.py

@@ -120,16 +120,48 @@ def dump_to_file(self) -> None:
     def load_from_file(self) -> None:
         """Load a config dict saved at .yaml file.
 
-        Note this will not automatically check the configs are valid,
-        this requires calling self.check_dict_values_raise_on_fail().
+        This will do a minimal backwards compatibility check and
+        add config keys to ensure backwards compatibility with new connection
+        methods added to Datashuttle.
+
+        However, this will not automatically check the configs are valid, this
+        requires calling self.check_dict_values_raise_on_fail()
         """
         with open(self.file_path) as config_file:
             config_dict = yaml.full_load(config_file)
 
         load_configs.convert_str_and_pathlib_paths(config_dict, "str_to_path")
 
+        self.update_config_for_backward_compatability_if_required(config_dict)
+
         self.data = config_dict
 
+    def update_config_for_backward_compatability_if_required(
+        self, config_dict: Dict
+    ):
+        """Add keys introduced in later Datashuttle versions if they are missing."""
+        canonical_config_keys_to_add = [
+            "gdrive_client_id",
+            "gdrive_root_folder_id",
+            "aws_access_key_id",
+            "aws_region",
+        ]
+
+        # All keys shall be missing for a backwards compatibility update
+        if not (
+            all(
+                key in config_dict.keys()
+                for key in canonical_config_keys_to_add
+            )
+        ):
+            assert not any(
+                key in config_dict.keys()
+                for key in canonical_config_keys_to_add
+            )
+
+            for key in canonical_config_keys_to_add:
+                config_dict[key] = None
+
     # -------------------------------------------------------------------------
     # Utils
     # -------------------------------------------------------------------------
@@ -186,6 +218,10 @@ def get_base_folder(
     ) -> Path:
         """Return the full base path for the given top-level folder.
 
+        If the connection method is `aws` or `drive`, the base path
+        might be `None` (e.g. if the Google Drive is the project folder).
+        In this case, the base path is ignored.
+
         Parameters
         ----------
         base
@@ -202,7 +238,12 @@ def get_base_folder(
         if base == "local":
             base_folder = self["local_path"] / top_level_folder
         elif base == "central":
-            base_folder = self["central_path"] / top_level_folder
+            if self["central_path"] is None:
+                # This path should never be triggered for local-only
+                assert self["connection_method"] in ["aws", "gdrive"]
+                base_folder = Path(top_level_folder)
+            else:
+                base_folder = self["central_path"] / top_level_folder
 
         return base_folder
 
@@ -299,8 +340,4 @@ def is_local_project(self):
         A project is 'local-only' if it has no `central_path` and `connection_method`.
         It can be used to make folders and validate, but not for transfer.
         """
-        canonical_configs.raise_on_bad_local_only_project_configs(self)
-
-        params_are_none = canonical_configs.local_only_configs_are_none(self)
-
-        return all(params_are_none)
+        return self["connection_method"] is None