diff --git a/linode_api4/groups/object_storage.py b/linode_api4/groups/object_storage.py index bbaf330d9..c42805ec1 100644 --- a/linode_api4/groups/object_storage.py +++ b/linode_api4/groups/object_storage.py @@ -1,6 +1,10 @@ +import re +import warnings from typing import List, Optional, Union from urllib import parse +from deprecated import deprecated + from linode_api4.errors import UnexpectedResponseError from linode_api4.groups import Group from linode_api4.objects import ( @@ -9,6 +13,7 @@ ObjectStorageACL, ObjectStorageBucket, ObjectStorageCluster, + ObjectStorageKeyPermission, ObjectStorageKeys, ) from linode_api4.util import drop_null_keys @@ -20,8 +25,14 @@ class ObjectStorageGroup(Group): available clusters, buckets, and managing keys and TLS/SSL certs, etc. """ + @deprecated( + reason="deprecated to use regions list API for listing available OJB clusters" + ) def clusters(self, *filters): """ + This endpoint will be deprecated to use the regions list API to list available OBJ clusters, + and a new access key API will directly expose the S3 endpoint hostname. + Returns a list of available Object Storage Clusters. You may filter this query to return only Clusters that are available in a specific region:: @@ -58,6 +69,7 @@ def keys_create( self, label: str, bucket_access: Optional[Union[dict, List[dict]]] = None, + regions: Optional[List[str]] = None, ): """ Creates a new Object Storage keypair that may be used to interact directly @@ -97,14 +109,16 @@ def keys_create( :param label: The label for this keypair, for identification only. :type label: str - :param bucket_access: One or a list of dicts with keys "cluster," - "permissions", and "bucket_name". If given, the - resulting Object Storage keys will only have the - requested level of access to the requested buckets, - if they exist and are owned by you. See the provided - :any:`bucket_access` function for a convenient way - to create these dicts. - :type bucket_access: dict or list of dict + :param bucket_access: One or a list of dicts with keys "cluster," "region", + "permissions", and "bucket_name". "cluster" key is + deprecated because multiple cluster can be placed + in the same region. Please consider switching to + regions. If given, the resulting Object Storage keys + will only have the requested level of access to the + requested buckets, if they exist and are owned by + you. See the provided :any:`bucket_access` function + for a convenient way to create these dicts. + :type bucket_access: Optional[Union[dict, List[dict]]] :returns: The new keypair, with the secret key populated. :rtype: ObjectStorageKeys @@ -115,22 +129,35 @@ def keys_create( if not isinstance(bucket_access, list): bucket_access = [bucket_access] - ba = [ - { - "permissions": c.get("permissions"), - "bucket_name": c.get("bucket_name"), - "cluster": ( - c.id - if "cluster" in c - and issubclass(type(c["cluster"]), Base) - else c.get("cluster") - ), + ba = [] + for access_rule in bucket_access: + access_rule_json = { + "permissions": access_rule.get("permissions"), + "bucket_name": access_rule.get("bucket_name"), } - for c in bucket_access - ] + + if "region" in access_rule: + access_rule_json["region"] = access_rule.get("region") + elif "cluster" in access_rule: + warnings.warn( + "'cluster' is a deprecated attribute, " + "please consider using 'region' instead.", + DeprecationWarning, + ) + access_rule_json["cluster"] = ( + access_rule.id + if "cluster" in access_rule + and issubclass(type(access_rule["cluster"]), Base) + else access_rule.get("cluster") + ) + + ba.append(access_rule_json) params["bucket_access"] = ba + if regions is not None: + params["regions"] = regions + result = self.client.post("/object-storage/keys", data=params) if not "id" in result: @@ -142,9 +169,74 @@ def keys_create( ret = ObjectStorageKeys(self.client, result["id"], result) return ret - def bucket_access(self, cluster, bucket_name, permissions): - return ObjectStorageBucket.access( - self, cluster, bucket_name, permissions + @classmethod + def bucket_access( + cls, + cluster_or_region: str, + bucket_name: str, + permissions: Union[str, ObjectStorageKeyPermission], + ): + """ + Returns a dict formatted to be included in the `bucket_access` argument + of :any:`keys_create`. See the docs for that method for an example of + usage. + + :param cluster_or_region: The region or Object Storage cluster to grant access in. + :type cluster_or_region: str + :param bucket_name: The name of the bucket to grant access to. + :type bucket_name: str + :param permissions: The permissions to grant. Should be one of "read_only" + or "read_write". + :type permissions: Union[str, ObjectStorageKeyPermission] + :param use_region: Whether to use region mode. + :type use_region: bool + + :returns: A dict formatted correctly for specifying bucket access for + new keys. + :rtype: dict + """ + + result = { + "bucket_name": bucket_name, + "permissions": permissions, + } + + if cls.is_cluster(cluster_or_region): + warnings.warn( + "Cluster ID for Object Storage APIs has been deprecated. " + "Please consider switch to a region ID (e.g., from `us-mia-1` to `us-mia`)", + DeprecationWarning, + ) + result["cluster"] = cluster_or_region + else: + result["region"] = cluster_or_region + + return result + + def buckets_in_region(self, region: str, *filters): + """ + Returns a list of Buckets in the region belonging to this Account. + + This endpoint is available for convenience. + It is recommended that instead you use the more fully-featured S3 API directly. + + API Documentation: https://www.linode.com/docs/api/object-storage/#object-storage-buckets-in-cluster-list + + :param filters: Any number of filters to apply to this query. + See :doc:`Filtering Collections` + for more details on filtering. + + :param region: The ID of an object storage region (e.g. `us-mia-1`). + :type region: str + + :returns: A list of Object Storage Buckets that in the requested cluster. + :rtype: PaginatedList of ObjectStorageBucket + """ + + return self.client._get_and_filter( + ObjectStorageBucket, + *filters, + endpoint=f"/object-storage/buckets/{region}", ) def cancel(self): @@ -197,10 +289,14 @@ def buckets(self, *filters): """ return self.client._get_and_filter(ObjectStorageBucket, *filters) + @staticmethod + def is_cluster(cluster_or_region: str): + return bool(re.match(r"^[a-z]{2}-[a-z]+-[0-9]+$", cluster_or_region)) + def bucket_create( self, - cluster, - label, + cluster_or_region: Union[str, ObjectStorageCluster], + label: str, acl: ObjectStorageACL = ObjectStorageACL.PRIVATE, cors_enabled=False, ): @@ -240,17 +336,30 @@ def bucket_create( :returns: A Object Storage Buckets that created by user. :rtype: ObjectStorageBucket """ - cluster_id = ( - cluster.id if isinstance(cluster, ObjectStorageCluster) else cluster + cluster_or_region_id = ( + cluster_or_region.id + if isinstance(cluster_or_region, ObjectStorageCluster) + else cluster_or_region ) params = { - "cluster": cluster_id, "label": label, "acl": acl, "cors_enabled": cors_enabled, } + if self.is_cluster(cluster_or_region_id): + warnings.warn( + "The cluster parameter has been deprecated for creating a object " + "storage bucket. Please consider switching to a region value. For " + "example, a cluster value of `us-mia-1` can be translated to a " + "region value of `us-mia`.", + DeprecationWarning, + ) + params["cluster"] = cluster_or_region_id + else: + params["region"] = cluster_or_region_id + result = self.client.post("/object-storage/buckets", data=params) if not "label" in result or not "cluster" in result: @@ -263,21 +372,21 @@ def bucket_create( self.client, result["label"], result["cluster"], result ) - def object_acl_config(self, cluster_id, bucket, name=None): + def object_acl_config(self, cluster_or_region_id: str, bucket, name=None): return ObjectStorageBucket( - self.client, bucket, cluster_id + self.client, bucket, cluster_or_region_id ).object_acl_config(name) def object_acl_config_update( - self, cluster_id, bucket, acl: ObjectStorageACL, name + self, cluster_or_region_id, bucket, acl: ObjectStorageACL, name ): return ObjectStorageBucket( - self.client, bucket, cluster_id + self.client, bucket, cluster_or_region_id ).object_acl_config_update(acl, name) def object_url_create( self, - cluster_id, + cluster_or_region_id, bucket, method, name, @@ -294,8 +403,8 @@ def object_url_create( API Documentation: https://www.linode.com/docs/api/object-storage/#object-storage-object-url-create - :param cluster_id: The ID of the cluster this bucket exists in. - :type cluster_id: str + :param cluster_or_region_id: The ID of the cluster or region this bucket exists in. + :type cluster_or_region_id: str :param bucket: The bucket name. :type bucket: str @@ -337,7 +446,7 @@ def object_url_create( result = self.client.post( "/object-storage/buckets/{}/{}/object-url".format( - parse.quote(str(cluster_id)), parse.quote(str(bucket)) + parse.quote(str(cluster_or_region_id)), parse.quote(str(bucket)) ), data=drop_null_keys(params), ) diff --git a/linode_api4/objects/object_storage.py b/linode_api4/objects/object_storage.py index 685925c9b..2cbcf59bd 100644 --- a/linode_api4/objects/object_storage.py +++ b/linode_api4/objects/object_storage.py @@ -1,6 +1,8 @@ from typing import Optional from urllib import parse +from deprecated import deprecated + from linode_api4.errors import UnexpectedResponseError from linode_api4.objects import ( Base, @@ -21,6 +23,11 @@ class ObjectStorageACL(StrEnum): CUSTOM = "custom" +class ObjectStorageKeyPermission(StrEnum): + READ_ONLY = "read_only" + READ_WRITE = "read_write" + + class ObjectStorageBucket(DerivedBase): """ A bucket where objects are stored in. @@ -28,12 +35,13 @@ class ObjectStorageBucket(DerivedBase): API documentation: https://www.linode.com/docs/api/object-storage/#object-storage-bucket-view """ - api_endpoint = "/object-storage/buckets/{cluster}/{label}" - parent_id_name = "cluster" + api_endpoint = "/object-storage/buckets/{region}/{label}" + parent_id_name = "region" id_attribute = "label" properties = { - "cluster": Property(identifier=True), + "region": Property(identifier=True), + "cluster": Property(), "created": Property(is_datetime=True), "hostname": Property(), "label": Property(identifier=True), @@ -57,8 +65,11 @@ def make_instance(cls, id, client, parent_id=None, json=None): """ if json is None: return None - if parent_id is None and json["cluster"]: - parent_id = json["cluster"] + + cluster_or_region = json.get("region") or json.get("cluster") + + if parent_id is None and cluster_or_region: + parent_id = cluster_or_region if parent_id: return super().make(id, client, cls, parent_id=parent_id, json=json) @@ -386,6 +397,13 @@ def object_acl_config_update(self, acl: ObjectStorageACL, name): return MappedObject(**result) + @deprecated( + reason=( + "'access' method has been deprecated in favor of the class method " + "'bucket_access' in ObjectStorageGroup, which can be accessed by " + "'client.object_storage.access'" + ) + ) def access(self, cluster, bucket_name, permissions): """ Returns a dict formatted to be included in the `bucket_access` argument @@ -411,8 +429,14 @@ def access(self, cluster, bucket_name, permissions): } +@deprecated( + reason="deprecated to use regions list API for viewing available OJB clusters" +) class ObjectStorageCluster(Base): """ + This class will be deprecated to use the regions list to view available OBJ clusters, + and a new access key API will directly expose the S3 endpoint hostname. + A cluster where Object Storage is available. API documentation: https://www.linode.com/docs/api/object-storage/#cluster-view @@ -428,6 +452,13 @@ class ObjectStorageCluster(Base): "static_site_domain": Property(), } + @deprecated( + reason=( + "'buckets_in_cluster' method has been deprecated, please consider " + "switching to 'buckets_in_region' in the object storage group (can " + "be accessed via 'client.object_storage.buckets_in_cluster')." + ) + ) def buckets_in_cluster(self, *filters): """ Returns a list of Buckets in this cluster belonging to this Account. @@ -470,4 +501,5 @@ class ObjectStorageKeys(Base): "secret_key": Property(), "bucket_access": Property(), "limited": Property(), + "regions": Property(unordered=True), } diff --git a/pyproject.toml b/pyproject.toml index 4e2c60f00..ea96865c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ] -dependencies = ["requests", "polling"] +dependencies = ["requests", "polling", "deprecated"] dynamic = ["version"] [project.optional-dependencies] diff --git a/test/fixtures/object-storage_buckets_us-east-1_example-bucket.json b/test/fixtures/object-storage_buckets_us-east-1_example-bucket.json index b8c9450b6..bb93ec99a 100644 --- a/test/fixtures/object-storage_buckets_us-east-1_example-bucket.json +++ b/test/fixtures/object-storage_buckets_us-east-1_example-bucket.json @@ -1,5 +1,6 @@ { "cluster": "us-east-1", + "region": "us-east", "created": "2019-01-01T01:23:45", "hostname": "example-bucket.us-east-1.linodeobjects.com", "label": "example-bucket", diff --git a/test/fixtures/object-storage_keys.json b/test/fixtures/object-storage_keys.json index da6c2278a..0a9181658 100644 --- a/test/fixtures/object-storage_keys.json +++ b/test/fixtures/object-storage_keys.json @@ -6,14 +6,40 @@ "id": 1, "label": "object-storage-key-1", "secret_key": "[REDACTED]", - "access_key": "testAccessKeyHere123" + "access_key": "testAccessKeyHere123", + "limited": false, + "regions": [ + { + "id": "us-east", + "s3_endpoint": "us-east-1.linodeobjects.com" + }, + { + "id": "us-west", + "s3_endpoint": "us-west-123.linodeobjects.com" + } + ] }, { "id": 2, "label": "object-storage-key-2", "secret_key": "[REDACTED]", - "access_key": "testAccessKeyHere456" + "access_key": "testAccessKeyHere456", + "limited": true, + "bucket_access": [ + { + "cluster": "us-mia-1", + "bucket_name": "example-bucket", + "permissions": "read_only", + "region": "us-mia" + } + ], + "regions": [ + { + "id": "us-mia", + "s3_endpoint": "us-mia-1.linodeobjects.com" + } + ] } ], "page": 1 -} +} \ No newline at end of file diff --git a/test/integration/models/object_storage/test_obj.py b/test/integration/models/object_storage/test_obj.py new file mode 100644 index 000000000..863eda129 --- /dev/null +++ b/test/integration/models/object_storage/test_obj.py @@ -0,0 +1,92 @@ +import time +from test.integration.conftest import get_region + +import pytest + +from linode_api4.linode_client import LinodeClient +from linode_api4.objects.object_storage import ( + ObjectStorageACL, + ObjectStorageBucket, + ObjectStorageKeyPermission, + ObjectStorageKeys, +) + + +@pytest.fixture(scope="session") +def region(test_linode_client: LinodeClient): + return get_region(test_linode_client, {"Object Storage"}).id + + +@pytest.fixture(scope="session") +def bucket(test_linode_client: LinodeClient, region: str): + bucket = test_linode_client.object_storage.bucket_create( + cluster_or_region=region, + label="bucket-" + str(time.time_ns()), + acl=ObjectStorageACL.PRIVATE, + cors_enabled=False, + ) + + yield bucket + bucket.delete() + + +@pytest.fixture(scope="session") +def obj_key(test_linode_client: LinodeClient): + key = test_linode_client.object_storage.keys_create( + label="obj-key-" + str(time.time_ns()), + ) + + yield key + key.delete() + + +@pytest.fixture(scope="session") +def obj_limited_key( + test_linode_client: LinodeClient, region: str, bucket: ObjectStorageBucket +): + key = test_linode_client.object_storage.keys_create( + label="obj-limited-key-" + str(time.time_ns()), + bucket_access=test_linode_client.object_storage.bucket_access( + cluster_or_region=region, + bucket_name=bucket.label, + permissions=ObjectStorageKeyPermission.READ_ONLY, + ), + regions=[region], + ) + + yield key + key.delete() + + +def test_keys( + test_linode_client: LinodeClient, + obj_key: ObjectStorageKeys, + obj_limited_key: ObjectStorageKeys, +): + loaded_key = test_linode_client.load(ObjectStorageKeys, obj_key.id) + loaded_limited_key = test_linode_client.load( + ObjectStorageKeys, obj_limited_key.id + ) + + assert loaded_key.label == obj_key.label + assert loaded_limited_key.label == obj_limited_key.label + + +def test_bucket( + test_linode_client: LinodeClient, + bucket: ObjectStorageBucket, +): + loaded_bucket = test_linode_client.load(ObjectStorageBucket, bucket.label) + + assert loaded_bucket.label == bucket.label + assert loaded_bucket.region == bucket.region + + +def test_bucket( + test_linode_client: LinodeClient, + bucket: ObjectStorageBucket, + region: str, +): + buckets = test_linode_client.object_storage.buckets_in_region(region=region) + assert len(buckets) >= 1 + assert any(b.label == bucket.label for b in buckets) diff --git a/test/unit/groups/__init__.py b/test/unit/groups/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/unit/groups/object_storage_test.py b/test/unit/groups/object_storage_test.py new file mode 100644 index 000000000..31c931498 --- /dev/null +++ b/test/unit/groups/object_storage_test.py @@ -0,0 +1,20 @@ +import pytest + +from linode_api4.groups.object_storage import ObjectStorageGroup + + +@pytest.mark.parametrize( + "cluster_or_region,is_cluster", + [ + ("us-east-1", True), + ("us-central-1", True), + ("us-mia-1", True), + ("us-iad-123", True), + ("us-east", False), + ("us-central", False), + ("us-mia", False), + ("us-iad", False), + ], +) +def test_is_cluster(cluster_or_region: str, is_cluster: bool): + assert ObjectStorageGroup.is_cluster(cluster_or_region) == is_cluster diff --git a/test/unit/linode_client_test.py b/test/unit/linode_client_test.py index 3facd2e95..081b27d09 100644 --- a/test/unit/linode_client_test.py +++ b/test/unit/linode_client_test.py @@ -980,6 +980,41 @@ def test_keys_create(self): self.assertEqual(m.call_url, "/object-storage/keys") self.assertEqual(m.call_data, {"label": "object-storage-key-1"}) + def test_limited_keys_create(self): + """ + Tests that you can create Object Storage Keys + """ + with self.mock_post("object-storage/keys/2") as m: + keys = self.client.object_storage.keys_create( + "object-storage-key-1", + self.client.object_storage.bucket_access( + "us-east", + "example-bucket", + "read_only", + ), + ["us-east"], + ) + + self.assertIsNotNone(keys) + self.assertEqual(keys.id, 2) + self.assertEqual(keys.label, "object-storage-key-2") + + self.assertEqual(m.call_url, "/object-storage/keys") + self.assertEqual( + m.call_data, + { + "label": "object-storage-key-1", + "bucket_access": [ + { + "permissions": "read_only", + "bucket_name": "example-bucket", + "region": "us-east", + } + ], + "regions": ["us-east"], + }, + ) + def test_transfer(self): """ Test that you can get the amount of outbound data transfer diff --git a/test/unit/objects/object_storage_test.py b/test/unit/objects/object_storage_test.py index 59317afa1..95d781a84 100644 --- a/test/unit/objects/object_storage_test.py +++ b/test/unit/objects/object_storage_test.py @@ -53,11 +53,11 @@ def test_bucket_access_modify(self): Test that you can modify bucket access settings. """ bucket_access_modify_url = ( - "/object-storage/buckets/us-east-1/example-bucket/access" + "/object-storage/buckets/us-east/example-bucket/access" ) with self.mock_post({}) as m: object_storage_bucket = ObjectStorageBucket( - self.client, "example-bucket", "us-east-1" + self.client, "example-bucket", "us-east" ) object_storage_bucket.access_modify(ObjectStorageACL.PRIVATE, True) self.assertEqual(