From ef89bac578c84dbaeebe3bc00121368bb4cb23fb Mon Sep 17 00:00:00 2001 From: Zhiwei Liang <121905282+zliang-akamai@users.noreply.github.com> Date: Thu, 19 Sep 2024 11:11:04 -0400 Subject: [PATCH 01/11] Add VPC Grant and Refactor `UserGrants` class (#455) * Add VPC grant; refactor UserGrants and make it serializable * Fix warning by adding `r` prior to a regex * Add `.DS_Store` into .gitignore --- .gitignore | 1 + linode_api4/objects/account.py | 85 ++++++++------ .../linode_client/test_linode_client.py | 2 +- test/unit/objects/account_test.py | 106 +++++++++++++++--- 4 files changed, 145 insertions(+), 49 deletions(-) diff --git a/.gitignore b/.gitignore index 6043f36e5..7beded74d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ docs/_build/* venv baked_version .vscode +.DS_Store diff --git a/linode_api4/objects/account.py b/linode_api4/objects/account.py index 9365a9127..4777ff1c4 100644 --- a/linode_api4/objects/account.py +++ b/linode_api4/objects/account.py @@ -5,21 +5,20 @@ import requests from linode_api4.errors import ApiError, UnexpectedResponseError -from linode_api4.objects import ( - DATE_FORMAT, - Base, - DerivedBase, - Domain, - Image, - Instance, - Property, - StackScript, - Volume, -) +from linode_api4.objects import DATE_FORMAT, Volume +from linode_api4.objects.base import Base, Property +from linode_api4.objects.database import Database +from linode_api4.objects.dbase import DerivedBase +from linode_api4.objects.domain import Domain +from linode_api4.objects.image import Image +from linode_api4.objects.linode import Instance, StackScript from linode_api4.objects.longview import LongviewClient, LongviewSubscription +from linode_api4.objects.networking import Firewall from linode_api4.objects.nodebalancer import NodeBalancer from linode_api4.objects.profile import PersonalAccessToken from linode_api4.objects.support import SupportTicket +from linode_api4.objects.volume import Volume +from linode_api4.objects.vpc import VPC class Account(Base): @@ -554,10 +553,6 @@ def get_obj_grants(): """ Returns Grant keys mapped to Object types. """ - from linode_api4.objects import ( # pylint: disable=import-outside-toplevel - Database, - Firewall, - ) return ( ("linode", Instance), @@ -569,6 +564,7 @@ def get_obj_grants(): ("longview", LongviewClient), ("database", Database), ("firewall", Firewall), + ("vpc", VPC), ) @@ -641,10 +637,47 @@ def _populate(self, json): self.global_grants = type("global_grants", (object,), json["global"]) for key, cls in get_obj_grants(): - lst = [] - for gdct in json[key]: - lst.append(Grant(self._client, cls, gdct)) - setattr(self, key, lst) + if key in json: + lst = [] + for gdct in json[key]: + lst.append(Grant(self._client, cls, gdct)) + setattr(self, key, lst) + + @property + def _global_grants_dict(self): + """ + The global grants stored in this object. + """ + return { + k: v + for k, v in vars(self.global_grants).items() + if not k.startswith("_") + } + + @property + def _grants_dict(self): + """ + The grants stored in this object. + """ + grants = {} + for key, _ in get_obj_grants(): + if hasattr(self, key): + lst = [] + for cg in getattr(self, key): + lst.append(cg._serialize()) + grants[key] = lst + + return grants + + def _serialize(self): + """ + Returns the user grants in as JSON the api will accept. + This is only relevant in the context of UserGrants.save + """ + return { + "global": self._global_grants_dict, + **self._grants_dict, + } def save(self): """ @@ -653,19 +686,7 @@ def save(self): API Documentation: https://techdocs.akamai.com/linode-api/reference/put-user-grants """ - req = { - "global": { - k: v - for k, v in vars(self.global_grants).items() - if not k.startswith("_") - }, - } - - for key, _ in get_obj_grants(): - lst = [] - for cg in getattr(self, key): - lst.append(cg._serialize()) - req[key] = lst + req = self._serialize() result = self._client.put( UserGrants.api_endpoint.format(username=self.username), data=req diff --git a/test/integration/linode_client/test_linode_client.py b/test/integration/linode_client/test_linode_client.py index 92224abd4..4f8f97f4a 100644 --- a/test/integration/linode_client/test_linode_client.py +++ b/test/integration/linode_client/test_linode_client.py @@ -222,7 +222,7 @@ def test_get_account_settings(test_linode_client): assert account_settings._populated == True assert re.search( - "'network_helper':\s*(True|False)", str(account_settings._raw_json) + r"'network_helper':\s*(True|False)", str(account_settings._raw_json) ) diff --git a/test/unit/objects/account_test.py b/test/unit/objects/account_test.py index 053cc3d0e..1f9da98fb 100644 --- a/test/unit/objects/account_test.py +++ b/test/unit/objects/account_test.py @@ -1,3 +1,5 @@ +from collections.abc import Iterable +from copy import deepcopy from datetime import datetime from test.unit.base import ClientBaseCase @@ -21,10 +23,12 @@ ServiceTransfer, StackScript, User, + UserGrants, Volume, get_obj_grants, ) from linode_api4.objects.account import ChildAccount +from linode_api4.objects.vpc import VPC class InvoiceTest(ClientBaseCase): @@ -204,22 +208,6 @@ def test_get_payment_method(self): self.assertTrue(paymentMethod.is_default) self.assertEqual(paymentMethod.type, "credit_card") - def test_get_user_grant(self): - """ - Tests that a user grant is loaded correctly - """ - grants = get_obj_grants() - - self.assertTrue(grants.count(("linode", Instance)) > 0) - self.assertTrue(grants.count(("domain", Domain)) > 0) - self.assertTrue(grants.count(("stackscript", StackScript)) > 0) - self.assertTrue(grants.count(("nodebalancer", NodeBalancer)) > 0) - self.assertTrue(grants.count(("volume", Volume)) > 0) - self.assertTrue(grants.count(("image", Image)) > 0) - self.assertTrue(grants.count(("longview", LongviewClient)) > 0) - self.assertTrue(grants.count(("database", Database)) > 0) - self.assertTrue(grants.count(("firewall", Firewall)) > 0) - def test_payment_method_make_default(self): """ Tests that making a payment method default creates the correct api request. @@ -309,3 +297,89 @@ def test_child_account_create_token(self): token = child_account.create_token() self.assertEqual(token.token, "abcdefghijklmnop") self.assertEqual(m.call_data, {}) + + +def test_get_user_grant(): + """ + Tests that a user grant is loaded correctly + """ + grants = get_obj_grants() + + assert grants.count(("linode", Instance)) > 0 + assert grants.count(("domain", Domain)) > 0 + assert grants.count(("stackscript", StackScript)) > 0 + assert grants.count(("nodebalancer", NodeBalancer)) > 0 + assert grants.count(("volume", Volume)) > 0 + assert grants.count(("image", Image)) > 0 + assert grants.count(("longview", LongviewClient)) > 0 + assert grants.count(("database", Database)) > 0 + assert grants.count(("firewall", Firewall)) > 0 + assert grants.count(("vpc", VPC)) > 0 + + +def test_user_grants_serialization(): + """ + Tests that user grants from JSON is serialized correctly + """ + user_grants_json = { + "database": [ + {"id": 123, "label": "example-entity", "permissions": "read_only"} + ], + "domain": [ + {"id": 123, "label": "example-entity", "permissions": "read_only"} + ], + "firewall": [ + {"id": 123, "label": "example-entity", "permissions": "read_only"} + ], + "global": { + "account_access": "read_only", + "add_databases": True, + "add_domains": True, + "add_firewalls": True, + "add_images": True, + "add_linodes": True, + "add_longview": True, + "add_nodebalancers": True, + "add_placement_groups": True, + "add_stackscripts": True, + "add_volumes": True, + "add_vpcs": True, + "cancel_account": False, + "child_account_access": True, + "longview_subscription": True, + }, + "image": [ + {"id": 123, "label": "example-entity", "permissions": "read_only"} + ], + "linode": [ + {"id": 123, "label": "example-entity", "permissions": "read_only"} + ], + "longview": [ + {"id": 123, "label": "example-entity", "permissions": "read_only"} + ], + "nodebalancer": [ + {"id": 123, "label": "example-entity", "permissions": "read_only"} + ], + "stackscript": [ + {"id": 123, "label": "example-entity", "permissions": "read_only"} + ], + "volume": [ + {"id": 123, "label": "example-entity", "permissions": "read_only"} + ], + "vpc": [ + {"id": 123, "label": "example-entity", "permissions": "read_only"} + ], + } + + expected_serialized_grants = deepcopy(user_grants_json) + + for grants in expected_serialized_grants.values(): + if isinstance(grants, Iterable): + for grant in grants: + if isinstance(grant, dict) and "label" in grant: + del grant["label"] + + assert ( + UserGrants(None, None, user_grants_json)._serialize() + == expected_serialized_grants + ) From 35416b569ecfb396f488538f11efe6a02d327324 Mon Sep 17 00:00:00 2001 From: Ye Chen <127243817+yec-akamai@users.noreply.github.com> Date: Tue, 24 Sep 2024 13:18:02 -0400 Subject: [PATCH 02/11] Refactor regions in image replicate tests; Add LA notice (#461) * refactor tests * add la notice * disable too-many-positional-arguments --- .pylintrc | 2 +- linode_api4/objects/image.py | 2 ++ test/integration/conftest.py | 10 ++++++-- test/integration/models/image/test_image.py | 28 ++++++++++++++------- 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/.pylintrc b/.pylintrc index 2084a0c5d..49a156351 100644 --- a/.pylintrc +++ b/.pylintrc @@ -50,7 +50,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=consider-using-dict-items,blacklisted-name,invalid-name,missing-docstring,empty-docstring,unneeded-not,singleton-comparison,misplaced-comparison-constant,unidiomatic-typecheck,consider-using-enumerate,consider-iterating-dictionary,bad-classmethod-argument,bad-mcs-method-argument,bad-mcs-classmethod-argument,single-string-used-for-slots,line-too-long,too-many-lines,trailing-whitespace,missing-final-newline,trailing-newlines,multiple-statements,superfluous-parens,bad-whitespace,mixed-line-endings,unexpected-line-ending-format,bad-continuation,wrong-spelling-in-comment,wrong-spelling-in-docstring,invalid-characters-in-docstring,multiple-imports,wrong-import-order,ungrouped-imports,wrong-import-position,old-style-class,len-as-condition,fatal,astroid-error,parse-error,method-check-failed,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,literal-comparison,no-self-use,no-classmethod-decorator,no-staticmethod-decorator,cyclic-import,duplicate-code,too-many-ancestors,too-many-instance-attributes,too-few-public-methods,too-many-public-methods,too-many-return-statements,too-many-branches,too-many-arguments,too-many-locals,too-many-statements,too-many-boolean-expressions,consider-merging-isinstance,too-many-nested-blocks,simplifiable-if-statement,redefined-argument-from-local,no-else-return,consider-using-ternary,trailing-comma-tuple,unreachable,dangerous-default-value,pointless-statement,pointless-string-statement,expression-not-assigned,unnecessary-pass,unnecessary-lambda,duplicate-key,deprecated-lambda,assign-to-new-keyword,useless-else-on-loop,exec-used,eval-used,confusing-with-statement,using-constant-test,lost-exception,assert-on-tuple,attribute-defined-outside-init,bad-staticmethod-argument,protected-access,arguments-differ,signature-differs,abstract-method,super-init-not-called,no-init,non-parent-init-called,useless-super-delegation,unnecessary-semicolon,bad-indentation,mixed-indentation,lowercase-l-suffix,wildcard-import,deprecated-module,relative-import,reimported,import-self,misplaced-future,fixme,invalid-encoded-data,global-variable-undefined,global-variable-not-assigned,global-statement,global-at-module-level,unused-import,unused-variable,unused-argument,unused-wildcard-import,redefined-outer-name,redefined-builtin,redefine-in-handler,undefined-loop-variable,cell-var-from-loop,bare-except,broad-except,duplicate-except,nonstandard-exception,binary-op-exception,property-on-old-class,logging-not-lazy,logging-format-interpolation,bad-format-string-key,unused-format-string-key,bad-format-string,missing-format-argument-key,unused-format-string-argument,format-combined-specification,missing-format-attribute,invalid-format-index,anomalous-backslash-in-string,anomalous-unicode-escape-in-string,bad-open-mode,boolean-datetime,redundant-unittest-assert,deprecated-method,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call,useless-object-inheritance,comparison-with-callable,bad-option-value,consider-using-f-string,unspecified-encoding,missing-timeout,unnecessary-dunder-call,no-value-for-parameter,c-extension-no-member,attribute-defined-outside-init,use-a-generator +disable=consider-using-dict-items,blacklisted-name,invalid-name,missing-docstring,empty-docstring,unneeded-not,singleton-comparison,misplaced-comparison-constant,unidiomatic-typecheck,consider-using-enumerate,consider-iterating-dictionary,bad-classmethod-argument,bad-mcs-method-argument,bad-mcs-classmethod-argument,single-string-used-for-slots,line-too-long,too-many-lines,trailing-whitespace,missing-final-newline,trailing-newlines,multiple-statements,superfluous-parens,bad-whitespace,mixed-line-endings,unexpected-line-ending-format,bad-continuation,wrong-spelling-in-comment,wrong-spelling-in-docstring,invalid-characters-in-docstring,multiple-imports,wrong-import-order,ungrouped-imports,wrong-import-position,old-style-class,len-as-condition,fatal,astroid-error,parse-error,method-check-failed,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,literal-comparison,no-self-use,no-classmethod-decorator,no-staticmethod-decorator,cyclic-import,duplicate-code,too-many-ancestors,too-many-instance-attributes,too-few-public-methods,too-many-public-methods,too-many-return-statements,too-many-branches,too-many-arguments,too-many-locals,too-many-statements,too-many-boolean-expressions,consider-merging-isinstance,too-many-nested-blocks,simplifiable-if-statement,redefined-argument-from-local,no-else-return,consider-using-ternary,trailing-comma-tuple,unreachable,dangerous-default-value,pointless-statement,pointless-string-statement,expression-not-assigned,unnecessary-pass,unnecessary-lambda,duplicate-key,deprecated-lambda,assign-to-new-keyword,useless-else-on-loop,exec-used,eval-used,confusing-with-statement,using-constant-test,lost-exception,assert-on-tuple,attribute-defined-outside-init,bad-staticmethod-argument,protected-access,arguments-differ,signature-differs,abstract-method,super-init-not-called,no-init,non-parent-init-called,useless-super-delegation,unnecessary-semicolon,bad-indentation,mixed-indentation,lowercase-l-suffix,wildcard-import,deprecated-module,relative-import,reimported,import-self,misplaced-future,fixme,invalid-encoded-data,global-variable-undefined,global-variable-not-assigned,global-statement,global-at-module-level,unused-import,unused-variable,unused-argument,unused-wildcard-import,redefined-outer-name,redefined-builtin,redefine-in-handler,undefined-loop-variable,cell-var-from-loop,bare-except,broad-except,duplicate-except,nonstandard-exception,binary-op-exception,property-on-old-class,logging-not-lazy,logging-format-interpolation,bad-format-string-key,unused-format-string-key,bad-format-string,missing-format-argument-key,unused-format-string-argument,format-combined-specification,missing-format-attribute,invalid-format-index,anomalous-backslash-in-string,anomalous-unicode-escape-in-string,bad-open-mode,boolean-datetime,redundant-unittest-assert,deprecated-method,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call,useless-object-inheritance,comparison-with-callable,bad-option-value,consider-using-f-string,unspecified-encoding,missing-timeout,unnecessary-dunder-call,no-value-for-parameter,c-extension-no-member,attribute-defined-outside-init,use-a-generator,too-many-positional-arguments # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/linode_api4/objects/image.py b/linode_api4/objects/image.py index b2c413f86..f3b8b3aaa 100644 --- a/linode_api4/objects/image.py +++ b/linode_api4/objects/image.py @@ -62,6 +62,8 @@ class Image(Base): def replicate(self, regions: Union[List[str], List[Region]]): """ + NOTE: Image replication may not currently be available to all users. + Replicate the image to other regions. Note: Image replication may not currently be available to all users. diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 220cd4093..0c73a4857 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -34,7 +34,7 @@ def get_random_label(): return label -def get_region( +def get_regions( client: LinodeClient, capabilities: Set[str] = None, site_type: str = None ): region_override = os.environ.get(ENV_REGION_OVERRIDE) @@ -53,7 +53,13 @@ def get_region( if site_type is not None: regions = [v for v in regions if v.site_type == site_type] - return random.choice(regions) + return regions + + +def get_region( + client: LinodeClient, capabilities: Set[str] = None, site_type: str = None +): + return random.choice(get_regions(client, capabilities, site_type)) def get_api_ca_file(): diff --git a/test/integration/models/image/test_image.py b/test/integration/models/image/test_image.py index 5c4025dfc..bfb958921 100644 --- a/test/integration/models/image/test_image.py +++ b/test/integration/models/image/test_image.py @@ -1,5 +1,5 @@ from io import BytesIO -from test.integration.conftest import get_region +from test.integration.conftest import get_region, get_regions from test.integration.helpers import ( delete_instance_with_test_kw, get_test_label, @@ -39,15 +39,19 @@ def test_uploaded_image(test_linode_client): label = get_test_label() + "_image" + regions = get_regions( + test_linode_client, capabilities={"Object Storage"}, site_type="core" + ) + image = test_linode_client.image_upload( label, - "us-east", + regions[1].id, BytesIO(test_image_content), description="integration test image upload", tags=["tests"], ) - yield image + yield image, regions image.delete() @@ -60,16 +64,20 @@ def test_get_image(test_linode_client, image_upload_url): def test_image_create_upload(test_linode_client, test_uploaded_image): - image = test_linode_client.load(Image, test_uploaded_image.id) + uploaded_image, _ = test_uploaded_image - assert image.label == test_uploaded_image.label + image = test_linode_client.load(Image, uploaded_image.id) + + assert image.label == uploaded_image.label assert image.description == "integration test image upload" assert image.tags[0] == "tests" @pytest.mark.smoke def test_image_replication(test_linode_client, test_uploaded_image): - image = test_linode_client.load(Image, test_uploaded_image.id) + uploaded_image, regions = test_uploaded_image + + image = test_linode_client.load(Image, uploaded_image.id) # wait for image to be available for replication def poll_func() -> bool: @@ -85,8 +93,10 @@ def poll_func() -> bool: except polling.TimeoutException: print("failed to wait for image status: timeout period expired.") - # image replication works stably in these two regions - image.replicate(["us-east", "eu-west"]) + replicate_regions = [r.id for r in regions[:2]] + image.replicate(replicate_regions) - assert image.label == test_uploaded_image.label + assert image.label == uploaded_image.label assert len(image.regions) == 2 + assert image.regions[0].region in replicate_regions + assert image.regions[1].region in replicate_regions From b91f188e321abb7ac7183a1245c93d199da3cbcc Mon Sep 17 00:00:00 2001 From: Lena Garber <114949949+lgarber-akamai@users.noreply.github.com> Date: Wed, 25 Sep 2024 11:41:42 -0400 Subject: [PATCH 03/11] fix: Ensure all arguments with None defaults are marked as optional (#459) * Ensure all arguments with None defaults are marked as optional * Fix import issue --- linode_api4/groups/image.py | 8 ++++---- linode_api4/groups/polling.py | 4 +++- linode_api4/linode_client.py | 10 +++++----- linode_api4/objects/image.py | 4 ++-- linode_api4/objects/lke.py | 8 ++++---- linode_api4/objects/region.py | 6 +++--- linode_api4/objects/vpc.py | 2 +- linode_api4/polling.py | 2 +- test/integration/conftest.py | 6 ++++-- test/unit/objects/image_test.py | 4 ++-- 10 files changed, 29 insertions(+), 25 deletions(-) diff --git a/linode_api4/groups/image.py b/linode_api4/groups/image.py index 451a73d19..e644dc169 100644 --- a/linode_api4/groups/image.py +++ b/linode_api4/groups/image.py @@ -32,8 +32,8 @@ def __call__(self, *filters): def create( self, disk: Union[Disk, int], - label: str = None, - description: str = None, + label: Optional[str] = None, + description: Optional[str] = None, cloud_init: bool = False, tags: Optional[List[str]] = None, ): @@ -82,7 +82,7 @@ def create_upload( self, label: str, region: str, - description: str = None, + description: Optional[str] = None, cloud_init: bool = False, tags: Optional[List[str]] = None, ) -> Tuple[Image, str]: @@ -132,7 +132,7 @@ def upload( label: str, region: str, file: BinaryIO, - description: str = None, + description: Optional[str] = None, tags: Optional[List[str]] = None, ) -> Image: """ diff --git a/linode_api4/groups/polling.py b/linode_api4/groups/polling.py index 7dff2d3d5..8ef2c4feb 100644 --- a/linode_api4/groups/polling.py +++ b/linode_api4/groups/polling.py @@ -1,3 +1,5 @@ +from typing import Optional + import polling from linode_api4.groups import Group @@ -13,7 +15,7 @@ def event_poller_create( self, entity_type: str, action: str, - entity_id: int = None, + entity_id: Optional[int] = None, ) -> EventPoller: """ Creates a new instance of the EventPoller class. diff --git a/linode_api4/linode_client.py b/linode_api4/linode_client.py index 66e3d45fe..1bbc631b7 100644 --- a/linode_api4/linode_client.py +++ b/linode_api4/linode_client.py @@ -3,7 +3,7 @@ import json import logging from importlib.metadata import version -from typing import BinaryIO, List, Tuple +from typing import BinaryIO, List, Optional, Tuple from urllib import parse import requests @@ -391,8 +391,8 @@ def image_create_upload( self, label: str, region: str, - description: str = None, - tags: List[str] = None, + description: Optional[str] = None, + tags: Optional[List[str]] = None, ) -> Tuple[Image, str]: """ .. note:: This method is an alias to maintain backwards compatibility. @@ -409,8 +409,8 @@ def image_upload( label: str, region: str, file: BinaryIO, - description: str = None, - tags: List[str] = None, + description: Optional[str] = None, + tags: Optional[List[str]] = None, ) -> Image: """ .. note:: This method is an alias to maintain backwards compatibility. diff --git a/linode_api4/objects/image.py b/linode_api4/objects/image.py index f3b8b3aaa..c9ac43863 100644 --- a/linode_api4/objects/image.py +++ b/linode_api4/objects/image.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List, Union +from typing import List, Optional, Union from linode_api4.objects import Base, Property, Region from linode_api4.objects.serializable import JSONObject, StrEnum @@ -25,7 +25,7 @@ class ImageRegion(JSONObject): """ region: str = "" - status: ReplicationStatus = None + status: Optional[ReplicationStatus] = None class Image(Base): diff --git a/linode_api4/objects/lke.py b/linode_api4/objects/lke.py index b6471553c..b0e628196 100644 --- a/linode_api4/objects/lke.py +++ b/linode_api4/objects/lke.py @@ -103,8 +103,8 @@ class LKEClusterControlPlaneACLAddresses(JSONObject): to access an LKE cluster's control plane. """ - ipv4: List[str] = None - ipv6: List[str] = None + ipv4: Optional[List[str]] = None + ipv6: Optional[List[str]] = None @dataclass @@ -117,7 +117,7 @@ class LKEClusterControlPlaneACL(JSONObject): """ enabled: bool = False - addresses: LKEClusterControlPlaneACLAddresses = None + addresses: Optional[LKEClusterControlPlaneACLAddresses] = None class LKENodePoolNode: @@ -351,7 +351,7 @@ def node_pool_create( self, node_type: Union[Type, str], node_count: int, - labels: Dict[str, str] = None, + labels: Optional[Dict[str, str]] = None, taints: List[Union[LKENodePoolTaint, Dict[str, Any]]] = None, **kwargs, ): diff --git a/linode_api4/objects/region.py b/linode_api4/objects/region.py index 6d8178eff..34577c336 100644 --- a/linode_api4/objects/region.py +++ b/linode_api4/objects/region.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List +from typing import List, Optional from linode_api4.errors import UnexpectedResponseError from linode_api4.objects.base import Base, JSONObject, Property @@ -59,6 +59,6 @@ class RegionAvailabilityEntry(JSONObject): API Documentation: https://techdocs.akamai.com/linode-api/reference/get-region-availability """ - region: str = None - plan: str = None + region: Optional[str] = None + plan: Optional[str] = None available: bool = False diff --git a/linode_api4/objects/vpc.py b/linode_api4/objects/vpc.py index 456bdcfbc..e4fe36c78 100644 --- a/linode_api4/objects/vpc.py +++ b/linode_api4/objects/vpc.py @@ -16,7 +16,7 @@ class VPCSubnetLinodeInterface(JSONObject): @dataclass class VPCSubnetLinode(JSONObject): id: int = 0 - interfaces: List[VPCSubnetLinodeInterface] = None + interfaces: Optional[List[VPCSubnetLinodeInterface]] = None class VPCSubnet(DerivedBase): diff --git a/linode_api4/polling.py b/linode_api4/polling.py index 947e59e47..7dc08d915 100644 --- a/linode_api4/polling.py +++ b/linode_api4/polling.py @@ -104,7 +104,7 @@ def __init__( client: "LinodeClient", entity_type: str, action: str, - entity_id: int = None, + entity_id: Optional[int] = None, ): self._client = client self._entity_type = entity_type diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 0c73a4857..9db01cb89 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -2,7 +2,7 @@ import os import random import time -from typing import Set +from typing import Optional, Set import pytest import requests @@ -35,7 +35,9 @@ def get_random_label(): def get_regions( - client: LinodeClient, capabilities: Set[str] = None, site_type: str = None + client: LinodeClient, + capabilities: Optional[Set[str]] = None, + site_type: Optional[str] = None, ): region_override = os.environ.get(ENV_REGION_OVERRIDE) diff --git a/test/unit/objects/image_test.py b/test/unit/objects/image_test.py index d4851e777..5d1ce42d5 100644 --- a/test/unit/objects/image_test.py +++ b/test/unit/objects/image_test.py @@ -1,7 +1,7 @@ from datetime import datetime from io import BytesIO from test.unit.base import ClientBaseCase -from typing import BinaryIO +from typing import BinaryIO, Optional from unittest.mock import patch from linode_api4.objects import Image, Region @@ -95,7 +95,7 @@ def test_image_upload(self): Test that an image can be uploaded. """ - def put_mock(url: str, data: BinaryIO = None, **kwargs): + def put_mock(url: str, data: Optional[BinaryIO] = None, **kwargs): self.assertEqual(url, "https://linode.com/") self.assertEqual(data.read(), TEST_IMAGE_CONTENT) From dddecc815f0491a25c207b854f66cabc5455e1e4 Mon Sep 17 00:00:00 2001 From: Youjung Kim <126618609+ykim-1@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:11:47 -0700 Subject: [PATCH 04/11] Adding retries for flaky tests (#458) --- test/integration/linode_client/test_linode_client.py | 1 + test/integration/models/image/test_image.py | 1 + test/integration/models/linode/test_linode.py | 3 +++ test/integration/models/lke/test_lke.py | 1 + test/integration/models/profile/test_profile.py | 4 ++++ tox.ini | 1 + 6 files changed, 11 insertions(+) diff --git a/test/integration/linode_client/test_linode_client.py b/test/integration/linode_client/test_linode_client.py index 4f8f97f4a..105535211 100644 --- a/test/integration/linode_client/test_linode_client.py +++ b/test/integration/linode_client/test_linode_client.py @@ -87,6 +87,7 @@ def test_get_regions(test_linode_client): @pytest.mark.smoke +@pytest.mark.flaky(reruns=3, reruns_delay=2) def test_image_create(setup_client_and_linode): client = setup_client_and_linode[0] linode = setup_client_and_linode[1] diff --git a/test/integration/models/image/test_image.py b/test/integration/models/image/test_image.py index bfb958921..4c2aa77d2 100644 --- a/test/integration/models/image/test_image.py +++ b/test/integration/models/image/test_image.py @@ -74,6 +74,7 @@ def test_image_create_upload(test_linode_client, test_uploaded_image): @pytest.mark.smoke +@pytest.mark.flaky(reruns=3, reruns_delay=2) def test_image_replication(test_linode_client, test_uploaded_image): uploaded_image, regions = test_uploaded_image diff --git a/test/integration/models/linode/test_linode.py b/test/integration/models/linode/test_linode.py index dd8907c0e..d6b272d9b 100644 --- a/test/integration/models/linode/test_linode.py +++ b/test/integration/models/linode/test_linode.py @@ -312,6 +312,7 @@ def test_linode_boot(create_linode): assert linode.status == "running" +@pytest.mark.flaky(reruns=3, reruns_delay=2) def test_linode_resize(create_linode_for_long_running_tests): linode = create_linode_for_long_running_tests @@ -590,6 +591,7 @@ def test_get_linode_types_overrides(test_linode_client): assert linode_type.region_prices[0].monthly >= 0 +@pytest.mark.flaky(reruns=3, reruns_delay=2) def test_save_linode_noforce(test_linode_client, create_linode): linode = create_linode old_label = linode.label @@ -601,6 +603,7 @@ def test_save_linode_noforce(test_linode_client, create_linode): assert old_label != linode.label +@pytest.mark.flaky(reruns=3, reruns_delay=2) def test_save_linode_force(test_linode_client, create_linode): linode = create_linode old_label = linode.label diff --git a/test/integration/models/lke/test_lke.py b/test/integration/models/lke/test_lke.py index eb31c8eb6..f4f92f921 100644 --- a/test/integration/models/lke/test_lke.py +++ b/test/integration/models/lke/test_lke.py @@ -277,6 +277,7 @@ def test_lke_cluster_acl(lke_cluster_with_acl): assert not cluster.control_plane_acl.enabled +@pytest.mark.flaky(reruns=3, reruns_delay=2) def test_lke_cluster_labels_and_taints(lke_cluster_with_labels_and_taints): pool = lke_cluster_with_labels_and_taints.pools[0] diff --git a/test/integration/models/profile/test_profile.py b/test/integration/models/profile/test_profile.py index cafec12ea..b57c8de17 100644 --- a/test/integration/models/profile/test_profile.py +++ b/test/integration/models/profile/test_profile.py @@ -1,3 +1,5 @@ +import pytest + from linode_api4.objects import PersonalAccessToken, Profile, SSHKey @@ -18,6 +20,7 @@ def test_get_personal_access_token_objects(test_linode_client): assert isinstance(personal_access_tokens[0], PersonalAccessToken) +@pytest.mark.flaky(reruns=3, reruns_delay=2) def test_get_sshkeys(test_linode_client, test_sshkey): client = test_linode_client @@ -29,6 +32,7 @@ def test_get_sshkeys(test_linode_client, test_sshkey): assert test_sshkey.label in ssh_labels +@pytest.mark.flaky(reruns=3, reruns_delay=2) def test_ssh_key_create(test_sshkey, ssh_key_gen): pub_key = ssh_key_gen[0] key = test_sshkey diff --git a/tox.ini b/tox.ini index 209db7170..266c26717 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ deps = mock pylint httpretty + pytest-rerunfailures commands = python -m pip install . coverage run --source linode_api4 -m pytest test/unit From 77dde133f2ba7b77e2edda217a2f3c5c66455c55 Mon Sep 17 00:00:00 2001 From: Youjung Kim <126618609+ykim-1@users.noreply.github.com> Date: Thu, 26 Sep 2024 10:52:28 -0700 Subject: [PATCH 05/11] test: Add cloud firewalls to migration tests and improve wait times to disk related tests (#460) * Add proper wait times in fixture for disk related tests; Add cloud firewall to migration test * unskipping migration test after LDE enabled on specified region --- test/integration/models/linode/test_linode.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/integration/models/linode/test_linode.py b/test/integration/models/linode/test_linode.py index d6b272d9b..f0745901e 100644 --- a/test/integration/models/linode/test_linode.py +++ b/test/integration/models/linode/test_linode.py @@ -107,12 +107,12 @@ def linode_for_disk_tests(test_linode_client, e2e_test_firewall): # Provisioning time wait_for_condition(10, 300, get_status, linode_instance, "running") - linode_instance.shutdown() + send_request_when_resource_available(300, linode_instance.shutdown) wait_for_condition(10, 100, get_status, linode_instance, "offline") # Now it allocates 100% disk space hence need to clear some space for tests - linode_instance.disks[1].delete() + send_request_when_resource_available(300, linode_instance.disks[1].delete) test_linode_client.polling.event_poller_create( "linode", "disk_delete", entity_id=linode_instance.id @@ -513,21 +513,25 @@ def test_linode_ips(create_linode): assert ips.ipv4.public[0].address == linode.ipv4[0] -def test_linode_initate_migration(test_linode_client): +def test_linode_initate_migration(test_linode_client, e2e_test_firewall): client = test_linode_client available_regions = client.regions() chosen_region = available_regions[4] label = get_test_label() + "_migration" linode, _ = client.linode.instance_create( - "g6-nanode-1", chosen_region, image="linode/debian12", label=label + "g6-nanode-1", + chosen_region, + image="linode/debian12", + label=label, + firewall=e2e_test_firewall, ) # Says it could take up to ~6 hrs for migration to fully complete send_request_when_resource_available( 300, linode.initiate_migration, - region="us-mia", + region="us-central", migration_type=MigrationType.COLD, ) From e2b8c46138ba6e1f380c1e91f17935412f8dcd0b Mon Sep 17 00:00:00 2001 From: Youjung Kim <126618609+ykim-1@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:10:08 -0700 Subject: [PATCH 06/11] test: Update smoke test coverage and improve nightly test workflow (#452) * update smoke test coverage and workflow * make format lint * Replace webhook with slack oauth token * Add fancy slack payload * Clean up slack payload syntax * Clean up slack payload syntax * fix invalid slack payload syntax * fix invalid slack payload syntax * setting payload variable in separate step * fix slack payload syntax * fix slack payload syntax * fix slack payload syntax * fix slack payload syntax * add repository name in slack payload * add slack notifications and conditions --- .github/workflows/e2e-test.yml | 66 +++++++++++++++++ .github/workflows/nightly-smoke-tests.yml | 71 +++++++++++++++++++ .../login_client/test_login_client.py | 1 + test/integration/models/lke/test_lke.py | 1 + .../models/nodebalancer/test_nodebalancer.py | 1 + .../models/object_storage/test_obj.py | 1 + .../models/placement/test_placement.py | 4 ++ .../models/profile/test_profile.py | 2 + test/integration/models/vpc/test_vpc.py | 3 + 9 files changed, 150 insertions(+) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 48cb55e13..5b6b08111 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -91,3 +91,69 @@ jobs: env: LINODE_CLI_OBJ_ACCESS_KEY: ${{ secrets.LINODE_CLI_OBJ_ACCESS_KEY }} LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }} + + notify-slack: + runs-on: ubuntu-latest + needs: [integration-tests] + if: always() && github.repository == 'linode/linode_api4-python' # Run even if integration tests fail and only on main repository + + steps: + - name: Notify Slack + uses: slackapi/slack-github-action@v1.27.0 + with: + channel-id: ${{ secrets.SLACK_CHANNEL_ID }} + payload: | + { + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":rocket: *${{ github.workflow }} Completed in: ${{ github.repository }}* :white_check_mark:" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Build Result:*\n${{ steps.integration-tests.outcome == 'success' && ':large_green_circle: Build Passed' || ':red_circle: Build Failed' }}" + }, + { + "type": "mrkdwn", + "text": "*Branch:*\n`${{ github.ref_name }}`" + } + ] + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Commit Hash:*\n<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>" + }, + { + "type": "mrkdwn", + "text": "*Run URL:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run Details>" + } + ] + }, + { + "type": "divider" + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "Triggered by: :bust_in_silhouette: `${{ github.actor }}`" + } + ] + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/nightly-smoke-tests.yml b/.github/workflows/nightly-smoke-tests.yml index b1a0fcbfd..c0b7b87c1 100644 --- a/.github/workflows/nightly-smoke-tests.yml +++ b/.github/workflows/nightly-smoke-tests.yml @@ -4,9 +4,17 @@ on: schedule: - cron: "0 0 * * *" workflow_dispatch: + inputs: + sha: + description: 'Commit SHA to test' + required: false + default: '' + type: string + jobs: smoke_tests: + if: github.repository == 'linode/linode_api4-python' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: @@ -29,7 +37,70 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run smoke tests + id: smoke_tests run: | make smoketest env: LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }} + + - name: Notify Slack + if: always() && github.repository == 'linode/linode_api4-python' + uses: slackapi/slack-github-action@v1.27.0 + with: + channel-id: ${{ secrets.SLACK_CHANNEL_ID }} + payload: | + { + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":rocket: *${{ github.workflow }} Completed in: ${{ github.repository }}* :white_check_mark:" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Build Result:*\n${{ steps.smoke_tests.outcome == 'success' && ':large_green_circle: Build Passed' || ':red_circle: Build Failed' }}" + }, + { + "type": "mrkdwn", + "text": "*Branch:*\n`${{ github.ref_name }}`" + } + ] + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Commit Hash:*\n<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>" + }, + { + "type": "mrkdwn", + "text": "*Run URL:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run Details>" + } + ] + }, + { + "type": "divider" + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "Triggered by: :bust_in_silhouette: `${{ github.actor }}`" + } + ] + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + diff --git a/test/integration/login_client/test_login_client.py b/test/integration/login_client/test_login_client.py index 7cb4246ea..ccbeb1976 100644 --- a/test/integration/login_client/test_login_client.py +++ b/test/integration/login_client/test_login_client.py @@ -27,6 +27,7 @@ def test_oauth_client_two(test_linode_client): oauth_client.delete() +@pytest.mark.smoke def test_get_oathclient(test_linode_client, test_oauth_client): client = test_linode_client diff --git a/test/integration/models/lke/test_lke.py b/test/integration/models/lke/test_lke.py index f4f92f921..bd0692dcc 100644 --- a/test/integration/models/lke/test_lke.py +++ b/test/integration/models/lke/test_lke.py @@ -127,6 +127,7 @@ def test_get_lke_clusters(test_linode_client, lke_cluster): assert cluster._raw_json == lke_cluster._raw_json +@pytest.mark.smoke def test_get_lke_pool(test_linode_client, lke_cluster): cluster = lke_cluster diff --git a/test/integration/models/nodebalancer/test_nodebalancer.py b/test/integration/models/nodebalancer/test_nodebalancer.py index a3c00cee9..5581c9029 100644 --- a/test/integration/models/nodebalancer/test_nodebalancer.py +++ b/test/integration/models/nodebalancer/test_nodebalancer.py @@ -78,6 +78,7 @@ def test_create_nb_node( assert "node_test" == node.label +@pytest.mark.smoke def test_get_nb_node(test_linode_client, create_nb_config): node = test_linode_client.load( NodeBalancerNode, diff --git a/test/integration/models/object_storage/test_obj.py b/test/integration/models/object_storage/test_obj.py index 3042f326a..82b2da022 100644 --- a/test/integration/models/object_storage/test_obj.py +++ b/test/integration/models/object_storage/test_obj.py @@ -93,6 +93,7 @@ def test_bucket( assert any(b.label == bucket.label for b in buckets) +@pytest.mark.smoke def test_list_obj_storage_bucket( test_linode_client: LinodeClient, bucket: ObjectStorageBucket, diff --git a/test/integration/models/placement/test_placement.py b/test/integration/models/placement/test_placement.py index 7919ef432..db570aa9e 100644 --- a/test/integration/models/placement/test_placement.py +++ b/test/integration/models/placement/test_placement.py @@ -1,6 +1,9 @@ +import pytest + from linode_api4 import PlacementGroup +@pytest.mark.smoke def test_get_pg(test_linode_client, create_placement_group): """ Tests that a Placement Group can be loaded. @@ -9,6 +12,7 @@ def test_get_pg(test_linode_client, create_placement_group): assert pg.id == create_placement_group.id +@pytest.mark.smoke def test_update_pg(test_linode_client, create_placement_group): """ Tests that a Placement Group can be updated successfully. diff --git a/test/integration/models/profile/test_profile.py b/test/integration/models/profile/test_profile.py index b57c8de17..6942eea38 100644 --- a/test/integration/models/profile/test_profile.py +++ b/test/integration/models/profile/test_profile.py @@ -3,6 +3,7 @@ from linode_api4.objects import PersonalAccessToken, Profile, SSHKey +@pytest.mark.smoke def test_user_profile(test_linode_client): client = test_linode_client @@ -20,6 +21,7 @@ def test_get_personal_access_token_objects(test_linode_client): assert isinstance(personal_access_tokens[0], PersonalAccessToken) +@pytest.mark.smoke @pytest.mark.flaky(reruns=3, reruns_delay=2) def test_get_sshkeys(test_linode_client, test_sshkey): client = test_linode_client diff --git a/test/integration/models/vpc/test_vpc.py b/test/integration/models/vpc/test_vpc.py index 6af3380b7..5dd14b502 100644 --- a/test/integration/models/vpc/test_vpc.py +++ b/test/integration/models/vpc/test_vpc.py @@ -5,12 +5,14 @@ from linode_api4 import VPC, ApiError, VPCSubnet +@pytest.mark.smoke def test_get_vpc(test_linode_client, create_vpc): vpc = test_linode_client.load(VPC, create_vpc.id) test_linode_client.vpcs() assert vpc.id == create_vpc.id +@pytest.mark.smoke def test_update_vpc(test_linode_client, create_vpc): vpc = create_vpc new_label = create_vpc.label + "-updated" @@ -33,6 +35,7 @@ def test_get_subnet(test_linode_client, create_vpc_with_subnet): assert loaded_subnet.id == subnet.id +@pytest.mark.smoke def test_update_subnet(test_linode_client, create_vpc_with_subnet): vpc, subnet = create_vpc_with_subnet new_label = subnet.label + "-updated" From 46d6d18a3a580c40b3a046339413027d74d65de0 Mon Sep 17 00:00:00 2001 From: Zhiwei Liang <121905282+zliang-akamai@users.noreply.github.com> Date: Tue, 8 Oct 2024 17:23:14 -0400 Subject: [PATCH 07/11] Resolve circular imports and restore top-level imports (#462) --- linode_api4/objects/linode.py | 29 ++++++++++++----------------- linode_api4/objects/networking.py | 5 ++++- linode_api4/objects/nodebalancer.py | 10 +++------- linode_api4/objects/volume.py | 4 +++- linode_api4/objects/vpc.py | 6 +----- 5 files changed, 23 insertions(+), 31 deletions(-) diff --git a/linode_api4/objects/linode.py b/linode_api4/objects/linode.py index 775def88a..12c75d85c 100644 --- a/linode_api4/objects/linode.py +++ b/linode_api4/objects/linode.py @@ -11,18 +11,19 @@ from linode_api4 import util from linode_api4.common import load_and_validate_keys from linode_api4.errors import UnexpectedResponseError -from linode_api4.objects import ( - Base, - DerivedBase, - Image, - JSONObject, - Property, - Region, -) -from linode_api4.objects.base import MappedObject +from linode_api4.objects.base import Base, MappedObject, Property +from linode_api4.objects.dbase import DerivedBase from linode_api4.objects.filtering import FilterableAttribute -from linode_api4.objects.networking import IPAddress, IPv6Range, VPCIPAddress -from linode_api4.objects.serializable import StrEnum +from linode_api4.objects.image import Image +from linode_api4.objects.networking import ( + Firewall, + IPAddress, + IPv6Range, + VPCIPAddress, +) +from linode_api4.objects.nodebalancer import NodeBalancer +from linode_api4.objects.region import Region +from linode_api4.objects.serializable import JSONObject, StrEnum from linode_api4.objects.vpc import VPC, VPCSubnet from linode_api4.paginated_list import PaginatedList @@ -1618,9 +1619,6 @@ def firewalls(self): :returns: A List of Firewalls of the Linode Instance. :rtype: List[Firewall] """ - from linode_api4.objects import ( # pylint: disable=import-outside-toplevel - Firewall, - ) result = self._client.get( "{}/firewalls".format(Instance.api_endpoint), model=self @@ -1640,9 +1638,6 @@ def nodebalancers(self): :returns: A List of Nodebalancers of the Linode Instance. :rtype: List[Nodebalancer] """ - from linode_api4.objects import ( # pylint: disable=import-outside-toplevel - NodeBalancer, - ) result = self._client.get( "{}/nodebalancers".format(Instance.api_endpoint), model=self diff --git a/linode_api4/objects/networking.py b/linode_api4/objects/networking.py index 9e19b2d92..c4fff1ac3 100644 --- a/linode_api4/objects/networking.py +++ b/linode_api4/objects/networking.py @@ -3,7 +3,10 @@ from linode_api4.common import Price, RegionPrice from linode_api4.errors import UnexpectedResponseError -from linode_api4.objects import Base, DerivedBase, JSONObject, Property, Region +from linode_api4.objects.base import Base, Property +from linode_api4.objects.dbase import DerivedBase +from linode_api4.objects.region import Region +from linode_api4.objects.serializable import JSONObject class IPv6Pool(Base): diff --git a/linode_api4/objects/nodebalancer.py b/linode_api4/objects/nodebalancer.py index 36d038bcb..d038b6998 100644 --- a/linode_api4/objects/nodebalancer.py +++ b/linode_api4/objects/nodebalancer.py @@ -3,14 +3,10 @@ from linode_api4.common import Price, RegionPrice from linode_api4.errors import UnexpectedResponseError -from linode_api4.objects import ( - Base, - DerivedBase, - MappedObject, - Property, - Region, -) +from linode_api4.objects.base import Base, MappedObject, Property +from linode_api4.objects.dbase import DerivedBase from linode_api4.objects.networking import Firewall, IPAddress +from linode_api4.objects.region import Region class NodeBalancerType(Base): diff --git a/linode_api4/objects/volume.py b/linode_api4/objects/volume.py index a79e3174c..6c8514f9f 100644 --- a/linode_api4/objects/volume.py +++ b/linode_api4/objects/volume.py @@ -1,6 +1,8 @@ from linode_api4.common import Price, RegionPrice from linode_api4.errors import UnexpectedResponseError -from linode_api4.objects import Base, Instance, Property, Region +from linode_api4.objects.base import Base, Property +from linode_api4.objects.linode import Instance, Region +from linode_api4.objects.region import Region class VolumeType(Base): diff --git a/linode_api4/objects/vpc.py b/linode_api4/objects/vpc.py index e4fe36c78..3c9a4aaba 100644 --- a/linode_api4/objects/vpc.py +++ b/linode_api4/objects/vpc.py @@ -3,6 +3,7 @@ from linode_api4.errors import UnexpectedResponseError from linode_api4.objects import Base, DerivedBase, Property, Region +from linode_api4.objects.networking import VPCIPAddress from linode_api4.objects.serializable import JSONObject from linode_api4.paginated_list import PaginatedList @@ -110,11 +111,6 @@ def ips(self) -> PaginatedList: :rtype: PaginatedList of VPCIPAddress """ - # need to avoid circular import - from linode_api4.objects import ( # pylint: disable=import-outside-toplevel - VPCIPAddress, - ) - return self._client._get_and_filter( VPCIPAddress, endpoint="/vpcs/{}/ips".format(self.id) ) From 9d63e07b11c881972d08a9673707a843dbd1398a Mon Sep 17 00:00:00 2001 From: Youjung Kim <126618609+ykim-1@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:46:16 -0700 Subject: [PATCH 08/11] test: Address pytest warnings; remove duplicate test helper (#463) * address pytest warnings, duplicate test helper * update description * update description * remove pytest.ini file; add pytest markers to toml * fix event test * add e2e firewall to test instance * add e2e firewall to test instance --- pyproject.toml | 7 +++++++ test/integration/helpers.py | 6 ------ test/integration/models/account/test_account.py | 16 ++++++++++++---- .../integration/models/firewall/test_firewall.py | 5 +++-- .../models/networking/test_networking.py | 4 ++-- 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6720a965c..b8b57880b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ dev = [ "sphinxcontrib-fulltoc>=1.2.0", "build>=0.10.0", "twine>=4.0.2", + "pytest-rerunfailures", ] doc = [ @@ -88,3 +89,9 @@ in-place = true recursive = true remove-all-unused-imports = true remove-duplicate-keys = false + +[tool.pytest.ini_options] +markers = [ + "smoke: mark a test as a smoke test", + "flaky: mark a test as a flaky test for rerun" +] \ No newline at end of file diff --git a/test/integration/helpers.py b/test/integration/helpers.py index e0aab06c4..e874ea7e2 100644 --- a/test/integration/helpers.py +++ b/test/integration/helpers.py @@ -12,12 +12,6 @@ def get_test_label(): return label -def get_rand_nanosec_test_label(): - unique_timestamp = str(time.time_ns())[:-3] - label = "test_" + unique_timestamp - return label - - def delete_instance_with_test_kw(paginated_list: PaginatedList): for i in paginated_list: try: diff --git a/test/integration/models/account/test_account.py b/test/integration/models/account/test_account.py index a9dce4a3a..ab20ee079 100644 --- a/test/integration/models/account/test_account.py +++ b/test/integration/models/account/test_account.py @@ -61,7 +61,7 @@ def test_get_account_settings(test_linode_client): @pytest.mark.smoke -def test_latest_get_event(test_linode_client): +def test_latest_get_event(test_linode_client, e2e_test_firewall): client = test_linode_client available_regions = client.regions() @@ -69,16 +69,24 @@ def test_latest_get_event(test_linode_client): label = get_test_label() linode, password = client.linode.instance_create( - "g6-nanode-1", chosen_region, image="linode/debian10", label=label + "g6-nanode-1", + chosen_region, + image="linode/debian10", + label=label, + firewall=e2e_test_firewall, ) events = client.load(Event, "") - latest_event = events._raw_json.get("data")[0] + latest_events = events._raw_json.get("data") linode.delete() - assert label in latest_event["entity"]["label"] + for event in latest_events[:15]: + if label == event["entity"]["label"]: + break + else: + assert False, f"Linode '{label}' not found in the last 15 events" def test_get_user(test_linode_client): diff --git a/test/integration/models/firewall/test_firewall.py b/test/integration/models/firewall/test_firewall.py index 7a7f58ff1..7f907cc2f 100644 --- a/test/integration/models/firewall/test_firewall.py +++ b/test/integration/models/firewall/test_firewall.py @@ -1,4 +1,5 @@ import time +from test.integration.helpers import get_test_label import pytest @@ -10,7 +11,7 @@ def linode_fw(test_linode_client): client = test_linode_client available_regions = client.regions() chosen_region = available_regions[4] - label = "linode_instance_fw_device" + label = get_test_label() linode_instance, password = client.linode.instance_create( "g6-nanode-1", chosen_region, image="linode/debian10", label=label @@ -79,6 +80,6 @@ def test_get_device(test_linode_client, test_firewall, linode_fw): FirewallDevice, firewall.devices.first().id, firewall.id ) - assert firewall_device.entity.label == "linode_instance_fw_device" + assert "test_" in firewall_device.entity.label assert firewall_device.entity.type == "linode" assert "/v4/linode/instances/" in firewall_device.entity.url diff --git a/test/integration/models/networking/test_networking.py b/test/integration/models/networking/test_networking.py index 3eb455cb4..a52f38ef2 100644 --- a/test/integration/models/networking/test_networking.py +++ b/test/integration/models/networking/test_networking.py @@ -1,4 +1,4 @@ -from test.integration.helpers import get_rand_nanosec_test_label +from test.integration.helpers import get_test_label import pytest @@ -22,7 +22,7 @@ def create_linode(test_linode_client): client = test_linode_client available_regions = client.regions() chosen_region = available_regions[4] - label = get_rand_nanosec_test_label() + label = get_test_label() linode_instance, _ = client.linode.instance_create( "g6-nanode-1", From 6fcc0695254c432c33f39e36ff145c958027525d Mon Sep 17 00:00:00 2001 From: Youjung Kim <126618609+ykim-1@users.noreply.github.com> Date: Wed, 9 Oct 2024 09:03:18 -0700 Subject: [PATCH 09/11] fix slack payload (#464) --- .github/workflows/e2e-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 5b6b08111..848154b55 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -120,7 +120,7 @@ jobs: "fields": [ { "type": "mrkdwn", - "text": "*Build Result:*\n${{ steps.integration-tests.outcome == 'success' && ':large_green_circle: Build Passed' || ':red_circle: Build Failed' }}" + "text": "*Build Result:*\n${{ needs.integration-tests.result == 'success' && ':large_green_circle: Build Passed' || ':red_circle: Build Failed' }}" }, { "type": "mrkdwn", From f7c6eef1c37949f7d4ddf7109df98db8ee98a4ce Mon Sep 17 00:00:00 2001 From: Erik Zilber Date: Mon, 14 Oct 2024 11:28:06 -0400 Subject: [PATCH 10/11] Added support for Block Storage Encryption (#453) * Implemented changes for Linode Disk Encryption * Added more test cases * Added LA note --- linode_api4/groups/volume.py | 4 ++- linode_api4/objects/linode.py | 1 + linode_api4/objects/volume.py | 1 + test/fixtures/volumes.json | 17 ++++++++++- test/integration/conftest.py | 29 +++++++++++++++++++ test/integration/models/linode/test_linode.py | 26 +++++++++++++++++ test/integration/models/volume/test_volume.py | 9 ++++++ test/unit/linode_client_test.py | 2 +- test/unit/objects/volume_test.py | 4 +++ 9 files changed, 90 insertions(+), 3 deletions(-) diff --git a/linode_api4/groups/volume.py b/linode_api4/groups/volume.py index dc0d7c601..3a30de762 100644 --- a/linode_api4/groups/volume.py +++ b/linode_api4/groups/volume.py @@ -45,7 +45,9 @@ def create(self, label, region=None, linode=None, size=20, **kwargs): tags included do not exist, they will be created as part of this operation. :type tags: list[str] - + :param encryption: Whether the new Volume should opt in or out of disk encryption. + :type encryption: str + Note: Block Storage Disk Encryption is not currently available to all users. :returns: The new Volume. :rtype: Volume """ diff --git a/linode_api4/objects/linode.py b/linode_api4/objects/linode.py index 12c75d85c..cb5c9d9af 100644 --- a/linode_api4/objects/linode.py +++ b/linode_api4/objects/linode.py @@ -679,6 +679,7 @@ class Instance(Base): "has_user_data": Property(), "disk_encryption": Property(), "lke_cluster_id": Property(), + "capabilities": Property(unordered=True), } @property diff --git a/linode_api4/objects/volume.py b/linode_api4/objects/volume.py index 6c8514f9f..58764e8d7 100644 --- a/linode_api4/objects/volume.py +++ b/linode_api4/objects/volume.py @@ -46,6 +46,7 @@ class Volume(Base): "filesystem_path": Property(), "hardware_type": Property(), "linode_label": Property(), + "encryption": Property(), } def attach(self, to_linode, config=None): diff --git a/test/fixtures/volumes.json b/test/fixtures/volumes.json index 18ba4f6da..2e8c86338 100644 --- a/test/fixtures/volumes.json +++ b/test/fixtures/volumes.json @@ -41,9 +41,24 @@ "filesystem_path": "this/is/a/file/path", "hardware_type": "nvme", "linode_label": "some_label" + }, + { + "id": 4, + "label": "block4", + "created": "2017-08-04T03:00:00", + "region": "ap-west-1a", + "linode_id": null, + "size": 40, + "updated": "2017-08-04T04:00:00", + "status": "active", + "tags": ["something"], + "filesystem_path": "this/is/a/file/path", + "hardware_type": "hdd", + "linode_label": null, + "encryption": "enabled" } ], - "results": 3, + "results": 4, "pages": 1, "page": 1 } diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 9db01cb89..cb1305d68 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -295,6 +295,35 @@ def test_volume(test_linode_client): raise e +@pytest.fixture(scope="session") +def test_volume_with_encryption(test_linode_client): + client = test_linode_client + timestamp = str(time.time_ns()) + region = get_region(client, {"Block Storage Encryption"}) + label = "TestSDK-" + timestamp + + volume = client.volume_create( + label=label, region=region, encryption="enabled" + ) + + yield volume + + timeout = 100 # give 100s for volume to be detached before deletion + + start_time = time.time() + + while time.time() - start_time < timeout: + try: + res = volume.delete() + if res: + break + else: + time.sleep(3) + except ApiError as e: + if time.time() - start_time > timeout: + raise e + + @pytest.fixture def test_tag(test_linode_client): client = test_linode_client diff --git a/test/integration/models/linode/test_linode.py b/test/integration/models/linode/test_linode.py index f0745901e..6d461cdf1 100644 --- a/test/integration/models/linode/test_linode.py +++ b/test/integration/models/linode/test_linode.py @@ -123,6 +123,25 @@ def linode_for_disk_tests(test_linode_client, e2e_test_firewall): linode_instance.delete() +@pytest.fixture +def linode_with_block_storage_encryption(test_linode_client, e2e_test_firewall): + client = test_linode_client + chosen_region = get_region(client, {"Linodes", "Block Storage Encryption"}) + label = get_test_label() + + linode_instance, password = client.linode.instance_create( + "g6-nanode-1", + chosen_region, + image="linode/alpine3.19", + label=label + "block-storage-encryption", + firewall=e2e_test_firewall, + ) + + yield linode_instance + + linode_instance.delete() + + @pytest.fixture def create_linode_for_long_running_tests(test_linode_client, e2e_test_firewall): client = test_linode_client @@ -440,6 +459,13 @@ def test_linode_with_disk_encryption_disabled(linode_with_disk_encryption): ) +def test_linode_with_block_storage_encryption( + linode_with_block_storage_encryption, +): + linode = linode_with_block_storage_encryption + assert "Block Storage Encryption" in linode.capabilities + + def wait_for_disk_status(disk: Disk, timeout): start_time = time.time() while True: diff --git a/test/integration/models/volume/test_volume.py b/test/integration/models/volume/test_volume.py index 820f7027a..19bc55c26 100644 --- a/test/integration/models/volume/test_volume.py +++ b/test/integration/models/volume/test_volume.py @@ -60,6 +60,15 @@ def test_get_volume(test_linode_client, test_volume): assert volume.id == test_volume.id +def test_get_volume_with_encryption( + test_linode_client, test_volume_with_encryption +): + volume = test_linode_client.load(Volume, test_volume_with_encryption.id) + + assert volume.id == test_volume_with_encryption.id + assert volume.encryption == "enabled" + + def test_update_volume_tag(test_linode_client, test_volume): volume = test_volume tag_1 = "volume_test_tag1" diff --git a/test/unit/linode_client_test.py b/test/unit/linode_client_test.py index 357826c0a..41cb9100d 100644 --- a/test/unit/linode_client_test.py +++ b/test/unit/linode_client_test.py @@ -150,7 +150,7 @@ def test_image_create(self): def test_get_volumes(self): v = self.client.volumes() - self.assertEqual(len(v), 3) + self.assertEqual(len(v), 4) self.assertEqual(v[0].label, "block1") self.assertEqual(v[0].region.id, "us-east-1a") self.assertEqual(v[1].label, "block2") diff --git a/test/unit/objects/volume_test.py b/test/unit/objects/volume_test.py index c18ac8d89..1344c2b94 100644 --- a/test/unit/objects/volume_test.py +++ b/test/unit/objects/volume_test.py @@ -31,6 +31,10 @@ def test_get_volume(self): self.assertEqual(volume.hardware_type, "hdd") self.assertEqual(volume.linode_label, None) + def test_get_volume_with_encryption(self): + volume = Volume(self.client, 4) + self.assertEqual(volume.encryption, "enabled") + def test_update_volume_tags(self): """ Tests that updating tags on an entity send the correct request From 0139b5135d2d64b27fa02a008533ebf12cb5756a Mon Sep 17 00:00:00 2001 From: Lena Garber <114949949+lgarber-akamai@users.noreply.github.com> Date: Mon, 14 Oct 2024 15:15:46 -0400 Subject: [PATCH 11/11] Add include_none_values ClassVar to JSONObject; apply to response-only classes (#466) * Add include_none_values ClassVar to JSONObject; apply to response-only structures * Add unit test case --- linode_api4/objects/image.py | 2 ++ linode_api4/objects/lke.py | 6 ++++++ linode_api4/objects/serializable.py | 8 +++++++- test/unit/objects/serializable_test.py | 21 +++++++++++++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/linode_api4/objects/image.py b/linode_api4/objects/image.py index c9ac43863..931ed4a31 100644 --- a/linode_api4/objects/image.py +++ b/linode_api4/objects/image.py @@ -24,6 +24,8 @@ class ImageRegion(JSONObject): The region and status of an image replica. """ + include_none_values = True + region: str = "" status: Optional[ReplicationStatus] = None diff --git a/linode_api4/objects/lke.py b/linode_api4/objects/lke.py index b0e628196..1c2ed3c1a 100644 --- a/linode_api4/objects/lke.py +++ b/linode_api4/objects/lke.py @@ -55,6 +55,8 @@ class LKENodePoolTaint(JSONObject): applied to a node pool. """ + include_none_values = True + key: Optional[str] = None value: Optional[str] = None effect: Optional[str] = None @@ -103,6 +105,8 @@ class LKEClusterControlPlaneACLAddresses(JSONObject): to access an LKE cluster's control plane. """ + include_none_values = True + ipv4: Optional[List[str]] = None ipv6: Optional[List[str]] = None @@ -116,6 +120,8 @@ class LKEClusterControlPlaneACL(JSONObject): NOTE: Control Plane ACLs may not currently be available to all users. """ + include_none_values = True + enabled: bool = False addresses: Optional[LKEClusterControlPlaneACLAddresses] = None diff --git a/linode_api4/objects/serializable.py b/linode_api4/objects/serializable.py index b0e7a2503..fea682f43 100644 --- a/linode_api4/objects/serializable.py +++ b/linode_api4/objects/serializable.py @@ -58,6 +58,12 @@ class JSONObject(metaclass=JSONFilterableMetaclass): ) """ + include_none_values: ClassVar[bool] = False + """ + If true, all None values for this class will be explicitly included in + the serialized output for instance of this class. + """ + always_include: ClassVar[Set[str]] = {} """ A set of keys corresponding to fields that should always be @@ -169,7 +175,7 @@ def should_include(key: str, value: Any) -> bool: Returns whether the given key/value pair should be included in the resulting dict. """ - if key in cls.always_include: + if cls.include_none_values or key in cls.always_include: return True hint = type_hints.get(key) diff --git a/test/unit/objects/serializable_test.py b/test/unit/objects/serializable_test.py index 579417e1c..a15f108b4 100644 --- a/test/unit/objects/serializable_test.py +++ b/test/unit/objects/serializable_test.py @@ -26,3 +26,24 @@ class Foo(JSONObject): assert foo["foo"] == "test" assert foo["bar"] == "test2" assert foo["baz"] == "test3" + + def test_serialize_optional_include_None(self): + @dataclass + class Foo(JSONObject): + include_none_values = True + + foo: Optional[str] = None + bar: Optional[str] = None + baz: str = None + + foo = Foo().dict + + assert foo["foo"] is None + assert foo["bar"] is None + assert foo["baz"] is None + + foo = Foo(foo="test", bar="test2", baz="test3").dict + + assert foo["foo"] == "test" + assert foo["bar"] == "test2" + assert foo["baz"] == "test3"