Apify storage client fixes and new docs groups

Author: vdusek

pyproject.toml

@@ -37,7 +37,7 @@ dependencies = [
     "apify-client>=1.12.0",
     "apify-shared>=1.3.0",
     "cachetools>=5.5.0",
-    "crawlee@git+https://github.com/apify/crawlee-python.git@0c4cfc9ada06e35f63213e6a937c4e85defcbecf",
+    "crawlee@git+https://github.com/apify/crawlee-python.git@master",
     "cryptography>=42.0.0",
     "httpx>=0.27.0",
     # TODO: ensure compatibility with the latest version of lazy-object-proxy

src/apify/_actor.py

@@ -54,7 +54,7 @@
 
 
 @docs_name('Actor')
-@docs_group('Classes')
+@docs_group('Actor')
 class _ActorType:
     """The class of `Actor`. Only make a new instance if you're absolutely sure you need to."""
 

src/apify/_charging.py

@@ -26,7 +26,7 @@
 run_validator = TypeAdapter[ActorRun | None](ActorRun | None)
 
 
-@docs_group('Interfaces')
+@docs_group('Charging')
 class ChargingManager(Protocol):
     """Provides fine-grained access to pay-per-event functionality."""
 
@@ -57,7 +57,7 @@ def get_pricing_info(self) -> ActorPricingInfo:
         """
 
 
-@docs_group('Data structures')
+@docs_group('Charging')
 @dataclass(frozen=True)
 class ChargeResult:
     """Result of the `ChargingManager.charge` method."""
@@ -72,7 +72,7 @@ class ChargeResult:
     """How many events of each known type can still be charged within the limit."""
 
 
-@docs_group('Data structures')
+@docs_group('Charging')
 @dataclass
 class ActorPricingInfo:
     """Result of the `ChargingManager.get_pricing_info` method."""

src/apify/_configuration.py

@@ -25,7 +25,7 @@ def _transform_to_list(value: Any) -> list[str] | None:
     return value if isinstance(value, list) else str(value).split(',')
 
 
-@docs_group('Classes')
+@docs_group('Configuration')
 class Configuration(CrawleeConfiguration):
     """A class for specifying the configuration of an Actor.
 

src/apify/_models.py

@@ -16,7 +16,7 @@
     from typing import TypeAlias
 
 
-@docs_group('Data structures')
+@docs_group('Other')
 class Webhook(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -35,14 +35,14 @@ class Webhook(BaseModel):
     ] = None
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunMeta(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
     origin: Annotated[MetaOrigin, Field()]
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunStats(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -63,7 +63,7 @@ class ActorRunStats(BaseModel):
     compute_units: Annotated[float, Field(alias='computeUnits')]
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunOptions(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -74,7 +74,7 @@ class ActorRunOptions(BaseModel):
     max_total_charge_usd: Annotated[Decimal | None, Field(alias='maxTotalChargeUsd')] = None
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunUsage(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -92,7 +92,7 @@ class ActorRunUsage(BaseModel):
     proxy_serps: Annotated[float | None, Field(alias='PROXY_SERPS')] = None
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRun(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 

src/apify/_platform_event_manager.py

@@ -31,13 +31,13 @@
 __all__ = ['EventManager', 'LocalEventManager', 'PlatformEventManager']
 
 
-@docs_group('Data structures')
+@docs_group('Event data')
 class PersistStateEvent(BaseModel):
     name: Literal[Event.PERSIST_STATE]
     data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))]
 
 
-@docs_group('Data structures')
+@docs_group('Event data')
 class SystemInfoEventData(BaseModel):
     mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')]
     mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')]
@@ -64,31 +64,31 @@ def to_crawlee_format(self, dedicated_cpus: float) -> EventSystemInfoData:
         )
 
 
-@docs_group('Data structures')
+@docs_group('Event data')
 class SystemInfoEvent(BaseModel):
     name: Literal[Event.SYSTEM_INFO]
     data: SystemInfoEventData
 
 
-@docs_group('Data structures')
+@docs_group('Event data')
 class MigratingEvent(BaseModel):
     name: Literal[Event.MIGRATING]
     data: Annotated[EventMigratingData, Field(default_factory=EventMigratingData)]
 
 
-@docs_group('Data structures')
+@docs_group('Event data')
 class AbortingEvent(BaseModel):
     name: Literal[Event.ABORTING]
     data: Annotated[EventAbortingData, Field(default_factory=EventAbortingData)]
 
 
-@docs_group('Data structures')
+@docs_group('Event data')
 class ExitEvent(BaseModel):
     name: Literal[Event.EXIT]
     data: Annotated[EventExitData, Field(default_factory=EventExitData)]
 
 
-@docs_group('Data structures')
+@docs_group('Event data')
 class EventWithoutData(BaseModel):
     name: Literal[
         Event.SESSION_RETIRED,
@@ -101,13 +101,13 @@ class EventWithoutData(BaseModel):
     data: Any = None
 
 
-@docs_group('Data structures')
+@docs_group('Event data')
 class DeprecatedEvent(BaseModel):
     name: Literal['cpuInfo']
     data: Annotated[dict[str, Any], Field(default_factory=dict)]
 
 
-@docs_group('Data structures')
+@docs_group('Event data')
 class UnknownEvent(BaseModel):
     name: str
     data: Annotated[dict[str, Any], Field(default_factory=dict)]
@@ -120,7 +120,7 @@ class UnknownEvent(BaseModel):
 )
 
 
-@docs_group('Classes')
+@docs_group('Event managers')
 class PlatformEventManager(EventManager):
     """A class for managing Actor events.
 

src/apify/_proxy_configuration.py

@@ -70,7 +70,7 @@ def _check(
         raise ValueError(f'{error_str} does not match pattern {pattern.pattern!r}')
 
 
-@docs_group('Classes')
+@docs_group('Configuration')
 @dataclass
 class ProxyInfo(CrawleeProxyInfo):
     """Provides information about a proxy connection that is used for requests."""
@@ -90,7 +90,7 @@ class ProxyInfo(CrawleeProxyInfo):
     """
 
 
-@docs_group('Classes')
+@docs_group('Configuration')
 class ProxyConfiguration(CrawleeProxyConfiguration):
     """Configures a connection to a proxy server with the provided options.
 

src/apify/_utils.py

@@ -30,7 +30,18 @@ def is_running_in_ipython() -> bool:
     return getattr(builtins, '__IPYTHON__', False)
 
 
-GroupName = Literal['Classes', 'Abstract classes', 'Interfaces', 'Data structures', 'Errors', 'Functions']
+# The order of the rendered API groups is defined in the docusaurus-plugin-typedoc-api.
+GroupName = Literal[
+    'Actor',
+    'Charging',
+    'Configuration',
+    'Event managers',
+    'Event data',
+    'Storage clients',
+    'Storage data',
+    'Storages',
+    'Other',
+]
 
 
 def docs_group(group_name: GroupName) -> Callable:  # noqa: ARG001

src/apify/storage_clients/_apify/_dataset_client.py

@@ -105,9 +105,6 @@ async def open(
                 f'(api_public_base_url={api_public_base_url}).'
             )
 
-        if id and name:
-            raise ValueError('Only one of "id" or "name" can be specified, not both.')
-
         # Create Apify client with the provided token and API URL.
         apify_client_async = ApifyClientAsync(
             token=token,
@@ -118,23 +115,40 @@ async def open(
         )
         apify_datasets_client = apify_client_async.datasets()
 
+        # If both id and name are provided, raise an error.
+        if id and name:
+            raise ValueError('Only one of "id" or "name" can be specified, not both.')
+
+        # If id is provided, get the storage by ID.
+        if id and name is None:
+            apify_dataset_client = apify_client_async.dataset(dataset_id=id)
+
         # If name is provided, get or create the storage by name.
-        if name is not None and id is None:
+        if name and id is None:
             id = DatasetMetadata.model_validate(
                 await apify_datasets_client.get_or_create(name=name),
             ).id
+            apify_dataset_client = apify_client_async.dataset(dataset_id=id)
 
         # If both id and name are None, try to get the default storage ID from environment variables.
         if id is None and name is None:
-            id = getattr(configuration, 'default_dataset_id', None)
+            id = configuration.default_dataset_id
+            apify_dataset_client = apify_client_async.dataset(dataset_id=id)
 
-        if id is None:
-            raise ValueError(
-                'Either "id" or "name" must be provided, or the storage ID must be set in environment variable.'
-            )
+        # Fetch its metadata.
+        metadata = await apify_dataset_client.get()
+
+        # If metadata is None, it means the storage does not exist, so we create it.
+        if metadata is None:
+            id = DatasetMetadata.model_validate(
+                await apify_datasets_client.get_or_create(),
+            ).id
+            apify_dataset_client = apify_client_async.dataset(dataset_id=id)
 
-        # Get the client for the specific storage by ID.
-        apify_dataset_client = apify_client_async.dataset(dataset_id=id)
+        # Verify that the storage exists by fetching its metadata again.
+        metadata = await apify_dataset_client.get()
+        if metadata is None:
+            raise ValueError(f'Opening dataset with id={id} and name={name} failed.')
 
         return cls(
             api_client=apify_dataset_client,

src/apify/storage_clients/_apify/_key_value_store_client.py

@@ -97,9 +97,6 @@ async def open(
                 f'(api_public_base_url={api_public_base_url}).'
             )
 
-        if id and name:
-            raise ValueError('Only one of "id" or "name" can be specified, not both.')
-
         # Create Apify client with the provided token and API URL.
         apify_client_async = ApifyClientAsync(
             token=token,
@@ -110,23 +107,40 @@ async def open(
         )
         apify_kvss_client = apify_client_async.key_value_stores()
 
+        # If both id and name are provided, raise an error.
+        if id and name:
+            raise ValueError('Only one of "id" or "name" can be specified, not both.')
+
+        # If id is provided, get the storage by ID.
+        if id and name is None:
+            apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
+
         # If name is provided, get or create the storage by name.
-        if name is not None and id is None:
+        if name and id is None:
             id = ApifyKeyValueStoreMetadata.model_validate(
                 await apify_kvss_client.get_or_create(name=name),
             ).id
+            apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
 
         # If both id and name are None, try to get the default storage ID from environment variables.
         if id is None and name is None:
-            id = getattr(configuration, 'default_key_value_store_id', None)
+            id = configuration.default_key_value_store_id
+            apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
 
-        if id is None:
-            raise ValueError(
-                'Either "id" or "name" must be provided, or the storage ID must be set in environment variable.'
-            )
+        # Fetch its metadata.
+        metadata = await apify_kvs_client.get()
+
+        # If metadata is None, it means the storage does not exist, so we create it.
+        if metadata is None:
+            id = ApifyKeyValueStoreMetadata.model_validate(
+                await apify_kvss_client.get_or_create(),
+            ).id
+            apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
 
-        # Get the client for the specific storage by ID.
-        apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
+        # Verify that the storage exists by fetching its metadata again.
+        metadata = await apify_kvs_client.get()
+        if metadata is None:
+            raise ValueError(f'Opening key-value store with id={id} and name={name} failed.')
 
         return cls(
             api_client=apify_kvs_client,