diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 8bdf0f78eed..f45958f4a00 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -14,6 +14,8 @@ New Features ~~~~~~~~~~~~ - Expose :py:class:`~xarray.indexes.RangeIndex`, and :py:class:`~xarray.indexes.CoordinateTransformIndex` as public api under the ``xarray.indexes`` namespace. By `Deepak Cherian `_. +- Support zarr-python's new ``.supports_consolidated_metadata`` store property (:pull:`10457``). + by Tom Nicholas `_. - Better error messages when encoding data to be written to disk fails (:pull:`10464`). By `Stephan Hoyer `_ diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index fff07063964..48405b906cd 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -1768,6 +1768,11 @@ def _get_open_params( else: missing_exc = zarr.errors.GroupNotFoundError + if _zarr_v3(): + # zarr 3.0.8 and earlier did not support this property - it was effectively assumed true + if not getattr(store, "supports_consolidated_metadata", True): + consolidated = consolidate_on_close = False + if consolidated in [None, True]: # open the root of the store, in case there is metadata consolidated there group = open_kwargs.pop("path") @@ -1825,6 +1830,7 @@ def _get_open_params( else: # this was the default for v2 and should apply to most existing Zarr data use_zarr_fill_value_as_mask = True + return ( zarr_group, consolidate_on_close, diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 2c1718cee7a..7bacaf58155 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -89,6 +89,7 @@ requires_scipy, requires_scipy_or_netCDF4, requires_zarr, + requires_zarr_v3, ) from xarray.tests.test_coding_times import ( _ALL_CALENDARS, @@ -117,6 +118,7 @@ if has_zarr_v3: from zarr.storage import MemoryStore as KVStore + from zarr.storage import WrapperStore ZARR_FORMATS = [2, 3] else: @@ -127,8 +129,11 @@ ) except ImportError: KVStore = None # type: ignore[assignment,misc,unused-ignore] + + WrapperStore = object # type: ignore[assignment,misc,unused-ignore] else: KVStore = None # type: ignore[assignment,misc,unused-ignore] + WrapperStore = object # type: ignore[assignment,misc,unused-ignore] ZARR_FORMATS = [] @@ -2400,12 +2405,13 @@ def test_read_non_consolidated_warning(self) -> None: self.save( expected, store_target=store, consolidated=False, **self.version_kwargs ) - with pytest.warns( - RuntimeWarning, - match="Failed to open Zarr store with consolidated", - ): - with xr.open_zarr(store, **self.version_kwargs) as ds: - assert_identical(ds, expected) + if getattr(store, "supports_consolidated_metadata", True): + with pytest.warns( + RuntimeWarning, + match="Failed to open Zarr store with consolidated", + ): + with xr.open_zarr(store, **self.version_kwargs) as ds: + assert_identical(ds, expected) def test_non_existent_store(self) -> None: with pytest.raises( @@ -3756,6 +3762,42 @@ def test_chunk_key_encoding_v2(self) -> None: assert actual["var1"].encoding["chunks"] == (2, 2) +class NoConsolidatedMetadataSupportStore(WrapperStore): + """ + Store that explicitly does not support consolidated metadata. + + Useful as a proxy for stores like Icechunk, see https://github.com/zarr-developers/zarr-python/pull/3119. + """ + + supports_consolidated_metadata = False + + def __init__( + self, + store, + *, + read_only: bool = False, + ) -> None: + self._store = store.with_read_only(read_only=read_only) + + def with_read_only( + self, read_only: bool = False + ) -> NoConsolidatedMetadataSupportStore: + return type(self)( + store=self._store, + read_only=read_only, + ) + + +@requires_zarr_v3 +class TestZarrNoConsolidatedMetadataSupport(ZarrBase): + @contextlib.contextmanager + def create_zarr_target(self): + # TODO the zarr version would need to be >3.08 for the supports_consolidated_metadata property to have any effect + yield NoConsolidatedMetadataSupportStore( + zarr.storage.MemoryStore({}, read_only=False) + ) + + @requires_zarr @pytest.mark.skipif( ON_WINDOWS,