From bf7f452b03e530fbe19c757b9d5f5311907fc756 Mon Sep 17 00:00:00 2001 From: Charles Turner Date: Mon, 3 Feb 2025 08:39:10 +0800 Subject: [PATCH 1/3] Add flaky decorator for tests with remote resources - should reduce incidence of cold start related test failures --- ci/environment-upstream-dev.yml | 1 + ci/environment.yml | 1 + tests/test_cat.py | 1 + tests/test_core.py | 3 +++ tests/test_tutorial.py | 2 ++ 5 files changed, 8 insertions(+) diff --git a/ci/environment-upstream-dev.yml b/ci/environment-upstream-dev.yml index 335b5a16..b68f789f 100644 --- a/ci/environment-upstream-dev.yml +++ b/ci/environment-upstream-dev.yml @@ -8,6 +8,7 @@ dependencies: - codecov - dask>=2024.12 - fastprogress>=1.0.0 + - flaky >= 3.8.0 - gcsfs >=2024.12 - h5netcdf>=0.8.1 - ipython diff --git a/ci/environment.yml b/ci/environment.yml index 4373d969..57ed4e61 100644 --- a/ci/environment.yml +++ b/ci/environment.yml @@ -7,6 +7,7 @@ dependencies: - codecov - dask>=2024.12 - fastprogress>=1.0.0 + - flaky >= 3.8.0 - fsspec>=2024.12 - gcsfs >=2024.12 - h5netcdf>=0.8.1 diff --git a/tests/test_cat.py b/tests/test_cat.py index bed00407..518a2f90 100644 --- a/tests/test_cat.py +++ b/tests/test_cat.py @@ -49,6 +49,7 @@ def test_assets_mutually_exclusive(): multi_variable_cat, ], ) +@pytest.mark.flaky(max_runs=3, min_passes=1) # Cold start related failures def test_esmcatmodel_load(file): cat = ESMCatalogModel.load(file) assert isinstance(cat, ESMCatalogModel) diff --git a/tests/test_core.py b/tests/test_core.py index 463df865..6a7ccbe6 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -71,6 +71,7 @@ def func_multivar(ds): (intake_esm.cat.ESMCatalogModel.load(cdf_cat_sample_cmip6), '.', None, None), ], ) +@pytest.mark.flaky(max_runs=3, min_passes=1) # Cold start related failures def test_catalog_init(capsys, obj, sep, read_csv_kwargs, columns_with_iterables): """Test that the catalog can be initialized.""" cat = intake.open_esm_datastore( @@ -355,6 +356,7 @@ def test_multi_variable_catalog_derived_cat(): (mixed_cat_sample_cmip6, dict(institution_id='BCC'), {}), ], ) +@pytest.mark.flaky(max_runs=3, min_passes=1) # Cold start related failures def test_to_dataset_dict(path, query, xarray_open_kwargs): cat = intake.open_esm_datastore(path) cat_sub = cat.search(**query) @@ -387,6 +389,7 @@ def test_to_dataset_dict(path, query, xarray_open_kwargs): ), ], ) +@pytest.mark.flaky(max_runs=3, min_passes=1) # Cold start related failures def test_to_datatree(path, query, xarray_open_kwargs): cat = intake.open_esm_datastore(path) cat_sub = cat.search(**query) diff --git a/tests/test_tutorial.py b/tests/test_tutorial.py index b098c8cf..4bfa683e 100644 --- a/tests/test_tutorial.py +++ b/tests/test_tutorial.py @@ -10,6 +10,7 @@ @pytest.mark.parametrize('name,url', tutorial_cats) +@pytest.mark.flaky(max_runs=3, min_passes=1) # Cold start related failures def test_get_url(name, url): cat_url = intake_esm.tutorial.get_url(name) assert isinstance(cat_url, str) @@ -18,6 +19,7 @@ def test_get_url(name, url): @pytest.mark.network @pytest.mark.parametrize('name,url', tutorial_cats) +@pytest.mark.flaky(max_runs=3, min_passes=1) # Cold start related failures def test_open_from_url(name, url): cat_url = intake_esm.tutorial.get_url(name) cat = intake.open_esm_datastore(cat_url) From fd60ed4ae860d9dce41c20988d93bf4478b13620 Mon Sep 17 00:00:00 2001 From: Charles Turner Date: Mon, 3 Feb 2025 09:25:27 +0800 Subject: [PATCH 2/3] Separate out expected failing tests from expected passing tests - the pytest.mark.flaky() decorator seems to work fine on my local machine but is causing CI issues - not quite sure why --- tests/test_tutorial.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/test_tutorial.py b/tests/test_tutorial.py index 4bfa683e..91fbe25a 100644 --- a/tests/test_tutorial.py +++ b/tests/test_tutorial.py @@ -4,9 +4,7 @@ import intake_esm from intake_esm.tutorial import DEFAULT_CATALOGS -tutorial_cats = list(zip(DEFAULT_CATALOGS.keys(), DEFAULT_CATALOGS.values())) + [ - pytest.param('bad_key', 'bad_url', marks=pytest.mark.xfail) -] +tutorial_cats = list(zip(DEFAULT_CATALOGS.keys(), DEFAULT_CATALOGS.values())) @pytest.mark.parametrize('name,url', tutorial_cats) @@ -17,6 +15,13 @@ def test_get_url(name, url): assert cat_url == url +@pytest.mark.xfail +def test_get_url_xfail(): + cat_url = intake_esm.tutorial.get_url('bad_key') + assert isinstance(cat_url, str) + assert 'bad_url' == 'bad_url' + + @pytest.mark.network @pytest.mark.parametrize('name,url', tutorial_cats) @pytest.mark.flaky(max_runs=3, min_passes=1) # Cold start related failures @@ -27,6 +32,15 @@ def test_open_from_url(name, url): assert cat == intake.open_esm_datastore(url) +@pytest.mark.network +@pytest.mark.xfail +def test_open_from_url_xfail(): + cat_url = intake_esm.tutorial.get_url('bad_url') + cat = intake.open_esm_datastore(cat_url) + assert isinstance(cat.esmcat, intake_esm.cat.ESMCatalogModel) + assert cat == intake.open_esm_datastore('bad_url') + + def test_get_available_cats(): cats = intake_esm.tutorial.get_available_cats() assert cats == list(DEFAULT_CATALOGS.keys()) From 86c05f970f3ad8394bd16c3857ecad8a2a95cbe8 Mon Sep 17 00:00:00 2001 From: Charles Turner Date: Mon, 3 Feb 2025 11:06:41 +0800 Subject: [PATCH 3/3] Pin zarr < 3.0 --- ci/environment-upstream-dev.yml | 2 +- ci/environment.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/environment-upstream-dev.yml b/ci/environment-upstream-dev.yml index b68f789f..af9a7514 100644 --- a/ci/environment-upstream-dev.yml +++ b/ci/environment-upstream-dev.yml @@ -33,7 +33,7 @@ dependencies: - scipy - xarray-datatree - xgcm - - zarr>=2.10 + - zarr>=2.10,<3.0 - pip: - git+https://github.com/intake/intake.git - git+https://github.com/pydata/xarray.git diff --git a/ci/environment.yml b/ci/environment.yml index 57ed4e61..fabda53a 100644 --- a/ci/environment.yml +++ b/ci/environment.yml @@ -30,5 +30,5 @@ dependencies: - scipy - xarray>=2024.10 - xarray-datatree - - zarr>=2.12 + - zarr>=2.12,<3.0 # - pytest-icdiff