Various test fixes

integration_tests/asserts.py

@@ -193,7 +193,7 @@ def check_datesets_page_datestring(
 
 
 def expect_values(
-    s: TimePeriodOverview,
+    s: TimePeriodOverview | None,
     dataset_count: int,
     footprint_count: int,
     time_range: Range,
@@ -204,6 +204,7 @@ def expect_values(
     size_bytes: int | None,
     region_dataset_counts: dict | None = None,
 ):
+    assert s is not None, "No overview for dataset"
     __tracebackhide__ = True
 
     was_timeline_error = False
@@ -271,6 +272,7 @@ def expect_values(
             was_regions_error = False
     except AssertionError:
         assert s.newest_dataset_creation_time is not None
+        assert s.time_range is not None
         print(
             f"""Got:
         dataset_count {s.dataset_count}

integration_tests/dumpdatasets.py

@@ -67,7 +67,9 @@ def dump_datasets(
 
 def _get_dumpable_doc(dc: Datacube, d: Dataset, include_sources=True):
     if include_sources:
-        return dc.index.datasets.get(d.id, include_sources=include_sources).metadata_doc
+        dataset = dc.index.datasets.get(d.id, include_sources=include_sources)
+        assert dataset is not None
+        return dataset.metadata_doc
     else:
         # Empty doc means "there are no sources", so we can load it easily.
         d.metadata.sources = {}

integration_tests/test_configurable_page_elements.py

@@ -126,7 +126,9 @@ def test_sister_sites(app_configured_client: FlaskClient) -> None:
     assert len(sister_instances) == 2
 
     for sister_instance in sister_instances:
-        assert "/about" in sister_instance.css_first("a.sister-link").attributes["href"]
+        href = sister_instance.css_first("a.sister-link").attributes["href"]
+        assert href is not None
+        assert "/about" in href
 
 
 def test_sister_sites_request_path(app_configured_client: FlaskClient) -> None:
@@ -136,18 +138,16 @@ def test_sister_sites_request_path(app_configured_client: FlaskClient) -> None:
     assert len(sister_instances) == 2
 
     for sister_instance in sister_instances:
-        assert (
-            "/products/ga_ls5t_ard_3"
-            in sister_instance.css_first("a.sister-link").attributes["href"]
-        )
+        href = sister_instance.css_first("a.sister-link").attributes["href"]
+        assert href is not None
+        assert "/products/ga_ls5t_ard_3" in href
 
     html = get_html(app_configured_client, "/products/ga_ls5t_ard_3/datasets")
 
     sister_instances = html.css("#sister-site-menu ul li")
     assert len(sister_instances) == 2
 
     for sister_instance in sister_instances:
-        assert (
-            "/products/ga_ls5t_ard_3/datasets"
-            in sister_instance.css_first("a.sister-link").attributes["href"]
-        )
+        href = sister_instance.css_first("a.sister-link").attributes["href"]
+        assert href is not None
+        assert "/products/ga_ls5t_ard_3/datasets" in href

integration_tests/test_dataset_listing.py

@@ -26,6 +26,7 @@ def test_parse_query_args(odc_test_db: Datacube) -> None:
     """
 
     product = odc_test_db.index.products.get_by_name("ga_ls7e_ard_3")
+    assert product is not None
     res = query_to_search(
         MultiDict(
             (
@@ -58,6 +59,7 @@ def test_default_args(dea_index: Index) -> None:
     assert DEFAULT_PLATFORM_END_DATE["LANDSAT_5"] == datetime(2011, 11, 30)
 
     product = dea_index.products.get_by_name("ard_ls5")
+    assert product is not None
 
     res = query_to_search(MultiDict(()), product)
 

integration_tests/test_eo3_support.py

@@ -88,10 +88,9 @@ def test_eo3_extents(eo3_index: Index) -> None:
     assert dataset_extent_row["creation_time"] == datetime(
         2020, 6, 5, 7, 15, 26, 599544, tzinfo=tz.tzutc()
     )
-    assert (
-        dataset_extent_row["product_ref"]
-        == eo3_index.products.get_by_name("ga_ls5t_ard_3").id
-    )
+    product = eo3_index.products.get_by_name("ga_ls5t_ard_3")
+    assert product is not None
+    assert dataset_extent_row["product_ref"] == product.id
 
     # This should be the geometry field of eo3, not the max/min bounds
     # that eo1 compatibility adds within `grid_spatial`.
@@ -194,6 +193,7 @@ def test_undo_eo3_doc_compatibility(eo3_index: Index) -> None:
     indexed_dataset = eo3_index.datasets.get(
         UUID("5b2f2c50-e618-4bef-ba1f-3d436d9aed14"), include_sources=True
     )
+    assert indexed_dataset is not None
     indexed_doc = with_parsed_datetimes(indexed_dataset.metadata_doc)
 
     # Undo the changes.
@@ -215,10 +215,11 @@ def test_undo_eo3_doc_compatibility(eo3_index: Index) -> None:
 
 def test_undo_eo3_compatibility_del_handling() -> None:
     doc = {"extent": "a", "lineage": {}}
-    assert _utils.undo_eo3_compatibility(doc) is None
+    _utils.undo_eo3_compatibility(doc)
+    assert "extent" not in doc
 
 
-def with_parsed_datetimes(v: dict, name=""):
+def with_parsed_datetimes(v: dict | str | datetime | list, name=""):
     """
     All date fields in eo3 metadata have names ending in 'datetime'. Return a doc
     with all of these fields parsed as actual dates.
@@ -228,7 +229,7 @@ def with_parsed_datetimes(v: dict, name=""):
     if not v:
         return v
 
-    if name.endswith("datetime"):
+    if name.endswith("datetime") and isinstance(v, str | datetime):
         dt = parse_time(v)
         # Strip/normalise timezone to match default yaml.load()
         if dt.tzinfo:
@@ -257,14 +258,18 @@ def test_can_search_eo3_items(eo3_index, client: FlaskClient) -> None:
     geojson = get_items(
         client, "http://localhost/stac/collections/ga_ls5t_ard_3/items?_full=false"
     )
-    assert len(geojson.get("features")) == 1
+    features = geojson.get("features")
+    assert features is not None
+    assert len(features) == 1
     assert "gqa:abs_iterative_mean_xy" not in geojson["features"][0]["properties"]
 
     # .... And full records
     geojson = get_items(
         client, "http://localhost/stac/collections/ga_ls5t_ard_3/items?_full=True"
     )
-    assert len(geojson.get("features")) == 1
+    features = geojson.get("features")
+    assert features is not None
+    assert len(features) == 1
     assert geojson["features"][0]["properties"][
         "gqa:abs_iterative_mean_xy"
     ] == pytest.approx(0.37)

integration_tests/test_page_loads.py

@@ -353,6 +353,14 @@ def test_storage_page(client: FlaskClient, odc_test_db) -> None:
     assert len(html.css(".data-table tbody tr")) == product_count
 
 
+def test_product_audit_redirects(client: FlaskClient) -> None:
+    assert_redirects_to(
+        client,
+        "/product-audit/day-times.txt",
+        "/audit/day-query-times.txt",
+    )
+
+
 @pytest.mark.skip(reason="TODO: fix out-of-date range return value")
 def test_out_of_date_range(client: FlaskClient) -> None:
     """
@@ -361,7 +369,7 @@ def test_out_of_date_range(client: FlaskClient) -> None:
     html = get_html(client, "/wofs_albers/2010")
 
     # The common error here is to say "No data: not yet summarised" rather than "0 datasets"
-    assert check_dataset_count(html, 0)
+    check_dataset_count(html, 0)
     assert "Historic Flood Mapping Water Observations from Space" in html.text()
 
 
@@ -709,9 +717,6 @@ def test_api_returns_timelines(client: FlaskClient) -> None:
     assert doc == {"2022-01-17T00:00:00": 1}
 
 
-pytest.mark.xfail(True, reason="telemetry data removed")
-
-
 @pytest.mark.parametrize("env_name", ("default",), indirect=True)
 def test_undisplayable_product(client: FlaskClient) -> None:
     """
@@ -944,7 +949,7 @@ def test_all_give_404s(client: FlaskClient) -> None:
     We should get 404 messages, not exceptions, for missing things.
     """
 
-    def expect_404(url: str, message_contains: str = None):
+    def expect_404(url: str, message_contains: str | None = None):
         __tracebackhide__ = True
         response = get_text_response(client, url, expect_status_code=404)
         if message_contains and message_contains not in response:
@@ -978,6 +983,8 @@ def expect_404(url: str, message_contains: str = None):
     expect_404(f"/dataset/{dataset_id}")
     expect_404(f"/dataset/{dataset_id}.odc-metadata.yaml")
 
+    expect_404("/api/dataset-timeline/non_existent/2025")
+
 
 def test_invalid_query_gives_400(client: FlaskClient) -> None:
     """

integration_tests/test_stores.py

@@ -21,10 +21,10 @@
 
 def _overview(
     product_name: str = "test_product",
-    year: int = None,
-    month: int = None,
-    day: int = None,
-):
+    year: int | None = None,
+    month: int | None = None,
+    day: int | None = None,
+) -> TimePeriodOverview:
     orig = TimePeriodOverview(
         product_name=product_name,
         year=year,
@@ -78,9 +78,11 @@ def test_add_period_list() -> None:
         "test_product", product_refresh_time, [_overview(), _overview(), total], 0.0
     )
     assert joined.dataset_count == _overview().dataset_count * 2
-    assert _overview().footprint_geometry.area == pytest.approx(
-        joined.footprint_geometry.area
-    )
+    geometry = _overview().footprint_geometry
+    assert geometry is not None
+    joined_geometry = joined.footprint_geometry
+    assert joined_geometry is not None
+    assert geometry.area == pytest.approx(joined_geometry.area)
 
     assert sum(joined.region_dataset_counts.values()) == joined.dataset_count
     assert sum(joined.timeline_dataset_counts.values()) == joined.dataset_count
@@ -103,14 +105,20 @@ def test_add_no_periods(summary_store: SummaryStore) -> None:
     """
     result, summary = summary_store.refresh("ga_ls8c_level1_3")
     assert result == GenerateResult.CREATED
+    assert summary is not None
     assert summary.dataset_count == 0
-    assert summary_store.get("ga_ls8c_level1_3", 2015, 7, 4).dataset_count == 0
+    dataset = summary_store.get("ga_ls8c_level1_3", 2015, 7, 4)
+    assert dataset is not None
+    assert dataset.dataset_count == 0
 
     result, summary = summary_store.refresh("ga_ls8c_level1_3")
     assert result == GenerateResult.NO_CHANGES
+    assert summary is not None
     assert summary.dataset_count == 0
 
-    assert summary_store.get("ga_ls8c_level1_3").dataset_count == 0
+    dataset = summary_store.get("ga_ls8c_level1_3")
+    assert dataset is not None
+    assert dataset.dataset_count == 0
     assert summary_store.get("ga_ls8c_level1_3", 2015, 7, None) is None
 
 

integration_tests/test_summarise_data.py

@@ -113,6 +113,7 @@ def test_generate_scene_all_time(run_generate, summary_store: SummaryStore) -> N
 
     # All time
     summary = summary_store.get("ls8_nbar_scene", year=None, month=None, day=None)
+    assert summary is not None
     assert (
         summary_store.index.datasets.count(product="ls8_nbar_scene")
         == summary.dataset_count
@@ -151,6 +152,7 @@ def test_generate_incremental_archivals(
 
     # When we have a summarised product...
     original_summary = summary_store.get("ga_ls9c_ard_3")
+    assert original_summary is not None
     original_dataset_count = original_summary.dataset_count
 
     # ... and we archive one dataset ...
@@ -161,18 +163,21 @@ def test_generate_incremental_archivals(
 
         # ... the next generation should catch it and update with one less dataset....
         run_generate("ga_ls9c_ard_3")
-        assert (
-            summary_store.get("ga_ls9c_ard_3").dataset_count
-            == original_dataset_count - 1
-        ), "Expected dataset count to decrease after archival"
+        dataset = summary_store.get("ga_ls9c_ard_3")
+        assert dataset is not None
+        assert dataset.dataset_count == original_dataset_count - 1, (
+            "Expected dataset count to decrease after archival"
+        )
     finally:
         # Now let's restore the dataset!
         index.datasets.restore([dataset_id])
 
     # It should be in the count again.
     # (this change should work because the new 'updated' column will be bumped on restore)
     run_generate("ga_ls9c_ard_3")
-    assert summary_store.get("ga_ls9c_ard_3").dataset_count == original_dataset_count, (
+    dataset = summary_store.get("ga_ls9c_ard_3")
+    assert dataset is not None
+    assert dataset.dataset_count == original_dataset_count, (
         "A dataset that was restored from archival was not refreshed by Explorer"
     )
 
@@ -203,12 +208,15 @@ def test_dataset_changing_product(run_generate, summary_store: SummaryStore) ->
 
     # When we have a summarised product...
     original_summary = summary_store.get("ga_ls9c_ard_3")
+    assert original_summary is not None
     original_dataset_count = original_summary.dataset_count
 
     try:
         # Move the dataset to another product
         _change_dataset_product(index, dataset_id, other_product)
-        assert index.datasets.get(dataset_id).product.name == "ga_ls8c_ard_3"
+        dataset = index.datasets.get(dataset_id)
+        assert dataset is not None
+        assert dataset.product.name == "ga_ls8c_ard_3"
 
         # Explorer should remove it too.
         print(f"Test dataset: {dataset_id}")
@@ -217,18 +225,20 @@ def test_dataset_changing_product(run_generate, summary_store: SummaryStore) ->
         #       but it's not in the product. And the incremental updater misses it.
         #       So we have to force the non-incremental updater.
         run_generate("ga_ls8c_ard_3", "ga_ls9c_ard_3", "--force-refresh")
-
-        assert (
-            summary_store.get("ga_ls9c_ard_3").dataset_count
-            == original_dataset_count - 1
-        ), "Expected dataset to be removed after product change"
+        dataset_summary = summary_store.get("ga_ls9c_ard_3")
+        assert dataset_summary is not None
+        assert dataset_summary.dataset_count == original_dataset_count - 1, (
+            "Expected dataset to be removed after product change"
+        )
 
     finally:
         # Now change it back
         _change_dataset_product(index, dataset_id, our_product)
 
     run_generate("ga_ls8c_ard_3", "ga_ls9c_ard_3", "--force-refresh")
-    assert summary_store.get("ga_ls9c_ard_3").dataset_count == original_dataset_count, (
+    dataset_summary = summary_store.get("ga_ls9c_ard_3")
+    assert dataset_summary is not None
+    assert dataset_summary.dataset_count == original_dataset_count, (
         "Expected dataset to be added again after the product changed back"
     )
 
@@ -272,8 +282,11 @@ def test_has_source_derived_product_links(
     run_generate()
 
     ls_fc_pc = summary_store.get_product_summary("ga_ls_fc_pc_cyear_3")
+    assert ls_fc_pc is not None
     ls_fc = summary_store.get_product_summary("ga_ls_fc_3")
+    assert ls_fc is not None
     ls8_ard = summary_store.get_product_summary("ga_ls8c_ard_3")
+    assert ls8_ard is not None
 
     print(repr([ls_fc_pc, ls_fc, ls8_ard]))
     assert ls_fc_pc.source_products == ["ga_ls_fc_3"]
@@ -291,8 +304,11 @@ def test_product_fixed_fields(run_generate, summary_store: SummaryStore) -> None
     run_generate()
 
     albers = summary_store.get_product_summary("ls8_nbar_albers")
+    assert albers is not None
     scene = summary_store.get_product_summary("ls8_nbar_scene")
+    assert scene is not None
     telem = summary_store.get_product_summary("ls8_satellite_telemetry_data")
+    assert telem is not None
 
     assert scene.fixed_metadata == {
         "platform": "LANDSAT_8",
@@ -329,9 +345,10 @@ def test_sampled_product_fixed_fields(summary_store: SummaryStore) -> None:
     # not big enough to trigger it in the `run_generate()` tests)
 
     # Tiled product, sampled
+    product = summary_store.index.products.get_by_name("ga_ls9c_ard_3")
+    assert product is not None
     fixed_fields = summary_store._find_product_fixed_metadata(
-        summary_store.index.products.get_by_name("ga_ls9c_ard_3"),
-        sample_datasets_size=5,
+        product, sample_datasets_size=5
     )
 
     assert fixed_fields == {
@@ -518,11 +535,12 @@ def test_calc_albers_summary_with_storage(summary_store: SummaryStore) -> None:
     )
 
     original = summary_store.get("ls8_nbar_albers", 2017)
-
+    assert original is not None
     # It should now return the same copy, not rebuild it.
     summary_store.refresh("ls8_nbar_albers")
 
     cached_s = summary_store.get("ls8_nbar_albers", 2017)
+    assert cached_s is not None
     assert original is not cached_s
     assert cached_s.dataset_count == original.dataset_count
     assert cached_s.summary_gen_time is not None