From 9ac1db9efa6f4dd8fca3be977deb73c4c7fa8ecd Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Tue, 2 Sep 2025 05:43:18 +0000 Subject: [PATCH 1/3] tweak stac link handling, add sat and sar extensions, add s1_nrb test --- datacube/metadata/_stacconverter.py | 122 ++-- tests/conftest.py | 22 + tests/data/eo3_s1_nrb.odc-type.yaml | 199 ++++++ tests/data/ga_s1_nrb_iw_hh_0.odc-product.yaml | 62 ++ ...003270-IW2_20180306T203033Z_stac-item.json | 613 ++++++++++++++++++ tests/test_eo3converter.py | 61 +- 6 files changed, 1013 insertions(+), 66 deletions(-) create mode 100644 tests/data/eo3_s1_nrb.odc-type.yaml create mode 100644 tests/data/ga_s1_nrb_iw_hh_0.odc-product.yaml create mode 100644 tests/data/ga_s1a_nrb_0-1-0_T002-003270-IW2_20180306T203033Z_stac-item.json diff --git a/datacube/metadata/_stacconverter.py b/datacube/metadata/_stacconverter.py index bbc1d4a0e..cd19f75dd 100644 --- a/datacube/metadata/_stacconverter.py +++ b/datacube/metadata/_stacconverter.py @@ -19,6 +19,8 @@ from pystac import Asset, Item, Link, MediaType from pystac.extensions.eo import Band, EOExtension from pystac.extensions.projection import ProjectionExtension +from pystac.extensions.sar import SarExtension +from pystac.extensions.sat import SatExtension from pystac.extensions.view import ViewExtension import datacube.utils.uris as dc_uris @@ -107,70 +109,84 @@ def _uri_resolve(location: str | None, path: str) -> str: def _stac_links( dataset: Dataset, - stac_url: str | None, + base_url: str | None, self_url: str | None, - collection_url: str | None, + ds_yaml_url: str | None, ) -> Generator[Link, Any, Any]: """ Add links for ODC product into a STAC Item """ # TODO: better logic for relative links - if dataset.uri: - if not self_url: - link = Link( - rel="self", - media_type=MediaType.JSON, - target=dataset.uri.replace("odc-metadata.yaml", "stac-item.json"), - ) - yield link - if dataset.uri.endswith("yaml"): - yield Link( - title="ODC Dataset YAML", - rel="odc_yaml", - media_type="text/yaml", - target=dataset.uri, - ) if self_url: yield Link( rel="self", media_type=MediaType.JSON, target=self_url, ) + elif base_url: + yield Link( + rel="self", + media_type=MediaType.JSON, + target=urljoin( + base_url, + f"/stac/collections/{dataset.product.name}/items/{dataset.id!s}", + ), + ) + else: + warnings.warn("Unable to determine self link for STAC Item.", stacklevel=2) - if collection_url: + if ds_yaml_url: yield Link( - rel="collection", - target=collection_url, + title="ODC Dataset YAML", + rel="odc_yaml", + media_type="text/yaml", + target=ds_yaml_url, ) - if stac_url: - if not collection_url: + + if base_url: + if not ds_yaml_url: yield Link( - rel="collection", - target=urljoin(stac_url, f"/stac/collections/{dataset.product.name}"), + title="ODC Dataset YAML", + rel="odc_yaml", + media_type="text/yaml", + target=urljoin(base_url, f"/dataset/{dataset.id}.odc-metadata.yaml"), ) + yield Link( + rel="collection", + target=urljoin(base_url, f"/stac/collections/{dataset.product.name}"), + ) yield Link( title="ODC Product Overview", rel="product_overview", media_type="text/html", - target=urljoin(stac_url, f"product/{dataset.product.name}"), + target=urljoin(base_url, f"product/{dataset.product.name}"), ) yield Link( title="ODC Dataset Overview", rel="alternative", media_type="text/html", - target=urljoin(stac_url, f"dataset/{dataset.id}"), + target=urljoin(base_url, f"dataset/{dataset.id}"), ) - - if not collection_url and not stac_url: + else: warnings.warn("No collection provided for STAC Item.", stacklevel=2) def ds2stac( dataset: Dataset, - stac_url: str | None = None, + base_url: str | None = None, self_url: str | None = None, - collection_url: str | None = None, + ds_yaml_url: str | None = None, + asset_location: str | None = None, ) -> Item: + """ + Convert an EO3-compatible ODC Dataset to a STAC Item. + :param base_url: The URL off which the Item links are determined + :param self_url: The Item self_link value + :param ds_yaml_url: URL for the ODC Dataset YAML + :param asset_location: Resolve Asset links against this URL. + Will default to the dataset location if not provided. + :return: pystac.Item + """ if dataset.extent is None: geometry = None bbox = None @@ -196,7 +212,7 @@ def ds2stac( ) # Add links - for link in _stac_links(dataset, stac_url, self_url, collection_url): + for link in _stac_links(dataset, base_url, self_url, ds_yaml_url): item.links.append(link) EOExtension.ext(item, add_if_missing=True) @@ -213,14 +229,22 @@ def ds2stac( if any(k.startswith("view:") for k in properties): ViewExtension.ext(item, add_if_missing=True) + if any(k.startswith("sar:") for k in properties): + SarExtension.ext(item, add_if_missing=True) + + if any(k.startswith("sat:") for k in properties): + SatExtension.ext(item, add_if_missing=True) + + # url against which asset href can be resolved + asset_location = asset_location or dataset.uri # Add assets that are data for name, measurement in dataset.measurements.items(): - if not dataset.uri and not measurement.get("path"): + if not measurement.get("path"): # No URL to link to. URL is mandatory for Stac validation. continue asset = Asset( - href=_uri_resolve(dataset.uri, measurement["path"]), + href=_uri_resolve(asset_location, measurement["path"]), media_type=_media_type(Path(measurement["path"])), title=name, roles=["data"], @@ -247,12 +271,12 @@ def ds2stac( # Add assets that are accessories for name, accessory in dataset.accessories.items(): - if not dataset.uri and not accessory.get("path"): + if not accessory.get("path"): # No URL to link to. URL is mandatory for Stac validation. continue asset = Asset( - href=_uri_resolve(dataset.uri, accessory["path"]), + href=_uri_resolve(asset_location, accessory["path"]), media_type=_media_type(Path(accessory["path"])), title=_asset_title_fields(name), roles=_asset_roles_fields(name), @@ -288,15 +312,29 @@ def infer_eo_product(metadata_doc: dict) -> Product: def ds_doc_to_stac( metadata_doc: dict, - uri: str | None = None, - stac_url: str | None = None, + ds_uri: str | None = None, + base_url: str | None = None, self_url: str | None = None, - collection_url: str | None = None, + ds_yaml_url: str | None = None, + asset_location: str | None = None, ) -> Item: + """ + Convert a raw dataset metadata document to a STAC Item. + + :metadata_doc: The raw ODC metadata document, loaded into a dict + :param ds_uri: The dataset uri. Will override the location value in the metadata doc if exists. + :param base_url: The URL off which the Item links are determined + :param self_url: The Item self_link value + :param ds_yaml_url: URL for the ODC Dataset YAML + :param asset_location: Resolve Asset links against this URL + :return: pystac.Item + """ warnings.warn("It is strongly preferred to use ds2stac if possible.", stacklevel=2) if is_doc_eo3(metadata_doc): product = infer_eo3_product(metadata_doc) - dataset = Dataset(product, prep_eo3(metadata_doc), uri=uri) + dataset = Dataset( + product, prep_eo3(metadata_doc), uri=ds_uri or metadata_doc.get("location") + ) else: warnings.warn( "Support for legacy eo datasets is deprecated and will require an " @@ -305,5 +343,7 @@ def ds_doc_to_stac( stacklevel=2, ) product = infer_eo_product(metadata_doc) - dataset = Dataset(product, metadata_doc, uri=uri) - return ds2stac(dataset, stac_url, self_url, collection_url) + dataset = Dataset( + product, metadata_doc, uri=ds_uri or metadata_doc.get("location") + ) + return ds2stac(dataset, base_url, self_url, ds_yaml_url, asset_location) diff --git a/tests/conftest.py b/tests/conftest.py index 3d98b5032..2240b6e0a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -373,6 +373,9 @@ def dask_client(): ODC_DATASET_FILE: str = "ga_ls8c_ard_3-1-0_088080_2020-05-25_final.odc-metadata.yaml" ODC_METADATA_FILE: str = "eo3_landsat_ard.odc-type.yaml" ODC_PRODUCT_FILE: str = "ard_ls8.odc-product.yaml" +S1_NRB_STAC: str = "ga_s1a_nrb_0-1-0_T002-003270-IW2_20180306T203033Z_stac-item.json" +S1_NRB_PRODUCT: str = "ga_s1_nrb_iw_hh_0.odc-product.yaml" +S1_NRB_METADATA_FILE: str = "eo3_s1_nrb.odc-type.yaml" @pytest.fixture @@ -486,3 +489,22 @@ def ds_ext_lineage(eo3_product, odc_dataset_doc) -> Dataset: ) ds.source_tree = LineageTree.from_eo3_doc(ds.metadata_doc, home="src_home") return ds + + +@pytest.fixture +def s1_nrb_metadata_type() -> MetadataType: + filepath = TEST_DATA_FOLDER.joinpath(S1_NRB_METADATA_FILE) + (_, doc), *_ = read_documents(filepath) + return metadata_from_doc(doc) + + +@pytest.fixture +def s1_nrb_stac() -> pystac.Item: + return pystac.item.Item.from_file(str(TEST_DATA_FOLDER.joinpath(S1_NRB_STAC))) + + +@pytest.fixture +def s1_nrb_product(s1_nrb_metadata_type) -> Product: + filepath = TEST_DATA_FOLDER.joinpath(S1_NRB_PRODUCT) + (_, doc), *_ = read_documents(filepath) + return Product(s1_nrb_metadata_type, doc) diff --git a/tests/data/eo3_s1_nrb.odc-type.yaml b/tests/data/eo3_s1_nrb.odc-type.yaml new file mode 100644 index 000000000..f8d25473f --- /dev/null +++ b/tests/data/eo3_s1_nrb.odc-type.yaml @@ -0,0 +1,199 @@ +--- +name: eo3_s1_nrb +description: EO3 for DEA Sentinel-1 Normalised Radar backscatter +dataset: + id: [id] + sources: [lineage, source_datasets] + + grid_spatial: [grid_spatial, projection] + measurements: [measurements] + creation_dt: [properties, 'odc:processing_datetime'] + label: [label] + format: [properties, 'odc:file_format'] + + search_fields: + platform: # Sentinel-1A/B/C/D + description: The platform code + offset: [properties, 'eo:platform'] # EO3 name for the field + indexed: false + + instrument: # Sentinel-1A/B/C/D CSAR + description: The instrument name + offset: [properties, 'eo:instrument'] + indexed: false + + product_family: # sar_ard + description: The product family code + offset: [properties, 'odc:product_family'] + indexed: false + + scene_id: # Scene name + description: The ESA scene ID (e.g. S1A_EW_GRDM_1SDH_20211219T152835_20211219T152939_041079_04E161_47D4) + offset: [properties, 'sarard:scene_id'] + indexed: false + type: string + + region_code: # Burst ID + description: > + The spatial reference code from the provider. + For Sentinel-1 normalised radar backscatter + processed with the OPERA-ISCE3 pipeline + the region code is comprised of three pieces of + ESA metadata, formatted as: "t{track_number}_{esa_burst_id}_{swath_id}". + In general it is a unique string identifier + that datasets covering roughly the same spatial + region share. + offset: [properties, 'odc:region_code'] + indexed: true + type: string + + crs_raw: + description: The raw CRS string as it appears in the metadata + offset: ['crs'] + indexed: false + + time: + description: Acquisition time range + type: datetime-range + min_offset: + - [properties, 'dtr:start_datetime'] + - [properties, datetime] + max_offset: + - [properties, 'dtr:end_datetime'] + - [properties, datetime] + + lon: + description: Longitude range + type: double-range + min_offset: + - [extent, lon, begin] + max_offset: + - [extent, lon, end] + + lat: + description: Latitude range + type: double-range + min_offset: + - [extent, lat, begin] + max_offset: + - [extent, lat, end] + + # Sentinel-1 specific properties + # Instrument mode + instrument_mode: + description: The acquisition mode (one of SM, IW, EW, WV) + offset: [properties, 'sar:instrument_mode'] + indexed: false + type: string + + # Observation direction + observation_direction: + description: The observation direction of the satellite relative to the flight path + offset: [properties, 'sar:observation_direction'] + indexed: false + type: string + + # Orbit state + orbit_state: + description: The orbit state (one of ascending or descending) + offset: [properties, 'sat:orbit_state'] + indexed: false + type: string + + # Absolute orbit + absolute_orbit: + description: The absolute orbit number of the satellite + offset: [properties, 'sat:absolute_orbit'] + indexed: false + type: integer + + # Relative orbit + relative_orbit: + description: The orbit number in the satellite's repeat cycle + offset: [properties, 'sat:relative_orbit'] + indexed: false + type: integer + + # Orbit cycle + orbit_cycle: + description: The number of days for the satellite to repeat its orbit track + offset: [properties, 'sat:orbit_cycle'] + indexed: false + type: integer + + # Orbit source + orbit_source: + description: The orbit file type + offset: [properties, 's1:orbit_source'] + indexed: false + type: string + + # Burst ID + burst_id: + description: The OPERA-ISCE3 Burst ID, formatted as "t{track_number}_{esa_burst_id}_{swath_id}" + offset: [properties, 'sarard:burst_id'] + indexed: true + type: string + + # Measurement type + measurement_type: + description: The backscatter measurement type (one of gamma_0, sigma_0, beta_0) + offset: [properties, 'sarard:measurement_type'] + indexed: false + type: string + + # Measurement convention + measurement_convention: + description: The convention used to measure backscatter + offset: [properties, 'sarard:measurement_convention'] + indexed: false + type: string + + # Conversion equation + conversion_eq: + description: The equation to convert product to decibels + offset: [properties, 'sarard:conversion_eq'] + indexed: false + type: string + + # Noise removal + noise_removal_applied: + description: Whether noise removal has been applied + offset: [properties, 'sarard:noise_removal_applied'] + indexed: false + type: string + + # Static tropospheric correction + static_tropospheric_correction_applied: + description: Whether static tropospheric correction has been applied + offset: [properties, 'sarard:static_tropospheric_correction_applied'] + indexed: false + type: string + + # Wet tropospheric correction + wet_tropospheric_correction_applied: + description: Whether wet tropospheric correction has been applied + offset: [properties, 'sarard:wet_tropospheric_correction_applied'] + indexed: false + type: string + + # Bistatic correction + bistatic_correction_applied: + description: Whether bistatic correction has been applied + offset: [properties, 'sarard:bistatic_correction_applied'] + indexed: false + type: string + + # Ionospheric correction + ionospheric_correction_applied: + description: Whether ionospheric correction has been applied + offset: [properties, 'sarard:ionospheric_correction_applied'] + indexed: false + type: string + + # Speckle filter + speckle_filter_applied: + description: Whether a speckle filter has been applied + offset: [properties, 'sarard:speckle_filter_applied'] + indexed: false + type: string diff --git a/tests/data/ga_s1_nrb_iw_hh_0.odc-product.yaml b/tests/data/ga_s1_nrb_iw_hh_0.odc-product.yaml new file mode 100644 index 000000000..5ae46ff7c --- /dev/null +++ b/tests/data/ga_s1_nrb_iw_hh_0.odc-product.yaml @@ -0,0 +1,62 @@ +--- +name: ga_s1_nrb_iw_hh_0 +description: Geoscience Australia Sentinel-1 Interferometric Wide Mode Normalised Radar Backscatter Gamma0 HH Linear Backscatter Collection 0. +metadata_type: eo3_s1_nrb + +license: CC-BY-4.0 + +metadata: + product: + name: ga_s1_nrb_iw_hh_0 + properties: + odc:product_family: sar_ard + +measurements: + - name: HH_gamma0 + aliases: + - hh_gamma0 + dtype: float32 + nodata: .nan + units: '1' + - name: mask + dtype: uint8 + nodata: 255 + units: '1' + flags_definition: + mask: + bits: [0,1,2,3,4,5,6,7] + values: + 1: shadow + 2: layover + 3: shadow and layover + 255: invalid sample + description: shadow layover data mask + - name: number_of_looks + aliases: + - n_looks + - nlooks + dtype: float32 + nodata: .nan + units: '1' + - name: gamma0_to_beta0_ratio + dtype: float32 + nodata: .nan + units: '1' + - name: gamma0_to_sigma0_ratio + dtype: float32 + nodata: .nan + units: '1' + - name: local_incidence_angle + aliases: + - lia + - LIA + dtype: float32 + nodata: .nan + units: '1' + - name: incidence_angle + aliases: + - ia + - IA + dtype: float32 + nodata: .nan + units: '1' diff --git a/tests/data/ga_s1a_nrb_0-1-0_T002-003270-IW2_20180306T203033Z_stac-item.json b/tests/data/ga_s1a_nrb_0-1-0_T002-003270-IW2_20180306T203033Z_stac-item.json new file mode 100644 index 000000000..6b075dc88 --- /dev/null +++ b/tests/data/ga_s1a_nrb_0-1-0_T002-003270-IW2_20180306T203033Z_stac-item.json @@ -0,0 +1,613 @@ +{ + "type": "Feature", + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/product/v0.1.0/schema.json", + "https://stac-extensions.github.io/sar/v1.1.0/schema.json", + "https://stac-extensions.github.io/projection/v2.0.0/schema.json", + "https://stac-extensions.github.io/sat/v1.1.0/schema.json", + "https://stac-extensions.github.io/sentinel-1/v0.2.0/schema.json", + "https://stac-extensions.github.io/processing/v1.2.0/schema.json", + "https://stac-extensions.github.io/storage/v2.0.0/schema.json", + "https://stac-extensions.github.io/ceos-ard/v0.2.0/schema.json" + ], + "id": "ga_s1a_nrb_0-1-0_T002-003270-IW2_20180306T203033Z", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 137.41483427834683, + -7.636146385818411 + ], + [ + 137.41431890797708, + -7.543817802484732 + ], + [ + 137.4138099866416, + -7.451488795654488 + ], + [ + 137.4133075076778, + -7.35915937041914 + ], + [ + 137.41281146451072, + -7.266829531873516 + ], + [ + 137.41232185065275, + -7.1744992851157905 + ], + [ + 137.32748784988001, + -7.17493724606201 + ], + [ + 137.24264863044976, + -7.175359562588836 + ], + [ + 137.15780438217303, + -7.175766230091816 + ], + [ + 137.07295529491805, + -7.1761572441365 + ], + [ + 136.98810155860846, + -7.1765326004585495 + ], + [ + 136.9032433632208, + -7.1768922949638565 + ], + [ + 136.81838089878258, + -7.177236323728659 + ], + [ + 136.73351435537, + -7.177564682999631 + ], + [ + 136.64864392310594, + -7.177877369194005 + ], + [ + 136.56376979215747, + -7.1781743788996435 + ], + [ + 136.47889215273412, + -7.178455708875146 + ], + [ + 136.4791925358277, + -7.270837397248897 + ], + [ + 136.4794968635057, + -7.363218697526912 + ], + [ + 136.4798051397536, + -7.455599604878236 + ], + [ + 136.48011736861082, + -7.547980114475703 + ], + [ + 136.48043355417053, + -7.640360221495966 + ], + [ + 136.56539957318753, + -7.640060587539001 + ], + [ + 136.65036206574916, + -7.639744253718228 + ], + [ + 136.73532084067307, + -7.63941122349016 + ], + [ + 136.8202757068197, + -7.639061500493378 + ], + [ + 136.9052264730944, + -7.63869508854843 + ], + [ + 136.9901729484496, + -7.638311991657716 + ], + [ + 137.07511494188725, + -7.637912214005405 + ], + [ + 137.16005226246057, + -7.637495759957298 + ], + [ + 137.24498471927663, + -7.637062634060726 + ], + [ + 137.3299121214984, + -7.6366128410444265 + ], + [ + 137.41483427834683, + -7.636146385818411 + ] + ] + ] + }, + "bbox": [ + 136.47889215273412, + -7.640360221495966, + 137.41483427834683, + -7.1744992851157905 + ], + "properties": { + "gsd": 20.0, + "constellation": "Sentinel-1", + "platform": "Sentinel-1A", + "instruments": [ + "Sentinel-1A CSAR" + ], + "created": "2025-08-26T07:50:33.135818Z", + "start_datetime": "2018-03-06T20:30:33.946912Z", + "end_datetime": "2018-03-06T20:30:37.036413Z", + "odc:product": "ga_s1_nrb_iw_vv_vh_0", + "odc:product_family": "sar_ard", + "odc:region_code": "t002_003270_iw2", + "odc:producer": "ga.gov.au", + "odc:dataset_version": "0-1-0", + "product:type": "RTC_S1", + "ceosard:type": "radar", + "ceosard:specification": "NRB", + "ceosard:specification_version": "5.5", + "proj:code": "EPSG:32753", + "proj:bbox": [ + 663300.0, + 9155180.0, + 766420.0, + 9206260.0 + ], + "sar:frequency_band": "C", + "sar:center_frequency": 5.40500045433435, + "sarard:center_frequency_unit": "GHz", + "sar:polarizations": [ + "VV", + "VH" + ], + "sar:observation_direction": "right", + "sar:beam_ids": [ + "IW2" + ], + "sar:instrument_mode": "IW", + "sat:orbit_state": "descending", + "sat:absolute_orbit": 20899, + "sat:relative_orbit": 2, + "sat:orbit_cycle": 12, + "s1:orbit_source": "POE precise orbit", + "processing:level": "L2", + "processing:facility": "Geoscience Australia", + "processing:datetime": "2025-08-26T07:50:33.135818Z", + "processing:version": "0-1-0", + "processing:software": { + "isce3": "0.24.4", + "s1Reader": "0.2.5", + "GeoscienceAustralia/RTC": "1.0.4", + "sar-pipeline": "0.3.1.dev0+g4b7eea64a.d20250825", + "dem-handler": "0.2.3" + }, + "sarard:source_id": [ + "S1A_IW_SLC__1SDV_20180306T203033_20180306T203100_020899_023DAA_F0D1.SAFE" + ], + "sarard:source_geometry": "slant range", + "sarard:scene_id": "S1A_IW_SLC__1SDV_20180306T203033_20180306T203100_020899_023DAA_F0D1", + "sarard:burst_id": "t002_003270_iw2", + "sarard:beam_id": "IW2", + "sarard:orbit_files": [ + "S1A_OPER_AUX_POEORB_OPOD_20210306T151055_V20180305T225942_20180307T005942.EOF" + ], + "sarard:UL_longitude": 136.47889215273412, + "sarard:UL_latitude": -7.1744992851157905, + "sarard:LR_longitude": 137.41483427834683, + "sarard:LR_latitude": -7.640360221495966, + "sarard:pixel_spacing_x": 20.0, + "sarard:pixel_spacing_y": 20.0, + "sarard:pixel_spacing_unit": "metre", + "sarard:resolution_x": 20.0, + "sarard:resolution_y": 20.0, + "sarard:resolution_unit": "metre", + "sarard:speckle_filter_applied": false, + "sarard:speckle_filter_type": "", + "sarard:speckle_filter_window": [], + "sarard:measurement_type": "Gamma-0", + "sarard:measurement_convention": "linear backscatter intensity", + "sarard:conversion_eq": "10*log10(backscatter_linear)", + "sarard:noise_removal_applied": true, + "sarard:static_tropospheric_correction_applied": true, + "sarard:wet_tropospheric_correction_applied": false, + "sarard:bistatic_correction_applied": true, + "sarard:ionospheric_correction_applied": false, + "sarard:geometric_accuracy_ALE": 2.94, + "sarard:geometric_accuracy_rmse": 3.08, + "sarard:geometric_accuracy_range": 1.63, + "sarard:geometric_accuracy_azimuth": 1.92, + "sarard:geometric_accuracy_unit": "metre", + "storage:schemes": { + "aws": { + "type": "aws-s3", + "platform": "https://{bucket}.s3.{region}.amazonaws.com", + "bucket": "deant-data-public-dev", + "region": "ap-southeast-2" + } + }, + "proj:shape": [ + 2554, + 5156 + ], + "proj:transform": [ + 20.0, + 0.0, + 663300.0, + 0.0, + -20.0, + 9206260.0, + 0.0, + 0.0, + 1.0 + ], + "sarard:number_of_lines": 2554, + "sarard:number_of_pixels_per_line": 5156, + "datetime": "2018-03-06T20:30:33.946912Z" + }, + "links": [ + { + "rel": "geoid-source", + "href": "https://aria-geoid.s3.us-west-2.amazonaws.com/us_nga_egm2008_1_4326__agisoft.tif", + "type": "image/tiff; application=geotiff" + }, + { + "rel": "ceos-ard-specification", + "href": "https://ceos.org/ard/files/PFS/SAR/v1.2/CEOS-ARD_PFS_Synthetic_Aperture_Radar_v1.2.pdf", + "type": "application/pdf" + }, + { + "rel": "derived-from", + "href": "https://datapool.asf.alaska.edu/SLC/SA/S1A_IW_SLC__1SDV_20180306T203033_20180306T203100_020899_023DAA_F0D1.zip" + }, + { + "rel": "dem-source", + "href": "https://registry.opendata.aws/copernicus-dem/", + "type": "text/html" + }, + { + "rel": "rtc-algorithm", + "href": "https://doi.org/10.1109/TGRS.2022.3147472", + "type": "text/html" + }, + { + "rel": "geocoding-algorithm", + "href": "https://doi.org/10.1109/TGRS.2022.3147472", + "type": "text/html" + }, + { + "rel": "noise-correction", + "href": "https://sentinels.copernicus.eu/documents/247904/2142675/Thermal-Denoising-of-Products-Generated-by-Sentinel-1-IPF.pdf", + "type": "application/pdf" + }, + { + "rel": "static-layers-stac-item", + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/for_zhengshu/ga_s1_nrb_iw_static_0/t002_003270_iw2/ga_s1_nrb-static_0-1-0_T002-003270-IW2_20140403_stac-item.json", + "type": "[None, 'application/geo+json', 'application/json']" + }, + { + "rel": "static-layers-browse", + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/index.html?prefix=experimental/for_zhengshu/ga_s1_nrb_iw_static_0/t002_003270_iw2", + "type": "text/html" + }, + { + "rel": "h5-metadata", + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/for_zhengshu/ga_s1_nrb_iw_vv_vh_0/t002_003270_iw2/2018/03/06/ga_s1a_nrb_0-1-0_T002-003270-IW2_20180306T203033Z_metadata.h5", + "type": "application/x-hdf5" + }, + { + "rel": "processing-config", + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/for_zhengshu/ga_s1_nrb_iw_vv_vh_0/t002_003270_iw2/2018/03/06/ga_s1a_nrb_0-1-0_T002-003270-IW2_20180306T203033Z_proc-config.yaml", + "type": "application/yaml" + }, + { + "rel": "browse", + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/index.html?prefix=experimental/for_zhengshu/ga_s1_nrb_iw_vv_vh_0/t002_003270_iw2/2018/03/06", + "type": "text/html" + }, + { + "rel": "self", + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/for_zhengshu/ga_s1_nrb_iw_vv_vh_0/t002_003270_iw2/2018/03/06/ga_s1a_nrb_0-1-0_T002-003270-IW2_20180306T203033Z_stac-item.json", + "type": "[None, 'application/geo+json', 'application/json']" + }, + { + "rel": "collection", + "href": "https://explorer.dev.dea.ga.gov.au/stac/collections/ga_s1_nrb_iw_vv_vh_0", + "type": "[None, 'application/geo+json', 'application/json']" + } + ], + "assets": { + "VV_gamma0": { + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/for_zhengshu/ga_s1_nrb_iw_vv_vh_0/t002_003270_iw2/2018/03/06/ga_s1a_nrb_0-1-0_T002-003270-IW2_20180306T203033Z_VV-gamma0.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "VV_gamma0", + "description": "VV polarised gamma0 linear backscatter", + "proj:shape": [ + 2554, + 5156 + ], + "proj:transform": [ + 20.0, + 0.0, + 663300.0, + 0.0, + -20.0, + 9206260.0, + 0.0, + 0.0, + 1.0 + ], + "proj:code": "EPSG:32753", + "raster:data_type": "float32", + "raster:sampling": "area", + "raster:nodata": "nan", + "raster:pixel_coordinate_convention": "pixel ULC", + "processing:level": "L2", + "roles": [ + "data", + "backscatter" + ] + }, + "VH_gamma0": { + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/for_zhengshu/ga_s1_nrb_iw_vv_vh_0/t002_003270_iw2/2018/03/06/ga_s1a_nrb_0-1-0_T002-003270-IW2_20180306T203033Z_VH-gamma0.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "VH_gamma0", + "description": "VH polarised gamma0 linear backscatter", + "proj:shape": [ + 2554, + 5156 + ], + "proj:transform": [ + 20.0, + 0.0, + 663300.0, + 0.0, + -20.0, + 9206260.0, + 0.0, + 0.0, + 1.0 + ], + "proj:code": "EPSG:32753", + "raster:data_type": "float32", + "raster:sampling": "area", + "raster:nodata": "nan", + "raster:pixel_coordinate_convention": "pixel ULC", + "processing:level": "L2", + "roles": [ + "data", + "backscatter" + ] + }, + "mask": { + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/for_zhengshu/ga_s1_nrb_iw_vv_vh_0/t002_003270_iw2/2018/03/06/ga_s1a_nrb_0-1-0_T002-003270-IW2_20180306T203033Z_mask.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "mask", + "description": "shadow layover data mask", + "proj:shape": [ + 2554, + 5156 + ], + "proj:transform": [ + 20.0, + 0.0, + 663300.0, + 0.0, + -20.0, + 9206260.0, + 0.0, + 0.0, + 1.0 + ], + "proj:code": "EPSG:32753", + "raster:data_type": "uint8", + "raster:sampling": "area", + "raster:nodata": 255.0, + "raster:pixel_coordinate_convention": "pixel ULC", + "raster:values": { + "shadow": 1, + "layover": 2, + "shadow_and_layover": 3, + "invalid_sample": 255 + }, + "roles": [ + "data", + "auxiliary", + "mask", + "shadow", + "layover" + ] + }, + "thumbnail": { + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/for_zhengshu/ga_s1_nrb_iw_static_0/t002_003270_iw2/ga_s1_nrb-static_0-1-0_T002-003270-IW2_20140403_thumbnail.png", + "type": "image/png", + "title": "thumbnail", + "description": "thumbnail image for backscatter", + "roles": [ + "thumbnail" + ] + }, + "number_of_looks": { + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/for_zhengshu/ga_s1_nrb_iw_static_0/t002_003270_iw2/ga_s1_nrb-static_0-1-0_T002-003270-IW2_20140403_number-of-looks.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "number_of_looks", + "description": "number of looks", + "proj:shape": [ + 2554, + 5156 + ], + "proj:transform": [ + 20.0, + 0.0, + 663300.0, + 0.0, + -20.0, + 9206260.0, + 0.0, + 0.0, + 1.0 + ], + "proj:code": "EPSG:32753", + "raster:data_type": "float32", + "raster:sampling": "area", + "raster:nodata": "nan", + "raster:pixel_coordinate_convention": "pixel ULC", + "roles": [ + "data", + "auxiliary" + ] + }, + "gamma0_to_beta0_ratio": { + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/for_zhengshu/ga_s1_nrb_iw_static_0/t002_003270_iw2/ga_s1_nrb-static_0-1-0_T002-003270-IW2_20140403_gamma0-to-beta0-ratio.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "gamma0_to_beta0_ratio", + "description": "backscatter conversion layer, gamma0 to beta0. Eq. beta0 = rtc_anf_gamma0_to_beta0*gamma0", + "proj:shape": [ + 2554, + 5156 + ], + "proj:transform": [ + 20.0, + 0.0, + 663300.0, + 0.0, + -20.0, + 9206260.0, + 0.0, + 0.0, + 1.0 + ], + "proj:code": "EPSG:32753", + "raster:data_type": "float32", + "raster:sampling": "area", + "raster:nodata": "nan", + "raster:pixel_coordinate_convention": "pixel ULC", + "roles": [ + "data", + "auxiliary", + "conversion" + ] + }, + "gamma0_to_sigma0_ratio": { + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/for_zhengshu/ga_s1_nrb_iw_static_0/t002_003270_iw2/ga_s1_nrb-static_0-1-0_T002-003270-IW2_20140403_gamma0-to-sigma0-ratio.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "gamma0_to_sigma0_ratio", + "description": "backscatter conversion layer, gamma0 to sigma0. Eq. sigma0 = rtc_anf_sigma0_to_sigma0*gamma0", + "proj:shape": [ + 2554, + 5156 + ], + "proj:transform": [ + 20.0, + 0.0, + 663300.0, + 0.0, + -20.0, + 9206260.0, + 0.0, + 0.0, + 1.0 + ], + "proj:code": "EPSG:32753", + "raster:data_type": "float32", + "raster:sampling": "area", + "raster:nodata": "nan", + "raster:pixel_coordinate_convention": "pixel ULC", + "roles": [ + "data", + "auxiliary", + "conversion" + ] + }, + "local_incidence_angle": { + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/for_zhengshu/ga_s1_nrb_iw_static_0/t002_003270_iw2/ga_s1_nrb-static_0-1-0_T002-003270-IW2_20140403_local-incidence-angle.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "local_incidence_angle", + "description": "local incidence angle (LIA)", + "proj:shape": [ + 2554, + 5156 + ], + "proj:transform": [ + 20.0, + 0.0, + 663300.0, + 0.0, + -20.0, + 9206260.0, + 0.0, + 0.0, + 1.0 + ], + "proj:code": "EPSG:32753", + "raster:data_type": "float32", + "raster:sampling": "area", + "raster:nodata": "nan", + "raster:pixel_coordinate_convention": "pixel ULC", + "roles": [ + "data", + "auxiliary" + ] + }, + "incidence_angle": { + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/for_zhengshu/ga_s1_nrb_iw_static_0/t002_003270_iw2/ga_s1_nrb-static_0-1-0_T002-003270-IW2_20140403_incidence-angle.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "incidence_angle", + "description": "incidence angle (IA)", + "proj:shape": [ + 2554, + 5156 + ], + "proj:transform": [ + 20.0, + 0.0, + 663300.0, + 0.0, + -20.0, + 9206260.0, + 0.0, + 0.0, + 1.0 + ], + "proj:code": "EPSG:32753", + "raster:data_type": "float32", + "raster:sampling": "area", + "raster:nodata": "nan", + "raster:pixel_coordinate_convention": "pixel ULC", + "roles": [ + "data", + "auxiliary" + ] + } + }, + "collection": "ga_s1_nrb_iw_vv_vh_0" +} diff --git a/tests/test_eo3converter.py b/tests/test_eo3converter.py index 109ddadd6..1a8673515 100644 --- a/tests/test_eo3converter.py +++ b/tests/test_eo3converter.py @@ -5,13 +5,14 @@ # pylint: disable=unused-argument,unused-variable,missing-module-docstring,wrong-import-position,import-error # pylint: disable=redefined-outer-name,protected-access,import-outside-toplevel -import os +import math import uuid from typing import Any import pystac import pytest from odc.geo.geom import Geometry +from odc.stac import load from odc.stac._mdtools import RasterCollectionMetadata, has_proj_ext, has_raster_ext from pystac.extensions.eo import EOExtension from pystac.extensions.item_assets import ItemAssetsExtension @@ -287,8 +288,11 @@ def test_accessories(sentinel_stac_ms: pystac.Item) -> None: def test_ds2stac(eo3_dataset: Dataset) -> None: - assert eo3_dataset.uri is not None - output_stac = ds2stac(eo3_dataset).to_dict() + output_stac = ds2stac( + eo3_dataset, + self_url="https://localhost/stac/eo3_dataset.json", + base_url="https://localhost/", + ).to_dict() assert output_stac["properties"]["instruments"] == ["oli", "tirs"] assert set(output_stac["assets"].keys()) == set( list(eo3_dataset.measurements.keys()) + list(eo3_dataset.accessories.keys()) @@ -296,32 +300,14 @@ def test_ds2stac(eo3_dataset: Dataset) -> None: assert output_stac["links"] == [ { "rel": "self", - "href": os.path.abspath(eo3_dataset.uri).replace( - "odc-metadata.yaml", "stac-item.json" - ), + "href": "https://localhost/stac/eo3_dataset.json", "type": "application/json", }, { + "title": "ODC Dataset YAML", "rel": "odc_yaml", - "href": eo3_dataset.uri, + "href": f"https://localhost/dataset/{eo3_dataset.id}.odc-metadata.yaml", "type": "text/yaml", - "title": "ODC Dataset YAML", - }, - ] - - -def test_ds2stac_links(eo3_dataset: Dataset) -> None: - eo3_dataset.uri = None - output_stac = ds2stac( - eo3_dataset, - self_url="https://localhost/stac/eo3_dataset.json", - stac_url="https://localhost/", - ).to_dict() - assert output_stac["links"] == [ - { - "rel": "self", - "href": "https://localhost/stac/eo3_dataset.json", - "type": "application/json", }, { "rel": "collection", @@ -434,6 +420,31 @@ def test_infer_eo3_product(odc_dataset_doc) -> None: def test_dsdoc_to_stac(odc_dataset_doc, eo3_dataset) -> None: - from_doc = ds_doc_to_stac(odc_dataset_doc, uri=eo3_dataset.uri) + from_doc = ds_doc_to_stac(odc_dataset_doc) from_ds = ds2stac(eo3_dataset) assert from_doc.to_dict() == from_ds.to_dict() + + +def test_s1_nrb(s1_nrb_stac, s1_nrb_product) -> None: + # simulate loading an indexed stac dataset by converting it to eo3 then back to stac + eo3_ds = next( + iter( + stac2ds([s1_nrb_stac], product_cache={"ga_s1_nrb_iw_hh_0": s1_nrb_product}) + ) + ) + dc_stac = ds2stac(eo3_ds) + assert ( + "https://stac-extensions.github.io/sat/v1.0.0/schema.json" + in dc_stac.stac_extensions + ) + assert ( + "https://stac-extensions.github.io/sar/v1.0.0/schema.json" + in dc_stac.stac_extensions + ) + stac_ds = load( + [dc_stac], + crs="EPSG:32753", + resolution=20, + bbox=(137.26307, -7.45486, 137.32457, -7.41362), + ) + assert not math.isnan(stac_ds.VV_gamma0.data[0][0][0]) From 12f430a199f2dad4a1ecbae690067fc043937c96 Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Tue, 2 Sep 2025 06:31:43 +0000 Subject: [PATCH 2/3] fix roundtrip test --- tests/test_eo3converter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_eo3converter.py b/tests/test_eo3converter.py index 1a8673515..dba6c8e56 100644 --- a/tests/test_eo3converter.py +++ b/tests/test_eo3converter.py @@ -339,7 +339,9 @@ def test_sources(ds_legacy_sources: Dataset, ds_ext_lineage: Dataset) -> None: def test_roundtrip(eo3_dataset: Dataset, eo3_product: Product) -> None: original = eo3_dataset - roundtrip = _item_to_ds(ds2stac(eo3_dataset), eo3_product) + roundtrip = _item_to_ds( + ds2stac(eo3_dataset, base_url="https://localhost/"), eo3_product + ) orig_doc = original.metadata_doc rt_doc = roundtrip.metadata_doc From b9c6d76234036a2f76d8fde546690f30b7a73023 Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Tue, 2 Sep 2025 06:36:50 +0000 Subject: [PATCH 3/3] combine properties loop for extensions --- datacube/metadata/_stacconverter.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/datacube/metadata/_stacconverter.py b/datacube/metadata/_stacconverter.py index cd19f75dd..4210aa7cd 100644 --- a/datacube/metadata/_stacconverter.py +++ b/datacube/metadata/_stacconverter.py @@ -226,14 +226,13 @@ def ds2stac( proj.apply(wkt2=dataset.crs.wkt, **_proj_fields(dataset.grids)) # To pass validation, only add 'view' extension when we're using it somewhere. - if any(k.startswith("view:") for k in properties): - ViewExtension.ext(item, add_if_missing=True) - - if any(k.startswith("sar:") for k in properties): - SarExtension.ext(item, add_if_missing=True) - - if any(k.startswith("sat:") for k in properties): - SatExtension.ext(item, add_if_missing=True) + for k in properties: + if k.startswith("view:"): + ViewExtension.ext(item, add_if_missing=True) + if k.startswith("sar:"): + SarExtension.ext(item, add_if_missing=True) + if k.startswith("sat:"): + SatExtension.ext(item, add_if_missing=True) # url against which asset href can be resolved asset_location = asset_location or dataset.uri