From eee71801210cd8ca4922895aeb5d0f16d643bf97 Mon Sep 17 00:00:00 2001 From: sudhamurthy Date: Thu, 13 Feb 2025 09:58:51 -0500 Subject: [PATCH 1/5] DAS-2300 - provides spatial subsetting support for 3d variables in SPL3SMP --- CHANGELOG.md | 8 + docker/service_version.txt | 2 +- hoss/coordinate_utilities.py | 56 +++++-- hoss/hoss_config.json | 64 +++++++- hoss/spatial.py | 9 +- tests/unit/test_coordinate_utilities.py | 190 ++++++++++++++++++++---- tests/unit/test_spatial.py | 12 +- tests/unit/test_subset.py | 8 +- 8 files changed, 289 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59fbdee..698ff57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v1.1.5 +### 2025-02-14 + +This version of HOSS adds support for 3D variables which +do not have the nominal order. This would provide support +for the 3D variables in SMAP - SPL3SMP with dimension order +information provided in the configurations file. + ## v1.1.4 ### 2025-02-12 diff --git a/docker/service_version.txt b/docker/service_version.txt index 65087b4..e25d8d9 100644 --- a/docker/service_version.txt +++ b/docker/service_version.txt @@ -1 +1 @@ -1.1.4 +1.1.5 diff --git a/hoss/coordinate_utilities.py b/hoss/coordinate_utilities.py index f7b21e7..6b7088b 100644 --- a/hoss/coordinate_utilities.py +++ b/hoss/coordinate_utilities.py @@ -136,8 +136,8 @@ def create_spatial_dimension_names_from_coordinates( if variable is not None: dimension_array_names = [ - f'{variable.group_path}/dim_y', - f'{variable.group_path}/dim_x', + f'{variable.group_path}/y_dim', + f'{variable.group_path}/x_dim', ] else: raise MissingVariable(variable_name) @@ -150,7 +150,8 @@ def create_dimension_arrays_from_coordinates( latitude_coordinate: VariableFromDmr, longitude_coordinate: VariableFromDmr, crs: CRS, - projected_dimension_names: list[str], + variable_name: str, + varinfo: VarInfoFromDmr, ) -> dict[str, np.ndarray]: """Generate artificial 1D dimensions scales for each 2D dimension or coordinate variable. @@ -159,8 +160,12 @@ def create_dimension_arrays_from_coordinates( 3) Generate the x-y dimscale array and return to the calling method """ - if len(projected_dimension_names) < 2: - raise InvalidDimensionNames(projected_dimension_names) + dimension_names = get_dimension_array_names(varinfo, variable_name) + if len(dimension_names) < 2: + raise InvalidDimensionNames(dimension_names) + + # check if the dimension names are configured in hoss_config + dimension_name_order = get_configured_dimension_order(varinfo, dimension_names) lat_arr = get_2d_coordinate_array( prefetch_dataset, @@ -171,10 +176,12 @@ def create_dimension_arrays_from_coordinates( longitude_coordinate.full_name_path, ) + # get the max spread x and y indices row_indices, col_indices = get_valid_sample_pts( lat_arr, lon_arr, latitude_coordinate, longitude_coordinate ) + # get the dimension order from the coordinate data dim_order_is_y_x, row_dim_values = get_dimension_order_and_dim_values( lat_arr, lon_arr, row_indices, crs, is_row=True ) @@ -188,18 +195,24 @@ def create_dimension_arrays_from_coordinates( lat_arr, lon_arr, dim_order_is_y_x ) + # calculate the dimension values y_dim = interpolate_dim_values_from_sample_pts( row_dim_values, np.transpose(row_indices)[0], row_size ) - x_dim = interpolate_dim_values_from_sample_pts( col_dim_values, np.transpose(col_indices)[1], col_size ) - projected_y, projected_x = ( - projected_dimension_names[-2], - projected_dimension_names[-1], - ) + # if it is not configured in the hoss_config.json + # assume it is nominal order [z,y,x] + if not dimension_name_order: + projected_y, projected_x = ( + dimension_names[-2], + dimension_names[-1], + ) + else: + projected_x = dimension_name_order['projection_x_coordinate'] + projected_y = dimension_name_order['projection_y_coordinate'] if dim_order_is_y_x: return {projected_y: y_dim, projected_x: x_dim} @@ -208,6 +221,23 @@ def create_dimension_arrays_from_coordinates( # return {projected_x: x_dim, projected_y: y_dim} +def get_configured_dimension_order( + varinfo: VarInfoFromDmr, dimension_names: list[str] +) -> dict[str, str]: + """This function returns the dimension order in a dictionary + with standard_names that is used to define the dimensions e.g. + 'projection_x_coordinate' and 'projection_y_coordinate' if they + are configured in hoss_config.json + + """ + dimension_name_order = {} + for dimension_name in dimension_names: + attrs = varinfo.get_missing_variable_attributes(dimension_name) + if 'standard_name' in attrs.keys(): + dimension_name_order[attrs['standard_name']] = dimension_name + return dimension_name_order + + def get_2d_coordinate_array( prefetch_dataset: Dataset, coordinate_name: str, @@ -445,10 +475,14 @@ def interpolate_dim_values_from_sample_pts( def create_dimension_arrays_from_geotransform( prefetch_dataset: Dataset, latitude_coordinate: VariableFromDmr, - projected_dimension_names: list[str], + variable_name: str, + varinfo: VarInfoFromDmr, geotranform, ) -> dict[str, np.ndarray]: """Generate artificial 1D dimensions scales from geotransform""" + + projected_dimension_names = get_dimension_array_names(varinfo, variable_name) + lat_arr = get_2d_coordinate_array( prefetch_dataset, latitude_coordinate.full_name_path, diff --git a/hoss/hoss_config.json b/hoss/hoss_config.json index a05f0a7..c50abe9 100644 --- a/hoss/hoss_config.json +++ b/hoss/hoss_config.json @@ -1,6 +1,6 @@ { "Identification": "hoss_config", - "Version": 21, + "Version": 22, "CollectionShortNamePath": [ "/HDF5_GLOBAL/short_name", "/NC_GLOBAL/short_name", @@ -442,6 +442,68 @@ ], "_Description": "SMAP L3 data are HDF5 and without dimension settings. Overrides here define the dimensions, a useful reference name, and critically, the dimension order." }, + { + "Applicability": { + "Mission": "SMAP", + "ShortNamePath": "SPL3SMP", + "VariablePattern": ".*/x_dim" + }, + "Attributes": [ + { + "Name": "dimensions", + "Value": "x_dim" + }, + { + "Name": "Units", + "Value": "m" + }, + { + "Name": "standard_name", + "Value": "projection_x_coordinate" + } + ], + "_Description": "The pseudo-dimension variable is here supplemented with variable attributes (as if it was a dimension variables) to fully specify the X dimension." + }, + { + "Applicability": { + "Mission": "SMAP", + "ShortNamePath": "SPL3SMP", + "VariablePattern": ".*/y_dim" + }, + "Attributes": [ + { + "Name": "dimensions", + "Value": "y_dim" + }, + { + "Name": "Units", + "Value": "m" + }, + { + "Name": "standard_name", + "Value": "projection_y_coordinate" + } + ], + "_Description": "The pseudo-dimension variable is here supplemented with variable attributes (as if it was a dimension variables) to fully specify the Y dimension." + }, + { + "Applicability": { + "Mission": "SMAP", + "ShortNamePath": "SPL3SMP", + "VariablePattern": ".*/am_pm" + }, + "Attributes": [ + { + "Name": "dimensions", + "Value": "am_pm" + }, + { + "Name": "long_name", + "Value": "AM-PM dimension of size 2, 0 => AM, 1=> PM" + } + ], + "_Description": "The pseudo-dimension variable is here supplemented with variable attributes (as if it was a dimension variables) to clarify the dimension name" + }, { "Applicability": { "Mission": "ICESat2", diff --git a/hoss/spatial.py b/hoss/spatial.py index 950fba7..c57ef11 100644 --- a/hoss/spatial.py +++ b/hoss/spatial.py @@ -248,14 +248,13 @@ def get_x_y_index_ranges_from_coordinates( """ crs = get_variable_crs(non_spatial_variable, varinfo) - - projected_dimension_names = get_dimension_array_names(varinfo, non_spatial_variable) master_geotransform = get_master_geotransform(non_spatial_variable, varinfo) if master_geotransform: dimension_arrays = create_dimension_arrays_from_geotransform( prefetch_coordinate_datasets, latitude_coordinate, - projected_dimension_names, + non_spatial_variable, + varinfo, master_geotransform, ) else: @@ -264,7 +263,9 @@ def get_x_y_index_ranges_from_coordinates( latitude_coordinate, longitude_coordinate, crs, - projected_dimension_names, + # projected_dimension_names, + non_spatial_variable, + varinfo, ) projected_y, projected_x = dimension_arrays.keys() diff --git a/tests/unit/test_coordinate_utilities.py b/tests/unit/test_coordinate_utilities.py index ac875f5..8d4b977 100644 --- a/tests/unit/test_coordinate_utilities.py +++ b/tests/unit/test_coordinate_utilities.py @@ -17,6 +17,7 @@ create_dimension_arrays_from_geotransform, create_spatial_dimension_names_from_coordinates, get_2d_coordinate_array, + get_configured_dimension_order, get_coordinate_variables, get_dimension_array_names, get_dimension_order_and_dim_values, @@ -528,8 +529,8 @@ def test_create_spatial_dimension_names_from_coordinates(self): """ expected_dimension_names = [ - '/Soil_Moisture_Retrieval_Data_AM/dim_y', - '/Soil_Moisture_Retrieval_Data_AM/dim_x', + '/Soil_Moisture_Retrieval_Data_AM/y_dim', + '/Soil_Moisture_Retrieval_Data_AM/x_dim', ] with self.subTest( @@ -569,12 +570,12 @@ def test_get_dimension_array_names(self): """ expected_override_dimensions_AM = [ - '/Soil_Moisture_Retrieval_Data_AM/dim_y', - '/Soil_Moisture_Retrieval_Data_AM/dim_x', + '/Soil_Moisture_Retrieval_Data_AM/y_dim', + '/Soil_Moisture_Retrieval_Data_AM/x_dim', ] expected_override_dimensions_PM = [ - '/Soil_Moisture_Retrieval_Data_PM/dim_y', - '/Soil_Moisture_Retrieval_Data_PM/dim_x', + '/Soil_Moisture_Retrieval_Data_PM/y_dim', + '/Soil_Moisture_Retrieval_Data_PM/x_dim', ] with self.subTest( @@ -636,6 +637,60 @@ def test_get_dimension_array_names(self): ], ) + def test_get_configured_dimension_order(self): + """Ensure that the expected x-y dimension name + order is returned correctly if configured in + hoss_config.json file + """ + with self.subTest( + 'Retrieves expected dimension order configured for not nominal order' + ): + configured_dimension_names_not_nominal = [ + '/Soil_Moisture_Retrieval_Data_AM/y_dim', + '/Soil_Moisture_Retrieval_Data_AM/x_dim', + '/Soil_Moisture_Retrieval_Data_AM/lc_type', + ] + expected_dimension_order = { + 'projection_y_coordinate': '/Soil_Moisture_Retrieval_Data_AM/y_dim', + 'projection_x_coordinate': '/Soil_Moisture_Retrieval_Data_AM/x_dim', + } + self.assertDictEqual( + get_configured_dimension_order( + self.varinfo, configured_dimension_names_not_nominal + ), + expected_dimension_order, + ) + + with self.subTest('Retrieves empty dictionary for not configured order'): + not_configured_dimension_names = [ + '/Soil_Moisture_Retrieval_Data_AM/dim_y', + '/Soil_Moisture_Retrieval_Data_AM/dim_x', + ] + expected_dimension_order = {} + self.assertDictEqual( + get_configured_dimension_order( + self.varinfo, + not_configured_dimension_names, + ), + expected_dimension_order, + ) + with self.subTest( + 'Retrieves expected dimension order configured for nominal order' + ): + configured_dimension_names_nominal = [ + '/Soil_Moisture_Retrieval_Data_AM/am_pm', + '/Soil_Moisture_Retrieval_Data_AM/y_dim', + '/Soil_Moisture_Retrieval_Data_AM/x_dim', + ] + expected_dimension_order = {} + + self.assertDictEqual( + get_configured_dimension_order( + self.smap_ftp_varinfo, configured_dimension_names_nominal + ), + expected_dimension_order, + ) + def test_get_row_col_sizes_from_coordinates(self): """Ensure that the correct row and column sizes are returned for the requested coordinates @@ -1139,12 +1194,12 @@ def test_create_dimension_arrays_from_coordinates( '/Soil_Moisture_Retrieval_Data_AM/longitude' ) projected_dimension_names_am = [ - '/Soil_Moisture_Retrieval_Data_AM/dim_y', - '/Soil_Moisture_Retrieval_Data_AM/dim_x', + '/Soil_Moisture_Retrieval_Data_AM/y_dim', + '/Soil_Moisture_Retrieval_Data_AM/x_dim', ] projected_dimension_names_pm = [ - '/Soil_Moisture_Retrieval_Data_PM/dim_y', - '/Soil_Moisture_Retrieval_Data_PM/dim_x', + '/Soil_Moisture_Retrieval_Data_PM/y_dim', + '/Soil_Moisture_Retrieval_Data_PM/x_dim', ] crs = CRS.from_cf( { @@ -1165,14 +1220,16 @@ def test_create_dimension_arrays_from_coordinates( latitude_coordinate, longitude_coordinate, crs, - projected_dimension_names_am, + '/Soil_Moisture_Retrieval_Data_AM/albedo', + self.varinfo, ) x_y_dim_pm = create_dimension_arrays_from_coordinates( smap_prefetch, latitude_coordinate, longitude_coordinate, crs, - projected_dimension_names_pm, + '/Soil_Moisture_Retrieval_Data_PM/albedo_pm', + self.varinfo, ) self.assertListEqual( @@ -1182,35 +1239,35 @@ def test_create_dimension_arrays_from_coordinates( list(x_y_dim_pm.keys()), projected_dimension_names_pm ) self.assertEqual( - x_y_dim_am['/Soil_Moisture_Retrieval_Data_AM/dim_y'][0], + x_y_dim_am['/Soil_Moisture_Retrieval_Data_AM/y_dim'][0], expected_ydim[0], ) self.assertEqual( - x_y_dim_am['/Soil_Moisture_Retrieval_Data_AM/dim_y'][-1], + x_y_dim_am['/Soil_Moisture_Retrieval_Data_AM/y_dim'][-1], expected_ydim[-1], ) self.assertEqual( - x_y_dim_am['/Soil_Moisture_Retrieval_Data_AM/dim_x'][0], + x_y_dim_am['/Soil_Moisture_Retrieval_Data_AM/x_dim'][0], expected_xdim[0], ) self.assertEqual( - x_y_dim_am['/Soil_Moisture_Retrieval_Data_AM/dim_x'][-1], + x_y_dim_am['/Soil_Moisture_Retrieval_Data_AM/x_dim'][-1], expected_xdim[-1], ) self.assertEqual( - x_y_dim_pm['/Soil_Moisture_Retrieval_Data_PM/dim_y'][0], + x_y_dim_pm['/Soil_Moisture_Retrieval_Data_PM/y_dim'][0], expected_ydim[0], ) self.assertEqual( - x_y_dim_pm['/Soil_Moisture_Retrieval_Data_PM/dim_y'][-1], + x_y_dim_pm['/Soil_Moisture_Retrieval_Data_PM/y_dim'][-1], expected_ydim[-1], ) self.assertEqual( - x_y_dim_pm['/Soil_Moisture_Retrieval_Data_PM/dim_x'][0], + x_y_dim_pm['/Soil_Moisture_Retrieval_Data_PM/x_dim'][0], expected_xdim[0], ) self.assertEqual( - x_y_dim_pm['/Soil_Moisture_Retrieval_Data_PM/dim_x'][-1], + x_y_dim_pm['/Soil_Moisture_Retrieval_Data_PM/x_dim'][-1], expected_xdim[-1], ) with self.subTest('Invalid data in coordinate datasets'): @@ -1236,7 +1293,8 @@ def test_create_dimension_arrays_from_coordinates( latitude_coordinate, longitude_coordinate, crs, - projected_dimension_names_am, + '/Soil_Moisture_Retrieval_Data_AM/albedo', + self.varinfo, ) self.assertEqual( context.exception.message, @@ -1265,7 +1323,8 @@ def test_create_dimension_arrays_from_coordinates( latitude_coordinate, longitude_coordinate, crs, - projected_dimension_names_am, + '/Soil_Moisture_Retrieval_Data_AM/albedo', + self.varinfo, ) self.assertEqual( context.exception.message, @@ -1282,15 +1341,16 @@ def test_create_dimension_arrays_from_coordinates( latitude_coordinate, longitude_coordinate, crs, - projected_dimension_names_am, + '/Soil_Moisture_Retrieval_Data_AM/albedo', + self.varinfo, ) - def test_create_dimension_arrays_from_3d_coordinates( + def test_create_dimension_arrays_from_nominal_3d_coordinates( self, ): """Ensure that the correct x and y dim arrays are returned from a lat/lon prefetch dataset and - crs provided. + crs provided for a nominal (z,y,x) order 3D variable """ latitude_coordinate = self.smap_ftp_varinfo.get_variable( @@ -1324,7 +1384,8 @@ def test_create_dimension_arrays_from_3d_coordinates( latitude_coordinate, longitude_coordinate, crs, - dimension_names_global, + '/Freeze_Thaw_Retrieval_Data_Global/landcover_class', + self.smap_ftp_varinfo, ) self.assertListEqual( @@ -1348,6 +1409,68 @@ def test_create_dimension_arrays_from_3d_coordinates( expected_xdim[-1], ) + def test_create_dimension_arrays_from_not_nominal_3d_coordinates(self): + """Ensure that the correct x and y dim arrays + are returned from a lat/lon prefetch dataset and + crs provided for a 3D variable that is (y,x,z) order and + not the nominal (z,y,x) order + """ + + latitude_coordinate = self.varinfo.get_variable( + '/Soil_Moisture_Retrieval_Data_AM/latitude' + ) + longitude_coordinate = self.varinfo.get_variable( + '/Soil_Moisture_Retrieval_Data_AM/longitude' + ) + dimension_names = [ + '/Soil_Moisture_Retrieval_Data_AM/y_dim', + '/Soil_Moisture_Retrieval_Data_AM/x_dim', + '/Soil_Moisture_Retrieval_Data_AM/lc_type', + ] + + crs = CRS.from_cf( + { + 'false_easting': 0.0, + 'false_northing': 0.0, + 'longitude_of_central_meridian': 0.0, + 'standard_parallel': 30.0, + 'grid_mapping_name': 'lambert_cylindrical_equal_area', + } + ) + expected_xdim = np.array([-17349514.353068016, 17349514.353068016]) + expected_ydim = np.array([7296524.6913595535, -7296524.691359556]) + with self.subTest('Projected x-y dim arrays from coordinate datasets'): + with Dataset(self.nc4file, 'r') as smap_prefetch: + x_y_dim = create_dimension_arrays_from_coordinates( + smap_prefetch, + latitude_coordinate, + longitude_coordinate, + crs, + '/Soil_Moisture_Retrieval_Data_AM/landcover_class', + self.varinfo, + ) + + self.assertListEqual( + list(x_y_dim.keys()), + [dimension_names[0], dimension_names[1]], + ) + self.assertEqual( + x_y_dim['/Soil_Moisture_Retrieval_Data_AM/y_dim'][0], + expected_ydim[0], + ) + self.assertEqual( + x_y_dim['/Soil_Moisture_Retrieval_Data_AM/y_dim'][-1], + expected_ydim[-1], + ) + self.assertEqual( + x_y_dim['/Soil_Moisture_Retrieval_Data_AM/x_dim'][0], + expected_xdim[0], + ) + self.assertEqual( + x_y_dim['/Soil_Moisture_Retrieval_Data_AM/x_dim'][-1], + expected_xdim[-1], + ) + def test_col_row_to_xy( self, ): @@ -1382,8 +1505,8 @@ def test_create_dimension_arrays_from_geotransform( '/Soil_Moisture_Retrieval_Data_AM/latitude' ) projected_dimension_names = [ - '/Soil_Moisture_Retrieval_Data_AM/dim_y', - '/Soil_Moisture_Retrieval_Data_AM/dim_x', + '/Soil_Moisture_Retrieval_Data_AM/y_dim', + '/Soil_Moisture_Retrieval_Data_AM/x_dim', ] geotransform = [-9000000, 3000, 0, 9000000, 0, -3000] expected_xdim = np.array([-8998500.0, -6109500.0]) @@ -1394,22 +1517,23 @@ def test_create_dimension_arrays_from_geotransform( x_y_dim = create_dimension_arrays_from_geotransform( smap_prefetch, latitude_coordinate, - projected_dimension_names, + '/Soil_Moisture_Retrieval_Data_AM/albedo', + smap_varinfo, geotransform, ) self.assertEqual( - x_y_dim['/Soil_Moisture_Retrieval_Data_AM/dim_x'][0], + x_y_dim['/Soil_Moisture_Retrieval_Data_AM/x_dim'][0], expected_xdim[0], ) self.assertEqual( - x_y_dim['/Soil_Moisture_Retrieval_Data_AM/dim_x'][-1], + x_y_dim['/Soil_Moisture_Retrieval_Data_AM/x_dim'][-1], expected_xdim[-1], ) self.assertEqual( - x_y_dim['/Soil_Moisture_Retrieval_Data_AM/dim_y'][0], + x_y_dim['/Soil_Moisture_Retrieval_Data_AM/y_dim'][0], expected_ydim[0], ) self.assertEqual( - x_y_dim['/Soil_Moisture_Retrieval_Data_AM/dim_y'][-1], + x_y_dim['/Soil_Moisture_Retrieval_Data_AM/y_dim'][-1], expected_ydim[-1], ) diff --git a/tests/unit/test_spatial.py b/tests/unit/test_spatial.py index a561752..19aa7d0 100644 --- a/tests/unit/test_spatial.py +++ b/tests/unit/test_spatial.py @@ -80,10 +80,10 @@ def test_get_spatial_index_ranges_projected_from_coordinates(self): '/Soil_Moisture_Retrieval_Data_PM/longitude_pm', } expected_index_ranges = { - '/Soil_Moisture_Retrieval_Data_AM/dim_x': (487, 594), - '/Soil_Moisture_Retrieval_Data_AM/dim_y': (9, 38), - '/Soil_Moisture_Retrieval_Data_PM/dim_x': (487, 594), - '/Soil_Moisture_Retrieval_Data_PM/dim_y': (9, 38), + '/Soil_Moisture_Retrieval_Data_AM/x_dim': (487, 594), + '/Soil_Moisture_Retrieval_Data_AM/y_dim': (9, 38), + '/Soil_Moisture_Retrieval_Data_PM/x_dim': (487, 594), + '/Soil_Moisture_Retrieval_Data_PM/y_dim': (9, 38), } index_ranges = get_spatial_index_ranges( required_variables, @@ -273,8 +273,8 @@ def test_get_x_y_index_ranges_from_coordinates( ) smap_file_path = 'tests/data/SC_SPL3SMP_008_prefetch.nc4' expected_index_ranges = { - '/Soil_Moisture_Retrieval_Data_AM/dim_x': (487, 595), - '/Soil_Moisture_Retrieval_Data_AM/dim_y': (9, 38), + '/Soil_Moisture_Retrieval_Data_AM/x_dim': (487, 595), + '/Soil_Moisture_Retrieval_Data_AM/y_dim': (9, 38), } bbox = BBox(2, 54, 42, 72) diff --git a/tests/unit/test_subset.py b/tests/unit/test_subset.py index a706683..ca6c775 100644 --- a/tests/unit/test_subset.py +++ b/tests/unit/test_subset.py @@ -1521,10 +1521,10 @@ def test_subset_granule_with_no_dimensions( '/Soil_Moisture_Retrieval_Data_PM/latitude_pm[9:38][487:595]', } expected_index_ranges = { - '/Soil_Moisture_Retrieval_Data_AM/dim_x': (487, 595), - '/Soil_Moisture_Retrieval_Data_AM/dim_y': (9, 38), - '/Soil_Moisture_Retrieval_Data_PM/dim_x': (487, 595), - '/Soil_Moisture_Retrieval_Data_PM/dim_y': (9, 38), + '/Soil_Moisture_Retrieval_Data_AM/x_dim': (487, 595), + '/Soil_Moisture_Retrieval_Data_AM/y_dim': (9, 38), + '/Soil_Moisture_Retrieval_Data_PM/x_dim': (487, 595), + '/Soil_Moisture_Retrieval_Data_PM/y_dim': (9, 38), } mock_get_varinfo.return_value = smap_varinfo From d65705388e2f51414c3378ac0c8ead69c1dc8350 Mon Sep 17 00:00:00 2001 From: sudhamurthy Date: Fri, 14 Feb 2025 11:56:05 -0500 Subject: [PATCH 2/5] DAS-2300 - PR feedback updates --- hoss/coordinate_utilities.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/hoss/coordinate_utilities.py b/hoss/coordinate_utilities.py index 6b7088b..759f359 100644 --- a/hoss/coordinate_utilities.py +++ b/hoss/coordinate_utilities.py @@ -204,16 +204,15 @@ def create_dimension_arrays_from_coordinates( ) # if it is not configured in the hoss_config.json - # assume it is nominal order [z,y,x] + # assume it is nominal order [z,y,x], if not dimension_name_order: + projected_y, projected_x = dimension_names[-2:] + else: + # if it is confgured e.g. [y,x,z] order projected_y, projected_x = ( - dimension_names[-2], - dimension_names[-1], + dimension_name_order['projection_y_coordinate'], + dimension_name_order['projection_x_coordinate'], ) - else: - projected_x = dimension_name_order['projection_x_coordinate'] - projected_y = dimension_name_order['projection_y_coordinate'] - if dim_order_is_y_x: return {projected_y: y_dim, projected_x: x_dim} raise UnsupportedDimensionOrder('x,y') From d9f816589f8105d12f8b66c5e51bd6f671148432 Mon Sep 17 00:00:00 2001 From: sudhamurthy Date: Sat, 15 Feb 2025 00:01:12 -0500 Subject: [PATCH 3/5] DAS-2300 - updates to PR feedback --- hoss/coordinate_utilities.py | 68 +++++----- hoss/dimension_utilities.py | 6 +- hoss/spatial.py | 8 +- tests/unit/test_coordinate_utilities.py | 161 ++++++++++++------------ 4 files changed, 119 insertions(+), 124 deletions(-) diff --git a/hoss/coordinate_utilities.py b/hoss/coordinate_utilities.py index 759f359..3f5600e 100644 --- a/hoss/coordinate_utilities.py +++ b/hoss/coordinate_utilities.py @@ -86,19 +86,20 @@ def any_absent_dimension_variables(varinfo: VarInfoFromDmr, variable: str) -> bo def get_dimension_array_names( varinfo: VarInfoFromDmr, variable_name: str, -) -> list[str]: +) -> dict[str:str]: """ Returns the dimensions names from coordinate variables or from configuration """ variable = varinfo.get_variable(variable_name) if variable is None: - return [] + return {} dimension_names = variable.dimensions + configured_dimensions = get_configured_dimension_order(varinfo, dimension_names) - if len(dimension_names) >= 2: - return dimension_names + if len(configured_dimensions) >= 2: + return configured_dimensions # creating dimension names from coordinates latitude_coordinates, longitude_coordinates = get_coordinate_variables( @@ -107,7 +108,7 @@ def get_dimension_array_names( # Given variable has coordinates: use latitude coordinate # to define variable spatial dimensions. if len(latitude_coordinates) == 1 and len(longitude_coordinates) == 1: - dimension_array_names = create_spatial_dimension_names_from_coordinates( + dimension_names = create_spatial_dimension_names_from_coordinates( varinfo, latitude_coordinates[0] ) @@ -115,18 +116,16 @@ def get_dimension_array_names( # but is itself a coordinate (latitude or longitude): # use as a coordinate to define spatial dimensions elif variable.is_latitude() or variable.is_longitude(): - dimension_array_names = create_spatial_dimension_names_from_coordinates( + dimension_names = create_spatial_dimension_names_from_coordinates( varinfo, variable_name ) - else: - dimension_array_names = [] - return dimension_array_names + return dimension_names def create_spatial_dimension_names_from_coordinates( varinfo: VarInfoFromDmr, variable_name: str -) -> str: +) -> dict[str:str]: """returns the x-y variable names that would match the group of the input variable. The 'dim_y' dimension and 'dim_x' names are returned with the group pathname @@ -135,14 +134,13 @@ def create_spatial_dimension_names_from_coordinates( variable = varinfo.get_variable(variable_name) if variable is not None: - dimension_array_names = [ - f'{variable.group_path}/y_dim', - f'{variable.group_path}/x_dim', - ] + dimension_names = { + 'projection_y_coordinate': f'{variable.group_path}/y_dim', + 'projection_x_coordinate': f'{variable.group_path}/x_dim', + } else: raise MissingVariable(variable_name) - - return dimension_array_names + return dimension_names def create_dimension_arrays_from_coordinates( @@ -150,8 +148,7 @@ def create_dimension_arrays_from_coordinates( latitude_coordinate: VariableFromDmr, longitude_coordinate: VariableFromDmr, crs: CRS, - variable_name: str, - varinfo: VarInfoFromDmr, + dimension_names: dict[str, str], ) -> dict[str, np.ndarray]: """Generate artificial 1D dimensions scales for each 2D dimension or coordinate variable. @@ -160,12 +157,12 @@ def create_dimension_arrays_from_coordinates( 3) Generate the x-y dimscale array and return to the calling method """ - dimension_names = get_dimension_array_names(varinfo, variable_name) + # dimension_names = get_dimension_array_names(varinfo, variable_name) if len(dimension_names) < 2: raise InvalidDimensionNames(dimension_names) # check if the dimension names are configured in hoss_config - dimension_name_order = get_configured_dimension_order(varinfo, dimension_names) + # dimension_name_order = get_configured_dimension_order(varinfo, dimension_names) lat_arr = get_2d_coordinate_array( prefetch_dataset, @@ -205,14 +202,13 @@ def create_dimension_arrays_from_coordinates( # if it is not configured in the hoss_config.json # assume it is nominal order [z,y,x], - if not dimension_name_order: - projected_y, projected_x = dimension_names[-2:] - else: - # if it is confgured e.g. [y,x,z] order - projected_y, projected_x = ( - dimension_name_order['projection_y_coordinate'], - dimension_name_order['projection_x_coordinate'], - ) + # if not dimension_name_order: + # projected_y, projected_x = dimension_names[-2:] + # else: + # if it is confgured e.g. [y,x,z] order + projected_y = dimension_names['projection_y_coordinate'] + projected_x = dimension_names['projection_x_coordinate'] + if dim_order_is_y_x: return {projected_y: y_dim, projected_x: x_dim} raise UnsupportedDimensionOrder('x,y') @@ -474,13 +470,12 @@ def interpolate_dim_values_from_sample_pts( def create_dimension_arrays_from_geotransform( prefetch_dataset: Dataset, latitude_coordinate: VariableFromDmr, - variable_name: str, - varinfo: VarInfoFromDmr, - geotranform, + projected_dimension_names: dict[str, str], + geotransform, ) -> dict[str, np.ndarray]: """Generate artificial 1D dimensions scales from geotransform""" - projected_dimension_names = get_dimension_array_names(varinfo, variable_name) + # projected_dimension_names = get_dimension_array_names(varinfo, variable_name) lat_arr = get_2d_coordinate_array( prefetch_dataset, @@ -489,16 +484,19 @@ def create_dimension_arrays_from_geotransform( # compute the x,y locations along a column and row column_dimensions = [ - col_row_to_xy(geotranform, col, 0) for col in range(lat_arr.shape[-1]) + col_row_to_xy(geotransform, col, 0) for col in range(lat_arr.shape[-1]) ] row_dimensions = [ - col_row_to_xy(geotranform, 0, row) for row in range(lat_arr.shape[-2]) + col_row_to_xy(geotransform, 0, row) for row in range(lat_arr.shape[-2]) ] # pull out dimension values x_values = np.array([x for x, y in column_dimensions], dtype=np.float64) y_values = np.array([y for x, y in row_dimensions], dtype=np.float64) - projected_y, projected_x = projected_dimension_names[-2:] + # projected_y, projected_x = projected_dimension_names[-2:] + + projected_y = projected_dimension_names['projection_y_coordinate'] + projected_x = projected_dimension_names['projection_x_coordinate'] return {projected_y: y_values, projected_x: x_values} diff --git a/hoss/dimension_utilities.py b/hoss/dimension_utilities.py index fc84b9b..5cf4e02 100644 --- a/hoss/dimension_utilities.py +++ b/hoss/dimension_utilities.py @@ -443,7 +443,11 @@ def add_index_range( else: # Anonymous dimensions, so check for dimension derived from coordinates # or from configuration - variable_dimensions = get_dimension_array_names(varinfo, variable_name) + variable_dimensions_dict = get_dimension_array_names(varinfo, variable_name) + if variable_dimensions_dict: + variable_dimensions = list(variable_dimensions_dict.values()) + else: + variable_dimensions = [] range_strings = get_range_strings(variable_dimensions, index_ranges) diff --git a/hoss/spatial.py b/hoss/spatial.py index c57ef11..6b2df90 100644 --- a/hoss/spatial.py +++ b/hoss/spatial.py @@ -246,6 +246,7 @@ def get_x_y_index_ranges_from_coordinates( points. """ + projected_dimension_names = get_dimension_array_names(varinfo, non_spatial_variable) crs = get_variable_crs(non_spatial_variable, varinfo) master_geotransform = get_master_geotransform(non_spatial_variable, varinfo) @@ -253,8 +254,7 @@ def get_x_y_index_ranges_from_coordinates( dimension_arrays = create_dimension_arrays_from_geotransform( prefetch_coordinate_datasets, latitude_coordinate, - non_spatial_variable, - varinfo, + projected_dimension_names, master_geotransform, ) else: @@ -263,9 +263,7 @@ def get_x_y_index_ranges_from_coordinates( latitude_coordinate, longitude_coordinate, crs, - # projected_dimension_names, - non_spatial_variable, - varinfo, + projected_dimension_names, ) projected_y, projected_x = dimension_arrays.keys() diff --git a/tests/unit/test_coordinate_utilities.py b/tests/unit/test_coordinate_utilities.py index 8d4b977..3e5d2c2 100644 --- a/tests/unit/test_coordinate_utilities.py +++ b/tests/unit/test_coordinate_utilities.py @@ -528,15 +528,15 @@ def test_create_spatial_dimension_names_from_coordinates(self): is returned for the coordinate variables """ - expected_dimension_names = [ - '/Soil_Moisture_Retrieval_Data_AM/y_dim', - '/Soil_Moisture_Retrieval_Data_AM/x_dim', - ] + expected_dimension_names = { + 'projection_y_coordinate': '/Soil_Moisture_Retrieval_Data_AM/y_dim', + 'projection_x_coordinate': '/Soil_Moisture_Retrieval_Data_AM/x_dim', + } with self.subTest( 'Retrieves expected projected dimension names for a science variable' ): - self.assertListEqual( + self.assertDictEqual( create_spatial_dimension_names_from_coordinates( self.varinfo, self.latitude ), @@ -546,7 +546,7 @@ def test_create_spatial_dimension_names_from_coordinates(self): with self.subTest( 'Retrieves expected dimension names for the longitude variable' ): - self.assertEqual( + self.assertDictEqual( create_spatial_dimension_names_from_coordinates( self.varinfo, self.longitude ), @@ -565,23 +565,23 @@ def test_create_spatial_dimension_names_from_coordinates(self): ) def test_get_dimension_array_names(self): - """Ensure that the expected projected dimension name - is returned for the coordinate variables + """Ensure that the expected projected dimension names + are returned for the requested variables """ - expected_override_dimensions_AM = [ - '/Soil_Moisture_Retrieval_Data_AM/y_dim', - '/Soil_Moisture_Retrieval_Data_AM/x_dim', - ] - expected_override_dimensions_PM = [ - '/Soil_Moisture_Retrieval_Data_PM/y_dim', - '/Soil_Moisture_Retrieval_Data_PM/x_dim', - ] + expected_override_dimensions_AM = { + 'projection_y_coordinate': '/Soil_Moisture_Retrieval_Data_AM/y_dim', + 'projection_x_coordinate': '/Soil_Moisture_Retrieval_Data_AM/x_dim', + } + expected_override_dimensions_PM = { + 'projection_y_coordinate': '/Soil_Moisture_Retrieval_Data_PM/y_dim', + 'projection_x_coordinate': '/Soil_Moisture_Retrieval_Data_PM/x_dim', + } with self.subTest( 'Retrieves expected override dimensions for the science variable' ): - self.assertListEqual( + self.assertDictEqual( get_dimension_array_names( self.varinfo, '/Soil_Moisture_Retrieval_Data_AM/surface_flag' ), @@ -591,7 +591,7 @@ def test_get_dimension_array_names(self): with self.subTest( 'Retrieves expected override dimensions for the longitude variable' ): - self.assertListEqual( + self.assertDictEqual( get_dimension_array_names(self.varinfo, self.longitude), expected_override_dimensions_AM, ) @@ -599,7 +599,7 @@ def test_get_dimension_array_names(self): with self.subTest( 'Retrieves expected override dimensions for the latitude variable' ): - self.assertListEqual( + self.assertDictEqual( get_dimension_array_names(self.varinfo, self.latitude), expected_override_dimensions_AM, ) @@ -607,7 +607,7 @@ def test_get_dimension_array_names(self): with self.subTest( 'Retrieves expected override dimensions science variable with a different grid' ): - self.assertListEqual( + self.assertDictEqual( get_dimension_array_names( self.varinfo, '/Soil_Moisture_Retrieval_Data_PM/surface_flag_pm' ), @@ -616,11 +616,14 @@ def test_get_dimension_array_names(self): with self.subTest( 'Retrieves empty dimensions list when science variable has no coordinates' ): - self.assertListEqual( - get_dimension_array_names( - self.varinfo, '/Soil_Moisture_Retrieval_Data_PM/surface_flag_pm' - ), - expected_override_dimensions_PM, + assert ( + len( + get_dimension_array_names( + self.test_varinfo, + '/Soil_Moisture_Retrieval_Data_AM/no_coordinate_variable', + ) + ) + == 0 ) with self.subTest( 'Retrieves expected dimension names for the 3D json configured dimensions' @@ -630,11 +633,10 @@ def test_get_dimension_array_names(self): ) self.assertEqual( dimension_names_3d, - [ - '/Freeze_Thaw_Retrieval_Data_Global/am_pm', - '/Freeze_Thaw_Retrieval_Data_Global/y_dim', - '/Freeze_Thaw_Retrieval_Data_Global/x_dim', - ], + { + 'projection_y_coordinate': '/Freeze_Thaw_Retrieval_Data_Global/y_dim', + 'projection_x_coordinate': '/Freeze_Thaw_Retrieval_Data_Global/x_dim', + }, ) def test_get_configured_dimension_order(self): @@ -678,17 +680,22 @@ def test_get_configured_dimension_order(self): 'Retrieves expected dimension order configured for nominal order' ): configured_dimension_names_nominal = [ - '/Soil_Moisture_Retrieval_Data_AM/am_pm', - '/Soil_Moisture_Retrieval_Data_AM/y_dim', - '/Soil_Moisture_Retrieval_Data_AM/x_dim', + '/Freeze_Thaw_Retrieval_Data_Global/am_pm', + '/Freeze_Thaw_Retrieval_Data_Global/y_dim', + '/Freeze_Thaw_Retrieval_Data_Global/x_dim', ] - expected_dimension_order = {} + expected_dimension_order = { + 'projection_y_coordinate': '/Freeze_Thaw_Retrieval_Data_Global/y_dim', + 'projection_x_coordinate': '/Freeze_Thaw_Retrieval_Data_Global/x_dim', + } - self.assertDictEqual( - get_configured_dimension_order( - self.smap_ftp_varinfo, configured_dimension_names_nominal - ), - expected_dimension_order, + assert ( + len( + get_configured_dimension_order( + self.smap_ftp_varinfo, configured_dimension_names_nominal + ) + ) + == 0 ) def test_get_row_col_sizes_from_coordinates(self): @@ -1193,14 +1200,14 @@ def test_create_dimension_arrays_from_coordinates( longitude_coordinate = smap_varinfo.get_variable( '/Soil_Moisture_Retrieval_Data_AM/longitude' ) - projected_dimension_names_am = [ - '/Soil_Moisture_Retrieval_Data_AM/y_dim', - '/Soil_Moisture_Retrieval_Data_AM/x_dim', - ] - projected_dimension_names_pm = [ - '/Soil_Moisture_Retrieval_Data_PM/y_dim', - '/Soil_Moisture_Retrieval_Data_PM/x_dim', - ] + projected_dimension_names_am = { + 'projection_y_coordinate': '/Soil_Moisture_Retrieval_Data_AM/y_dim', + 'projection_x_coordinate': '/Soil_Moisture_Retrieval_Data_AM/x_dim', + } + projected_dimension_names_pm = { + 'projection_y_coordinate': '/Soil_Moisture_Retrieval_Data_PM/y_dim', + 'projection_x_coordinate': '/Soil_Moisture_Retrieval_Data_PM/x_dim', + } crs = CRS.from_cf( { 'false_easting': 0.0, @@ -1220,23 +1227,21 @@ def test_create_dimension_arrays_from_coordinates( latitude_coordinate, longitude_coordinate, crs, - '/Soil_Moisture_Retrieval_Data_AM/albedo', - self.varinfo, + projected_dimension_names_am, ) x_y_dim_pm = create_dimension_arrays_from_coordinates( smap_prefetch, latitude_coordinate, longitude_coordinate, crs, - '/Soil_Moisture_Retrieval_Data_PM/albedo_pm', - self.varinfo, + projected_dimension_names_pm, ) self.assertListEqual( - list(x_y_dim_am.keys()), projected_dimension_names_am + list(x_y_dim_am.keys()), list(projected_dimension_names_am.values()) ) self.assertListEqual( - list(x_y_dim_pm.keys()), projected_dimension_names_pm + list(x_y_dim_pm.keys()), list(projected_dimension_names_pm.values()) ) self.assertEqual( x_y_dim_am['/Soil_Moisture_Retrieval_Data_AM/y_dim'][0], @@ -1293,8 +1298,7 @@ def test_create_dimension_arrays_from_coordinates( latitude_coordinate, longitude_coordinate, crs, - '/Soil_Moisture_Retrieval_Data_AM/albedo', - self.varinfo, + projected_dimension_names_am, ) self.assertEqual( context.exception.message, @@ -1323,8 +1327,7 @@ def test_create_dimension_arrays_from_coordinates( latitude_coordinate, longitude_coordinate, crs, - '/Soil_Moisture_Retrieval_Data_AM/albedo', - self.varinfo, + projected_dimension_names_am, ) self.assertEqual( context.exception.message, @@ -1341,8 +1344,7 @@ def test_create_dimension_arrays_from_coordinates( latitude_coordinate, longitude_coordinate, crs, - '/Soil_Moisture_Retrieval_Data_AM/albedo', - self.varinfo, + projected_dimension_names_am, ) def test_create_dimension_arrays_from_nominal_3d_coordinates( @@ -1359,11 +1361,10 @@ def test_create_dimension_arrays_from_nominal_3d_coordinates( longitude_coordinate = self.smap_ftp_varinfo.get_variable( '/Freeze_Thaw_Retrieval_Data_Global/longitude' ) - dimension_names_global = [ - '/Freeze_Thaw_Retrieval_Data_Global/am_pm', - '/Freeze_Thaw_Retrieval_Data_Global/y_dim', - '/Freeze_Thaw_Retrieval_Data_Global/x_dim', - ] + dimension_names_global = { + 'projection_y_coordinate': '/Freeze_Thaw_Retrieval_Data_Global/y_dim', + 'projection_x_coordinate': '/Freeze_Thaw_Retrieval_Data_Global/x_dim', + } crs = CRS.from_cf( { @@ -1384,13 +1385,11 @@ def test_create_dimension_arrays_from_nominal_3d_coordinates( latitude_coordinate, longitude_coordinate, crs, - '/Freeze_Thaw_Retrieval_Data_Global/landcover_class', - self.smap_ftp_varinfo, + dimension_names_global, ) self.assertListEqual( - list(x_y_dim_global.keys()), - [dimension_names_global[1], dimension_names_global[2]], + list(x_y_dim_global.keys()), list(dimension_names_global.values()) ) self.assertEqual( x_y_dim_global['/Freeze_Thaw_Retrieval_Data_Global/y_dim'][0], @@ -1422,11 +1421,10 @@ def test_create_dimension_arrays_from_not_nominal_3d_coordinates(self): longitude_coordinate = self.varinfo.get_variable( '/Soil_Moisture_Retrieval_Data_AM/longitude' ) - dimension_names = [ - '/Soil_Moisture_Retrieval_Data_AM/y_dim', - '/Soil_Moisture_Retrieval_Data_AM/x_dim', - '/Soil_Moisture_Retrieval_Data_AM/lc_type', - ] + dimension_names = { + 'projection_y_coordinate': '/Soil_Moisture_Retrieval_Data_AM/y_dim', + 'projection_x_coordinate': '/Soil_Moisture_Retrieval_Data_AM/x_dim', + } crs = CRS.from_cf( { @@ -1446,13 +1444,11 @@ def test_create_dimension_arrays_from_not_nominal_3d_coordinates(self): latitude_coordinate, longitude_coordinate, crs, - '/Soil_Moisture_Retrieval_Data_AM/landcover_class', - self.varinfo, + dimension_names, ) self.assertListEqual( - list(x_y_dim.keys()), - [dimension_names[0], dimension_names[1]], + list(x_y_dim.keys()), list(dimension_names.values()) ) self.assertEqual( x_y_dim['/Soil_Moisture_Retrieval_Data_AM/y_dim'][0], @@ -1504,10 +1500,10 @@ def test_create_dimension_arrays_from_geotransform( latitude_coordinate = smap_varinfo.get_variable( '/Soil_Moisture_Retrieval_Data_AM/latitude' ) - projected_dimension_names = [ - '/Soil_Moisture_Retrieval_Data_AM/y_dim', - '/Soil_Moisture_Retrieval_Data_AM/x_dim', - ] + projected_dimension_names = { + 'projection_y_coordinate': '/Soil_Moisture_Retrieval_Data_AM/y_dim', + 'projection_x_coordinate': '/Soil_Moisture_Retrieval_Data_AM/x_dim', + } geotransform = [-9000000, 3000, 0, 9000000, 0, -3000] expected_xdim = np.array([-8998500.0, -6109500.0]) expected_ydim = np.array([8998500.0, 7783500.0]) @@ -1517,8 +1513,7 @@ def test_create_dimension_arrays_from_geotransform( x_y_dim = create_dimension_arrays_from_geotransform( smap_prefetch, latitude_coordinate, - '/Soil_Moisture_Retrieval_Data_AM/albedo', - smap_varinfo, + projected_dimension_names, geotransform, ) self.assertEqual( From a420d75c9cdc428bd7f1515eb6638b853581d9e2 Mon Sep 17 00:00:00 2001 From: sudhamurthy Date: Sat, 15 Feb 2025 00:31:59 -0500 Subject: [PATCH 4/5] DAS-2300 - removed comments and minor updates --- hoss/coordinate_utilities.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/hoss/coordinate_utilities.py b/hoss/coordinate_utilities.py index 3f5600e..cf5f0db 100644 --- a/hoss/coordinate_utilities.py +++ b/hoss/coordinate_utilities.py @@ -95,11 +95,11 @@ def get_dimension_array_names( if variable is None: return {} - dimension_names = variable.dimensions - configured_dimensions = get_configured_dimension_order(varinfo, dimension_names) + configured_dimensions = variable.dimensions + dimension_names = get_configured_dimension_order(varinfo, configured_dimensions) - if len(configured_dimensions) >= 2: - return configured_dimensions + if len(dimension_names) >= 2: + return dimension_names # creating dimension names from coordinates latitude_coordinates, longitude_coordinates = get_coordinate_variables( @@ -119,7 +119,8 @@ def get_dimension_array_names( dimension_names = create_spatial_dimension_names_from_coordinates( varinfo, variable_name ) - + else: + dimension_names = {} return dimension_names @@ -162,8 +163,6 @@ def create_dimension_arrays_from_coordinates( raise InvalidDimensionNames(dimension_names) # check if the dimension names are configured in hoss_config - # dimension_name_order = get_configured_dimension_order(varinfo, dimension_names) - lat_arr = get_2d_coordinate_array( prefetch_dataset, latitude_coordinate.full_name_path, @@ -200,12 +199,6 @@ def create_dimension_arrays_from_coordinates( col_dim_values, np.transpose(col_indices)[1], col_size ) - # if it is not configured in the hoss_config.json - # assume it is nominal order [z,y,x], - # if not dimension_name_order: - # projected_y, projected_x = dimension_names[-2:] - # else: - # if it is confgured e.g. [y,x,z] order projected_y = dimension_names['projection_y_coordinate'] projected_x = dimension_names['projection_x_coordinate'] @@ -475,8 +468,6 @@ def create_dimension_arrays_from_geotransform( ) -> dict[str, np.ndarray]: """Generate artificial 1D dimensions scales from geotransform""" - # projected_dimension_names = get_dimension_array_names(varinfo, variable_name) - lat_arr = get_2d_coordinate_array( prefetch_dataset, latitude_coordinate.full_name_path, @@ -493,7 +484,6 @@ def create_dimension_arrays_from_geotransform( # pull out dimension values x_values = np.array([x for x, y in column_dimensions], dtype=np.float64) y_values = np.array([y for x, y in row_dimensions], dtype=np.float64) - # projected_y, projected_x = projected_dimension_names[-2:] projected_y = projected_dimension_names['projection_y_coordinate'] projected_x = projected_dimension_names['projection_x_coordinate'] From e838e4659bd57909610c485a60d92ce2316ceb70 Mon Sep 17 00:00:00 2001 From: sudhamurthy Date: Mon, 17 Feb 2025 17:42:22 -0500 Subject: [PATCH 5/5] DAS-2300 - updates to comments based on PR feedback --- hoss/coordinate_utilities.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/hoss/coordinate_utilities.py b/hoss/coordinate_utilities.py index cf5f0db..3da1527 100644 --- a/hoss/coordinate_utilities.py +++ b/hoss/coordinate_utilities.py @@ -25,8 +25,8 @@ def get_coordinate_variables( ) -> tuple[list[str], list[str]]: """This function returns latitude and longitude variable names from latitude and longitude variables listed in the CF-Convention coordinates - metadata attribute. It returns them in a specific - order [latitude_name, longitude_name]" + metadata attribute. It checks that the variables exist in the file and then + returns the lists in a specific order: [latitude_names], [longitude_names] """ coordinate_variables = varinfo.get_references_for_attribute( @@ -88,8 +88,9 @@ def get_dimension_array_names( variable_name: str, ) -> dict[str:str]: """ - Returns the dimensions names from coordinate variables or from - configuration + Returns the dimension names from coordinate variables or from configuration. + VarInfo implements pulling dimension names from configuration, which is + used for some collections with anonymous dimensions. """ variable = varinfo.get_variable(variable_name) if variable is None: @@ -328,7 +329,9 @@ def get_max_spread_pts( valid_indices = np.ma.array(arr_indices, mask=valid_geospatial_mask) elif valid_geospatial_mask.ndim == 3: # use just 2 of the dimensions - # mask arr_ind to hide the invalid data points + # This assumes that the first dimension is the "extra" non-spatial dimension, + # Currently we define the dimensions and their order in the configuration file, + # ToDo When the configuration entry is dropped, this needs to be reconsidered. valid_indices = np.ma.array(arr_indices, mask=valid_geospatial_mask[0, :, :]) else: raise NotImplementedError @@ -365,7 +368,7 @@ def get_dimension_order_and_dim_values( projected y or projected_x values are varying across row or column. Also returns a 1-D array of dimension values for the requested projected spatial dimension. The input lat lon arrays and dimension - indices are assumed to be 2D in this implementation of the function. + indices are assumed to be 1D or 2D in this implementation of the function. """ if lat_array_points.ndim == 1 and lon_array_points.ndim == 1: lat_arr_values = lat_array_points