DAS-2238 - updates to enable 3D variables without dimension scales to work

Author: sudha-murthy

CHANGELOG.md

@@ -1,3 +1,11 @@
+## v1.1.3
+### 2025-01-29
+
+This version of HOSS supports configuration updates to hoss_config.json to
+add dimension configurations for 3D variables. Functions were updated to provide the
+ability to spatial subset 3D variables for products like SMAP L3 which did
+not have dimension arrays.
+
 ## v1.1.2
 ### 2025-01-20
 
@@ -13,8 +21,10 @@
 This version of HOSS merges the feature branch that contains V1.1.0 to the main branch.
 Additional updates included code quality improvements with additional unit tests, revised methodology
 in functions that selected the data points from the coordinate datasets and calculation of the dimension
-arrays. Functions were added to determine the dimension order for 2D variables.
-
+arrays. Functions were added to determine the dimension order for 2D variables. These updates,
+V1.1.1 and V1.1.0 are entirely to support SMAP L3 data - in particular SPL2SMAP_S, SPL3SMAP, SPL3SMA -
+all of which have “anonymous” dimensions (without dimension names and dimension variables).  No functional
+updates are present, nor changes that effect existing collection support.
 
 ## v1.1.0
 ### 2024-11-25

hoss/coordinate_utilities.py

@@ -60,8 +60,13 @@ def get_variables_with_anonymous_dims(
     return set(
         variable
         for variable in variables
-        if (len(varinfo.get_variable(variable).dimensions) == 0)
-        or (any_absent_dimension_variables(varinfo, variable))
+        if (
+            varinfo.get_variable(variable)
+            and (
+                (len(varinfo.get_variable(variable).dimensions) == 0)
+                or (any_absent_dimension_variables(varinfo, variable))
+            )
+        )
     )
 
 
@@ -70,42 +75,56 @@ def any_absent_dimension_variables(varinfo: VarInfoFromDmr, variable: str) -> bo
     that have been created by opendap, but are not really
     dimension variables
     """
+
     return any(
         varinfo.get_variable(dimension) is None
         for dimension in varinfo.get_variable(variable).dimensions
     )
 
 
-def get_dimension_array_names_from_coordinate_variables(
+def get_dimension_array_names(
     varinfo: VarInfoFromDmr,
     variable_name: str,
 ) -> list[str]:
     """
-    Returns the dimensions names from coordinate variables
+    Returns the dimensions names from coordinate variables or from
+    configuration
     """
+    variable = varinfo.get_variable(variable_name)
+    dimension_names = varinfo.get_variable(variable_name).dimensions
+
+    if len(dimension_names) >= 2:
+        # if the dimension is not a variable, was added as a
+        # configuration entry,
+        if variable is not None:
+            return dimension_names
 
     latitude_coordinates, longitude_coordinates = get_coordinate_variables(
         varinfo, [variable_name]
     )
 
     # for one variable, the coordinate array length will always be 1 or 0
     if len(latitude_coordinates) == 1 and len(longitude_coordinates) == 1:
-        dimension_array_names = get_dimension_array_names(
+        dimension_array_names = get_dimension_array_names_from_coordinates(
             varinfo, latitude_coordinates[0]
         )
     # if variable does not have coordinates (len = 0)
     elif (
         varinfo.get_variable(variable_name).is_latitude()
         or varinfo.get_variable(variable_name).is_longitude()
     ):
-        dimension_array_names = get_dimension_array_names(varinfo, variable_name)
+        dimension_array_names = get_dimension_array_names_from_coordinates(
+            varinfo, variable_name
+        )
     else:
         dimension_array_names = []
 
     return dimension_array_names
 
 
-def get_dimension_array_names(varinfo: VarInfoFromDmr, variable_name: str) -> str:
+def get_dimension_array_names_from_coordinates(
+    varinfo: VarInfoFromDmr, variable_name: str
+) -> str:
     """returns the x-y variable names that would
     match the group of the input variable. The 'dim_y' dimension
     and 'dim_x' names are returned with the group pathname
@@ -172,13 +191,19 @@ def create_dimension_arrays_from_coordinates(
         col_dim_values, np.transpose(col_indices)[1], col_size
     )
 
-    projected_y, projected_x = tuple(projected_dimension_names)
+    if len(projected_dimension_names) >= 2:
+        projected_y, projected_x = (
+            projected_dimension_names[-2],
+            projected_dimension_names[-1],
+        )
+
+        if dim_order_is_y_x:
+            return {projected_y: y_dim, projected_x: x_dim}
+        raise UnsupportedDimensionOrder('x,y')
+        # this is not currently supported in the calling function in spatial.py
+        # return {projected_x: x_dim, projected_y: y_dim}
 
-    if dim_order_is_y_x:
-        return {projected_y: y_dim, projected_x: x_dim}
-    raise UnsupportedDimensionOrder('x,y')
-    # this is not currently supported in the calling function in spatial.py
-    # return {projected_x: x_dim, projected_y: y_dim}
+    raise UnsupportedDimensionOrder(projected_dimension_names)
 
 
 def get_2d_coordinate_array(
@@ -322,8 +347,16 @@ def get_dimension_order_and_dim_values(
     projected spatial dimension. The input lat lon arrays and dimension
     indices are assumed to be 2D in this implementation of the function.
     """
-    lat_arr_values = [lat_array_points[i][j] for i, j in grid_dimension_indices]
-    lon_arr_values = [lon_array_points[i][j] for i, j in grid_dimension_indices]
+    if lat_array_points.ndim == 1 and lon_array_points.ndim == 1:
+        lat_arr_values = lat_array_points
+        lon_arr_values = lon_array_points
+    elif lat_array_points.ndim == 2 and lon_array_points.ndim == 2:
+        lat_arr_values = [lat_array_points[i][j] for i, j in grid_dimension_indices]
+        lon_arr_values = [lon_array_points[i][j] for i, j in grid_dimension_indices]
+    else:
+        # assuming a nominal z,y,x order
+        lat_arr_values = [lat_array_points[0][i][j] for i, j in grid_dimension_indices]
+        lon_arr_values = [lon_array_points[0][i][j] for i, j in grid_dimension_indices]
 
     from_geo_transformer = Transformer.from_crs(4326, crs)
     x_values, y_values = (  # pylint: disable=unpacking-non-sequence

hoss/dimension_utilities.py

@@ -24,7 +24,7 @@
 
 from hoss.coordinate_utilities import (
     get_coordinate_variables,
-    get_dimension_array_names_from_coordinate_variables,
+    get_dimension_array_names,
 )
 from hoss.exceptions import (
     InvalidIndexSubsetRequest,
@@ -441,17 +441,17 @@ def add_index_range(
     if variable.dimensions:
         variable_dimensions = variable.dimensions
     else:
-        # Anonymous dimensions, so check for dimension derived from coordinates:
-        variable_dimensions = get_dimension_array_names_from_coordinate_variables(
-            varinfo, variable_name
-        )
+        # Anonymous dimensions, so check for dimension derived from coordinates
+        # or from configuration
+        variable_dimensions = get_dimension_array_names(varinfo, variable_name)
 
     range_strings = get_range_strings(variable_dimensions, index_ranges)
 
     if all(range_string == '[]' for range_string in range_strings):
         indices_string = ''
     else:
         indices_string = ''.join(range_strings)
+
     return f'{variable_name}{indices_string}'
 
 

hoss/hoss_config.json

@@ -1,6 +1,6 @@
 {
   "Identification": "hoss_config",
-  "Version": 19,
+  "Version": 20,
   "CollectionShortNamePath": [
     "/HDF5_GLOBAL/short_name",
     "/NC_GLOBAL/short_name",
@@ -314,6 +314,48 @@
       ],
       "_Description": "Ensure variables in /Soil_Moisture_Retrieval_Data_Polar_PM group point to correct coordinate variables."
     },
+    {
+      "Applicability": {
+        "Mission": "SMAP",
+        "ShortNamePath": "SPL3FT(A|P|P_E)",
+        "VariablePattern": "^/Freeze_Thaw_Retrieval_Data_(Global|Polar)/(transition_direction$|transition_state_flag$)"
+      },
+      "Attributes": [
+        {
+          "Name": "dimensions",
+          "Value": "y_dim x_dim"
+        }
+      ],
+      "_Description": "Only these fully referenced variables are 2D, with this rule overriding the 3D rule defined broadly for all variables"
+    },
+    {
+      "Applicability": {
+        "Mission": "SMAP",
+        "ShortNamePath": "SPL3FT(A|P|P_E)",
+        "VariablePattern": "^/Freeze_Thaw_Retrieval_Data_(Global|Polar)/((?!transition_state_flag$)(?!transition_direction$).)*$|/Radar_Data/.*"
+      },
+      "Attributes": [
+        {
+          "Name": "dimensions",
+          "Value": "am_pm y_dim x_dim"
+        }
+      ],
+      "_Description": "SMAP L3 data are HDF5 and without dimension settings. Overrides here define the dimensions, a useful reference name, and critically, the dimension order."
+    },
+    {
+      "Applicability": {
+        "Mission": "SMAP",
+        "ShortNamePath": "SPL3SMP",
+        "VariablePattern": ".*landcover.*"
+      },
+      "Attributes": [
+        {
+          "Name": "dimensions",
+          "Value": "y_dim x_dim lc_type"
+        }
+      ],
+      "_Description": "SMAP L3 data are HDF5 and without dimension settings. Overrides here define the dimensions, a useful reference name, and critically, the dimension order."
+    },
     {
       "Applicability": {
         "Mission": "ICESat2",

hoss/spatial.py

@@ -38,7 +38,7 @@
 from hoss.coordinate_utilities import (
     create_dimension_arrays_from_coordinates,
     get_coordinate_variables,
-    get_dimension_array_names_from_coordinate_variables,
+    get_dimension_array_names,
     get_variables_with_anonymous_dims,
 )
 from hoss.dimension_utilities import (
@@ -247,9 +247,7 @@ def get_x_y_index_ranges_from_coordinates(
 
     crs = get_variable_crs(non_spatial_variable, varinfo)
 
-    projected_dimension_names = get_dimension_array_names_from_coordinate_variables(
-        varinfo, non_spatial_variable
-    )
+    projected_dimension_names = get_dimension_array_names(varinfo, non_spatial_variable)
 
     dimension_arrays = create_dimension_arrays_from_coordinates(
         prefetch_coordinate_datasets,