fix the nodata issue when default to nan with float dtype

Author: Emma Ai

odc/stats/plugins/_base.py

@@ -2,6 +2,7 @@
 from typing import Mapping, Optional, Sequence, Tuple
 
 import xarray as xr
+import numpy as np
 from datacube.model import Dataset
 from datacube.utils.geometry import GeoBox
 from odc.algo import to_rgba
@@ -47,6 +48,12 @@ def measurements(self) -> Tuple[str, ...]:
         pass
 
     def native_transform(self, xx: xr.Dataset) -> xr.Dataset:
+        for var in xx.data_vars:
+            if (
+                xx[var].attrs.get("nodata") is None
+                and np.dtype(xx[var].dtype).kind == "f"
+            ):
+                xx[var].attrs["nodata"] = np.nan
         return xx
 
     def fuser(self, xx: xr.Dataset) -> xr.Dataset:

odc/stats/plugins/lc_level3.py

@@ -21,16 +21,22 @@ def measurements(self) -> Tuple[str, ...]:
         _measurements = ["level3_class"]
         return _measurements
 
+    def fuser(self, xx: xr.Dataset) -> xr.Dataset:
+        return xx
+
     def reduce(self, xx: xr.Dataset) -> xr.Dataset:
 
         # Cultivated pipeline applies a mask which feeds only terrestrial veg (110) to the model
-        # Just exclude no data (255) and apply the cultivated results
+        # Just exclude no data (255 or nan) and apply the cultivated results
+        # 255: load with product definition; nan: load without
+        # hence accormmodate both
+
         res = expr_eval(
-            "where(a<nodata, a, b)",
+            "where(((a==a)&(nodata!=nodata))|((a<nodata)&(nodata==nodata)), a, b)",
             {"a": xx.cultivated_class.data, "b": xx.classes_l3_l4.data},
             name="mask_cultivated",
             dtype="float32",
-            **{"nodata": NODATA},
+            **{"nodata": xx.cultivated_class.attrs.get("nodata")},
         )
 
         # Mask urban results with bare sfc (210)
@@ -42,10 +48,22 @@ def reduce(self, xx: xr.Dataset) -> xr.Dataset:
                 "b": xx.urban_classes.data,
             },
             name="mark_urban",
-            dtype="uint8",
+            dtype="float32",
             **{"_u": 210},
         )
 
+        # Mark nodata to 255 in case any nan
+
+        res = expr_eval(
+            "where(a==a, a, nodata)",
+            {
+                "a": res,
+            },
+            name="mark_nodata",
+            dtype="uint8",
+            **{"nodata": NODATA},
+        )
+
         attrs = xx.attrs.copy()
         attrs["nodata"] = NODATA
         dims = xx.classes_l3_l4.dims[1:]

odc/stats/plugins/lc_veg_class_a1.py

@@ -94,9 +94,6 @@ def measurements(self) -> Tuple[str, ...]:
         _measurements = ["classes_l3_l4", "water_seasonality"]
         return _measurements
 
-    def native_transform(self, xx):
-        return xx
-
     def fuser(self, xx):
         return xx
 
@@ -175,12 +172,16 @@ def l3_class(self, xx: xr.Dataset):
                     )
                 elif b == "canopy_cover_class":
                     # aquatic_veg: (mangroves > 0) & (mangroves != nodata)
+                    # mangroves.nodata = 255 or nan
                     l3_mask = expr_eval(
-                        "where((a>0)&(a<nodata), m, b)",
+                        "where(((a>0)&(a<nodata)&(nodata==nodata))|((a>0)&(nodata!=nodata)), m, b)",
                         {"a": xx[b].data, "b": l3_mask},
                         name="mark_mangroves",
                         dtype="uint8",
-                        **{"nodata": NODATA, "m": self.output_classes["aquatic_veg"]},
+                        **{
+                            "nodata": xx[b].attrs["nodata"],
+                            "m": self.output_classes["aquatic_veg"],
+                        },
                     )
 
         # all unmarked values (0) is terretrial veg
@@ -199,15 +200,15 @@ def l3_class(self, xx: xr.Dataset):
 
         # Mask nans with NODATA
         l3_mask = expr_eval(
-            "where((a!=a)|(b>=nodata), nodata, e)",
+            "where((a!=a)|((b>=_n)&(_n==_n))|(b!=b), nodata, e)",
             {
                 "a": si5,
                 "b": xx.veg_frequency.data,
                 "e": l3_mask,
             },
             name="mark_nodata",
             dtype="uint8",
-            **{"nodata": NODATA},
+            **{"_n": xx.veg_frequency.attrs["nodata"], "nodata": NODATA},
         )
 
         # Now add the water frequency