Skip to content

Commit af02544

Browse files
author
Emma Ai
committed
normalize water frequency
1 parent 6c33ccd commit af02544

File tree

2 files changed

+117
-92
lines changed

2 files changed

+117
-92
lines changed

‎odc/stats/plugins/lc_fc_wo_a0.py‎

Lines changed: 101 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -52,102 +52,117 @@ def native_transform(self, xx):
5252
5. Drop the WOfS band
5353
"""
5454

55-
# clear and dry pixels not mask against bit 4: terrain high slope,
55+
# valid and dry pixels not mask against bit 4: terrain high slope,
5656
# bit 3: terrain shadow, and
5757
# bit 2: low solar angle
58-
valid = (xx["water"] & ~((1 << 4) | (1 << 3) | (1 << 2))) == 0
58+
valid = (xx["water"].data & ~((1 << 4) | (1 << 3) | (1 << 2))) == 0
5959

60-
# clear and wet pixels not mask against bit 2: low solar angle
61-
wet = (xx["water"] & ~(1 << 2)) == 128
60+
# clear wet pixels not mask against bit 2: low solar angle
61+
wet = (xx["water"].data & ~(1 << 2)) == 128
6262

63-
# dilate both 'valid' and 'water'
64-
for key, val in self.BAD_BITS_MASK.items():
65-
if self.cloud_filters.get(key) is not None:
66-
raw_mask = (xx["water"] & val) > 0
67-
raw_mask = mask_cleanup(
68-
raw_mask, mask_filters=self.cloud_filters.get(key)
69-
)
70-
valid &= ~raw_mask
71-
wet &= ~raw_mask
72-
73-
xx = xx.drop_vars(["water"])
63+
# clear dry pixels
64+
clear = xx["water"].data == 0
7465

75-
# get valid wo pixels, both dry and wet
76-
wet = expr_eval(
66+
# get "valid" wo pixels, both dry and wet used in veg_frequency
67+
wet_valid = expr_eval(
7768
"where(a|b, a, _nan)",
78-
{"a": wet.data, "b": valid.data},
69+
{"a": wet, "b": valid},
7970
name="get_valid_pixels",
8071
dtype="float32",
8172
**{"_nan": np.nan},
8273
)
8374

84-
# pick all valid fc pixels
85-
xx = keep_good_only(xx, valid, nodata=NODATA)
86-
xx = to_float(xx, dtype="float32")
87-
88-
# get high ue valid pixels
89-
ue = expr_eval(
90-
"where(a>=_v, 1, _nan)",
91-
{"a": xx["ue"].data},
92-
name="get_high_ue",
75+
# get "clear" wo pixels, both dry and wet used in water_frequency
76+
wet_clear = expr_eval(
77+
"where(a|b, a, _nan)",
78+
{"a": wet, "b": clear},
79+
name="get_clear_pixels",
9380
dtype="float32",
94-
**{
95-
"_v": self.ue_threshold,
96-
"_nan": np.nan,
97-
},
81+
**{"_nan": np.nan},
9882
)
9983

100-
# get low ue valid pixels
84+
# dilate both 'valid' and 'water'
85+
for key, val in self.BAD_BITS_MASK.items():
86+
if self.cloud_filters.get(key) is not None:
87+
raw_mask = (xx["water"] & val) > 0
88+
raw_mask = mask_cleanup(
89+
raw_mask, mask_filters=self.cloud_filters.get(key)
90+
)
91+
valid = expr_eval(
92+
"where(b>0, 0, a)",
93+
{"a": valid, "b": raw_mask.data},
94+
name="get_valid_pixels",
95+
dtype="uint8",
96+
)
97+
wet_clear = expr_eval(
98+
"where(b>0, _nan, a)",
99+
{"a": wet_clear, "b": raw_mask.data},
100+
name="get_lear_pixels",
101+
dtype="float32",
102+
**{"_nan": np.nan},
103+
)
104+
wet_valid = expr_eval(
105+
"where(b>0, _nan, a)",
106+
{"a": wet_valid, "b": raw_mask.data},
107+
name="get_valid_pixels",
108+
dtype="float32",
109+
**{"_nan": np.nan},
110+
)
111+
xx = xx.drop_vars(["water"])
112+
113+
# Pick out the fc pixels that have an unmixing error of less than the threshold
101114
valid = expr_eval(
102-
"where(b<_v, 1, 0)",
103-
{"b": xx["ue"].data},
104-
name="get_valid_pixels",
115+
"where(b<_v, a, 0)",
116+
{"a": valid, "b": xx["ue"].data},
117+
name="get_low_ue",
105118
dtype="bool",
106119
**{"_v": self.ue_threshold},
107120
)
108121
xx = xx.drop_vars(["ue"])
109122
valid = xr.DataArray(valid, dims=xx["pv"].dims, coords=xx["pv"].coords)
110-
xx = keep_good_only(xx, valid, nodata=np.nan)
111123

112-
xx["wet"] = xr.DataArray(wet, dims=xx["pv"].dims, coords=xx["pv"].coords)
113-
xx["ue"] = xr.DataArray(ue, dims=xx["pv"].dims, coords=xx["pv"].coords)
124+
xx = keep_good_only(xx, valid, nodata=NODATA)
125+
xx = to_float(xx, dtype="float32")
126+
127+
xx["wet_valid"] = xr.DataArray(
128+
wet_valid, dims=xx["pv"].dims, coords=xx["pv"].coords
129+
)
130+
xx["wet_clear"] = xr.DataArray(
131+
wet_clear, dims=xx["pv"].dims, coords=xx["pv"].coords
132+
)
133+
114134
return xx
115135

116136
def fuser(self, xx):
117137

118-
wet = xx["wet"]
119-
ue = xx["ue"]
138+
wet_valid = xx["wet_valid"]
139+
wet_clear = xx["wet_clear"]
120140

121141
xx = _xr_fuse(
122-
xx.drop_vars(["wet", "ue"]),
142+
xx.drop_vars(["wet_valid", "wet_clear"]),
123143
partial(_fuse_mean_np, nodata=np.nan),
124-
"fuse_fc",
144+
"",
125145
)
126146

127-
xx["wet"] = _nodata_fuser(wet, nodata=np.nan)
128-
xx["ue"] = _nodata_fuser(ue, nodata=np.nan)
147+
xx["wet_valid"] = _nodata_fuser(wet_valid, nodata=np.nan)
148+
xx["wet_clear"] = _nodata_fuser(wet_clear, nodata=np.nan)
129149

130150
return xx
131151

132152
def _veg_or_not(self, xx: xr.Dataset):
133-
# pv or npv > bs: 1
153+
# either pv or npv > bs: 1
134154
# otherwise 0
135155
data = expr_eval(
136156
"where((a>b)|(c>b), 1, 0)",
137-
{
138-
"a": xx["pv"].data,
139-
"c": xx["npv"].data,
140-
"b": xx["bs"].data,
141-
},
157+
{"a": xx["pv"].data, "c": xx["npv"].data, "b": xx["bs"].data},
142158
name="get_veg",
143159
dtype="uint8",
144160
)
145161

146-
# mark nans only if not valid & low ue
147-
# if any high ue valid (ue is not nan): 0
162+
# mark nans
148163
data = expr_eval(
149-
"where((a!=a)&(c!=c), nodata, b)",
150-
{"a": xx["pv"].data, "c": xx["ue"].data, "b": data},
164+
"where(a!=a, nodata, b)",
165+
{"a": xx["pv"].data, "b": data},
151166
name="get_veg",
152167
dtype="uint8",
153168
**{"nodata": int(NODATA)},
@@ -156,7 +171,7 @@ def _veg_or_not(self, xx: xr.Dataset):
156171
# mark water freq >= 0.5 as 0
157172
data = expr_eval(
158173
"where(a>0, 0, b)",
159-
{"a": xx["wet"].data, "b": data},
174+
{"a": xx["wet_valid"].data, "b": data},
160175
name="get_veg",
161176
dtype="uint8",
162177
)
@@ -167,25 +182,25 @@ def _water_or_not(self, xx: xr.Dataset):
167182
# mark water freq > 0.5 as 1
168183
data = expr_eval(
169184
"where(a>0.5, 1, 0)",
170-
{"a": xx["wet"].data},
185+
{"a": xx["wet_clear"].data},
171186
name="get_water",
172187
dtype="uint8",
173188
)
174189

175190
# mark nans
176191
data = expr_eval(
177192
"where(a!=a, nodata, b)",
178-
{"a": xx["wet"].data, "b": data},
193+
{"a": xx["wet_clear"].data, "b": data},
179194
name="get_water",
180195
dtype="uint8",
181196
**{"nodata": int(NODATA)},
182197
)
183198
return data
184199

185-
def _max_consecutive_months(self, data, nodata):
186-
nan_mask = da.ones(data.shape[1:], chunks=data.chunks[1:], dtype="bool")
200+
def _max_consecutive_months(self, data, nodata, normalize=False):
187201
tmp = da.zeros(data.shape[1:], chunks=data.chunks[1:], dtype="uint8")
188202
max_count = da.zeros(data.shape[1:], chunks=data.chunks[1:], dtype="uint8")
203+
total = da.zeros(data.shape[1:], chunks=data.chunks[1:], dtype="uint8")
189204

190205
for t in data:
191206
# +1 if not nodata
@@ -213,23 +228,34 @@ def _max_consecutive_months(self, data, nodata):
213228
dtype="uint8",
214229
)
215230

216-
# mark nodata
217-
nan_mask = expr_eval(
218-
"where(a==nodata, b, False)",
219-
{"a": t, "b": nan_mask},
220-
name="mark_nodata",
221-
dtype="bool",
231+
# total valid
232+
total = expr_eval(
233+
"where(a==nodata, b, b+1)",
234+
{"a": t, "b": total},
235+
name="get_total_valid",
236+
dtype="uint8",
222237
**{"nodata": nodata},
223238
)
224239

225240
# mark nodata
226-
max_count = expr_eval(
227-
"where(a, nodata, b)",
228-
{"a": nan_mask, "b": max_count},
229-
name="mark_nodata",
230-
dtype="uint8",
231-
**{"nodata": int(nodata)},
232-
)
241+
if normalize:
242+
max_count = expr_eval(
243+
"where(a<=0, nodata, b/a*12)",
244+
{"a": total, "b": max_count},
245+
name="normalize_max_count",
246+
dtype="float32",
247+
**{"nodata": int(nodata)},
248+
)
249+
max_count = da.ceil(max_count).astype("uint8")
250+
else:
251+
max_count = expr_eval(
252+
"where(a<=0, nodata, b)",
253+
{"a": total, "b": max_count},
254+
name="mark_nodata",
255+
dtype="uint8",
256+
**{"nodata": int(nodata)},
257+
)
258+
233259
return max_count
234260

235261
def reduce(self, xx: xr.Dataset) -> xr.Dataset:
@@ -240,15 +266,15 @@ def reduce(self, xx: xr.Dataset) -> xr.Dataset:
240266
max_count_veg = self._max_consecutive_months(data, NODATA)
241267

242268
data = self._water_or_not(xx)
243-
max_count_water = self._max_consecutive_months(data, NODATA)
269+
max_count_water = self._max_consecutive_months(data, NODATA, normalize=True)
244270

245271
attrs = xx.attrs.copy()
246272
attrs["nodata"] = int(NODATA)
247273
data_vars = {
248-
k: xr.DataArray(v, dims=xx["wet"].dims[1:], attrs=attrs)
274+
k: xr.DataArray(v, dims=xx["pv"].dims[1:], attrs=attrs)
249275
for k, v in zip(self.measurements, [max_count_veg, max_count_water])
250276
}
251-
coords = dict((dim, xx.coords[dim]) for dim in xx["wet"].dims[1:])
277+
coords = dict((dim, xx.coords[dim]) for dim in xx["pv"].dims[1:])
252278
return xr.Dataset(data_vars=data_vars, coords=coords, attrs=xx.attrs)
253279

254280

‎tests/test_landcover_plugin_a0.py‎

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -327,12 +327,12 @@ def test_native_transform(fc_wo_dataset, bits):
327327
np.array([1, 1, 3, 5, 6, 2, 6, 2, 2, 5, 6, 0, 0, 2, 3]),
328328
np.array([0, 3, 2, 1, 3, 5, 6, 1, 4, 5, 6, 0, 2, 4, 2]),
329329
)
330-
result = np.where(out_xx["wet"].data == out_xx["wet"].data)
330+
result = np.where(out_xx["wet_valid"].data == out_xx["wet_valid"].data)
331331
for a, b in zip(expected_valid, result):
332332
assert (a == b).all()
333333

334334
expected_valid = (np.array([1, 2, 3]), np.array([6, 2, 0]), np.array([6, 1, 2]))
335-
result = np.where(out_xx["wet"].data == 1)
335+
result = np.where(out_xx["wet_valid"].data == 1)
336336

337337
for a, b in zip(expected_valid, result):
338338
assert (a == b).all()
@@ -391,11 +391,11 @@ def test_water_or_not(fc_wo_dataset):
391391
xx = xx.groupby("solar_day").map(partial(StatsVegCount.fuser, None))
392392
yy = stats_veg._water_or_not(xx).compute()
393393
valid_index = (
394-
np.array([0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 2]),
395-
np.array([1, 1, 3, 5, 6, 2, 6, 0, 0, 2, 2, 3, 5, 6]),
396-
np.array([0, 3, 2, 1, 3, 5, 6, 0, 2, 1, 4, 2, 5, 6]),
394+
np.array([0, 0, 1, 1, 2, 2, 2]),
395+
np.array([3, 6, 2, 6, 0, 2, 2]),
396+
np.array([2, 3, 5, 6, 2, 1, 4]),
397397
)
398-
expected_value = np.array([0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0])
398+
expected_value = np.array([0, 0, 0, 1, 1, 1, 0])
399399
i = 0
400400
for idx in zip(*valid_index):
401401
assert yy[idx] == expected_value[i]
@@ -410,27 +410,26 @@ def test_reduce(fc_wo_dataset):
410410
expected_value = np.array(
411411
[
412412
[1, 255, 0, 255, 255, 255, 255],
413-
[1, 255, 255, 0, 255, 255, 255],
414-
[255, 0, 255, 255, 1, 0, 255],
413+
[1, 255, 255, 255, 255, 255, 255],
414+
[255, 0, 255, 255, 1, 255, 255],
415415
[255, 255, 1, 255, 255, 255, 255],
416416
[255, 255, 255, 255, 255, 255, 255],
417-
[255, 1, 255, 255, 255, 0, 255],
418-
[255, 255, 255, 0, 255, 255, 0],
419-
],
420-
dtype="uint8",
417+
[255, 1, 255, 255, 255, 255, 255],
418+
[255, 255, 255, 255, 255, 255, 0],
419+
]
421420
)
422421

423422
assert (xx.veg_frequency.data == expected_value).all()
424423

425424
expected_value = np.array(
426425
[
427-
[0, 255, 1, 255, 255, 255, 255],
428-
[0, 255, 255, 0, 255, 255, 255],
429-
[255, 1, 255, 255, 0, 0, 255],
426+
[255, 255, 12, 255, 255, 255, 255],
427+
[255, 255, 255, 255, 255, 255, 255],
428+
[255, 12, 255, 255, 0, 0, 255],
430429
[255, 255, 0, 255, 255, 255, 255],
431430
[255, 255, 255, 255, 255, 255, 255],
432-
[255, 0, 255, 255, 255, 0, 255],
433-
[255, 255, 255, 0, 255, 255, 1],
431+
[255, 255, 255, 255, 255, 255, 255],
432+
[255, 255, 255, 0, 255, 255, 12],
434433
],
435434
dtype="uint8",
436435
)

0 commit comments

Comments
 (0)