Skip to content

Refactor resampler module to submodules #3124

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 34 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 27 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
e873d3b
Refactor resampler module to submodules
lahtinep May 12, 2025
b65098a
Update resample imports
lahtinep May 12, 2025
f3712ab
Update and move resampling docstring
lahtinep May 12, 2025
0b59ca8
Fix DOS line-endings, fix get_area_def import
lahtinep May 13, 2025
9051519
Fix imports to use new resampler modules
lahtinep May 13, 2025
41f5ef0
Simplify prepare_resampler()
lahtinep May 13, 2025
43f4b33
Simplify add_xy_coords()
lahtinep May 13, 2025
80b16e1
Simplify BucketResamplerBase.resample()
lahtinep May 13, 2025
b16e834
Simplify KDTreeResampler.load_neighbour_info()
lahtinep May 13, 2025
d68afb5
Simplify NativeResampler.compute()
lahtinep May 13, 2025
ea5d24b
Simplify native._rechunk_if_nonfactor_chunks()
lahtinep May 13, 2025
c71f33d
Simplify _check_crs_units and add_xy_coords() more
lahtinep May 13, 2025
86a4def
Reduce cyclomatic complecity of NativeResampler._expand_reduce()
lahtinep May 13, 2025
44ee39b
Make base.hash_dict a private function
lahtinep May 13, 2025
e33ea4b
Make get_area_file and update_resampled_coords private functions
lahtinep May 16, 2025
412d848
Fix import
lahtinep May 16, 2025
d7e4e9e
Move get_area_def and get_area_file to a new module satpy.area_utils
lahtinep May 16, 2025
125e691
Create satpy.coords_utils module
lahtinep May 16, 2025
f24507f
Fix get_area_def patching
lahtinep May 16, 2025
28e02ca
Merge branch 'main' into refactor-resample
lahtinep May 16, 2025
9371f45
Fix bucket doc references, again
lahtinep May 16, 2025
5a73b6e
Merge branch 'main' into refactor-resample
pnuu May 28, 2025
e5a4db7
Use helper function instead of if/elif/else in satpy.resample imports
pnuu May 28, 2025
3fcbe6d
Fix resample imports and doc references
pnuu May 28, 2025
5fe8f06
Remove unnecessary pyproj<2.0 check
pnuu May 28, 2025
ea17ec0
Merge branch 'main' into refactor-resample
pnuu May 28, 2025
bfdb334
Fix resampling related invalid sphinx references
djhoese May 28, 2025
9d5a3a7
Move BaseResampler import from top level to functions
pnuu May 30, 2025
9f9a385
Refactor collection of resampler classes
pnuu May 30, 2025
561101b
Merge branch 'refactor-resample' of https://github.com/pnuu/satpy int…
pnuu May 30, 2025
5bbd4f1
Rename area_utils.py to area.py
pnuu May 30, 2025
0b2be45
Rename coords_utils.py to coords.py
pnuu May 30, 2025
7ff4f6d
Move resample_dataset and prepare_rempler import to functions
pnuu May 30, 2025
80296ee
Add a test for warnings issued when importing directly from satpy.res…
pnuu May 30, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions doc/source/generate_area_def_list.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
"""Generate the area definition list restructuredtext document.

This should be run once before generating the sphinx documentation to
produce the ``area_def_list.rst`` file referenced by ``satpy/resample.py``.
produce the ``area_def_list.rst`` file referenced by
``satpy/resample/__init__.py``.

"""
import logging
Expand All @@ -19,7 +20,7 @@
from pyresample.utils.proj4 import ignore_pyproj_proj_warnings
from reader_table import rst_table_header, rst_table_row

from satpy.resample import get_area_file
from satpy.area_utils import get_area_file

logger = logging.getLogger(__name__)

Expand Down
43 changes: 43 additions & 0 deletions satpy/area_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#!/usr/bin/env python
# Copyright (c) 2015-2025 Satpy developers
#
# This file is part of satpy.
#
# satpy is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# satpy. If not, see <http://www.gnu.org/licenses/>.
"""Utility functions for area definitions."""
from ._config import config_search_paths, get_config_path


def get_area_file():
"""Find area file(s) to use.

The files are to be named `areas.yaml` or `areas.def`.
"""
paths = config_search_paths("areas.yaml")
if paths:
return paths
else:
return get_config_path("areas.def")

Check warning on line 30 in satpy/area_utils.py

View check run for this annotation

Codecov / codecov/patch

satpy/area_utils.py#L30

Added line #L30 was not covered by tests


def get_area_def(area_name):
"""Get the definition of *area_name* from file.

The file is defined to use is to be placed in the $SATPY_CONFIG_PATH
directory, and its name is defined in satpy's configuration file.
"""
try:
from pyresample import parse_area_file
except ImportError:
from pyresample.utils import parse_area_file

Check warning on line 42 in satpy/area_utils.py

View check run for this annotation

Codecov / codecov/patch

satpy/area_utils.py#L41-L42

Added lines #L41 - L42 were not covered by tests
return parse_area_file(get_area_file(), area_name)[0]
2 changes: 1 addition & 1 deletion satpy/composites/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1597,7 +1597,7 @@ def __init__(self, name, filename=None, url=None, known_hash=None, area=None, #
self._known_hash = known_hash
self.area = None
if area is not None:
from satpy.resample import get_area_def
from satpy.area_utils import get_area_def
self.area = get_area_def(area)

super(StaticImageCompositor, self).__init__(name, **kwargs)
Expand Down
131 changes: 131 additions & 0 deletions satpy/coords_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
#!/usr/bin/env python
# Copyright (c) 2015-2025 Satpy developers
#
# This file is part of satpy.
#
# satpy is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# satpy. If not, see <http://www.gnu.org/licenses/>.
"""Utility functions for coordinates."""

import logging

import xarray as xr

LOG = logging.getLogger(__name__)


def add_xy_coords(data_arr, area, crs=None):
"""Assign x/y coordinates to DataArray from provided area.

If 'x' and 'y' coordinates already exist then they will not be added.

Args:
data_arr (xarray.DataArray): data object to add x/y coordinates to
area (pyresample.geometry.AreaDefinition): area providing the
coordinate data.
crs (pyproj.crs.CRS or None): CRS providing additional information
about the area's coordinate reference system if available.
Requires pyproj 2.0+.

Returns (xarray.DataArray): Updated DataArray object

"""
if "x" in data_arr.coords and "y" in data_arr.coords:
# x/y coords already provided
return data_arr
if "x" not in data_arr.dims or "y" not in data_arr.dims:
# no defined x and y dimensions
return data_arr
if not hasattr(area, "get_proj_vectors"):
return data_arr
x, y = area.get_proj_vectors()

# convert to DataArrays
y_attrs = {}
x_attrs = {}

_check_crs_units(crs, x_attrs, y_attrs)

y = xr.DataArray(y, dims=("y",), attrs=y_attrs)
x = xr.DataArray(x, dims=("x",), attrs=x_attrs)
return data_arr.assign_coords(y=y, x=x)


def _check_crs_units(crs, x_attrs, y_attrs):
if crs is None:
return
units = crs.axis_info[0].unit_name
# fix udunits/CF standard units
units = units.replace("metre", "meter")
if units == "degree":
y_attrs["units"] = "degrees_north"
x_attrs["units"] = "degrees_east"
else:
y_attrs["units"] = units
x_attrs["units"] = units


def add_crs_xy_coords(data_arr, area):
"""Add :class:`pyproj.crs.CRS` and x/y or lons/lats to coordinates.

For SwathDefinition or GridDefinition areas this will add a
`crs` coordinate and coordinates for the 2D arrays of `lons` and `lats`.

For AreaDefinition areas this will add a `crs` coordinate and the
1-dimensional `x` and `y` coordinate variables.

Args:
data_arr (xarray.DataArray): DataArray to add the 'crs'
coordinate.
area (pyresample.geometry.AreaDefinition): Area to get CRS
information from.

"""
from pyresample.geometry import SwathDefinition

crs, data_arr = _add_crs(area, data_arr)

# Add x/y coordinates if possible
if isinstance(area, SwathDefinition):
# add lon/lat arrays for swath definitions
# SwathDefinitions created by Satpy should be assigning DataArray
# objects as the lons/lats attributes so use those directly to
# maintain original .attrs metadata (instead of converting to dask
# array).
lons = area.lons
lats = area.lats
lons.attrs.setdefault("standard_name", "longitude")
lons.attrs.setdefault("long_name", "longitude")
lons.attrs.setdefault("units", "degrees_east")
lats.attrs.setdefault("standard_name", "latitude")
lats.attrs.setdefault("long_name", "latitude")
lats.attrs.setdefault("units", "degrees_north")
# See https://github.com/pydata/xarray/issues/3068
# data_arr = data_arr.assign_coords(longitude=lons, latitude=lats)
else:
# Gridded data (AreaDefinition/StackedAreaDefinition)
data_arr = add_xy_coords(data_arr, area, crs=crs)
return data_arr


def _add_crs(area, data_arr):
from pyproj import CRS

if hasattr(area, "crs"):
crs = area.crs
else:
# default lat/lon projection
latlon_proj = "+proj=latlong +datum=WGS84 +ellps=WGS84"
proj_str = getattr(area, "proj_str", latlon_proj)
crs = CRS.from_string(proj_str)

Check warning on line 129 in satpy/coords_utils.py

View check run for this annotation

Codecov / codecov/patch

satpy/coords_utils.py#L127-L129

Added lines #L127 - L129 were not covered by tests
data_arr = data_arr.assign_coords(crs=crs)
return crs, data_arr
2 changes: 1 addition & 1 deletion satpy/modifiers/parallax.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
from pyresample.geometry import SwathDefinition

from satpy.modifiers import ModifierBase
from satpy.resample import resample_dataset
from satpy.resample.base import resample_dataset
from satpy.utils import get_satpos, lonlat2xyz, xyz2lonlat

logger = logging.getLogger(__name__)
Expand Down
2 changes: 1 addition & 1 deletion satpy/readers/cmsaf_claas2.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

import datetime

from satpy.area_utils import get_area_def
from satpy.readers.core.netcdf import NetCDF4FileHandler
from satpy.resample import get_area_def


def _is_georef_offset_present(date):
Expand Down
3 changes: 2 additions & 1 deletion satpy/readers/core/yaml_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,11 @@

from satpy import DatasetDict
from satpy._compat import cache
from satpy.area_utils import get_area_def
from satpy.aux_download import DataDownloadMixin
from satpy.coords_utils import add_crs_xy_coords
from satpy.dataset import DataID, DataQuery, get_key
from satpy.dataset.dataid import default_co_keys_config, default_id_keys_config, get_keys_from_config
from satpy.resample import add_crs_xy_coords, get_area_def
from satpy.utils import recursive_dict_update

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -735,7 +736,7 @@
def _load_dataset_data(self, file_handlers, dsid, **kwargs):
ds_info = self.all_ids[dsid]
proj = self._load_dataset(dsid, ds_info, file_handlers, **kwargs)
# FIXME: areas could be concatenated here

Check notice on line 739 in satpy/readers/core/yaml_reader.py

View check run for this annotation

codefactor.io / CodeFactor

satpy/readers/core/yaml_reader.py#L739

Unresolved comment '# FIXME: areas could be concatenated here'. (C100)
# Update the metadata
proj.attrs["start_time"] = file_handlers[0].start_time
proj.attrs["end_time"] = file_handlers[-1].end_time
Expand Down
2 changes: 1 addition & 1 deletion satpy/readers/eum_l2_bufr.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,11 @@
import numpy as np
import xarray as xr

from satpy.area_utils import get_area_def
from satpy.readers.core._geos_area import get_geos_area_naming
from satpy.readers.core.eum import get_service_mode, recarray2dict
from satpy.readers.core.file_handlers import BaseFileHandler
from satpy.readers.core.seviri import mpef_product_header
from satpy.resample import get_area_def
from satpy.utils import get_legacy_chunk_size

try:
Expand Down
2 changes: 1 addition & 1 deletion satpy/readers/fci_l2_nc.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@
from pyresample import geometry

from satpy._compat import cached_property
from satpy.area_utils import get_area_def
from satpy.readers.core._geos_area import get_geos_area_naming, make_ext
from satpy.readers.core.eum import get_service_mode
from satpy.readers.core.fci import platform_name_translate
from satpy.readers.core.file_handlers import BaseFileHandler
from satpy.resample import get_area_def
from satpy.utils import get_legacy_chunk_size

logger = logging.getLogger(__name__)
Expand Down
2 changes: 1 addition & 1 deletion satpy/readers/gerb_l2_hr_h5.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
import datetime as dt
import logging

from satpy.area_utils import get_area_def
from satpy.readers.core.hdf5 import HDF5FileHandler
from satpy.resample import get_area_def

LOG = logging.getLogger(__name__)

Expand Down
2 changes: 1 addition & 1 deletion satpy/readers/hsaf_h5.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@
import numpy as np
import xarray as xr

from satpy.area_utils import get_area_def
from satpy.readers.core.file_handlers import BaseFileHandler
from satpy.resample import get_area_def
from satpy.utils import get_legacy_chunk_size

LOG = logging.getLogger(__name__)
Expand Down
2 changes: 1 addition & 1 deletion satpy/readers/li_l2_nc.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,8 @@
import numpy as np
import xarray as xr

from satpy.area_utils import get_area_def
from satpy.readers.core.li_nc import LINCFileHandler
from satpy.resample import get_area_def
from satpy.utils import get_legacy_chunk_size

logger = logging.getLogger(__name__)
Expand Down
Loading
Loading