diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b38870cb3..c29709086 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,181 +9,190 @@ on: # Everything should be merged through PRs anyway. branches: - develop + - develop-1.9 - test-ci-* - pypi/publish paths: - - '**' - - '!notebooks/**' - - '!docs/**' - - '!old/**' - - '!README.md' + - "**" + - "!notebooks/**" + - "!docs/**" + - "!old/**" + - "!README.md" jobs: build-wheels: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.9 - - - uses: actions/cache@v3 - id: wheels_cache - with: - path: ./wheels - key: wheels-${{ github.sha }} - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools - python -m pip install --upgrade \ - toml \ - wheel \ - packaging \ - twine - python -m pip freeze - - - name: Build Clean Packages - run: | - mkdir -p ./wheels/clean - ./scripts/build-wheels.sh ./wheels/clean - find ./wheels/clean -type f - - - name: Patch Package Versions - run: | - grep -R --files-with-matches --include '*.py' '__version__ =' \ - | xargs python ./scripts/patch_version.py ${GITHUB_RUN_NUMBER:-0} - - - name: Build Dev Packages - run: | - mkdir -p ./wheels/dev - ./scripts/build-wheels.sh ./wheels/dev - find ./wheels/dev -type f + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - uses: actions/cache@v4 + id: wheels_cache + with: + path: ./wheels + key: wheels-${{ github.sha }} + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools + python -m pip install --upgrade \ + toml \ + wheel \ + packaging \ + twine + python -m pip freeze + + - name: Build Clean Packages + run: | + mkdir -p ./wheels/clean + ./scripts/build-wheels.sh ./wheels/clean + find ./wheels/clean -type f + + - name: Patch Package Versions + run: | + grep -R --files-with-matches --include '*.py' '__version__ =' \ + | xargs python ./scripts/patch_version.py ${GITHUB_RUN_NUMBER:-0} + + - name: Build Dev Packages + run: | + mkdir -p ./wheels/dev + ./scripts/build-wheels.sh ./wheels/dev + find ./wheels/dev -type f build-test-env-base: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - - uses: actions/cache@v3 - id: conda_cache - with: - path: | - tests/env - key: ${{ runner.os }}-test-env-${{ hashFiles('tests/test-env.yml') }} - - - uses: conda-incubator/setup-miniconda@v2 - if: steps.conda_cache.outputs.cache-hit != 'true' - with: - channels: conda-forge,defaults - channel-priority: true - activate-environment: "" - # mamba-version: "*" - use-mamba: true - miniforge-variant: Mambaforge - - - name: Dump Conda Environment Info - shell: bash -l {0} - if: steps.conda_cache.outputs.cache-hit != 'true' - run: | + - uses: actions/checkout@v4 + + - uses: actions/cache@v4 + id: conda_cache + with: + path: | + tests/env + key: ${{ runner.os }}-test-env-${{ hashFiles('tests/test-env.yml') }} + + - uses: conda-incubator/setup-miniconda@v3 + if: steps.conda_cache.outputs.cache-hit != 'true' + with: + channels: conda-forge,defaults + channel-priority: true + activate-environment: "" + mamba-version: "*" + use-mamba: true + + - name: Dump Conda Environment Info + shell: bash -l {0} + if: steps.conda_cache.outputs.cache-hit != 'true' + run: | conda info conda list - mamba -V conda config --show-sources conda config --show printenv | sort - - name: Build Python Environment for Testing - shell: bash -l {0} - if: steps.conda_cache.outputs.cache-hit != 'true' - run: | - mamba env create -f tests/test-env.yml -p tests/env - - - name: Check Python Env - shell: bash -l {0} - if: steps.conda_cache.outputs.cache-hit != 'true' - run: | - mamba env export -p tests/env + - name: Build Python Environment for Testing + shell: bash -l {0} + if: steps.conda_cache.outputs.cache-hit != 'true' + run: | + mamba env create -f tests/test-env.yml -p tests/env + - name: Check Python Env + shell: bash -l {0} + if: steps.conda_cache.outputs.cache-hit != 'true' + run: | + mamba env export -p tests/env test-with-coverage: runs-on: ubuntu-latest needs: - build-test-env-base - + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres + # Provide the password for postgres + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: datacube + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 steps: - - uses: actions/checkout@v2 - - - name: Get Conda Environment from Cache - uses: actions/cache@v3 - id: conda_cache - with: - path: | - tests/env - key: ${{ runner.os }}-test-env-${{ hashFiles('tests/test-env.yml') }} - - - name: Update PATH - shell: bash - run: | - echo "$(pwd)/tests/env/bin" >> $GITHUB_PATH - export PATH="$(pwd)/tests/env/bin" - - - name: Install in Edit mode - shell: bash - run: | - which python - which createdb - which datacube - - ./scripts/dev-install.sh --no-deps - - - name: Start Test DB - shell: bash - run: | - echo "Launching test db" - pgdata=$(pwd)/.dbdata - initdb -D ${pgdata} --auth-host=md5 --encoding=UTF8 - pg_ctl -D ${pgdata} -l "${pgdata}/pg.log" start - createdb datacube - datacube system init - pip list --format=freeze - - env: - DATACUBE_DB_URL: postgresql:///datacube - - - name: Run Tests - shell: bash - run: | - datacube system check - datacube metadata add "https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/product_metadata/eo3_sentinel_ard.odc-type.yaml" - - echo "Running Tests" - pytest --cov=. \ - --cov-report=html \ - --cov-report=xml:coverage.xml \ - --timeout=30 \ - libs apps - - env: - AWS_DEFAULT_REGION: us-west-2 - DASK_TEMPORARY_DIRECTORY: /tmp/dask - DATACUBE_DB_URL: postgresql:///datacube - - - name: Upload Coverage - if: | - github.repository == 'opendatacube/odc-tools' - - uses: codecov/codecov-action@v1 - with: - fail_ci_if_error: false - verbose: false - + - uses: actions/checkout@v4 + + - name: Get Conda Environment from Cache + uses: actions/cache@v4 + id: conda_cache + with: + path: | + tests/env + key: ${{ runner.os }}-test-env-${{ hashFiles('tests/test-env.yml') }} + + - name: Update PATH + shell: bash + run: | + echo "$(pwd)/tests/env/bin" >> $GITHUB_PATH + export PATH="$(pwd)/tests/env/bin" + + - name: Install in Edit mode + shell: bash + run: | + which python + which datacube + + ./scripts/dev-install.sh --no-deps + + - name: Setup Test DB + shell: bash + run: | + echo "Launching test db" + datacube --version + datacube system init + pip list --format=freeze + + env: + ODC_DEFAULT_DB_URL: postgresql://postgres:postgres@localhost/datacube + + - name: Run Tests + shell: bash + run: | + datacube system check + + echo "Running Tests" + pytest --cov=. \ + --cov-report=html \ + --cov-report=xml:coverage.xml \ + --timeout=30 \ + libs apps + + env: + AWS_DEFAULT_REGION: us-west-2 + DASK_TEMPORARY_DIRECTORY: /tmp/dask + ODC_DEFAULT_DB_URL: postgresql://postgres:postgres@localhost/datacube + + - name: Upload Coverage + if: | + github.repository == 'opendatacube/odc-tools' + + uses: codecov/codecov-action@v5 + with: + fail_ci_if_error: false + verbose: false test-wheels: runs-on: ubuntu-latest @@ -192,81 +201,95 @@ jobs: - build-test-env-base - build-wheels + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres + # Provide the password for postgres + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: datacube + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 steps: - - uses: actions/checkout@v2 - - - name: Get Wheels from Cache - uses: actions/cache@v3 - id: wheels_cache - with: - path: ./wheels - key: wheels-${{ github.sha }} - - - name: Get Conda Environment from Cache - uses: actions/cache@v3 - id: conda_cache - with: - path: | - tests/env - key: ${{ runner.os }}-test-env-${{ hashFiles('tests/test-env.yml') }} - - - name: Update PATH - shell: bash - run: | - echo "$(pwd)/tests/env/bin" >> $GITHUB_PATH - - - name: Install wheels for testing - shell: bash - run: | - which python - which createdb - which datacube - - ls -lh wheels/clean - python -m pip install --no-deps wheels/clean/*whl - python -m pip check || true - - - name: Start Test DB - shell: bash - run: | - echo "Launching test db" - pgdata=$(pwd)/.dbdata - initdb -D ${pgdata} --auth-host=md5 --encoding=UTF8 - pg_ctl -D ${pgdata} -l "${pgdata}/pg.log" start - createdb datacube - datacube system init - - env: - DATACUBE_DB_URL: postgresql:///datacube - - - name: Run Tests - shell: bash - run: | - datacube system check - datacube metadata add "https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/product_metadata/eo3_sentinel_ard.odc-type.yaml" - - echo "Running Tests" - pytest --timeout=30 libs apps - - env: - AWS_DEFAULT_REGION: us-west-2 - DASK_TEMPORARY_DIRECTORY: /tmp/dask - DATACUBE_DB_URL: postgresql:///datacube + - uses: actions/checkout@v4 + + - name: Get Wheels from Cache + uses: actions/cache@v4 + id: wheels_cache + with: + path: ./wheels + key: wheels-${{ github.sha }} + + - name: Get Conda Environment from Cache + uses: actions/cache@v4 + id: conda_cache + with: + path: | + tests/env + key: ${{ runner.os }}-test-env-${{ hashFiles('tests/test-env.yml') }} + + - name: Update PATH + shell: bash + run: | + echo "$(pwd)/tests/env/bin" >> $GITHUB_PATH + + - name: Install wheels for testing + shell: bash + run: | + which python + which datacube + + ls -lh wheels/clean + python -m pip install --no-deps wheels/clean/*whl + python -m pip check || true + + - name: Start Test DB + shell: bash + run: | + echo "Launching test db" + datacube system init + + env: + ODC_DEFAULT_DB_URL: postgresql://postgres:postgres@localhost/datacube + + - name: Run Tests + shell: bash + run: | + datacube system check + # from "https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/product_metadata/eo3_sentinel_ard.odc-type.yaml" + datacube metadata add apps/dc_tools/tests/data/eo3_sentinel_ard.odc-type.yaml + + echo "Running Tests" + pytest --timeout=30 libs apps + + env: + AWS_DEFAULT_REGION: us-west-2 + DASK_TEMPORARY_DIRECTORY: /tmp/dask + ODC_DEFAULT_DB_URL: postgresql://postgres:postgres@localhost/datacube publish-pypi: if: | - github.event_name == 'push' - && github.repository == 'opendatacube/odc-tools' - && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/pypi/publish') + github.event_name == 'push' + && github.repository == 'opendatacube/odc-tools' + && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/pypi/publish') strategy: matrix: pkg: - - odc-cloud - - odc-io - - odc-ui - - odc-apps-cloud - - odc-apps-dc-tools + - odc-cloud + - odc-io + - odc-ui + - odc-apps-cloud + - odc-apps-dc-tools needs: - build-wheels @@ -275,57 +298,57 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - - name: Config - if: | - github.event_name == 'push' - && github.repository == 'opendatacube/odc-tools' - && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/pypi/publish') - id: cfg - env: - PKG: ${{ matrix.pkg }} - run: | - tk="pypi_token_${PKG//-/_}" - echo "tk=${tk}" >> $GITHUB_OUTPUT - echo "publish=yes" >> $GITHUB_OUTPUT - - - name: Setup Python - if: steps.cfg.outputs.publish == 'yes' - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - - name: Install Twine - if: steps.cfg.outputs.publish == 'yes' - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools - python -m pip install --upgrade \ - toml \ - wheel \ - twine - python -m pip freeze - - uses: actions/cache@v3 - id: wheels_cache - if: steps.cfg.outputs.publish == 'yes' - with: - path: ./wheels - key: wheels-${{ github.sha }} - - - name: Prepare for upload - if: steps.cfg.outputs.publish == 'yes' - run: | - mkdir -p ./pips - ./scripts/mk-pip-tree.sh ./wheels/clean ./pips - find ./pips -type f - - name: Upload to PyPI - if: steps.cfg.outputs.publish == 'yes' - env: - TWINE_PASSWORD: ${{ secrets[ steps.cfg.outputs.tk ] }} - TWINE_USERNAME: __token__ - PKG: ${{ matrix.pkg }} - - run: | - ls pips/${PKG} - twine upload --non-interactive --skip-existing pips/${PKG}/* + - uses: actions/checkout@v4 + + - name: Config + if: | + github.event_name == 'push' + && github.repository == 'opendatacube/odc-tools' + && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/pypi/publish') + id: cfg + env: + PKG: ${{ matrix.pkg }} + run: | + tk="pypi_token_${PKG//-/_}" + echo "tk=${tk}" >> $GITHUB_OUTPUT + echo "publish=yes" >> $GITHUB_OUTPUT + + - name: Setup Python + if: steps.cfg.outputs.publish == 'yes' + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Twine + if: steps.cfg.outputs.publish == 'yes' + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools + python -m pip install --upgrade \ + toml \ + wheel \ + twine + python -m pip freeze + - uses: actions/cache@v4 + id: wheels_cache + if: steps.cfg.outputs.publish == 'yes' + with: + path: ./wheels + key: wheels-${{ github.sha }} + + - name: Prepare for upload + if: steps.cfg.outputs.publish == 'yes' + run: | + mkdir -p ./pips + ./scripts/mk-pip-tree.sh ./wheels/clean ./pips + find ./pips -type f + - name: Upload to PyPI + if: steps.cfg.outputs.publish == 'yes' + env: + TWINE_PASSWORD: ${{ secrets[ steps.cfg.outputs.tk ] }} + TWINE_USERNAME: __token__ + PKG: ${{ matrix.pkg }} + + run: | + ls pips/${PKG} + twine upload --non-interactive --skip-existing pips/${PKG}/* diff --git a/.gitignore b/.gitignore index 7a54edea4..056042b4b 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,6 @@ docs/notebooks/ .run/ docker/wheels/ + +.jj +.ropeproject diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 906efd859..27b80ef77 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: - id: yamllint args: ['-c', '.yamllint'] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: end-of-file-fixer - id: check-docstring-first @@ -26,14 +26,14 @@ repos: # name: isort (python) # args: [ "--profile", "black", "--filter-files" ] - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 24.10.0 hooks: - id: black - repo: https://github.com/PyCQA/pylint - rev: 'v3.2.3' # Use the sha / tag you want to point at + rev: 'v3.3.2' # Use the sha / tag you want to point at hooks: - id: pylint - repo: https://github.com/PyCQA/flake8 - rev: '7.0.0' + rev: '7.1.1' hooks: - id: flake8 diff --git a/apps/dc_tools/odc/apps/dc_tools/_stac.py b/apps/dc_tools/odc/apps/dc_tools/_stac.py index 27e8e77a2..6fe14820a 100644 --- a/apps/dc_tools/odc/apps/dc_tools/_stac.py +++ b/apps/dc_tools/odc/apps/dc_tools/_stac.py @@ -9,8 +9,9 @@ import numpy from datacube.model import Dataset -from datacube.utils.geometry import Geometry, box +from odc.geo.geom import Geometry, box from eodatasets3.stac import to_stac_item +from eodatasets3.serialise import from_doc from toolz import get_in from urllib.parse import urlparse @@ -452,7 +453,6 @@ def transform_geom_json_coordinates_to_list(geom_json): def ds_to_stac(ds: Dataset) -> dict: """Get STAC document from dataset with eo3 metadata""" - from eodatasets3.serialise import from_doc if ds.is_eo3: if not ds.uris: raise ValueError("Can't find dataset location") diff --git a/apps/dc_tools/odc/apps/dc_tools/_version.py b/apps/dc_tools/odc/apps/dc_tools/_version.py index 927de2582..0a0a43a57 100644 --- a/apps/dc_tools/odc/apps/dc_tools/_version.py +++ b/apps/dc_tools/odc/apps/dc_tools/_version.py @@ -1 +1 @@ -__version__ = "0.2.18" +__version__ = "1.9.0" diff --git a/apps/dc_tools/odc/apps/dc_tools/add_update_products.py b/apps/dc_tools/odc/apps/dc_tools/add_update_products.py index 56287e5d9..9b3cf92a0 100644 --- a/apps/dc_tools/odc/apps/dc_tools/add_update_products.py +++ b/apps/dc_tools/odc/apps/dc_tools/add_update_products.py @@ -14,6 +14,8 @@ import datacube from datacube import Datacube +from datacube.cfg import ODCEnvironment +from datacube.ui.click import environment_option, pass_config from odc.apps.dc_tools.utils import ( update_if_exists_flag, statsd_gauge_reporting, @@ -123,12 +125,16 @@ def add_update_products( @click.command("dc-sync-products") +@environment_option +@pass_config @click.argument("csv-path", nargs=1) @update_if_exists_flag @statsd_setting -def cli(csv_path: str, update_if_exists: bool, statsd_setting: str): +def cli( + cfg_env: ODCEnvironment, csv_path: str, update_if_exists: bool, statsd_setting: str +): # Check we can connect to the Datacube - dc = datacube.Datacube(app="add_update_products") + dc = datacube.Datacube(app="add_update_products", env=cfg_env) logging.info( "Starting up: connected to Datacube, and update-if-exists is: %s", update_if_exists, diff --git a/apps/dc_tools/odc/apps/dc_tools/azure_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/azure_to_dc.py index 99ce6e058..f1a18e804 100644 --- a/apps/dc_tools/odc/apps/dc_tools/azure_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/azure_to_dc.py @@ -10,7 +10,9 @@ import click from datacube import Datacube +from datacube.cfg import ODCEnvironment from datacube.index.hl import Doc2Dataset +from datacube.ui.click import environment_option, pass_config from odc.apps.dc_tools._stac import stac_transform from odc.apps.dc_tools.utils import ( SkippedException, @@ -128,6 +130,8 @@ def dump_list_to_odc( @click.command("azure-to-dc") +@environment_option +@pass_config @update_flag @update_if_exists_flag @allow_unsafe @@ -151,6 +155,7 @@ def dump_list_to_odc( @click.argument("suffix", type=str, nargs=1) @rename_product def cli( + cfg_env: ODCEnvironment, update: bool, update_if_exists: bool, allow_unsafe: bool, @@ -166,7 +171,7 @@ def cli( rename_product: str, ): # Set up the datacube first, to ensure we have a connection - dc = Datacube() + dc = Datacube(env=cfg_env) print(f"Opening AZ Container {container_name} on {account_url}") print(f"Searching on prefix '{prefix}' for files matching suffix '{suffix}'") yaml_urls = find_blobs( diff --git a/apps/dc_tools/odc/apps/dc_tools/cop_dem_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/cop_dem_to_dc.py index 0acd4b788..7123730a7 100644 --- a/apps/dc_tools/odc/apps/dc_tools/cop_dem_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/cop_dem_to_dc.py @@ -14,6 +14,7 @@ from datacube import Datacube from datacube.index.hl import Doc2Dataset +from datacube.ui.click import environment_option, pass_config from datacube.utils import read_documents from odc.apps.dc_tools.utils import ( SkippedException, @@ -202,6 +203,8 @@ def cop_dem_to_dc( @click.command("cop-dem-to-dc") +@environment_option +@pass_config @limit @update_if_exists_flag @bbox @@ -226,6 +229,7 @@ def cop_dem_to_dc( help="Number of threads to use to process, default 20", ) def cli( + cfg_env, limit, update_if_exists, bbox, @@ -244,7 +248,7 @@ def cli( f"Unknown product {product}, must be one of {' '.join(PRODUCTS)}" ) - dc = Datacube() + dc = Datacube(env=cfg_env) if add_product: add_cop_dem_product(dc, product) diff --git a/apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py index 733515e81..567a6361e 100644 --- a/apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py @@ -14,6 +14,7 @@ from datacube import Datacube from datacube.index.hl import Doc2Dataset +from datacube.ui.click import environment_option, pass_config from datacube.utils import read_documents from odc.apps.dc_tools.utils import ( bbox, @@ -112,6 +113,7 @@ def get_tile_uris(bounding_box: str) -> Tuple[str, str]: ) +# pylint: disable=too-many-positional-arguments def process_uri_tile( uri_tile: Tuple[str, str, str], dc: Datacube, @@ -160,6 +162,7 @@ def select_map_version(version: str): map_version["algo"] = "v200" +# pylint: disable=too-many-positional-arguments def esa_wc_to_dc( dc: Datacube, bounding_box, @@ -204,6 +207,7 @@ def esa_wc_to_dc( sys.stdout.write(f"\rAdded {success} datasets...") except rasterio.errors.RasterioIOError: logging.info("Couldn't read file %s", uri, exc_info=True) + failure += 1 except Exception: # pylint:disable=broad-except logging.exception("Failed to handle uri %s", uri) failure += 1 @@ -213,6 +217,8 @@ def esa_wc_to_dc( @click.command("esa-wc-to-dc") +@environment_option +@pass_config @limit @update_if_exists_flag @bbox @@ -237,7 +243,8 @@ def esa_wc_to_dc( type=str, help="Select version of world cover map, default 2020", ) -def cli( +def cli( # pylint: disable=too-many-positional-arguments + cfg_env, limit, update_if_exists, bbox, @@ -255,7 +262,7 @@ def cli( # Select map version select_map_version(version) - dc = Datacube() + dc = Datacube(env=cfg_env) if add_product: add_odc_product(dc) diff --git a/apps/dc_tools/odc/apps/dc_tools/fs_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/fs_to_dc.py index f72e77e2a..18e8ede62 100755 --- a/apps/dc_tools/odc/apps/dc_tools/fs_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/fs_to_dc.py @@ -6,6 +6,7 @@ import datacube from datacube.index.hl import Doc2Dataset +from datacube.ui.click import environment_option, pass_config from odc.apps.dc_tools._stac import stac_transform from odc.apps.dc_tools.utils import ( allow_unsafe, @@ -26,6 +27,8 @@ @click.command("fs-to-dc") +@environment_option +@pass_config @click.argument("input_directory", type=str, nargs=1) @update_if_exists_flag @allow_unsafe @@ -39,6 +42,7 @@ help="File system glob to use, defaults to **/*.yaml or **/*.json for STAC.", ) def cli( + cfg_env, input_directory, update_if_exists, allow_unsafe, @@ -48,7 +52,7 @@ def cli( archive_less_mature, publish_action, ): - dc = datacube.Datacube() + dc = datacube.Datacube(env=cfg_env) doc2ds = Doc2Dataset(dc.index) if glob is None: diff --git a/apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py index cf3dc0fab..1aed0f6c7 100755 --- a/apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py @@ -10,6 +10,7 @@ from datacube import Datacube from datacube.index.hl import Doc2Dataset +from datacube.ui.click import environment_option, pass_config from odc.apps.dc_tools._docs import parse_doc_stream from odc.apps.dc_tools._stac import stac_transform from odc.apps.dc_tools.utils import ( @@ -62,6 +63,9 @@ def dump_to_odc( found_docs = False for uri, metadata in uris_docs: + if metadata is None: + ds_skipped += 1 + continue found_docs = True stac_doc = None if transform: @@ -93,6 +97,8 @@ def dump_to_odc( @click.command("s3-to-dc") +@environment_option +@pass_config @click.option( "--log", type=click.Choice( @@ -118,6 +124,7 @@ def dump_to_odc( @click.argument("uris", nargs=-1) @click.argument("product", type=str, nargs=1, required=False) def cli( + cfg_env, log, skip_lineage, fail_on_missing_lineage, @@ -156,7 +163,7 @@ def cli( if request_payer: opts["RequestPayer"] = "requester" - dc = Datacube() + dc = Datacube(env=cfg_env) # if it's a uri, a product wasn't provided, and 'product' is actually another uri if product.startswith("s3://"): diff --git a/apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py index a6c945eec..0c1dd8d39 100644 --- a/apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py @@ -19,6 +19,7 @@ from datacube import Datacube from datacube.index.hl import Doc2Dataset +from datacube.ui.click import environment_option, pass_config from datacube.utils import documents from odc.apps.dc_tools.utils import ( IndexingException, @@ -306,6 +307,8 @@ def queue_to_odc( @click.command("sqs-to-dc") +@environment_option +@pass_config @skip_lineage @fail_on_missing_lineage @verify_lineage @@ -342,6 +345,7 @@ def queue_to_odc( @click.argument("queue_name", type=str, nargs=1) @click.argument("product", type=str, nargs=1) def cli( + cfg_env, skip_lineage, fail_on_missing_lineage, verify_lineage, @@ -369,7 +373,7 @@ def cli( queue = sqs.get_queue_by_name(QueueName=queue_name) # Do the thing - dc = Datacube() + dc = Datacube(env=cfg_env) success, failed, skipped = queue_to_odc( queue, dc, diff --git a/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py index b1f357928..3d84f2ae9 100644 --- a/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py @@ -10,7 +10,7 @@ import click from datacube import Datacube from datacube.model import Dataset - +from datacube.ui.click import environment_option, pass_config from odc.stac.eo3 import stac2ds from odc.apps.dc_tools.utils import ( @@ -90,7 +90,9 @@ def item_to_meta_uri( "Couldn't find matching product for product name: %s", product_name_sanitised, ) - raise SkippedException(f"Couldn't find matching product for product name: {product_name_sanitised}") + raise SkippedException( + f"Couldn't find matching product for product name: {product_name_sanitised}" + ) # Convert the STAC Item to a Dataset dataset = next(stac2ds([item])) @@ -183,6 +185,8 @@ def stac_api_to_odc( @click.command("stac-to-dc") +@environment_option +@pass_config @limit @update_if_exists_flag @allow_unsafe @@ -216,6 +220,7 @@ def stac_api_to_odc( @publish_action @statsd_setting def cli( + cfg_env, limit, update_if_exists, allow_unsafe, @@ -249,7 +254,7 @@ def cli( config["max_items"] = limit # Do the thing - dc = Datacube() + dc = Datacube(env=cfg_env) added, failed, skipped = stac_api_to_odc( dc, update_if_exists, diff --git a/apps/dc_tools/odc/apps/dc_tools/thredds_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/thredds_to_dc.py index 44a2c15f3..6c4fc4a9c 100644 --- a/apps/dc_tools/odc/apps/dc_tools/thredds_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/thredds_to_dc.py @@ -8,6 +8,8 @@ from typing import List, Tuple from datacube import Datacube +from datacube.cfg import ODCEnvironment +from datacube.ui.click import environment_option, pass_config from odc.apps.dc_tools.utils import statsd_gauge_reporting, statsd_setting from ._docs import from_yaml_doc_stream @@ -48,6 +50,8 @@ def dump_list_to_odc( @click.command("thredds-to-dc") +@environment_option +@pass_config @click.option( "--skip-lineage", is_flag=True, @@ -73,6 +77,7 @@ def dump_list_to_odc( @click.argument("uri", type=str, nargs=1) @click.argument("product", type=str, nargs=1) def cli( + cfg_env: ODCEnvironment, skip_lineage: bool, fail_on_missing_lineage: bool, verify_lineage: bool, @@ -91,7 +96,7 @@ def cli( yaml_contents = download_yamls(yaml_urls) # Consume generator and fetch YAML's - dc = Datacube() + dc = Datacube(env=cfg_env) added, failed = dump_list_to_odc( yaml_contents, dc, diff --git a/apps/dc_tools/odc/apps/dc_tools/utils.py b/apps/dc_tools/odc/apps/dc_tools/utils.py index 1d346c7fa..2eae5ed1d 100644 --- a/apps/dc_tools/odc/apps/dc_tools/utils.py +++ b/apps/dc_tools/odc/apps/dc_tools/utils.py @@ -6,7 +6,7 @@ from datacube import Datacube from datacube.model import Dataset from datacube.index.hl import Doc2Dataset -from datacube.utils import changes +from datacube.utils import changes, jsonify_document from datadog import initialize, statsd from odc.aws.queue import publish_to_topic @@ -41,12 +41,11 @@ class SkippedException(Exception): ) fail_on_missing_lineage = click.option( - "--fail-on-missing-lineage/--auto-add-lineage", + "--fail-on-missing-lineage", is_flag=True, - default=True, help=( - "Default is to fail if lineage documents not present in the database. " - "Set auto add to try to index lineage documents." + "Default is to permit unindexed/external lineage documents. " + "Set flag to fail if lineage documents are not present in the database." ), ) @@ -220,15 +219,18 @@ def index_update_dataset( """ # Make sure we can create a dataset first if not isinstance(dataset, Dataset): - print("Not a dataset: ", dataset) try: if doc2ds is None: doc2ds = Doc2Dataset(dc.index) - dataset, _ = doc2ds(dataset, uri) + dataset, err = doc2ds(jsonify_document(dataset), uri) except ValueError as e: raise IndexingException( f"Exception thrown when trying to create dataset: '{e}'\n The URI was {uri}" ) from e + if dataset is None: + raise IndexingException( + f"Failed to create dataset with error {err}\n The URI was {uri}" + ) with dc.index.transaction(): # Process in a transaction diff --git a/apps/dc_tools/setup.cfg b/apps/dc_tools/setup.cfg index ab37ec7e1..7686c1693 100644 --- a/apps/dc_tools/setup.cfg +++ b/apps/dc_tools/setup.cfg @@ -16,11 +16,13 @@ url = https://github.com/opendatacube/odc-tools/ include_package_data = true zip_safe = false packages = find_namespace: -python_requires = >= 3.9 +python_requires = >= 3.10 tests_require = pytest + pytest_httpserver deepdiff docker + psycopg2 install_requires = click @@ -28,16 +30,16 @@ install_requires = pystac-client>=0.2.0 toolz pyyaml - datacube>=1.8.15 + datacube>=1.9.0 odc_io odc-cloud[ASYNC]>=0.2.3 + odc-geo odc-stac pystac>=1.0.0 rio-stac urlpath datadog - eodatasets3 - importlib_resources>=6.0 + eodatasets3>=1.9 [options.extras_require] tests = diff --git a/apps/dc_tools/tests/conftest.py b/apps/dc_tools/tests/conftest.py index 6db93f3fd..32372f348 100644 --- a/apps/dc_tools/tests/conftest.py +++ b/apps/dc_tools/tests/conftest.py @@ -10,6 +10,8 @@ import yaml from click.testing import CliRunner from datacube import Datacube +from datacube.cfg import ODCConfig, ODCEnvironment +from datacube.drivers.postgis import _core as pgis_core from datacube.drivers.postgres import _core as pgres_core from datacube.index import index_connect from datacube.model import MetadataType @@ -228,13 +230,15 @@ def postgresql_server(): # If we're running inside docker already, don't attempt to start a container! # Hopefully we're using the `with-test-db` script and can use *that* database. - if Path("/.dockerenv").exists() and os.environ.get("DATACUBE_DB_URL"): + if Path("/.dockerenv").exists() and ( + os.environ.get("ODC_DATACUBE_DB_URL") or os.environ.get("ODC_CONFIG_PATH") + ): yield GET_DB_FROM_ENV else: client = docker.from_env() container = client.containers.run( - "postgres:alpine", + "postgis/postgis:16-3.4", # "postgres:alpine", auto_remove=True, remove=True, detach=True, @@ -259,31 +263,38 @@ def postgresql_server(): "db_port": host_port, "db_database": "odc_tools_test", "db_password": "badpassword", - "index_driver": "default", } # 'f"postgresql://odc_tools_test:badpassword@localhost:{host_port}/odc_tools_test", finally: container.remove(v=True, force=True) -@pytest.fixture +@pytest.fixture(scope="module") def odc_test_db( - postgresql_server, tmp_path, monkeypatch + postgresql_server, tmp_path_factory, request ): # pytest: disable=inconsistent-return-statements if postgresql_server == GET_DB_FROM_ENV: - return os.environ["DATACUBE_DB_URL"] + yield None # return os.environ["ODC_DATACUBE_DB_URL"] else: - temp_datacube_config_file = tmp_path / "test_datacube.conf" + temp_datacube_config_file = ( + tmp_path_factory.mktemp("odc") / "test_datacube.conf" + ) config = configparser.ConfigParser() config["default"] = postgresql_server + config["default"]["index_driver"] = "default" + config["postgis"] = postgresql_server + config["postgis"]["index_driver"] = "postgis" with open(temp_datacube_config_file, "w", encoding="utf8") as fout: config.write(fout) + # Use pytest.MonkeyPatch instead of the monkeypatch fixture + # to enable this fixture to not be function scoped + mp = pytest.MonkeyPatch() # This environment variable points to the configuration file, and is used by the odc-tools CLI apps # as well as direct ODC API access, eg creating `Datacube()` - monkeypatch.setenv( - "DATACUBE_CONFIG_PATH", + mp.setenv( + "ODC_CONFIG_PATH", str(temp_datacube_config_file.absolute()), ) # This environment is used by the `datacube ...` CLI tools, which don't obey the same environment variables @@ -293,19 +304,36 @@ def odc_test_db( postgres_url = "postgresql://{db_username}:{db_password}@{db_hostname}:{db_port}/{db_database}".format( **postgresql_server ) - monkeypatch.setenv("DATACUBE_DB_URL", postgres_url) + new_db_database = request.module.__name__.replace(".", "_") + new_db_database = new_db_database.replace("-", "_") while True: try: - with psycopg2.connect(postgres_url): - break + conn = psycopg2.connect(postgres_url) + conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + + with conn.cursor() as cur: + cur.execute(f"CREATE DATABASE {new_db_database};") + break except psycopg2.OperationalError: print("Waiting for PostgreSQL to become available") time.sleep(1) - return postgres_url + yield postgres_url + mp.undo() + + +@pytest.fixture(scope="module", params=["default", "postgis"]) +def env_name(request) -> str: + return request.param + + +@pytest.fixture(scope="module") +def cfg_env(odc_test_db, env_name) -> ODCEnvironment: + """Provides a :class:`ODCEnvironment` configured with suitable config file paths.""" + return ODCConfig()[env_name] @pytest.fixture -def odc_db(odc_test_db): +def odc_db(cfg_env): """ Provide a temporary PostgreSQL server initialised by ODC, usable as the default ODC DB by setting environment variables. @@ -313,7 +341,8 @@ def odc_db(odc_test_db): :return: Datacube instance """ - index = index_connect(validate_connection=False) + # if done using with-test-db, wouldn't we already have an index? + index = index_connect(cfg_env, validate_connection=False) index.init_db() dc = Datacube(index=index) @@ -330,13 +359,20 @@ def odc_db(odc_test_db): yield dc dc.close() - pgres_core.drop_db(index._db._engine) # pylint:disable=protected-access - # We need to run this as well, I think because SQLAlchemy grabs them into it's MetaData, - # and attempts to recreate them. WTF TODO FIX - remove_postgres_dynamic_indexes() - # with psycopg2.connect(odc_test_db) as conn: - # with conn.cursor() as cur: - # cur.execute("DROP SCHEMA IF EXISTS agdc CASCADE;") + + with index._db._engine.begin() as conn: # pylint:disable=protected-access + if index.name == "pg_index": + pgres_core.drop_db(conn) + # We need to run this as well, I think because SQLAlchemy grabs them into it's MetaData, + # and attempts to recreate them. WTF TODO FIX + remove_postgres_dynamic_indexes() + # with psycopg2.connect(odc_test_db) as conn: + # with conn.cursor() as cur: + # cur.execute("DROP SCHEMA IF EXISTS agdc CASCADE;") + else: + pgis_core.drop_db(conn) + + remove_postgis_dynamic_indexes() def remove_postgres_dynamic_indexes(): @@ -350,6 +386,16 @@ def remove_postgres_dynamic_indexes(): ) +def remove_postgis_dynamic_indexes(): + """ + Clear any dynamically created postgis indexes from the schema. + """ + # Our normal indexes start with "ix_", dynamic indexes with "dix_" + # for table in pgis_core.METADATA.tables.values(): + # table.indexes.intersection_update([i for i in table.indexes if not i.name.startswith('dix_')]) + # Dynamic indexes disabled. + + @pytest.fixture def odc_test_db_with_products(odc_db: Datacube): local_csv = str(Path(__file__).parent / "data/example_product_list.csv") @@ -370,7 +416,7 @@ def s2am_dsid(): @pytest.fixture -def odc_db_for_archive(odc_test_db_with_products: Datacube): +def odc_db_for_archive(odc_test_db_with_products: Datacube, env_name): """Create a temporary test database with some pre-indexed datasets.""" # pylint:disable=import-outside-toplevel from odc.apps.dc_tools.fs_to_dc import cli as fs_to_dc_cli @@ -380,7 +426,8 @@ def odc_db_for_archive(odc_test_db_with_products: Datacube): "ga_s2am_ard_3-2-1_49JFM_2016-12-14_final.stac-item.json", ): result = CliRunner().invoke( - fs_to_dc_cli, ["--stac", "--glob", filename, str(TEST_DATA_FOLDER)] + fs_to_dc_cli, + ["--stac", "--glob", filename, str(TEST_DATA_FOLDER), "--env", env_name], ) print(result.output) assert result.exit_code == 0 diff --git a/apps/dc_tools/tests/data/eo3_sentinel_ard.odc-type.yaml b/apps/dc_tools/tests/data/eo3_sentinel_ard.odc-type.yaml index 410b8a849..67b823b32 100644 --- a/apps/dc_tools/tests/data/eo3_sentinel_ard.odc-type.yaml +++ b/apps/dc_tools/tests/data/eo3_sentinel_ard.odc-type.yaml @@ -1,361 +1,359 @@ --- -# Metadata Type -# url: https://explorer.sandbox.dea.ga.gov.au/metadata-types/eo3_sentinel_ard.odc-type.yaml name: eo3_sentinel_ard description: EO3 for Sentinel 2 ARD dataset: - id: - - id - label: - - label - format: - - properties - - odc:file_format - sources: - - lineage - - source_datasets - creation_dt: - - properties - - odc:processing_datetime - grid_spatial: - - grid_spatial - - projection - measurements: - - measurements + id: [id] # No longer configurable in newer ODCs. + sources: [lineage, source_datasets] # No longer configurable in newer ODCs. + + grid_spatial: [grid_spatial, projection] + measurements: [measurements] + creation_dt: [properties, 'odc:processing_datetime'] + label: [label] + format: [properties, 'odc:file_format'] + search_fields: - lat: - type: double-range - max_offset: - - - extent - - lat - - end + platform: + description: Platform code + offset: [properties, 'eo:platform'] + indexed: false + + instrument: + description: Instrument name + offset: [properties, 'eo:instrument'] + indexed: false + + product_family: + description: Product family code + offset: [properties, 'odc:product_family'] + indexed: false + + region_code: + description: > + Spatial reference code from the provider. + + For Sentinel it is MGRS code. + + offset: [properties, 'odc:region_code'] + + crs_raw: + description: | + The raw CRS string as it appears in metadata + + (e.g. ‘epsg:32654’) + offset: ['crs'] + indexed: false + + dataset_maturity: + description: One of - final|interim|nrt (near real time) + offset: [properties, 'dea:dataset_maturity'] + indexed: false + + cloud_cover: + description: | + The proportion (from 0 to 100) of the dataset's valid data area that contains cloud pixels. + + For these ARD products, this value comes from the Fmask algorithm. + type: double + offset: [properties, 'eo:cloud_cover'] + + time: + description: Acquisition time range + type: datetime-range min_offset: - - - extent - - lat - - begin - description: Latitude range + - [properties, 'dtr:start_datetime'] + - [properties, datetime] + max_offset: + - [properties, 'dtr:end_datetime'] + - [properties, datetime] + + # LonLat bounding box, generated on the fly from: + # `grids`, `crs` and `geometry` of the new metadata format + # + # Bounding box is defined by two ranges: + # [lon.begin, lon.end] -- Longitude + # [lat.begin, lat.end] -- Latitude + # + # Note that STAC is using `bbox` for the same thing as following: + # + # bbox: [left, bottom, right, top] + # 0 1 2 3 + # lon lat lon lat + # + # But MetadataType does not support integer index keys, so... + # BoundingBox: [lon.begin, lat.begin, lon.end, lat.end] + lon: + description: Longitude range type: double-range - max_offset: - - - extent - - lon - - end min_offset: - - - extent - - lon - - begin - description: Longitude range - time: - type: datetime-range + - [extent, lon, begin] max_offset: - - - properties - - dtr:end_datetime - - - properties - - datetime + - [extent, lon, end] + + lat: + description: Latitude range + type: double-range min_offset: - - - properties - - dtr:start_datetime - - - properties - - datetime - description: Acquisition time range + - [extent, lat, begin] + max_offset: + - [extent, lat, end] + + # semi-auto generated below + # The proportion (from 0 to 100) of the dataset's valid data area that contains clear land pixels according to the Fmask algorithm + eo_gsd: - type: double - offset: - - properties - - eo:gsd + # The nominal distance between pixel centers available, in meters. + description: | + Ground sampling distance of the sensor’s best resolution band + in metres; represents the size (or spatial resolution) of one pixel. indexed: false - description: "Ground sampling distance of the sensor’s best resolution band\n\ - in metres; represents the size (or spatial resolution) of one pixel.\n" - crs_raw: offset: - - crs + - properties + - eo:gsd + type: double + eo_sun_azimuth: + description: | + The azimuth angle of the sun at the moment of acquisition, in degree units measured clockwise from due north indexed: false - description: "The raw CRS string as it appears in metadata\n\n(e.g. ‘epsg:32654’)\n" - platform: offset: - - properties - - eo:platform - indexed: false - description: Platform code - gqa_abs_x: + - properties + - eo:sun_azimuth type: double - offset: - - properties - - gqa:abs_x + eo_sun_elevation: + description: | + The elevation angle of the sun at the moment of acquisition, in degree units relative to the horizon. indexed: false - description: "Absolute value of the x-axis (east-to-west) GCP residuals, in\ - \ pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5\ - \ metres)\n" - gqa_abs_y: - type: double offset: - - properties - - gqa:abs_y - indexed: false - description: "Absolute value of the y-axis (north-to-south) GCP residuals, in\ - \ pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5\ - \ metres)\n" - gqa_cep90: + - properties + - eo:sun_elevation type: double - offset: - - properties - - gqa:cep90 + + + # Sentinel-specific. + sentinel_product_name: + description: | + ESA product URI, with the '.SAFE' ending removed. + + (e.g. 'S2A_MSIL1C_20220303T000731_N0400_R073_T56LNM_20220303T012845') indexed: false - description: "Circular error probable (90%) of the values of the GCP residuals,\ - \ in pixel units based on a 25 metre resolution reference image (i.e. 0.2\ - \ = 5 metres)\n" - fmask_snow: - type: double offset: - - properties - - fmask:snow + - properties + - sentinel:product_name + sentinel_tile_id: + description: | + Granule ID according to the ESA naming convention + + (e.g. ‘S2A_OPER_MSI_L1C_TL_SGS__20161214T040601_A007721_T53KRB_N02.04’) indexed: false - description: "The proportion (from 0 to 100) of the dataset's valid data area\ - \ that contains clear snow pixels according to the Fmask algorithm\n" - gqa_abs_xy: - type: double offset: - - properties - - gqa:abs_xy + - properties + - sentinel:sentinel_tile_id + sentinel_datastrip_id: + description: | + Unique identifier for a datastrip relative to a given Datatake. + + (e.g. ‘S2A_OPER_MSI_L1C_DS_SGS__20161214T040601_S20161214T005840_N02.04’) indexed: false - description: "Absolute value of the total GCP residuals, in pixel units based\ - \ on a 25 metre resolution reference image (i.e. 0.2 = 5 metres)\n" - gqa_mean_x: - type: double offset: - - properties - - gqa:mean_x + - properties + - sentinel:datastrip_id + + # ARD-specific + fmask_clear: + description: | + The proportion (from 0 to 100) of the dataset's valid data area that contains clear land pixels according to the Fmask algorithm indexed: false - description: "Mean of the values of the x-axis (east-to-west) GCP residuals,\ - \ in pixel units based on a 25 metre resolution reference image (i.e. 0.2\ - \ = 5 metres)\n" - gqa_mean_y: - type: double offset: - - properties - - gqa:mean_y + - properties + - fmask:clear + type: double + fmask_cloud_shadow: + description: | + The proportion (from 0 to 100) of the dataset's valid data area that contains cloud shadow pixels according to the Fmask algorithm indexed: false - description: "Mean of the values of the y-axis (north-to-south) GCP residuals,\ - \ in pixel units based on a 25 metre resolution reference image (i.e. 0.2\ - \ = 5 metres)\n" - instrument: offset: - - properties - - eo:instrument - indexed: false - description: Instrument name - cloud_cover: + - properties + - fmask:cloud_shadow type: double + fmask_snow: + description: | + The proportion (from 0 to 100) of the dataset's valid data area that contains clear snow pixels according to the Fmask algorithm + indexed: false offset: - - properties - - eo:cloud_cover - description: "The proportion (from 0 to 100) of the dataset's valid data area\ - \ that contains cloud pixels.\n\nFor these ARD products, this value comes\ - \ from the Fmask algorithm.\n" - fmask_clear: + - properties + - fmask:snow type: double - offset: - - properties - - fmask:clear - indexed: false - description: "The proportion (from 0 to 100) of the dataset's valid data area\ - \ that contains clear land pixels according to the Fmask algorithm\n" fmask_water: - type: double - offset: - - properties - - fmask:water + description: | + The proportion (from 0 to 100) of the dataset's valid data area that contains clear water pixels according to the Fmask algorithm indexed: false - description: "The proportion (from 0 to 100) of the dataset's valid data area\ - \ that contains clear water pixels according to the Fmask algorithm\n" - gqa_mean_xy: - type: double offset: - - properties - - gqa:mean_xy + - properties + - fmask:water + type: double + gqa_abs_iterative_mean_x: + description: | + Mean of the absolute values of the x-axis (east-to-west) GCP residuals after removal of outliers, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) indexed: false - description: "Mean of the values of the GCP residuals, in pixel units based\ - \ on a 25 metre resolution reference image (i.e. 0.2 = 5 metres)\n" - region_code: offset: - - properties - - odc:region_code - description: "Spatial reference code from the provider.\nFor Sentinel it is\ - \ MGRS code.\n" - gqa_stddev_x: + - properties + - gqa:abs_iterative_mean_x type: double - offset: - - properties - - gqa:stddev_x + gqa_abs_iterative_mean_xy: + description: | + Mean of the absolute values of the GCP residuals after removal of outliers, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) indexed: false - description: "Standard Deviation of the values of the x-axis (east-to-west)\ - \ GCP residuals, in pixel units based on a 25 metre resolution reference image\ - \ (i.e. 0.2 = 5 metres)\n" - gqa_stddev_y: - type: double offset: - - properties - - gqa:stddev_y - indexed: false - description: "Standard Deviation of the values of the y-axis (north-to-south)\ - \ GCP residuals, in pixel units based on a 25 metre resolution reference image\ - \ (i.e. 0.2 = 5 metres)\n" - gqa_stddev_xy: + - properties + - gqa:abs_iterative_mean_xy type: double - offset: - - properties - - gqa:stddev_xy + gqa_abs_iterative_mean_y: + description: | + Mean of the absolute values of the y-axis (north-to-south) GCP residuals after removal of outliers, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) indexed: false - description: "Standard Deviation of the values of the GCP residuals, in pixel\ - \ units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres)\n" - eo_sun_azimuth: - type: double offset: - - properties - - eo:sun_azimuth + - properties + - gqa:abs_iterative_mean_y + type: double + gqa_abs_x: + description: | + Absolute value of the x-axis (east-to-west) GCP residuals, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) indexed: false - description: "The azimuth angle of the sun at the moment of acquisition, in\ - \ degree units measured clockwise from due north\n" - product_family: offset: - - properties - - odc:product_family + - properties + - gqa:abs_x + type: double + gqa_abs_xy: + description: | + Absolute value of the total GCP residuals, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) indexed: false - description: Product family code - dataset_maturity: offset: - - properties - - dea:dataset_maturity - indexed: false - description: One of - final|interim|nrt (near real time) - eo_sun_elevation: + - properties + - gqa:abs_xy type: double - offset: - - properties - - eo:sun_elevation + gqa_abs_y: + description: | + Absolute value of the y-axis (north-to-south) GCP residuals, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) indexed: false - description: "The elevation angle of the sun at the moment of acquisition, in\ - \ degree units relative to the horizon.\n" - sentinel_tile_id: offset: - - properties - - sentinel:sentinel_tile_id - indexed: false - description: "Granule ID according to the ESA naming convention\n\n(e.g. ‘S2A_OPER_MSI_L1C_TL_SGS__20161214T040601_A007721_T53KRB_N02.04’)\n" - s2cloudless_clear: + - properties + - gqa:abs_y type: double + gqa_cep90: + description: | + Circular error probable (90%) of the values of the GCP residuals, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) + indexed: false offset: - - properties - - s2cloudless:clear - description: "The proportion (from 0 to 100) of the dataset's valid data area\ - \ that contains clear land pixels according to s3cloudless\n" - s2cloudless_cloud: + - properties + - gqa:cep90 type: double + gqa_iterative_mean_x: + description: | + Mean of the values of the x-axis (east-to-west) GCP residuals after removal of outliers, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) + indexed: false offset: - - properties - - s2cloudless:cloud - description: "The proportion (from 0 to 100) of the dataset's valid data area\ - \ that contains cloud land pixels according to s3cloudless\n" - fmask_cloud_shadow: + - properties + - gqa:iterative_mean_x type: double - offset: - - properties - - fmask:cloud_shadow + gqa_iterative_mean_xy: + description: | + Mean of the values of the GCP residuals after removal of outliers, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) indexed: false - description: "The proportion (from 0 to 100) of the dataset's valid data area\ - \ that contains cloud shadow pixels according to the Fmask algorithm\n" - gqa_iterative_mean_x: - type: double offset: - - properties - - gqa:iterative_mean_x - indexed: false - description: "Mean of the values of the x-axis (east-to-west) GCP residuals\ - \ after removal of outliers, in pixel units based on a 25 metre resolution\ - \ reference image (i.e. 0.2 = 5 metres)\n" - gqa_iterative_mean_y: + - properties + - gqa:iterative_mean_xy type: double - offset: - - properties - - gqa:iterative_mean_y + gqa_iterative_mean_y: + description: | + Mean of the values of the y-axis (north-to-south) GCP residuals after removal of outliers, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) indexed: false - description: "Mean of the values of the y-axis (north-to-south) GCP residuals\ - \ after removal of outliers, in pixel units based on a 25 metre resolution\ - \ reference image (i.e. 0.2 = 5 metres)\n" - gqa_iterative_mean_xy: - type: double offset: - - properties - - gqa:iterative_mean_xy + - properties + - gqa:iterative_mean_y + type: double + gqa_iterative_stddev_x: + description: | + Standard Deviation of the values of the x-axis (east-to-west) GCP residuals after removal of outliers, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) indexed: false - description: "Mean of the values of the GCP residuals after removal of outliers,\ - \ in pixel units based on a 25 metre resolution reference image (i.e. 0.2\ - \ = 5 metres)\n" - sentinel_datastrip_id: offset: - - properties - - sentinel:datastrip_id + - properties + - gqa:iterative_stddev_x + type: double + gqa_iterative_stddev_xy: + description: | + Standard Deviation of the values of the GCP residuals after removal of outliers, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) indexed: false - description: "Unique identifier for a datastrip relative to a given Datatake.\n\ - \n(e.g. ‘S2A_OPER_MSI_L1C_DS_SGS__20161214T040601_S20161214T005840_N02.04’)\n" - sentinel_product_name: offset: - - properties - - sentinel:product_name - indexed: false - description: "ESA product URI, with the '.SAFE' ending removed.\n\n(e.g. 'S2A_MSIL1C_20220303T000731_N0400_R073_T56LNM_20220303T012845')\n" - gqa_iterative_stddev_x: + - properties + - gqa:iterative_stddev_xy type: double - offset: - - properties - - gqa:iterative_stddev_x - indexed: false - description: "Standard Deviation of the values of the x-axis (east-to-west)\ - \ GCP residuals after removal of outliers, in pixel units based on a 25 metre\ - \ resolution reference image (i.e. 0.2 = 5 metres)\n" gqa_iterative_stddev_y: + description: | + Standard Deviation of the values of the y-axis (north-to-south) GCP residuals after removal of outliers, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) + indexed: false + offset: + - properties + - gqa:iterative_stddev_y type: double + gqa_mean_x: + description: | + Mean of the values of the x-axis (east-to-west) GCP residuals, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) + indexed: false offset: - - properties - - gqa:iterative_stddev_y + - properties + - gqa:mean_x + type: double + gqa_mean_xy: + description: | + Mean of the values of the GCP residuals, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) indexed: false - description: "Standard Deviation of the values of the y-axis (north-to-south)\ - \ GCP residuals after removal of outliers, in pixel units based on a 25 metre\ - \ resolution reference image (i.e. 0.2 = 5 metres)\n" - gqa_iterative_stddev_xy: + offset: + - properties + - gqa:mean_xy type: double + gqa_mean_y: + description: | + Mean of the values of the y-axis (north-to-south) GCP residuals, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) + indexed: false offset: - - properties - - gqa:iterative_stddev_xy + - properties + - gqa:mean_y + type: double + gqa_stddev_x: + description: | + Standard Deviation of the values of the x-axis (east-to-west) GCP residuals, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) indexed: false - description: "Standard Deviation of the values of the GCP residuals after removal\ - \ of outliers, in pixel units based on a 25 metre resolution reference image\ - \ (i.e. 0.2 = 5 metres)\n" - gqa_abs_iterative_mean_x: + offset: + - properties + - gqa:stddev_x type: double + gqa_stddev_xy: + description: | + Standard Deviation of the values of the GCP residuals, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) + indexed: false offset: - - properties - - gqa:abs_iterative_mean_x + - properties + - gqa:stddev_xy + type: double + gqa_stddev_y: + description: | + Standard Deviation of the values of the y-axis (north-to-south) GCP residuals, in pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres) indexed: false - description: "Mean of the absolute values of the x-axis (east-to-west) GCP residuals\ - \ after removal of outliers, in pixel units based on a 25 metre resolution\ - \ reference image (i.e. 0.2 = 5 metres)\n" - gqa_abs_iterative_mean_y: + offset: + - properties + - gqa:stddev_y type: double + s2cloudless_clear: + type: double + description: | + The proportion (from 0 to 100) of the dataset's valid data area that contains clear land pixels according to s2cloudless offset: - properties - - gqa:abs_iterative_mean_y - indexed: false - description: "Mean of the absolute values of the y-axis (north-to-south) GCP\ - \ residuals after removal of outliers, in pixel units based on a 25 metre\ - \ resolution reference image (i.e. 0.2 = 5 metres)\n" - gqa_abs_iterative_mean_xy: + - "s2cloudless:clear" + s2cloudless_cloud: + description: | + The proportion (from 0 to 100) of the dataset's valid data area that contains cloud land pixels according to s2cloudless type: double offset: - properties - - gqa:abs_iterative_mean_xy - indexed: false - description: "Mean of the absolute values of the GCP residuals after removal\ - \ of outliers, in pixel units based on a 25 metre resolution reference image\ - \ (i.e. 0.2 = 5 metres)\n" -... + - "s2cloudless:cloud" diff --git a/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N03E003_Map.tif b/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N03E003_Map.tif new file mode 100644 index 000000000..0bb79a977 Binary files /dev/null and b/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N03E003_Map.tif differ diff --git a/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N03E006_Map.tif b/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N03E006_Map.tif new file mode 100644 index 000000000..b7583511c Binary files /dev/null and b/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N03E006_Map.tif differ diff --git a/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N06E003_Map.tif b/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N06E003_Map.tif new file mode 100644 index 000000000..e45750d3a Binary files /dev/null and b/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N06E003_Map.tif differ diff --git a/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N06E006_Map.tif b/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N06E006_Map.tif new file mode 100644 index 000000000..ce9290423 Binary files /dev/null and b/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N06E006_Map.tif differ diff --git a/apps/dc_tools/tests/test_add_update_products.py b/apps/dc_tools/tests/test_add_update_products.py index 023dcffed..6885945de 100644 --- a/apps/dc_tools/tests/test_add_update_products.py +++ b/apps/dc_tools/tests/test_add_update_products.py @@ -32,7 +32,7 @@ def test_load_product_def(remote_product): assert products[0]["name"] == "s2_l2a" -def test_add_products(local_csv, odc_db): +def test_add_products(local_csv, odc_db, env_name): runner = CliRunner() # This will fail if requester pays is enabled result = runner.invoke( @@ -40,6 +40,8 @@ def test_add_products(local_csv, odc_db): [ local_csv, "--update-if-exists", + "--env", + env_name, ], ) print(f"CLI Output: {result.output}") diff --git a/apps/dc_tools/tests/test_cop_dem_to_dc.py b/apps/dc_tools/tests/test_cop_dem_to_dc.py index f33101f9d..d3f03549b 100644 --- a/apps/dc_tools/tests/test_cop_dem_to_dc.py +++ b/apps/dc_tools/tests/test_cop_dem_to_dc.py @@ -42,7 +42,7 @@ def test_complex_bbox(bbox_africa): # Test the actual process @pytest.mark.parametrize("product", PRODUCTS) -def test_indexing_cli(bbox, product, odc_db): +def test_indexing_cli(bbox, product, odc_db, env_name): runner = CliRunner() result = runner.invoke( cop_dem_to_dc_cli, @@ -52,6 +52,8 @@ def test_indexing_cli(bbox, product, odc_db): bbox, "--product", product, + "--env", + env_name, ], ) assert result.exit_code == 0 @@ -67,6 +69,8 @@ def test_indexing_cli(bbox, product, odc_db): bbox, "--product", product, + "--env", + env_name, ], ) assert result.exit_code == 0 diff --git a/apps/dc_tools/tests/test_esa_worldcover_to_dc.py b/apps/dc_tools/tests/test_esa_worldcover_to_dc.py index 4452dcb47..92307b8f0 100644 --- a/apps/dc_tools/tests/test_esa_worldcover_to_dc.py +++ b/apps/dc_tools/tests/test_esa_worldcover_to_dc.py @@ -1,7 +1,14 @@ +from pathlib import Path + import pytest from click.testing import CliRunner -from odc.apps.dc_tools.esa_worldcover_to_dc import _unpack_bbox, cli, get_tile_uris +from odc.apps.dc_tools.esa_worldcover_to_dc import ( + _unpack_bbox, + cli, + get_tile_uris, + URI_TEMPLATE, +) @pytest.fixture @@ -43,6 +50,7 @@ def test_get_dem_tile_uris(bbox): "v100/2020/map/ESA_WorldCover_10m_2020_v100_N03E003_Map.tif" ) + print(uris) assert len(uris) == 4 @@ -52,16 +60,31 @@ def test_complex_bbox(bbox_africa): assert len(uris) == 899 -# Test the actual process -def test_indexing_cli(bbox, odc_test_db_with_products): +@pytest.fixture +def mock_esa_worldcover_datasets(monkeypatch): + """Replace the fetching of remote ESA WorldCover datasets with local downsampled versions""" + fname_template = URI_TEMPLATE.split("/")[-1] + local_template = ( + "file://" + + str(Path(__file__).parent.absolute()) + + f"/data/esa_worldcover/{fname_template}" + ) + monkeypatch.setattr( + "odc.apps.dc_tools.esa_worldcover_to_dc.URI_TEMPLATE", local_template + ) + + +def test_indexing_cli( + bbox, odc_test_db_with_products, mock_esa_worldcover_datasets, env_name +): runner = CliRunner() result = runner.invoke( cli, [ "--bbox", bbox, - "--statsd-setting", - "localhost:8125", + "--env", + env_name, ], ) assert result.exit_code == 0 diff --git a/apps/dc_tools/tests/test_fs_to_dc.py b/apps/dc_tools/tests/test_fs_to_dc.py index 49edadef9..86184bf40 100644 --- a/apps/dc_tools/tests/test_fs_to_dc.py +++ b/apps/dc_tools/tests/test_fs_to_dc.py @@ -6,7 +6,7 @@ TEST_DATA_FOLDER: Path = Path(__file__).parent.joinpath("data") -def test_fs_to_fc_yaml(test_data_dir, odc_test_db_with_products): +def test_fs_to_fc_yaml(test_data_dir, env_name, odc_test_db_with_products): runner = CliRunner() result = runner.invoke( fs_to_dc_cli, @@ -14,13 +14,15 @@ def test_fs_to_fc_yaml(test_data_dir, odc_test_db_with_products): test_data_dir, "--stac", "--glob=**/NASADEM_HGT_s56w072.stac-item.json", + "--env", + env_name, ], catch_exceptions=False, ) assert result.exit_code == 0 -def test_archive_less_mature(odc_db, test_data_dir, nrt_dsid, final_dsid): +def test_archive_less_mature(odc_db, env_name, test_data_dir, nrt_dsid, final_dsid): dc = odc_db runner = CliRunner() @@ -31,6 +33,8 @@ def test_archive_less_mature(odc_db, test_data_dir, nrt_dsid, final_dsid): test_data_dir, "--glob=**/maturity-nrt.odc-metadata.yaml", "--archive-less-mature", + "--env", + env_name, ], ) assert result.exit_code == 0 @@ -44,6 +48,8 @@ def test_archive_less_mature(odc_db, test_data_dir, nrt_dsid, final_dsid): test_data_dir, "--glob=**/maturity-final.odc-metadata.yaml", "--archive-less-mature", + "--env", + env_name, ], ) assert result.exit_code == 0 @@ -51,7 +57,9 @@ def test_archive_less_mature(odc_db, test_data_dir, nrt_dsid, final_dsid): assert dc.index.datasets.get(nrt_dsid).archived_time is not None -def test_dont_archive_less_mature(odc_db, test_data_dir, nrt_dsid, final_dsid): +def test_dont_archive_less_mature( + odc_db, env_name, test_data_dir, nrt_dsid, final_dsid +): # no archiving should be done if --archive-less-mature is not set dc = odc_db runner = CliRunner() @@ -62,6 +70,8 @@ def test_dont_archive_less_mature(odc_db, test_data_dir, nrt_dsid, final_dsid): [ test_data_dir, "--glob=**/maturity-nrt.odc-metadata.yaml", + "--env", + env_name, ], ) assert result.exit_code == 0 @@ -74,6 +84,8 @@ def test_dont_archive_less_mature(odc_db, test_data_dir, nrt_dsid, final_dsid): [ test_data_dir, "--glob=**/maturity-final.odc-metadata.yaml", + "--env", + env_name, ], ) assert result.exit_code == 0 @@ -81,7 +93,7 @@ def test_dont_archive_less_mature(odc_db, test_data_dir, nrt_dsid, final_dsid): assert dc.index.datasets.get(nrt_dsid).archived_time is None -def test_keep_more_mature(odc_db, test_data_dir, nrt_dsid, final_dsid): +def test_keep_more_mature(odc_db, env_name, test_data_dir, nrt_dsid, final_dsid): dc = odc_db runner = CliRunner() @@ -92,6 +104,8 @@ def test_keep_more_mature(odc_db, test_data_dir, nrt_dsid, final_dsid): test_data_dir, "--glob=**/maturity-final.odc-metadata.yaml", "--archive-less-mature", + "--env", + env_name, ], ) assert result.exit_code == 0 @@ -105,6 +119,8 @@ def test_keep_more_mature(odc_db, test_data_dir, nrt_dsid, final_dsid): test_data_dir, "--glob=**/maturity-nrt.odc-metadata.yaml", "--archive-less-mature", + "--env", + env_name, ], ) assert result.exit_code == 0 diff --git a/apps/dc_tools/tests/test_s3_to_dc.py b/apps/dc_tools/tests/test_s3_to_dc.py index 7c1a177f9..a393b8c4b 100644 --- a/apps/dc_tools/tests/test_s3_to_dc.py +++ b/apps/dc_tools/tests/test_s3_to_dc.py @@ -7,7 +7,7 @@ def test_s3_to_dc_skips_already_indexed_datasets( - mocked_s3_datasets, odc_test_db_with_products + mocked_s3_datasets, odc_test_db_with_products, env_name ): runner = CliRunner() # This will fail if requester pays is enabled @@ -18,6 +18,8 @@ def test_s3_to_dc_skips_already_indexed_datasets( "--no-sign-request", "s3://odc-tools-test/cemp_insar/**/*.yaml", "cemp_insar_alos_displacement", + "--env", + env_name, ], ) for _ in range(1, 3) @@ -38,7 +40,9 @@ def test_s3_to_dc_skips_already_indexed_datasets( ) -def test_s3_to_dc_stac(mocked_s3_datasets, aws_env, odc_test_db_with_products): +def test_s3_to_dc_stac( + mocked_s3_datasets, aws_env, odc_test_db_with_products, env_name +): result = CliRunner().invoke( s3_to_dc, [ @@ -46,6 +50,8 @@ def test_s3_to_dc_stac(mocked_s3_datasets, aws_env, odc_test_db_with_products): "--stac", "s3://odc-tools-test/sentinel-s2-l2a-cogs/31/Q/GB/2020/8/S2B_31QGB_20200831_0_L2A/*_L2A.json", "s2_l2a", + "--env", + env_name, ], catch_exceptions=False, ) @@ -55,7 +61,9 @@ def test_s3_to_dc_stac(mocked_s3_datasets, aws_env, odc_test_db_with_products): ) -def test_s3_to_dc_stac_update_if_exist(mocked_s3_datasets, odc_test_db_with_products): +def test_s3_to_dc_stac_update_if_exist( + mocked_s3_datasets, odc_test_db_with_products, env_name +): result = CliRunner().invoke( s3_to_dc, [ @@ -64,6 +72,8 @@ def test_s3_to_dc_stac_update_if_exist(mocked_s3_datasets, odc_test_db_with_prod "--update-if-exists", "s3://odc-tools-test/sentinel-s2-l2a-cogs/31/Q/GB/2020/8/S2B_31QGB_20200831_0_L2A/*_L2A.json", "s2_l2a", + "--env", + env_name, ], ) assert result.exit_code == 0 @@ -73,7 +83,7 @@ def test_s3_to_dc_stac_update_if_exist(mocked_s3_datasets, odc_test_db_with_prod def test_s3_to_dc_stac_update_if_exist_allow_unsafe( - mocked_s3_datasets, odc_test_db_with_products + mocked_s3_datasets, odc_test_db_with_products, env_name ): runner = CliRunner() result = runner.invoke( @@ -85,6 +95,8 @@ def test_s3_to_dc_stac_update_if_exist_allow_unsafe( "--allow-unsafe", "s3://odc-tools-test/sentinel-s2-l2a-cogs/31/Q/GB/2020/8/S2B_31QGB_20200831_0_L2A/*_L2A.json", "s2_l2a", + "--env", + env_name, ], ) print(f"s3-to-dc exit_code: {result.exit_code}, output:{result.output}") @@ -95,15 +107,17 @@ def test_s3_to_dc_stac_update_if_exist_allow_unsafe( def test_s3_to_dc_fails_to_index_non_dataset_yaml( - mocked_s3_datasets, odc_test_db_with_products + mocked_s3_datasets, odc_test_db_with_products, env_name ): runner = CliRunner() result = runner.invoke( s3_to_dc, [ "--no-sign-request", - "s3://dea-public-data/derivative/ga_ls5t_nbart_gm_cyear_3/3-0-0/x08/y23/1994--P1Y/ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final.proc-info.yaml", + "s3://odc-tools-test/baseline/ga_s2am_ard_3/49/JFM/2016/12/14/20161214T092514/ga_s2am_ard_3-2-1_49JFM_2016-12-14_final.odc-metadata.yaml", "ga_ls5t_nbart_gm_cyear_3", + "--env", + env_name, ], catch_exceptions=False, ) @@ -114,7 +128,7 @@ def test_s3_to_dc_fails_to_index_non_dataset_yaml( def test_s3_to_dc_partially_succeeds_when_given_invalid_and_valid_dataset_yamls( - mocked_s3_datasets, odc_test_db_with_products + mocked_s3_datasets, odc_test_db_with_products, env_name ): runner = CliRunner() result = runner.invoke( @@ -125,6 +139,8 @@ def test_s3_to_dc_partially_succeeds_when_given_invalid_and_valid_dataset_yamls( # This folder contains two yaml one valid dataset yaml and another non dataset yaml "s3://odc-tools-test/derivative/ga_ls5t_nbart_gm_cyear_3/3-0-0/x08/y23/1994--P1Y/*.yaml", "ga_ls5t_nbart_gm_cyear_3", + "--env", + env_name, ], ) assert result.exit_code == 1 @@ -133,7 +149,9 @@ def test_s3_to_dc_partially_succeeds_when_given_invalid_and_valid_dataset_yamls( ) -def test_s3_to_dc_list_absolute_urls(mocked_s3_datasets, odc_test_db_with_products): +def test_s3_to_dc_list_absolute_urls( + mocked_s3_datasets, odc_test_db_with_products, env_name +): # provide mulitple uris, as absolute URLs runner = CliRunner() result = runner.invoke( @@ -144,6 +162,8 @@ def test_s3_to_dc_list_absolute_urls(mocked_s3_datasets, odc_test_db_with_produc "s3://odc-tools-test/cemp_insar/04/01/alos_cumul_2010-04-01.yaml", "s3://odc-tools-test/cemp_insar/08/11/alos_cumul_2010-08-11.yaml", "cemp_insar_alos_displacement", + "--env", + env_name, ], ) assert result.exit_code == 0 @@ -152,7 +172,7 @@ def test_s3_to_dc_list_absolute_urls(mocked_s3_datasets, odc_test_db_with_produc ) -def test_s3_to_dc_no_product(mocked_s3_datasets, odc_test_db_with_products): +def test_s3_to_dc_no_product(mocked_s3_datasets, odc_test_db_with_products, env_name): # product should not need to be specified runner = CliRunner() result = runner.invoke( @@ -160,6 +180,8 @@ def test_s3_to_dc_no_product(mocked_s3_datasets, odc_test_db_with_products): [ "--no-sign-request", "s3://odc-tools-test/cemp_insar/01/07/alos_cumul_2010-01-07.yaml", + "--env", + env_name, ], catch_exceptions=False, ) @@ -175,6 +197,8 @@ def test_s3_to_dc_no_product(mocked_s3_datasets, odc_test_db_with_products): "--no-sign-request", "--stac", "s3://odc-tools-test/sentinel-s2-l2a-cogs/31/Q/GB/2020/8/S2B_31QGB_20200831_0_L2A/*_L2A.json", + "--env", + env_name, ], catch_exceptions=False, ) diff --git a/apps/dc_tools/tests/test_sns_publishing.py b/apps/dc_tools/tests/test_sns_publishing.py index b456925ad..f8ed9b74f 100644 --- a/apps/dc_tools/tests/test_sns_publishing.py +++ b/apps/dc_tools/tests/test_sns_publishing.py @@ -71,7 +71,7 @@ def sns_setup(aws_credentials, aws_env): def test_s3_publishing_action_from_stac( - mocked_s3_datasets, odc_test_db_with_products, s2am_dsid, sns_setup + mocked_s3_datasets, odc_test_db_with_products, env_name, s2am_dsid, sns_setup ): _, _, output_topic_arn, sqs, _, output_queue_url = sns_setup @@ -90,6 +90,8 @@ def test_s3_publishing_action_from_stac( f"--publish-action={output_topic_arn}", "s3://odc-tools-test/baseline/ga_s2am_ard_3/49/JFM/2016/12/14/20161214T092514/*stac-item.json", "ga_s2am_ard_3", + "--env", + env_name, ], catch_exceptions=False, ) @@ -109,7 +111,7 @@ def test_s3_publishing_action_from_stac( def test_s3_publishing_action_from_eo3( - mocked_s3_datasets, odc_test_db_with_products, s2am_dsid, sns_setup + mocked_s3_datasets, odc_test_db_with_products, env_name, s2am_dsid, sns_setup ): """Same as above but requiring stac to eo3 conversion""" _, _, output_topic_arn, sqs, _, output_queue_url = sns_setup @@ -127,6 +129,8 @@ def test_s3_publishing_action_from_eo3( f"--publish-action={output_topic_arn}", "s3://odc-tools-test/baseline/ga_s2am_ard_3/49/JFM/2016/12/14/20161214T092514/*odc-metadata.yaml", "ga_s2am_ard_3", + "--env", + env_name, ], catch_exceptions=False, ) @@ -156,7 +160,7 @@ def stac_doc(): def test_sqs_publishing( - aws_credentials, aws_env, stac_doc, odc_test_db_with_products, sns_setup + aws_credentials, aws_env, stac_doc, odc_test_db_with_products, env_name, sns_setup ): """Test that actions are published with sqs_to_dc""" ( @@ -195,6 +199,8 @@ def test_sqs_publishing( "--update-if-exists", "--stac", f"--publish-action={output_topic_arn}", + "--env", + env_name, ], catch_exceptions=False, ) @@ -213,7 +219,13 @@ def test_sqs_publishing( def test_sqs_publishing_archive_flag( - aws_credentials, aws_env, stac_doc, odc_db_for_archive, ls5t_dsid, sns_setup + aws_credentials, + aws_env, + stac_doc, + odc_db_for_archive, + env_name, + ls5t_dsid, + sns_setup, ): """Test that an ARCHIVE SNS message is published when the --archive flag is used.""" ( @@ -246,6 +258,8 @@ def test_sqs_publishing_archive_flag( "--stac", "--archive", f"--publish-action={output_topic_arn}", + "--env", + env_name, ], catch_exceptions=False, ) @@ -263,7 +277,13 @@ def test_sqs_publishing_archive_flag( def test_sqs_publishing_archive_attribute( - aws_credentials, aws_env, stac_doc, odc_db_for_archive, ls5t_dsid, sns_setup + aws_credentials, + aws_env, + stac_doc, + odc_db_for_archive, + env_name, + ls5t_dsid, + sns_setup, ): """Test that archiving occurs when ARCHIVED is in the message attributes""" ( @@ -297,6 +317,8 @@ def test_sqs_publishing_archive_attribute( "--update-if-exists", "--stac", f"--publish-action={output_topic_arn}", + "--env", + env_name, ], catch_exceptions=False, ) @@ -316,6 +338,7 @@ def test_with_archive_less_mature( aws_credentials, aws_env, odc_db, + env_name, nrt_dsid, final_dsid, sns_setup, @@ -333,6 +356,8 @@ def test_with_archive_less_mature( "--glob=**/maturity-nrt.odc-metadata.yaml", "--archive-less-mature", f"--publish-action={output_topic_arn}", + "--env", + env_name, ], catch_exceptions=False, ) @@ -360,6 +385,8 @@ def test_with_archive_less_mature( "--glob=**/maturity-final.odc-metadata.yaml", "--archive-less-mature", f"--publish-action={output_topic_arn}", + "--env", + env_name, ], catch_exceptions=False, ) diff --git a/apps/dc_tools/tests/test_sqs_to_dc.py b/apps/dc_tools/tests/test_sqs_to_dc.py index 13ca3a7f5..a320249c6 100644 --- a/apps/dc_tools/tests/test_sqs_to_dc.py +++ b/apps/dc_tools/tests/test_sqs_to_dc.py @@ -98,12 +98,14 @@ def aws_credentials(): os.environ["AWS_SESSION_TOKEN"] = "testing" -def test_extract_metadata_from_message(aws_credentials, odc_test_db_with_products): +def test_extract_metadata_from_message( + aws_credentials, odc_test_db_with_products, cfg_env +): with mock_aws(): TEST_QUEUE_NAME = "a_test_queue" sqs_resource = boto3.resource("sqs") - dc = Datacube() + dc = Datacube(env=cfg_env) a_queue = sqs_resource.create_queue(QueueName=TEST_QUEUE_NAME) assert int(a_queue.attributes.get("ApproximateNumberOfMessages")) == 0 diff --git a/docker/Dockerfile b/docker/Dockerfile index d07ef6f3f..643beda9b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ #syntax=docker/dockerfile:1.2 -ARG V_PG=14 -ARG V_PGIS=14-postgis-3 +ARG V_PG=16 +ARG V_PGIS=16-postgis-3 FROM osgeo/gdal:ubuntu-small-3.5.3 ENV LC_ALL=C.UTF-8 diff --git a/docker/assets/with-test-db b/docker/assets/with-test-db index 8fd52c5b9..86b28b17d 100755 --- a/docker/assets/with-test-db +++ b/docker/assets/with-test-db @@ -36,7 +36,20 @@ _prep_db () { } } -export DATACUBE_DB_URL="postgresql:///datacube" +cat < $HOME/.odc_tools_datacube.conf +[datacube] +db_hostname: +db_database: odc_tools_test +index_driver: default + +[postgis] +db_hostname: +db_database: odc_tools_test +index_driver: postgis + +EOL + +export ODC_CONFIG_PATH="${HOME}/.odc_tools_datacube.conf" [ -z "${1:-}" ] || { case "$1" in diff --git a/docker/constraints.txt b/docker/constraints.txt index 7e74a79b6..48be46b6e 100644 --- a/docker/constraints.txt +++ b/docker/constraints.txt @@ -1,235 +1,183 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --no-annotate --no-build-isolation --output-file=constraints.txt --strip-extras requirements.in # -affine==2.3.0 -aiobotocore==2.4.0 -aiohttp==3.8.1 -aioitertools==0.9.0 -aiosignal==1.2.0 -alabaster==0.7.12 -argon2-cffi==21.3.0 -argon2-cffi-bindings==21.2.0 -astroid==2.9.3 -asttokens==2.0.5 -async-timeout==4.0.2 -attrs==21.4.0 -autodocsumm==0.2.11 -awscli==1.25.60 -azure-core==1.22.1 -azure-storage-blob==12.9.0 -babel==2.9.1 -backcall==0.2.0 -beautifulsoup4==4.12.2 -black==22.1.0 -bleach==4.1.0 -bokeh==2.4.2 -boltons==21.0.0 -boto3==1.24.59 -botocore==1.27.59 -bottleneck==1.3.5 -build==0.8.0 -cachetools==5.0.0 -cattrs==22.2.0 -certifi==2021.10.8 -cffi==1.15.0 -cftime==1.5.2 -charset-normalizer==2.0.12 -ciso8601==2.2.0 -click==8.0.3 +affine==2.4.0 +aiobotocore==2.16.0 +aiohappyeyeballs==2.4.4 +aiohttp==3.11.11 +aioitertools==0.12.0 +aiosignal==1.3.2 +alembic==1.14.0 +antimeridian==0.4.0 +asttokens==3.0.0 +async-timeout==5.0.1 +attrs==24.3.0 +awscli==1.36.22 +azure-core==1.32.0 +azure-storage-blob==12.24.0 +blinker==1.9.0 +bokeh==3.6.2 +boltons==24.1.0 +boto3==1.35.81 +botocore==1.35.81 +bottleneck==1.4.2 +branca==0.8.1 +build==1.2.2.post1 +cachetools==5.5.0 +cattrs==24.1.2 +certifi==2024.12.14 +cffi==1.17.1 +charset-normalizer==3.4.0 +ciso8601==2.3.2 +click==8.1.8 click-plugins==1.1.1 cligj==0.7.2 -cloudpickle==2.0.0 -colorama==0.4.3 -commonmark==0.9.1 -coverage==6.3.1 -cryptography==36.0.1 -dask==2022.2.0 -dask-image==2021.12.0 -datacube==1.8.15 -datadog==0.44.0 -debugpy==1.5.1 +cloudpickle==3.1.0 +colorama==0.4.6 +comm==0.2.2 +contourpy==1.3.1 +cryptography==44.0.0 +dask==2024.10.0 +dask-expr==1.1.16 +dask-image==2024.5.3 +datacube==1.9.0 +datadog==0.50.2 decorator==5.1.1 -deepdiff==5.7.0 +deepdiff==8.1.1 defusedxml==0.7.1 -deprecat==2.1.1 -distributed==2022.02.0 -docker==5.0.3 -docutils==0.15.2 -entrypoints==0.4 -eodatasets3==0.29.5 -exceptiongroup==1.0.1 -executing==0.8.2 -flask==2.2.2 -flask-cors==3.0.10 -frozenlist==1.3.0 -fsspec==2022.1.0 -geoalchemy2==0.12.5 -google-api-core==2.5.0 -google-auth==2.6.0 -google-cloud-core==2.2.2 -google-cloud-storage==2.1.0 -google-crc32c==1.3.0 -google-resumable-media==2.2.1 -googleapis-common-protos==1.54.0 -greenlet==1.1.2 +deprecat==2.1.3 +distributed==2024.10.0 +docker==7.1.0 +docutils==0.16 +eodatasets3==1.9.0 +exceptiongroup==1.2.2 +executing==2.1.0 +fiona==1.10.1 +flask==3.1.0 +flask-cors==5.0.0 +frozenlist==1.5.0 +fsspec==2024.12.0 +geoalchemy2==0.16.0 +google-api-core==2.24.0 +google-auth==2.37.0 +google-cloud-core==2.4.1 +google-cloud-storage==2.19.0 +google-crc32c==1.6.0 +google-resumable-media==2.7.2 +googleapis-common-protos==1.66.0 +greenlet==3.1.1 +h5py==3.12.1 hdstats==0.2.1 -heapdict==1.0.1 -hypothesis==6.36.2 -idna==3.3 -imageio==2.16.0 -imagesize==1.3.0 -importlib-metadata==5.0.0 -importlib-resources==6.0.0 -iniconfig==1.1.1 -ipykernel==6.9.1 -ipyleaflet==0.15.0 -ipython==8.0.1 -ipython-genutils==0.2.0 -ipywidgets==7.6.5 -isodate==0.6.1 -isort==5.10.1 -itsdangerous==2.1.2 -jedi==0.18.1 -jinja2==3.0.3 -jmespath==0.10.0 -jsonschema==3.2.0 -jupyter-client==7.1.2 -jupyter-core==4.9.2 -jupyter-ui-poll==0.2.1 -jupyterlab-pygments==0.1.2 -jupyterlab-widgets==1.0.2 -lark==1.1.2 -lazy-object-proxy==1.7.1 -locket==0.2.1 -lxml==4.7.1 -markupsafe==2.1.1 -matplotlib-inline==0.1.3 -mccabe==0.6.1 -mistune==0.8.4 -moto==4.0.9 -msgpack==1.0.3 -msrest==0.6.21 -multidict==6.0.2 -mypy-extensions==0.4.3 -nbclient==0.5.11 -nbconvert==6.4.2 -nbformat==5.1.3 -nbsphinx==0.9.2 -nest-asyncio==1.5.4 -netcdf4==1.5.8 -networkx==2.6.3 -notebook==6.4.8 -numexpr==2.8.1 -numpy==1.23.3 -oauthlib==3.2.0 -ordered-set==4.0.2 -packaging==21.3 -pandas==1.4.1 -pandocfilters==1.5.0 -parso==0.8.3 -partd==1.2.0 -pathspec==0.9.0 -pep517==0.12.0 -pexpect==4.8.0 -pickleshare==0.7.5 -pillow==9.0.1 -pims==0.5 -pip-tools==6.9.0 -platformdirs==2.5.0 -pluggy==1.0.0 -prometheus-client==0.13.1 -prompt-toolkit==3.0.28 -protobuf==3.19.4 -psutil==5.9.0 -psycopg2==2.9.3 +idna==3.10 +imageio==2.36.1 +importlib-metadata==8.5.0 +iniconfig==2.0.0 +ipyleaflet==0.19.2 +ipython==8.31.0 +ipywidgets==8.1.5 +isodate==0.7.2 +itsdangerous==2.2.0 +jedi==0.19.2 +jinja2==3.1.5 +jmespath==1.0.1 +jsonschema==4.23.0 +jsonschema-specifications==2024.10.1 +jupyter-leaflet==0.19.2 +jupyter-ui-poll==1.0.0 +jupyterlab-widgets==3.0.13 +lark==1.2.2 +lazy-loader==0.4 +locket==1.0.0 +lxml==5.3.0 +lz4==4.3.3 +mako==1.3.8 +markupsafe==3.0.2 +matplotlib-inline==0.1.7 +moto==5.0.24 +msgpack==1.1.0 +multidict==6.1.0 +networkx==3.4.2 +numexpr==2.10.2 +numpy==1.26.4 +odc-geo==0.4.8 +orderly-set==5.2.3 +packaging==24.2 +pandas==2.2.3 +parso==0.8.4 +partd==1.4.2 +pexpect==4.9.0 +pillow==11.0.0 +pims==0.7 +pip-tools==7.4.1 +pluggy==1.5.0 +prompt-toolkit==3.0.48 +propcache==0.2.1 +proto-plus==1.25.0 +protobuf==5.29.2 +psutil==6.1.1 ptyprocess==0.7.0 -pure-eval==0.2.2 -py==1.11.0 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 -pycodestyle==2.8.0 -pycparser==2.21 -pydata-sphinx-theme==0.9.0 -pygments==2.11.2 -pylint==2.12.2 -pyparsing==3.0.7 -pyproj==3.1.0 -pyrsistent==0.18.1 -pystac==1.2.0 -pystac-client==0.3.2 -pytest==7.0.1 -pytest-cov==3.0.0 -pytest-httpserver==1.0.4 -pytest-timeout==2.1.0 -python-dateutil==2.8.2 -python-rapidjson==1.9 -pytz==2021.3 -pywavelets==1.2.0 -pyyaml==5.4.1 -pyzmq==22.3.0 -rasterio==1.3.2 -recommonmark==0.7.1 -requests==2.27.1 -requests-oauthlib==1.3.1 -responses==0.18.0 -rio-stac==0.3.2 +pure-eval==0.2.3 +pyarrow==18.1.0 +pyasn1==0.6.1 +pyasn1-modules==0.4.1 +pycparser==2.22 +pygments==2.18.0 +pyparsing==3.2.0 +pyproj==3.7.0 +pyproject-hooks==1.2.0 +pystac==1.11.0 +pystac-client==0.8.5 +pytest==8.3.4 +python-dateutil==2.9.0.post0 +python-rapidjson==1.20 +pytz==2024.2 +pyyaml==6.0.2 +rasterio==1.4.3 +referencing==0.35.1 +requests==2.32.3 +responses==0.25.3 +rio-stac==0.10.1 +rpds-py==0.22.3 rsa==4.7.2 -ruamel-yaml==0.17.21 -ruamel-yaml-clib==0.2.7 -s3transfer==0.6.0 -scikit-image==0.19.1 -scipy==1.8.0 -send2trash==1.8.0 -shapely==2.0.1 -six==1.16.0 -slicerator==1.0.0 -snowballstemmer==2.2.0 -snuggs==1.4.7 +ruamel-yaml==0.18.6 +ruamel-yaml-clib==0.2.12 +s3transfer==0.10.4 +scikit-image==0.25.0 +scipy==1.14.1 +shapely==2.0.6 +six==1.17.0 +slicerator==1.1.0 sortedcontainers==2.4.0 -soupsieve==2.4.1 -sphinx==4.4.0 -sphinx-autodoc-typehints==1.17.0 -sphinx-click==3.1.0 -sphinxcontrib-applehelp==1.0.2 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.0 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -sqlalchemy==1.4.31 -stack-data==0.2.0 -structlog==22.1.0 -tblib==1.7.0 -terminado==0.13.1 -testpath==0.5.0 +sqlalchemy==2.0.36 +stack-data==0.6.3 +structlog==24.4.0 +tblib==3.0.0 thredds-crawler==1.5.4 -tifffile==2022.2.9 -toml==0.10.2 -tomli==2.0.1 -toolz==0.11.2 -tornado==6.1 -tqdm==4.62.3 -traitlets==5.1.1 +tifffile==2024.12.12 +tomli==2.2.1 +toolz==1.0.0 +tornado==6.4.2 +tqdm==4.67.1 +traitlets==5.14.3 traittypes==0.2.1 -typing-extensions==4.1.1 -urllib3==1.26.8 +typing-extensions==4.12.2 +tzdata==2024.2 +urllib3==2.3.0 urlpath==1.2.0 -wcwidth==0.2.5 -webencodings==0.5.1 -websocket-client==1.2.3 -werkzeug==2.2.2 -wheel==0.37.1 -widgetsnbextension==3.5.2 -wrapt==1.13.3 -xarray==2022.3.0 -xmltodict==0.12.0 -xyzservices==2022.2.0 -yarl==1.7.2 -zict==2.0.0 -zipp==3.10.0 +wcwidth==0.2.13 +werkzeug==3.1.3 +wheel==0.45.1 +widgetsnbextension==4.0.13 +wrapt==1.17.0 +xarray==2024.11.0 +xmltodict==0.14.2 +xyzservices==2024.9.0 +yarl==1.18.3 +zict==3.0.0 +zipp==3.21.0 # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/docker/requirements.in b/docker/requirements.in index 3116e240f..eaaffc0e6 100644 --- a/docker/requirements.in +++ b/docker/requirements.in @@ -21,15 +21,15 @@ bottleneck>=1.3.5 click # Make sure dask has all the features enabled -dask[complete] -dask_image +dask[complete]>=2023.2.0,<2024.11.0 +dask_image>=2023.2.0,<2024.11.0 -datacube[dev]>=1.8.15 +datacube[dev]>=1.9.0 datadog # things needed for tests that might not be referenced in setup.py deepdiff docker -eodatasets3 +eodatasets3>=1.9.0 fsspec google-cloud-storage hdstats>=0.1.7.post5 @@ -38,9 +38,9 @@ ipywidgets jinja2 jupyter_ui_poll numexpr -numpy>=1.23.0 +numpy>=1.24.0,<2 pip-tools -pyproj +pyproj>=3.5 pystac>=1.0.0 pystac-client>=0.2.0 pytest @@ -53,8 +53,7 @@ thredds_crawler toolz tqdm urlpath -xarray==2022.3.0 -importlib_resources>=6.0 +xarray>=2023.9.0 # Moto and Flask diff --git a/libs/cloud/odc/aws/queue.py b/libs/cloud/odc/aws/queue.py index 773d80021..7c60c0fed 100644 --- a/libs/cloud/odc/aws/queue.py +++ b/libs/cloud/odc/aws/queue.py @@ -5,6 +5,10 @@ from typing import Any, Iterable, Mapping, Optional +class ODCSQSException(Exception): + """Something wrong with ODC/AWS SQS handling""" + + def redrive_queue( queue_name: str, to_queue_name: Optional[str] = None, @@ -36,11 +40,11 @@ def post_messages(to_queue, messages): else: source_queues = list(dead_queue.dead_letter_source_queues.all()) if len(source_queues) == 0: - raise Exception( + raise ODCSQSException( "No alive queue found for the deadletter queue, please check your configuration." ) if len(source_queues) > 1: - raise Exception( + raise ODCSQSException( "Deadletter queue has more than one source, please specify the target queue name." ) alive_queue = source_queues[0] @@ -178,7 +182,7 @@ def get_messages( return messages if limit < 1: - raise Exception(f"Limit {limit} is not valid.") + raise ODCSQSException(f"Limit {limit} is not valid.") return itertools.islice(messages, limit) diff --git a/libs/ui/odc/ui/_images.py b/libs/ui/odc/ui/_images.py index 2dd2d00a3..b811800c0 100644 --- a/libs/ui/odc/ui/_images.py +++ b/libs/ui/odc/ui/_images.py @@ -10,8 +10,8 @@ from odc.algo import is_rgb, to_rgba from typing import Optional, Tuple, Union -from datacube.utils.geometry import box -from datacube.testutils.geom import epsg4326 +from odc.geo.geom import box +from odc.geo.testutils import epsg4326 def image_shape(d): diff --git a/libs/ui/odc/ui/_map.py b/libs/ui/odc/ui/_map.py index 954262929..2321aca83 100644 --- a/libs/ui/odc/ui/_map.py +++ b/libs/ui/odc/ui/_map.py @@ -7,8 +7,8 @@ def dss_to_geojson(dss, bbox=False, simplify=True, tolerance=0.001): - from datacube.testutils.geom import epsg4326 - from datacube.utils.geometry import bbox_union + from odc.geo.testutils import epsg4326 + from odc.geo.geom import bbox_union geoms = [ds.extent.to_crs(epsg4326) for ds in dss] @@ -244,8 +244,8 @@ def select_on_a_map(m=None, **kwargs): display(m_2) def extract_geometry(state): - from datacube.testutils.geom import epsg4326 - from datacube.utils.geometry import Geometry + from odc.geo.testutils import epsg4326 + from odc.geo.geom import Geometry return Geometry(state.selection, epsg4326) diff --git a/libs/ui/setup.cfg b/libs/ui/setup.cfg index 625fa37e4..bc5bb76b3 100644 --- a/libs/ui/setup.cfg +++ b/libs/ui/setup.cfg @@ -19,7 +19,7 @@ packages = find_namespace: python_requires = >=3.9 tests_require = pytest install_requires = - datacube + datacube>=1.9.0-rc9 ipyleaflet ipython ipywidgets>=8.0 @@ -27,6 +27,7 @@ install_requires = matplotlib numpy odc-algo + odc-geo pandas rasterio xarray diff --git a/pylintrc b/pylintrc index 7b1bbec66..e8b549194 100644 --- a/pylintrc +++ b/pylintrc @@ -33,7 +33,8 @@ disable=duplicate-code, # and `odc.io`, whereby you can't put an __init__.py in `libs/io/odc/`, so # pylint thinks the `io` directory is actually the python built in `io` package # and it's really confusing - cyclic-import + cyclic-import, + too-many-positional-arguments # We have this all over the place, lots of click CLI arguments, and elsewhere [BASIC] @@ -197,4 +198,4 @@ max-public-methods=20 # Exceptions that will emit a warning when being caught. Defaults to # "Exception" -overgeneral-exceptions=Exception +overgeneral-exceptions=builtins.Exception diff --git a/scripts/setup-test-db.sh b/scripts/setup-test-db.sh index f7e053208..e871fbb92 100755 --- a/scripts/setup-test-db.sh +++ b/scripts/setup-test-db.sh @@ -12,7 +12,7 @@ function start_db() { if [[ "$(<${pgdata}/pg.log)" =~ .*"Address already in use".* ]]; then sudo kill -9 $(sudo lsof -i:5432 | awk 'NR==2 {print $2}') pg_ctl -D ${pgdata} -l "${pgdata}/pg.log" start || cat "${pgdata}/pg.log" - else + else cat "${pgdata}/pg.log" exit 1 fi @@ -23,11 +23,12 @@ if [ -d "$(pwd)/.dbdata" ]; then rm -rf "$(pwd)/.dbdata" fi -export DATACUBE_DB_URL=postgresql:///datacube +export ODC_DATACUBE_DB_URL=postgresql:///datacube pgdata=$(pwd)/.dbdata initdb -D ${pgdata} --auth-host=md5 --encoding=UTF8 start_db createdb datacube datacube system init # add any new metadata types -datacube metadata add "https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/product_metadata/eo3_sentinel_ard.odc-type.yaml" +# datacube metadata add "https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/product_metadata/eo3_sentinel_ard.odc-type.yaml" +datacube metadata add apps/dc_tools/tests/data/eo3_sentinel_ard.odc-type.yaml diff --git a/tests/test-env.yml b/tests/test-env.yml index 0ec8b2224..fa7dde6eb 100644 --- a/tests/test-env.yml +++ b/tests/test-env.yml @@ -1,5 +1,5 @@ # Conda environment for running tests in odc-tools -# conda env create -f test-env.yml +# mamba env create -f test-env.yml # conda activate odc-tools-tests name: odc-tools-tests @@ -7,11 +7,11 @@ channels: - conda-forge dependencies: - - python=3.9 + - python=3.12 # Datacube - - datacube>=1.8.15 - - sqlalchemy<2.0.0 + # - datacube[postgres]>=1.9.0 + - sqlalchemy>=2.0 # odc.ui - ipywidgets>=8.0 @@ -27,8 +27,10 @@ dependencies: - urlpath - datadog - docker-py - - eodatasets3 + # - eodatasets3>=1.9.0 - importlib_resources>=6.0 + - odc-geo + - odc-stac # odc.{aws,aio}: aiobotocore/boto3 # pin aiobotocore for easier resolution of dependencies @@ -49,11 +51,15 @@ dependencies: - sphinx-autodoc-typehints - nbsphinx - - pip=23 + - pip=24 - pip: + - datacube[postgres]>=1.9.0 # odc.apps.dc-tools - thredds-crawler - rio-stac + # - psycopg2-binary + - eodatasets3>=1.9.0 + - docker # odc.ui - jupyter-ui-poll>=0.2.0a