Get ready for 1.9 release

.github/workflows/main.yml

@@ -9,6 +9,7 @@ on:
     # Everything should be merged through PRs anyway.
     branches:
       - develop
+      - develop-1.9
       - test-ci-*
       - pypi/publish
     paths:
@@ -23,14 +24,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Setup Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.9
+        python-version: '3.10'
 
-    - uses: actions/cache@v3
+    - uses: actions/cache@v4
       id: wheels_cache
       with:
         path: ./wheels
@@ -68,24 +69,24 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
-    - uses: actions/cache@v3
+    - uses: actions/cache@v4
       id: conda_cache
       with:
         path: |
           tests/env
         key: ${{ runner.os }}-test-env-${{ hashFiles('tests/test-env.yml') }}
 
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: conda-incubator/setup-miniconda@v3
       if: steps.conda_cache.outputs.cache-hit != 'true'
       with:
         channels: conda-forge,defaults
         channel-priority: true
         activate-environment: ""
           # mamba-version: "*"
         use-mamba: true
-        miniforge-variant: Mambaforge
+        miniforge-variant: Miniforge3
 
     - name: Dump Conda Environment Info
       shell: bash -l {0}
@@ -118,10 +119,10 @@ jobs:
       - build-test-env-base
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Get Conda Environment from Cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: conda_cache
       with:
         path: |
@@ -148,15 +149,20 @@ jobs:
       run: |
         echo "Launching test db"
         pgdata=$(pwd)/.dbdata
-        initdb -D ${pgdata} --auth-host=md5 --encoding=UTF8
-        pg_ctl -D ${pgdata} -l "${pgdata}/pg.log" start
-        createdb datacube
+        sudo chmod 777 /var/run/postgresql
+        /usr/lib/postgresql/16/bin/initdb -D ${pgdata} --auth-host=md5 --encoding=UTF8
+        /usr/lib/postgresql/16/bin/pg_ctl -D ${pgdata} -l "${pgdata}/pg.log" start
+        /usr/lib/postgresql/16/bin/createdb datacube
         datacube system init
         pip list --format=freeze
 
       env:
         ODC_DATACUBE_DB_URL: postgresql:///datacube
 
+    # - name: Setup tmate session
+    #   if: ${{ failure() }}
+    #   uses: mxschmitt/action-tmate@v3
+
     - name: Run Tests
       shell: bash
       run: |
@@ -179,7 +185,7 @@ jobs:
       if: |
         github.repository == 'opendatacube/odc-tools'
 
-      uses: codecov/codecov-action@v1
+      uses: codecov/codecov-action@v5
       with:
         fail_ci_if_error: false
         verbose: false
@@ -193,17 +199,17 @@ jobs:
       - build-wheels
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Get Wheels from Cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: wheels_cache
       with:
         path: ./wheels
         key: wheels-${{ github.sha }}
 
     - name: Get Conda Environment from Cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: conda_cache
       with:
         path: |
@@ -231,9 +237,10 @@ jobs:
       run: |
         echo "Launching test db"
         pgdata=$(pwd)/.dbdata
-        initdb -D ${pgdata} --auth-host=md5 --encoding=UTF8
-        pg_ctl -D ${pgdata} -l "${pgdata}/pg.log" start
-        createdb datacube
+        sudo chmod 777 /var/run/postgresql
+        /usr/lib/postgresql/16/bin/initdb -D ${pgdata} --auth-host=md5 --encoding=UTF8
+        /usr/lib/postgresql/16/bin/pg_ctl -D ${pgdata} -l "${pgdata}/pg.log" start
+        /usr/lib/postgresql/16/bin/createdb datacube
         datacube system init
 
       env:
@@ -275,7 +282,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Config
       if: |
@@ -292,9 +299,9 @@ jobs:
 
     - name: Setup Python
       if: steps.cfg.outputs.publish == 'yes'
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.8
+        python-version: '3.10'
 
     - name: Install Twine
       if: steps.cfg.outputs.publish == 'yes'

apps/dc_tools/odc/apps/dc_tools/add_update_products.py

@@ -14,6 +14,8 @@
 
 import datacube
 from datacube import Datacube
+from datacube.cfg import ODCEnvironment
+from datacube.ui.click import environment_option, pass_config
 from odc.apps.dc_tools.utils import (
     update_if_exists_flag,
     statsd_gauge_reporting,
@@ -123,12 +125,16 @@ def add_update_products(
 
 
 @click.command("dc-sync-products")
+@environment_option
+@pass_config
 @click.argument("csv-path", nargs=1)
 @update_if_exists_flag
 @statsd_setting
-def cli(csv_path: str, update_if_exists: bool, statsd_setting: str):
+def cli(
+    cfg_env: ODCEnvironment, csv_path: str, update_if_exists: bool, statsd_setting: str
+):
     # Check we can connect to the Datacube
-    dc = datacube.Datacube(app="add_update_products")
+    dc = datacube.Datacube(app="add_update_products", env=cfg_env)
     logging.info(
         "Starting up: connected to Datacube, and update-if-exists is: %s",
         update_if_exists,

apps/dc_tools/odc/apps/dc_tools/azure_to_dc.py

@@ -10,7 +10,9 @@
 
 import click
 from datacube import Datacube
+from datacube.cfg import ODCEnvironment
 from datacube.index.hl import Doc2Dataset
+from datacube.ui.click import environment_option, pass_config
 from odc.apps.dc_tools._stac import stac_transform
 from odc.apps.dc_tools.utils import (
     SkippedException,
@@ -128,6 +130,8 @@ def dump_list_to_odc(
 
 
 @click.command("azure-to-dc")
+@environment_option
+@pass_config
 @update_flag
 @update_if_exists_flag
 @allow_unsafe
@@ -151,6 +155,7 @@ def dump_list_to_odc(
 @click.argument("suffix", type=str, nargs=1)
 @rename_product
 def cli(
+    cfg_env: ODCEnvironment,
     update: bool,
     update_if_exists: bool,
     allow_unsafe: bool,
@@ -166,7 +171,7 @@ def cli(
     rename_product: str,
 ):
     # Set up the datacube first, to ensure we have a connection
-    dc = Datacube()
+    dc = Datacube(env=cfg_env)
     print(f"Opening AZ Container {container_name} on {account_url}")
     print(f"Searching on prefix '{prefix}' for files matching suffix '{suffix}'")
     yaml_urls = find_blobs(

apps/dc_tools/odc/apps/dc_tools/cop_dem_to_dc.py

@@ -14,6 +14,7 @@
 
 from datacube import Datacube
 from datacube.index.hl import Doc2Dataset
+from datacube.ui.click import environment_option, pass_config
 from datacube.utils import read_documents
 from odc.apps.dc_tools.utils import (
     SkippedException,
@@ -202,6 +203,8 @@ def cop_dem_to_dc(
 
 
 @click.command("cop-dem-to-dc")
+@environment_option
+@pass_config
 @limit
 @update_if_exists_flag
 @bbox
@@ -226,6 +229,7 @@ def cop_dem_to_dc(
     help="Number of threads to use to process, default 20",
 )
 def cli(
+    cfg_env,
     limit,
     update_if_exists,
     bbox,
@@ -244,7 +248,7 @@ def cli(
             f"Unknown product {product}, must be one of {' '.join(PRODUCTS)}"
         )
 
-    dc = Datacube()
+    dc = Datacube(env=cfg_env)
 
     if add_product:
         add_cop_dem_product(dc, product)

apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py

@@ -14,6 +14,7 @@
 
 from datacube import Datacube
 from datacube.index.hl import Doc2Dataset
+from datacube.ui.click import environment_option, pass_config
 from datacube.utils import read_documents
 from odc.apps.dc_tools.utils import (
     bbox,
@@ -213,6 +214,8 @@ def esa_wc_to_dc(
 
 
 @click.command("esa-wc-to-dc")
+@environment_option
+@pass_config
 @limit
 @update_if_exists_flag
 @bbox
@@ -238,6 +241,7 @@ def esa_wc_to_dc(
     help="Select version of world cover map, default 2020",
 )
 def cli(
+    cfg_env,
     limit,
     update_if_exists,
     bbox,
@@ -255,7 +259,7 @@ def cli(
     # Select map version
     select_map_version(version)
 
-    dc = Datacube()
+    dc = Datacube(env=cfg_env)
 
     if add_product:
         add_odc_product(dc)

apps/dc_tools/odc/apps/dc_tools/fs_to_dc.py

@@ -6,6 +6,7 @@
 
 import datacube
 from datacube.index.hl import Doc2Dataset
+from datacube.ui.click import environment_option, pass_config
 from odc.apps.dc_tools._stac import stac_transform
 from odc.apps.dc_tools.utils import (
     allow_unsafe,
@@ -26,6 +27,8 @@
 
 
 @click.command("fs-to-dc")
+@environment_option
+@pass_config
 @click.argument("input_directory", type=str, nargs=1)
 @update_if_exists_flag
 @allow_unsafe
@@ -39,6 +42,7 @@
     help="File system glob to use, defaults to **/*.yaml or **/*.json for STAC.",
 )
 def cli(
+    cfg_env,
     input_directory,
     update_if_exists,
     allow_unsafe,
@@ -48,7 +52,7 @@ def cli(
     archive_less_mature,
     publish_action,
 ):
-    dc = datacube.Datacube()
+    dc = datacube.Datacube(env=cfg_env)
     doc2ds = Doc2Dataset(dc.index)
 
     if glob is None:

apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py

@@ -10,6 +10,7 @@
 
 from datacube import Datacube
 from datacube.index.hl import Doc2Dataset
+from datacube.ui.click import environment_option, pass_config
 from odc.apps.dc_tools._docs import parse_doc_stream
 from odc.apps.dc_tools._stac import stac_transform
 from odc.apps.dc_tools.utils import (
@@ -93,6 +94,8 @@ def dump_to_odc(
 
 
 @click.command("s3-to-dc")
+@environment_option
+@pass_config
 @click.option(
     "--log",
     type=click.Choice(
@@ -118,6 +121,7 @@ def dump_to_odc(
 @click.argument("uris", nargs=-1)
 @click.argument("product", type=str, nargs=1, required=False)
 def cli(
+    cfg_env,
     log,
     skip_lineage,
     fail_on_missing_lineage,
@@ -156,7 +160,7 @@ def cli(
     if request_payer:
         opts["RequestPayer"] = "requester"
 
-    dc = Datacube()
+    dc = Datacube(env=cfg_env)
 
     # if it's a uri, a product wasn't provided, and 'product' is actually another uri
     if product.startswith("s3://"):

apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py

@@ -19,6 +19,7 @@
 
 from datacube import Datacube
 from datacube.index.hl import Doc2Dataset
+from datacube.ui.click import environment_option, pass_config
 from datacube.utils import documents
 from odc.apps.dc_tools.utils import (
     IndexingException,
@@ -306,6 +307,8 @@ def queue_to_odc(
 
 
 @click.command("sqs-to-dc")
+@environment_option
+@pass_config
 @skip_lineage
 @fail_on_missing_lineage
 @verify_lineage
@@ -342,6 +345,7 @@ def queue_to_odc(
 @click.argument("queue_name", type=str, nargs=1)
 @click.argument("product", type=str, nargs=1)
 def cli(
+    cfg_env,
     skip_lineage,
     fail_on_missing_lineage,
     verify_lineage,
@@ -369,7 +373,7 @@ def cli(
     queue = sqs.get_queue_by_name(QueueName=queue_name)
 
     # Do the thing
-    dc = Datacube()
+    dc = Datacube(env=cfg_env)
     success, failed, skipped = queue_to_odc(
         queue,
         dc,