Pattern

.github/workflows/mopper-conda-release.yaml

@@ -2,6 +2,8 @@ name: Build of mopper conda package for new release
 
 # Controls when the action will run.
 on:
+  release:
+    types: ['released', 'prereleased']
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 

ACDD_conf.yaml

@@ -4,7 +4,8 @@
 cmor:
     # If test true it will just run the setup but not launch the job automatically
     test: false
-    appdir:  /g/data/ua8/Working/packages/ACCESS-MOPPeR 
+    # working directory if default uses current directory
+    appdir:  default 
     # output directory for all generated data (CMORISED files & logs)
     # if default it is set to /scratch/$project/$user/MOPPER_OUTPUT<exp>
     outpath: default
@@ -29,6 +30,11 @@ cmor:
     access_version: CM2 
     # reference date for time units (set as 'default' to use start_date)
     reference_date: 1970-01-01             
+    # Path and file templates can be changed based on the experiment.
+    # The example below should be considered a minimum requirement. 
+    # Consider adding 'table_id" if using the "all tables" option to list
+    # the variables to process as variables can be present at same frequency
+    # in more than one tables  
     path_template: "{product_version}/{frequency}"
     # date_range is automatically added at the end of filename
     file_template: "{variable_id}_{source_id}_{experiment_id}_{frequency}"
@@ -41,6 +47,7 @@ cmor:
     shuffle: 1
     # Variables to CMORise: 
     # CMOR table/variable to process; default is 'all'. 
+    # 'all' will use all the tables listed in the mapping file
     # Or create a yaml file listing variables to process (VAR_SUBSET[_LIST]).
     # each line: <table: [var1, var2, var3 ..]>
     tables: CMIP6_Amon
@@ -81,16 +88,21 @@ cmor:
     _AXIS_ENTRY_FILE: "ACDD_coordinate.json"
     _FORMULA_VAR_FILE: "ACDD_formula_terms.json"
     grids: "ACDD_grids.json"
-  # Additional NCI information:
+# Additional NCI information:
     # NCI project to charge compute; $PROJECT = your default project
     project: v45 
-    # additional NCI projects to be included in the storage flags
+    # additional NCI projects to be included in the storage flags, comma separated list
     addprojs: []
     # queue and memory (GB) per CPU (depends on queue),
-    # hugemem is reccomended for high reoslution data and/or derived variables 
+    # hugemem is recommended for high reoslution data and/or derived variables 
     # hugemem requires a minimum of 6 cpus this is handled by the code
     queue: hugemem
     mem_per_cpu: 32 
+    max_cpus: 24
+    # Mopper uses multiprocessing to produce files in parallel, usually 1 cpu per worker
+    # is a good compromise, occasionally you might want to pass a higher number
+    # if running out of memory
+    cpuxworker: 1 
     # walltime in "hh:mm:ss"
     walltime: '8:00:00'
     mode: custom
@@ -99,7 +111,7 @@ cmor:
     # you can override that by supplying the env to pass to "source"
     # Ex 
     # conda_env: <custom-env-path>/bin/activate
-    # or you can set "test: true" and modify mopper_job.sh manually
+    # to allow other settings use "test: true" and modify mopper_job.sh manually
     conda_env: default
 
 #
@@ -174,4 +186,4 @@ attrs:
     parent: !!bool false 
     # CMOR will add a tracking_id if you want to define a prefix add here
     tracking_id_prefix: 
-    comment: "post-processed using ACCESS-MOPPeR v1.0.0 https://doi.org/10.5281/zenodo.10346216"
+    comment: "post-processed using ACCESS-MOPPeR v1.1.0 https://doi.org/10.5281/zenodo.13841181"

CMIP6_conf.yaml

@@ -4,7 +4,8 @@
 cmor:
     # If test true it will just run the setup but not launch the job automatically
     test: false
-    appdir:  /g/data/ua8/Working/packages/ACCESS-MOPPeR
+    # working directory if default uses current directory
+    appdir:  default
     # output directory for all generated data (CMORISED files & logs)
     # if default it is set to /scratch/$project/$user/MOPPER_OUTPUT<exp>
     outpath: default
@@ -38,6 +39,7 @@ cmor:
 
     # Variables to CMORise: 
     # CMOR table/variable to process; default is 'all'.
+    # 'all' will use all the tables listed in the mapping file
     # Or create a yaml file listing variables to process (VAR_SUBSET[_LIST]).
     # each line: <table: [var1, var2, var3 ..]>
     tables: CMIP6_Amon 
@@ -79,19 +81,29 @@ cmor:
     grids: CMIP6_grids.json
   # Additional NCI information:
     # NCI project to charge compute; $PROJECT = your default project
-    # NCI queue to use; hugemem is recommended
     project: v45
-    # additional NCI projects to be included in the storage flags
+    # additional NCI projects to be included in the storage flags, comma separated list
     addprojs: []
     # queue and memory (GB) per CPU (depends on queue) 
-    # hugemem is reccomended for high reoslution data and/or derived variables
+    # hugemem is recommended for high resolution data and/or derived variables
     # hugemem requires a minimum of 6 cpus this is handled by the code
     queue: hugemem
     mem_per_cpu: 30
+    max_cpus: 24
+    # Mopper uses multiprocessing to produce files in parallel, usually 1 cpu per worker
+    # is a good compromise, occasionally you might want to pass a higher number
+    # if running out of memory
+    cpuxworker: 1
     # walltime in "hh:mm:ss"
     walltime: '8:00:00'
     mode: cmip6
+    # if default uses hh5 conda env
+    # conda_env to use by default hh5 analysis3-unstable
+    # as this has the code and all dependecies installed
+    # you can override that by supplying the env to pass to "source"
+    # Ex
     # conda_env: <custom-env-path>/bin/activate
+    # to allow other settings use "test: true" and modify mopper_job.sh manually
     conda_env: default
 
 # Global attributes: these will be added to each files comment unwanted ones

README.md

@@ -1,6 +1,6 @@
 # [ACCESS Model Output Post-Processor (MOPPeR)](https://access-mopper.readthedocs.io/en/latest)
 [![Read the docs](https://readthedocs.org/projects/access-mopper/badge/?version=latest)](https://access-mopper.readthedocs.io/en/latest/)
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.12747219.svg)](https://doi.org/10.5281/zenodo.12747219)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.13841181.svg)](https://doi.org/10.5281/zenodo.13841181)
 
 This code is derived from the [APP4](https://doi.org/10.5281/zenodo.7703469), initially created by Peter Uhe for CMIP5, and further developed for CMIP6-era by Chloe Mackallah from CSIRO, O&A Aspendale.
 
@@ -36,6 +36,6 @@ If you want to install an unstable version or a different branch:
 MOPPeR is pre-installed into a Conda environment at NCI. Load it with::
 
     module use /g/data3/hh5/public/modules
-    module load conda/analysis3
+    module load conda/analysis3-unstable
 
   NB. You need to be a member of the hh5 project to load the modules.

conda/environment.yaml

@@ -2,7 +2,6 @@ name: mopper_env
 channels:
   - conda-forge
   - coecms
-  - default
 
 dependencies:
   - anaconda-client

conda/meta.yaml

@@ -1,16 +1,17 @@
 package:
     name: mopper 
-    version: "{{ environ.get('GIT_DESCRIBE_TAG', '1.0') }}"
-
+    #version: "{{ environ.get('GIT_DESCRIBE_TAG', '1.0') }}"
+    version: '1.1.0'
 #source:
-#    path: ./
+# path: ./
 
 source:
   #url: https://github.com/ACCESS-Hive/ACCESS-MOPPeR/archive/refs/tags/{{version}}.tar.gz
   git_url: https://github.com/ACCESS-Hive/ACCESS-MOPPeR.git
-  #git_tag: prerelease
-  git_rev: "{{ version }}" 
+  git_tag: prerelease
+  #git_rev: "1.1.0"
   #git_depth: 1 # (Defaults to -1/not shallow)
+  #path: ../.
 
 build:
     number: 1

conda/testenv.yaml

@@ -5,7 +5,7 @@ channels:
 dependencies:
   - cmor
   - click
-  - xarray
+  - conda-forge::xarray
   - numpy
   - dask
   - pyyaml
@@ -16,3 +16,5 @@ dependencies:
   - codecov
   - pyfakefs
   - ruff
+  - metpy
+  - gsw

docs/calculations.rst

@@ -0,0 +1,78 @@
+Calculating derived variables
+=============================
+
+Calculations are used to derive a variable from one or multiple inputs, to resample a variable to a new frequency or generally to modify a variable so it will match fully the corresponding definition in a CMOR table.
+
+How calculations work
+---------------------
+Calculations are defined in the mapping file under the filed by the same name. The `calculation` string gets literally evaluated by the tool using python eval() function.
+As an example 
+ simple calculation could be summing  avariable across all its vertical levels:
+
+.. code-block:: bash
+
+    mrso;fld_s08i223;var[0].sum(dim='depth')
+
+`var` represents the list of input variables, in this case there's only one which var[0] in the calculation string. In this case the calculation is very simple and can be fully defined in the mapping itself. If the calculation is more complex it's easier to use a pre-defined function, for example:
+
+.. code-block:: bash
+
+    hus24;fld_s00i010 fld_s00i408;plevinterp(var[0], var[1], 24)
+
+Here plevinterp is called to interpolate specific humidity from model levels to pressure levels, this function takes three input arguments, the variable to interpolate, pressure at model levels and finally the number of pressure levels, which corresponds to a specific definition of the pressure levels coordinate.
+Already available functions are listed below.
+
+.. note::
+
+    When more than one variable is used as input, if the variables are not all in the same file, more than one file pattern can be specified in the mapping row.   
+
+Resample
+^^^^^^^^
+If a variable is available in the raw model output but not at the desired frequency, the tool will try to see if a higher frequency is available to be resampled. For example, if a user is interested in daily surface temperature but this is available only as hourly data, during the `mop setup` phase the tool will add a `resample` attribute with value 'D' to the variable and this will used as argument for the resample function. Which kind of statistics to use for the function is defined based on the `timeshot` attribute, so if a variable is defined as a maximum, minimum or sum these are used in the resample instead of the mean.
+
+Contributing
+------------
+TBA
+
+
+Available functions
+-------------------
+
+Atmosphere and aerosol
+^^^^^^^^^^^^^^^^^^^^^^
+.. automodule:: mopper.calc_atmos
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Ocean
+^^^^^
+.. automodule:: mopper.calc_ocean
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+SeaIce
+^^^^^^
+.. automodule:: mopper.calc_seaice
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Land
+^^^^
+.. automodule:: mopper.calc_land
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Other
+^^^^^
+.. automodule:: mopper.calc_utils
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/cmor.rst

@@ -48,7 +48,7 @@ There are custom tables for CM2 variables not yet included in the CMIP6 tables a
 Experiment input file
 ---------------------
 
-This provides user-supplied metadata and configuration directives used by CMOR, in cluding which controlled vocabulary (CV), grids and coordinate definitions to use and values for the attributes describing the model and simulation.
+This provides user-supplied metadata and configuration directives used by CMOR, including which controlled vocabulary (CV), grids and coordinate definitions to use and values for the attributes describing the model and simulation.
 
 We simplified this process so the user only has to pass one configuration file to control all the necessary inputs.
 The `mop setup` command will then create an experiment file as expected by CMOR based on this and the selected CV file. This is described in the :ref:`Getting started section <conf-file>`.

docs/cmor_conf.yaml

@@ -39,8 +39,9 @@ cmor:
     # shuffle 0: off 1:on Shuffle reduces size without impacting speed
     deflate_level: 4
     shuffle: 1
-    # Variables to CMORise: 
-    # CMOR table/variable to process; default is 'all'. 
+    # Variables to CMORise:
+    # CMOR table/variable to process; default is 'all'.
+    # 'all' will use all the tables listed in the mapping file
     # Or create a yaml file listing variables to process (VAR_SUBSET[_LIST]).
     # each line: <table: [var1, var2, var3 ..]>
     tables: CMIP6_Amon
@@ -85,12 +86,25 @@ cmor:
     # NCI project to charge compute; $PROJECT = your default project
     # NCI queue to use; hugemem is recommended
     project: v45 
-    # additional NCI projects to be included in the storage flags
+    # additional NCI projects to be included in the storage flags, comma separated list
     addprojs: []
-    # queue and memory (GB) per CPU (depends on queue) 
+    # queue and memory (GB) per CPU (depends on queue),
+    # hugemem is recommended for high reoslution data and/or derived variables
+    # hugemem requires a minimum of 6 cpus this is handled by the code
     queue: hugemem
-    mem_per_cpu: 32 
+    mem_per_cpu: 32
+    max_cpus: 24
+    # Mopper uses multiprocessing to produce files in parallel, usually 1 cpu per worker
+    # is a good compromise, occasionally you might want to pass a higher number
+    # if running out of memory
+    cpuxworker: 1
     # walltime in "hh:mm:ss"
     walltime: '8:00:00'
     mode: custom
-    conda_env: /g/data/.../mopper_env/bin/activate
+    # conda_env to use by default hh5 analysis3-unstable
+    # as this has the code and all dependecies installed
+    # you can override that by supplying the env to pass to "source"
+    # Ex
+    # conda_env: <custom-env-path>/bin/activate
+    # to allow other settings use "test: true" and modify mopper_job.sh manually
+    conda_env: default

docs/conf.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# clef documentation build configuration file, created by
-# sphinx-quickstart on Fri May 13 10:16:43 2016.
+# ACCESS-MOPPeR documentation build configuration file.
 #
 # This file is execfile()d with the current directory set to its
 # containing dir.
@@ -12,11 +11,15 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
+import sys
+import os
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #sys.path.insert(0, os.path.abspath('.'))
+sys.path.insert(0, os.path.abspath('../src')) 
+
 
 # -- General configuration ------------------------------------------------
 
@@ -48,6 +51,10 @@
 # The master toctree document.
 master_doc = 'index'
 
+# adding this to avoid failure of autodoc because of missing packages
+autodoc_mock_imports = ['numpy', 'xarray', 'dask', 'logging', 'click',
+                        'yaml', 'json', 'importlib']
+
 # General information about the project.
 project = u'ACCESS-MOPPeR'
 copyright = u'2023, Paola Petrelli, Sam Green'

docs/customising.rst

@@ -39,10 +39,10 @@ These variables will be listed in a section labelled:
 
    # Variables with different frequency: Use with caution!
 
-The frequency listed in the mapping template is taken directly from the file, make sure it matches the one of the listed cmor table. If they differ the `cmor table` should be updated with one that has a definition with the correct frequency.
+The frequency listed in the mapping template is taken directly from the file, make sure it matches the one of the listed CMOR table. If they differ the `cmor table` should be updated with one that has a definition with the correct frequency.
 
-.. warning:: 
-   Pay attention if the variable is an instantaneous value or not (i.e. time: point vs mean, sum, max etc in the cell_methods).    This should match the frequency in the cmor table definition.
+.. warning::
+   Pay attention if the variable is an instantaneous value or not (i.e. time: point vs mean, sum, max etc in the cell_methods). This should match the frequency in the CMOR table definition.
    If it doesn't a new variable with the correct frequency, cell_methods and time _axis should be defined, see below.
 
 A similar message precedes all the variables mapped from a different version from the selected (or default) one: 
@@ -66,7 +66,7 @@ For both steps existing identical records will be overwritten. Once new records
 
 While it's possible to modify an existing CMOR table, it's probably better to do so only for a custom table as CMIP6 has strict standards and it's important to keep these tables the same as their official version. Other things to pay attention to are:
 
- * Check that the table to modify doesn't contain a conflicting variable, for example a variable that uses the same output name. In most cases the output name and the variable name used as key for the record are the same. However, the key name is what is used to point to the correct variable definition in the mapping table and can be different from the output name. This allows two variables with the same output name to be part of the same cmor table.
+ * Check that the table to modify doesn't contain a conflicting variable, for example a variable that uses the same output name. In most cases the output name and the variable name used as key for the record are the same. However, the key name is what is used to point to the correct variable definition in the mapping table and can be different from the output name. This allows two variables with the same output name to be part of the same CMOR table.
  * When adding a variable, a sub-hourly frequency the frequency in the table should be `subhr` or most often `subhrPt`. This is because of the way CMOR3 is structured it only accept a defined set of frequencies. As mopper uses the frequencies to estimate the file sizes, if working with sub-hourly data you need to then specify what the exact interval is in the configuration file using the `subhr` field and `min` as units. 
 
 .. warning:: 
@@ -94,13 +94,14 @@ Here we're showing how the pressure level calculation is defined for air tempera
 
 For context this is the function definition:
 
-.. code-block:: ipython3
+.. code-block:: python
 
    def plevinterp(ctx, var, pmod, levnum):
 
 where `ctx` is the `context` of the specific file including information on the original variable. This is automatically passed to the function and should not be included in the function call.
 
 .. note::
 
+   All currently available calculations are listed in `Available functions`_.
    We are planning to provide a simplified way to introduce new calculations and to update the central database with user provided mappings and variable definitions. For the moment open a `new issue on github <https://github.com/ACCESS-Community-Hub/ACCESS-MOPPeR/issues/new>`_ so we can review the updates and add them to the official version.