Skip to content
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
cfdf480
Allow template value in !glob.
christinaholtNOAA Sep 9, 2025
bf60ee1
Use Copier and Linker in driver.
christinaholtNOAA Sep 9, 2025
a7a0cef
Use Copier and Linker for staging
christinaholtNOAA Sep 9, 2025
565d079
Add copy fallback. Factor out FileStager mixin.
christinaholtNOAA Sep 10, 2025
04a8125
Add stager module.
christinaholtNOAA Sep 10, 2025
43f1214
Adding tests for stager.
christinaholtNOAA Sep 10, 2025
ad96f4c
Factor out files_to_* documentation.
christinaholtNOAA Sep 10, 2025
74fe8fe
Successful docs build.
christinaholtNOAA Sep 10, 2025
12cec19
A why comment.
christinaholtNOAA Sep 10, 2025
85b7f33
Mock the failure to test the fallback.
christinaholtNOAA Sep 10, 2025
e8a5e4f
Apply suggestions from Paul's code review
christinaholtNOAA Sep 10, 2025
ba7346e
Apply suggestion for code review.
christinaholtNOAA Sep 10, 2025
0d61935
Merge branch 'glob_staging' of https://github.com/christinaholtNOAA/u…
christinaholtNOAA Sep 10, 2025
b70782a
Regenerate docs.
christinaholtNOAA Sep 10, 2025
1e785fa
Merge remote-tracking branch 'origin/main' into glob_staging
christinaholtNOAA Sep 11, 2025
126454f
Reproducible notebooks.
christinaholtNOAA Sep 11, 2025
02bc4a2
Merge branch 'main' into glob_staging
christinaholtNOAA Sep 11, 2025
ec9521c
Fix dep.
christinaholtNOAA Sep 11, 2025
d72c573
Merge branch 'glob_staging' of https://github.com/christinaholtNOAA/u…
christinaholtNOAA Sep 11, 2025
3e50198
Make freezegun optional to pass tests.
christinaholtNOAA Sep 11, 2025
82a59c7
Update src/uwtools/drivers/stager.py
christinaholtNOAA Sep 11, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/sections/user_guide/cli/drivers/fv3/help.out
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ Positional arguments:
The field_table file
files_copied
Files copied for run
files_hardlinked
Files hard-linked for run
files_linked
Files linked for run
model_configure
Expand Down
2 changes: 2 additions & 0 deletions docs/sections/user_guide/cli/drivers/ioda/help.out
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ Positional arguments:
The executable's YAML configuration file
files_copied
Files copied for run
files_hardlinked
Files hard-linked for run
files_linked
Files linked for run
provisioned_rundir
Expand Down
2 changes: 2 additions & 0 deletions docs/sections/user_guide/cli/drivers/jedi/help.out
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ Positional arguments:
The executable's YAML configuration file
files_copied
Files copied for run
files_hardlinked
Files hard-linked for run
files_linked
Files linked for run
provisioned_rundir
Expand Down
2 changes: 2 additions & 0 deletions docs/sections/user_guide/cli/drivers/mpas/help.out
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ Positional arguments:
Boundary files
files_copied
Files copied for run
files_hardlinked
Files hard-linked for run
files_linked
Files linked for run
namelist_file
Expand Down
2 changes: 2 additions & 0 deletions docs/sections/user_guide/cli/drivers/mpas_init/help.out
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ Positional arguments:
Boundary files
files_copied
Files copied for run
files_hardlinked
Files hard-linked for run
files_linked
Files linked for run
namelist_file
Expand Down
2 changes: 2 additions & 0 deletions docs/sections/user_guide/cli/drivers/mpassit/help.out
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ Positional arguments:
TASK
files_copied
Files copied for run
files_hardlinked
Files hard-linked for run
files_linked
Files linked for run
namelist_file
Expand Down
4 changes: 4 additions & 0 deletions docs/sections/user_guide/cli/drivers/orog/help.out
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ Optional arguments:

Positional arguments:
TASK
files_copied
Files copied for run
files_hardlinked
Files hard-linked for run
files_linked
Files linked for run
grid_file
Expand Down
2 changes: 2 additions & 0 deletions docs/sections/user_guide/cli/drivers/upp/help.out
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ Positional arguments:
The GRIB control file
files_copied
Files copied for run
files_hardlinked
Files hard-linked for run
files_linked
Files linked for run
namelist_file
Expand Down
2 changes: 2 additions & 0 deletions docs/sections/user_guide/cli/drivers/upp_assets/help.out
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ Positional arguments:
The GRIB control file
files_copied
Files copied for run
files_hardlinked
Files hard-linked for run
files_linked
Files linked for run
namelist_file
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"minProperties": 1,
"type": "object"
},
"files_to_link": {
"files_to_hardlink": {
"additionalProperties": {
"type": "fs_src"
},
Expand Down
6 changes: 3 additions & 3 deletions docs/sections/user_guide/cli/tools/fs/hardlink-help.out
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ usage: uw fs hardlink [-h] [--version] [--config-file PATH]
[--target-dir PATH] [--cycle CYCLE]
[--leadtime LEADTIME] [--dry-run]
[--key-path KEY[.KEY...]] [--report] [--quiet]
[--verbose] [--symlink-fallback]
[--verbose] [--fallback {copy,symlink}]

Create hardlinks

Expand All @@ -29,5 +29,5 @@ Optional arguments:
Print no logging messages
--verbose, -v
Print all logging messages
--symlink-fallback
Symlink if hardlink fails
--fallback {copy,symlink}
Alternative if hardlink fails
10 changes: 1 addition & 9 deletions docs/sections/user_guide/yaml/components/fv3.rst
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,7 @@ field_table:

The path to a :weather-model-io:`valid field-table file<field-table-file>` to be copied into the run directory.

files_to_copy:
^^^^^^^^^^^^^^

See :ref:`this page <files_yaml>` for details.

files_to_link:
^^^^^^^^^^^^^^

Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies.
.. include:: /shared/stager.rst

lateral_boundary_conditions:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Expand Down
10 changes: 1 addition & 9 deletions docs/sections/user_guide/yaml/components/ioda.rst
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,7 @@ configuration_file:

Supports ``base_file:`` and ``update_values:`` blocks (see :ref:`updating_values` for details).

files_to_copy:
^^^^^^^^^^^^^^

See :ref:`this page <files_yaml>` for details.

files_to_link:
^^^^^^^^^^^^^^

Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies.
.. include:: /shared/stager.rst

rundir:
^^^^^^^
Expand Down
10 changes: 1 addition & 9 deletions docs/sections/user_guide/yaml/components/jedi.rst
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,7 @@ configuration_file:

Supports ``base_file:`` and ``update_values:`` blocks (see :ref:`updating_values` for details).

files_to_copy:
^^^^^^^^^^^^^^

See :ref:`this page <files_yaml>` for details.

files_to_link:
^^^^^^^^^^^^^^

Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies.
.. include:: /shared/stager.rst

rundir:
^^^^^^^
Expand Down
12 changes: 2 additions & 10 deletions docs/sections/user_guide/yaml/components/mpas.rst
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,9 @@ length:

The length of the forecast in integer hours.

files_to_copy:
^^^^^^^^^^^^^^
.. include:: /shared/stager.rst

See :ref:`this page <files_yaml>` for details.

To reduce duplication of information in this section, it may be helpful to template the file that depends on the number of cores being used to run the executable. For example, instead of:
To reduce duplication of information in these sections, it may be helpful to template the file that depends on the number of cores being used to run the executable. For example, instead of:

.. code-block:: text

Expand All @@ -70,11 +67,6 @@ Jinja2 expressions can be used to reference the number of cores used in executio
files_to_copy:
conus.graph.info.part.{{mpas.execution["batchargs"]["cores"]}}: /path/to/conus.graph.info.part.{{mpas.execution["batchargs"]["cores"]}}

files_to_link:
^^^^^^^^^^^^^^

Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies.

namelist:
^^^^^^^^^

Expand Down
12 changes: 2 additions & 10 deletions docs/sections/user_guide/yaml/components/mpas_init.rst
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,9 @@ execution:

See :ref:`this page <execution_yaml>` for details.

files_to_copy:
^^^^^^^^^^^^^^
.. include:: /shared/stager.rst

See :ref:`this page <files_yaml>` for details.

To reduce duplication of information in this section, it may be helpful to template the file that depends on the number of cores being used to run the executable. For example, instead of:
To reduce duplication of information in these sections, it may be helpful to template the file that depends on the number of cores being used to run the executable. For example, instead of:

.. code-block:: text

Expand All @@ -64,11 +61,6 @@ Jinja2 expressions can be used to reference the number of cores used in executio
files_to_copy:
conus.graph.info.part.{{mpas_init.execution["batchargs"]["cores"]}}: /path/to/conus.graph.info.part.{{mpas_init.execution["batchargs"]["cores"]}}

files_to_link:
^^^^^^^^^^^^^^

Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies.

namelist:
^^^^^^^^^

Expand Down
12 changes: 2 additions & 10 deletions docs/sections/user_guide/yaml/components/mpassit.rst
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,9 @@ execution:

See :ref:`this page <execution_yaml>` for details.

files_to_copy:
^^^^^^^^^^^^^^
.. include:: /shared/stager.rst

See :ref:`this page <files_yaml>` for details.

To reduce duplication of information in this section, it may be helpful to reference the config value that specifies the number of cores used to run the executable. For example, instead of:
To reduce duplication of information in these sections, it may be helpful to reference the config value that specifies the number of cores used to run the executable. For example, instead of:

.. code-block:: text

Expand All @@ -42,11 +39,6 @@ Jinja2 expressions can be used to reference the number of cores used for executi
files_to_copy:
x1.999.graph.info.part.{{ mpassit.execution.batchargs.cores }}: /path/to/x1.999.graph.info.part.{{ mpassit.execution.batchargs.cores }}

files_to_link:
^^^^^^^^^^^^^^

Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies.

namelist:
^^^^^^^^^

Expand Down
10 changes: 1 addition & 9 deletions docs/sections/user_guide/yaml/components/orog.rst
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,7 @@ execution:

See :ref:`here <execution_yaml>` for details.

files_to_copy:
^^^^^^^^^^^^^^

See :ref:`this page <files_yaml>` for details.

files_to_link:
^^^^^^^^^^^^^^

Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies.
.. include:: /shared/stager.rst

mask:
^^^^^
Expand Down
10 changes: 1 addition & 9 deletions docs/sections/user_guide/yaml/components/upp.rst
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,7 @@ execution:

See :ref:`this page <execution_yaml>` for details.

files_to_copy:
^^^^^^^^^^^^^^

See :ref:`this page <files_yaml>` for details.

files_to_link:
^^^^^^^^^^^^^^

Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies.
.. include:: /shared/stager.rst

namelist:
^^^^^^^^^
Expand Down
10 changes: 1 addition & 9 deletions docs/sections/user_guide/yaml/components/upp_assets.rst
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,7 @@ Here is a prototype UW YAML ``upp_assets:`` block, explained in detail below:
UW YAML for the ``upp_assets:`` Block
-------------------------------------

files_to_copy:
^^^^^^^^^^^^^^

See :ref:`this page <files_yaml>` for details.

files_to_link:
^^^^^^^^^^^^^^

Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies.
.. include:: /shared/stager.rst

namelist:
^^^^^^^^^
Expand Down
14 changes: 14 additions & 0 deletions docs/shared/stager.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
files_to_copy:
^^^^^^^^^^^^^^

See :ref:`this page <files_yaml>` for details.

files_to_hardlink:
^^^^^^^^^^^^^^^^^^

Identical to ``files_to_copy:`` except that hard links will be created in the run directory instead of copies, when possible. Otherwise a copy will be created.

files_to_link:
^^^^^^^^^^^^^^

Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies.
8 changes: 4 additions & 4 deletions src/uwtools/api/fs.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,14 @@ def link(
key_path: list[YAMLKey] | None = None,
dry_run: bool = False,
stdin_ok: bool = False,
symlink_fallback: bool = False,
fallback: str | None = None,
) -> dict[str, list[str]]:
"""
Create links to filesystem items.

When ``hardlink`` is ``False`` (the default), links may target files, hardlinks, symlinks, and
directories; when ``True``, links may not be made across filesystems, or to directories. When
``symlink_fallback`` is ``True``, a symlink will be created, if possible, if a hardlink cannot
``fallback`` is set, a copy or symlink will be created, if possible, if a hardlink cannot
be created.

:param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).
Expand All @@ -77,7 +77,7 @@ def link(
:param key_path: Path of keys to config block to use.
:param dry_run: Do not link files.
:param stdin_ok: OK to read from ``stdin``?
:param symlink_fallback: Symlink if hardlink fails when hardlink=True?
:param fallback: copy or symlink if hardlink fails when hardlink=True?
:return: A report on files linked / not linked.
"""
stager = Linker(
Expand All @@ -87,7 +87,7 @@ def link(
hardlink=hardlink,
leadtime=leadtime,
key_path=key_path,
symlink_fallback=symlink_fallback,
fallback=fallback,
)
assets = cast(list, stager.go(dry_run=dry_run).assets)
ready = lambda state: [str(asset.ref) for asset in assets if asset.ready() is state]
Expand Down
13 changes: 7 additions & 6 deletions src/uwtools/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ def _add_subparser_fs_hardlink(subparsers: Subparsers) -> ActionChecks:
"""
parser = _add_subparser(subparsers, STR.hardlink, "Create hardlinks")
checks, optional = _add_subparser_fs_common(parser)
_add_arg_symlink_fallback(optional)
_add_arg_fallback(optional)
return checks


Expand Down Expand Up @@ -439,7 +439,7 @@ def _dispatch_fs_hardlink(args: Args) -> bool:
key_path=args[STR.keypath],
dry_run=args[STR.dryrun],
stdin_ok=True,
symlink_fallback=args[STR.symlinkfallback],
fallback=args[STR.fallback],
)
return _dispatch_fs_report(report=report if args[STR.report] else None)

Expand Down Expand Up @@ -912,11 +912,12 @@ def _add_arg_report(group: Group) -> None:
)


def _add_arg_symlink_fallback(group: Group) -> None:
def _add_arg_fallback(group: Group) -> None:
group.add_argument(
_switch(STR.symlinkfallback),
action="store_true",
help="Symlink if hardlink fails",
_switch(STR.fallback),
choices=[STR.copy, STR.symlink],
help="Alternative if hardlink fails",
type=str,
)


Expand Down
4 changes: 4 additions & 0 deletions src/uwtools/config/jinja2.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,10 @@ def dereference(
deref_debug("Rendering", val.value)
val.value = _deref_render(val.value, context, local)
rendered = _deref_convert(val)
elif isinstance(val, UWYAMLGlob):
deref_debug("Rendering", val.value)
val.value = _deref_render(val.value, context, local)
rendered = val
else:
deref_debug("Accepting", val)
rendered = val
Expand Down
Loading