Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
cfdf480
Allow template value in !glob.
christinaholtNOAA Sep 9, 2025
bf60ee1
Use Copier and Linker in driver.
christinaholtNOAA Sep 9, 2025
a7a0cef
Use Copier and Linker for staging
christinaholtNOAA Sep 9, 2025
565d079
Add copy fallback. Factor out FileStager mixin.
christinaholtNOAA Sep 10, 2025
04a8125
Add stager module.
christinaholtNOAA Sep 10, 2025
43f1214
Adding tests for stager.
christinaholtNOAA Sep 10, 2025
ad96f4c
Factor out files_to_* documentation.
christinaholtNOAA Sep 10, 2025
74fe8fe
Successful docs build.
christinaholtNOAA Sep 10, 2025
12cec19
A why comment.
christinaholtNOAA Sep 10, 2025
85b7f33
Mock the failure to test the fallback.
christinaholtNOAA Sep 10, 2025
e8a5e4f
Apply suggestions from Paul's code review
christinaholtNOAA Sep 10, 2025
ba7346e
Apply suggestion for code review.
christinaholtNOAA Sep 10, 2025
0d61935
Merge branch 'glob_staging' of https://github.com/christinaholtNOAA/u…
christinaholtNOAA Sep 10, 2025
b70782a
Regenerate docs.
christinaholtNOAA Sep 10, 2025
1e785fa
Merge remote-tracking branch 'origin/main' into glob_staging
christinaholtNOAA Sep 11, 2025
126454f
Reproducible notebooks.
christinaholtNOAA Sep 11, 2025
02bc4a2
Merge branch 'main' into glob_staging
christinaholtNOAA Sep 11, 2025
ec9521c
Fix dep.
christinaholtNOAA Sep 11, 2025
d72c573
Merge branch 'glob_staging' of https://github.com/christinaholtNOAA/u…
christinaholtNOAA Sep 11, 2025
3e50198
Make freezegun optional to pass tests.
christinaholtNOAA Sep 11, 2025
82a59c7
Update src/uwtools/drivers/stager.py
christinaholtNOAA Sep 11, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/uwtools/config/jinja2.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,10 @@ def dereference(
deref_debug("Rendering", val.value)
val.value = _deref_render(val.value, context, local)
rendered = _deref_convert(val)
elif isinstance(val, UWYAMLGlob):
deref_debug("Rendering", val.value)
val.value = _deref_render(val.value, context, local)
rendered = val
else:
deref_debug("Accepting", val)
rendered = val
Expand Down
22 changes: 16 additions & 6 deletions src/uwtools/drivers/fv3.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from uwtools.config.formats.yaml import YAMLConfig
from uwtools.drivers.driver import DriverCycleBased
from uwtools.drivers.support import set_driver_docstring
from uwtools.fs import Copier, Linker
from uwtools.strings import STR
from uwtools.utils.tasks import file, filecopy, symlink

Expand Down Expand Up @@ -79,9 +80,21 @@ def files_copied(self):
Files copied for run.
"""
yield self.taskname("files copied")
yield [Copier(config=self.config.get("files_to_copy", {}), target_dir=self.rundir).go()]

@tasks
def files_hardlinked(self):
"""
Files hard-linked for run.
"""
yield self.taskname("files hard-linked")
yield [
filecopy(src=Path(src), dst=self.rundir / dst)
for dst, src in self.config.get("files_to_copy", {}).items()
Linker(
config=self.config.get("files_to_hardlink", {}),
target_dir=self.rundir,
hardlink=True,
symlink_fallback=True,
).go()
]

@tasks
Expand All @@ -90,10 +103,7 @@ def files_linked(self):
Files linked for run.
"""
yield self.taskname("files linked")
yield [
symlink(target=Path(target), linkname=self.rundir / linkname)
for linkname, target in self.config.get("files_to_link", {}).items()
]
yield [Linker(config=self.config.get("files_to_link", {}), target_dir=self.rundir).go()]

@task
def model_configure(self):
Expand Down
24 changes: 17 additions & 7 deletions src/uwtools/drivers/jedi_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@

from uwtools.config.formats.yaml import YAMLConfig
from uwtools.drivers.driver import DriverCycleBased
from uwtools.fs import Copier, Linker
from uwtools.strings import STR
from uwtools.utils.tasks import file, filecopy, symlink
from uwtools.utils.tasks import file


class JEDIBase(DriverCycleBased):
Expand Down Expand Up @@ -43,9 +44,21 @@ def files_copied(self):
Files copied for run.
"""
yield self.taskname("files copied")
yield [Copier(config=self.config.get("files_to_copy", {}), target_dir=self.rundir).go()]

@tasks
def files_hardlinked(self):
"""
Files hard-linked for run.
"""
yield self.taskname("files hard-linked")
yield [
filecopy(src=Path(src), dst=self.rundir / dst)
for dst, src in self.config.get("files_to_copy", {}).items()
Linker(
config=self.config.get("files_to_hardlink", {}),
target_dir=self.rundir,
hardlink=True,
symlink_fallback=True,
).go()
]

@tasks
Expand All @@ -54,10 +67,7 @@ def files_linked(self):
Files linked for run.
"""
yield self.taskname("files linked")
yield [
symlink(target=Path(target), linkname=self.rundir / linkname)
for linkname, target in self.config.get("files_to_link", {}).items()
]
yield [Linker(config=self.config.get("files_to_link", {}), target_dir=self.rundir).go()]

@tasks
@abstractmethod
Expand Down
29 changes: 20 additions & 9 deletions src/uwtools/drivers/mpas_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,18 @@
from datetime import datetime, timezone
from functools import reduce
from itertools import islice
from pathlib import Path
from typing import cast
from typing import TYPE_CHECKING, cast

from dateutil.relativedelta import relativedelta
from iotaa import asset, task, tasks
from lxml import etree
from lxml.etree import Element, SubElement

from uwtools.drivers.driver import DriverCycleBased
from uwtools.utils.tasks import filecopy, symlink
from uwtools.fs import Copier, Linker

if TYPE_CHECKING:
from pathlib import Path


class MPASBase(DriverCycleBased):
Expand All @@ -41,9 +43,21 @@ def files_copied(self):
Files copied for run.
"""
yield self.taskname("files copied")
yield [Copier(config=self.config.get("files_to_copy", {}), target_dir=self.rundir).go()]

@tasks
def files_hardlinked(self):
"""
Files hard-linked for run.
"""
yield self.taskname("files hard-linked")
yield [
filecopy(src=Path(src), dst=self.rundir / dst)
for dst, src in self.config.get("files_to_copy", {}).items()
Linker(
config=self.config.get("files_to_hardlink", {}),
target_dir=self.rundir,
hardlink=True,
symlink_fallback=True,
).go()
]

@tasks
Expand All @@ -52,10 +66,7 @@ def files_linked(self):
Files linked for run.
"""
yield self.taskname("files linked")
yield [
symlink(target=Path(target), linkname=self.rundir / linkname)
for linkname, target in self.config.get("files_to_link", {}).items()
]
yield [Linker(config=self.config.get("files_to_link", {}), target_dir=self.rundir).go()]

@task
@abstractmethod
Expand Down
24 changes: 17 additions & 7 deletions src/uwtools/drivers/mpassit.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@
from uwtools.api.config import get_nml_config
from uwtools.config.formats.nml import NMLConfig
from uwtools.drivers.driver import DriverCycleLeadtimeBased
from uwtools.fs import Copier, Linker
from uwtools.strings import STR
from uwtools.utils.tasks import file, filecopy, symlink
from uwtools.utils.tasks import file


class MPASSIT(DriverCycleLeadtimeBased):
Expand All @@ -29,9 +30,21 @@ def files_copied(self):
Files copied for run.
"""
yield self.taskname("files copied")
yield [Copier(config=self.config.get("files_to_copy", {}), target_dir=self.rundir).go()]

@tasks
def files_hardlinked(self):
"""
Files hard-linked for run.
"""
yield self.taskname("files hard-linked")
yield [
filecopy(src=Path(src), dst=self.rundir / dst)
for dst, src in self.config.get("files_to_copy", {}).items()
Linker(
config=self.config.get("files_to_hardlink", {}),
target_dir=self.rundir,
hardlink=True,
symlink_fallback=True,
).go()
]

@tasks
Expand All @@ -40,10 +53,7 @@ def files_linked(self):
Files linked for run.
"""
yield self.taskname("files linked")
yield [
symlink(target=Path(target), linkname=self.rundir / linkname)
for linkname, target in self.config.get("files_to_link", {}).items()
]
yield [Linker(config=self.config.get("files_to_link", {}), target_dir=self.rundir).go()]

@task
def namelist_file(self):
Expand Down
36 changes: 33 additions & 3 deletions src/uwtools/drivers/orog.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@

from uwtools.drivers.driver import DriverTimeInvariant
from uwtools.drivers.support import set_driver_docstring
from uwtools.fs import Copier, Linker
from uwtools.strings import STR
from uwtools.utils.file import writable
from uwtools.utils.tasks import symlink


class Orog(DriverTimeInvariant):
Expand All @@ -20,15 +20,45 @@ class Orog(DriverTimeInvariant):

# Workflow tasks

@tasks
def files_copied(self):
"""
Files copied for run.
"""
yield self.taskname("files copied")
yield [
Copier(
config=self.config.get("files_to_copy", {}),
target_dir=self.rundir,
).go()
]

@tasks
def files_hardlinked(self):
"""
Files hard-linked for run.
"""
yield self.taskname("files hard-linked")
yield [
Linker(
config=self.config.get("files_to_hardlink", {}),
target_dir=self.rundir,
hardlink=True,
symlink_fallback=True,
).go()
]

@tasks
def files_linked(self):
"""
Files linked for run.
"""
yield self.taskname("files linked")
yield [
symlink(target=Path(target), linkname=self.rundir / linkname)
for linkname, target in self.config.get("files_to_link", {}).items()
Linker(
config=self.config.get("files_to_link", {}),
target_dir=self.rundir,
).go()
]

@external
Expand Down
21 changes: 14 additions & 7 deletions src/uwtools/drivers/upp_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@
from uwtools.config.formats.nml import NMLConfig
from uwtools.drivers.upp_assets import UPPAssets
from uwtools.exceptions import UWConfigError
from uwtools.fs import Copier, Linker
from uwtools.strings import STR
from uwtools.utils.tasks import file, filecopy, symlink
from uwtools.utils.tasks import file, filecopy

if TYPE_CHECKING:
from uwtools.drivers.upp import UPP
Expand All @@ -34,18 +35,24 @@ def control_file(obj: UPP | UPPAssets):

def files_copied(obj: UPP | UPPAssets):
yield obj.taskname("files copied")
yield [Copier(config=obj.config.get("files_to_copy", {}), target_dir=obj.rundir).go()]


def files_hardlinked(obj: UPP | UPPAssets):
yield obj.taskname("files hard-linked")
yield [
filecopy(src=Path(src), dst=obj.rundir / dst)
for dst, src in obj.config.get("files_to_copy", {}).items()
Linker(
config=obj.config.get("files_to_hardlink", {}),
target_dir=obj.rundir,
hardlink=True,
symlink_fallback=True,
).go()
]


def files_linked(obj: UPP | UPPAssets):
yield obj.taskname("files linked")
yield [
symlink(target=Path(target), linkname=obj.rundir / linkname)
for linkname, target in obj.config.get("files_to_link", {}).items()
]
yield [Linker(config=obj.config.get("files_to_link", {}), target_dir=obj.rundir).go()]


def namelist_file(obj: UPP | UPPAssets):
Expand Down
3 changes: 3 additions & 0 deletions src/uwtools/resources/jsonschema/fv3.jsonschema
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@
"files_to_copy": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_hardlink": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_link": {
"$ref": "urn:uwtools:files-to-stage"
},
Expand Down
3 changes: 3 additions & 0 deletions src/uwtools/resources/jsonschema/ioda.jsonschema
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@
"files_to_copy": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_hardlink": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_link": {
"$ref": "urn:uwtools:files-to-stage"
},
Expand Down
3 changes: 3 additions & 0 deletions src/uwtools/resources/jsonschema/jedi.jsonschema
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@
"files_to_copy": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_hardlink": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_link": {
"$ref": "urn:uwtools:files-to-stage"
},
Expand Down
3 changes: 3 additions & 0 deletions src/uwtools/resources/jsonschema/mpas-init.jsonschema
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@
"files_to_copy": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_hardlink": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_link": {
"$ref": "urn:uwtools:files-to-stage"
},
Expand Down
3 changes: 3 additions & 0 deletions src/uwtools/resources/jsonschema/mpas.jsonschema
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@
"files_to_copy": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_hardlink": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_link": {
"$ref": "urn:uwtools:files-to-stage"
},
Expand Down
3 changes: 3 additions & 0 deletions src/uwtools/resources/jsonschema/mpassit.jsonschema
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
"files_to_copy": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_hardlink": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_link": {
"$ref": "urn:uwtools:files-to-stage"
},
Expand Down
6 changes: 6 additions & 0 deletions src/uwtools/resources/jsonschema/orog.jsonschema
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@
"execution": {
"$ref": "urn:uwtools:execution-parallel"
},
"files_to_copy": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_hardlink": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_link": {
"$ref": "urn:uwtools:files-to-stage"
},
Expand Down
3 changes: 3 additions & 0 deletions src/uwtools/resources/jsonschema/upp-assets.jsonschema
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
"files_to_copy": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_hardlink": {
"$ref": "urn:uwtools:files-to-stage"
},
"files_to_link": {
"$ref": "urn:uwtools:files-to-stage"
},
Expand Down
Loading
Loading