From 80c80a5ff92f192078878e1063a04e16cdf1316c Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 18 Jul 2025 16:38:56 +0000 Subject: [PATCH 01/69] Wire up CLI --- src/uwtools/api/rocoto.py | 19 +++++++++++++ src/uwtools/cli.py | 56 +++++++++++++++++++++++++++++++++++++-- src/uwtools/rocoto.py | 13 ++++++++- src/uwtools/strings.py | 2 ++ 4 files changed, 87 insertions(+), 3 deletions(-) diff --git a/src/uwtools/api/rocoto.py b/src/uwtools/api/rocoto.py index ec80f5be1..1eeb1d24e 100644 --- a/src/uwtools/api/rocoto.py +++ b/src/uwtools/api/rocoto.py @@ -7,11 +7,13 @@ from typing import TYPE_CHECKING from uwtools.rocoto import realize_rocoto_xml as _realize +from uwtools.rocoto import run_workflow as _run from uwtools.rocoto import validate_rocoto_xml_file as _validate from uwtools.utils.api import ensure_data_source as _ensure_data_source from uwtools.utils.file import str2path as _str2path if TYPE_CHECKING: + from datetime import datetime from pathlib import Path from uwtools.config.formats.yaml import YAMLConfig as _YAMLConfig @@ -40,6 +42,23 @@ def realize( return True +def run(cycle: datetime, database: Path | str, task: str, workflow: Path | str) -> bool: + """ + Run the specified Rocoto workflow to completion (or failure). + + :param cycle: A datetime object to make available for use in the config. + :param database: Path to the Rocoto database file. + :param task: The workflow task to run. + :param workflow: Path to the Rocoto XML workflow document. + """ + return _run( + cycle=cycle, + database=_ensure_data_source(_str2path(database), stdin_ok=False), + task=task, + workflow=_ensure_data_source(_str2path(workflow), stdin_ok=False), + ) + + def validate( xml_file: Path | str | None = None, stdin_ok: bool = False, diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 5af596de1..f8e903b83 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -462,6 +462,7 @@ def _add_subparser_rocoto(subparsers: Subparsers) -> ModeChecks: subparsers = _add_subparsers(parser, STR.action, STR.action.upper()) return { STR.realize: _add_subparser_rocoto_realize(subparsers), + STR.run: _add_subparser_rocoto_run(subparsers), STR.validate: _add_subparser_rocoto_validate(subparsers), } @@ -479,6 +480,22 @@ def _add_subparser_rocoto_realize(subparsers: Subparsers) -> ActionChecks: return _add_args_verbosity(optional) +def _add_subparser_rocoto_run(subparsers: Subparsers) -> ActionChecks: + """ + Add subparser for mode: rocoto run. + + :param subparsers: Parent parser's subparsers, to add this subparser to. + """ + parser = _add_subparser(subparsers, STR.run, "Run a Rocoto workflow") + required = parser.add_argument_group(TITLE_REQ_ARG) + _add_arg_cycle(required) + _add_arg_database(required) + _add_arg_task(required) + _add_arg_workflow(required) + optional = _basic_setup(parser) + return _add_args_verbosity(optional) + + def _add_subparser_rocoto_validate(subparsers: Subparsers) -> ActionChecks: """ Add subparser for mode: rocoto validate. @@ -499,6 +516,7 @@ def _dispatch_rocoto(args: Args) -> bool: """ actions = { STR.realize: _dispatch_rocoto_realize, + STR.run: _dispatch_rocoto_run, STR.validate: _dispatch_rocoto_validate, } return actions[args[STR.action]](args) @@ -506,7 +524,7 @@ def _dispatch_rocoto(args: Args) -> bool: def _dispatch_rocoto_realize(args: Args) -> bool: """ - Define dispatch logic for rocoto realize action. Validate input and output. + Define dispatch logic for rocoto realize action. :param args: Parsed command-line args. """ @@ -517,6 +535,20 @@ def _dispatch_rocoto_realize(args: Args) -> bool: ) +def _dispatch_rocoto_run(args: Args) -> bool: + """ + Define dispatch logic for rocoto run action. + + :param args: Parsed command-line args. + """ + return uwtools.api.rocoto.run( + cycle=args[STR.cycle], + database=args[STR.database], + task=args[STR.task], + workflow=args[STR.workflow], + ) + + def _dispatch_rocoto_validate(args: Args) -> bool: """ Define dispatch logic for rocoto validate action. @@ -674,6 +706,16 @@ def _add_arg_cycle(group: Group, required: bool = False) -> None: ) +def _add_arg_database(group: Group) -> None: + group.add_argument( + _switch(STR.database), + "-d", + help="The Rocoto database file", + required=True, + type=Path, + ) + + def _add_arg_dry_run(group: Group) -> None: group.add_argument( _switch(STR.dryrun), @@ -858,7 +900,7 @@ def _add_arg_target_dir(group: Group, required: bool = False, helpmsg: str | Non def _add_arg_task(group: Group) -> None: group.add_argument( _switch(STR.task), - help="Driver task to execute", + help="Task to execute", required=True, type=str, ) @@ -930,6 +972,16 @@ def _add_arg_verbose(group: Group) -> None: ) +def _add_arg_workflow(group: Group) -> None: + group.add_argument( + _switch(STR.workflow), + "-w", + help="The Rocoto XML file", + required=True, + type=Path, + ) + + # Support diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index ae7ecd640..43dd57359 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from math import log10 from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any from lxml import etree from lxml.builder import E # type: ignore[import-not-found] @@ -20,6 +20,9 @@ from uwtools.logging import log from uwtools.utils.file import readable, resource_path, writable +if TYPE_CHECKING: + from datetime import datetime + def realize_rocoto_xml(config: YAMLConfig | Path | None, output_file: Path | None = None) -> str: """ @@ -40,6 +43,14 @@ def realize_rocoto_xml(config: YAMLConfig | Path | None, output_file: Path | Non return xml +def run_workflow(cycle: datetime, database: Path, task: str, workflow: Path) -> bool: + assert cycle + assert database + assert task + assert workflow + return True + + def validate_rocoto_xml_file(xml_file: Path | None) -> bool: """ Validate purported Rocoto XML file against its schema. diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index 7670237c0..2877ae3c0 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -73,6 +73,7 @@ class STR: config: str = "config" copy: str = "copy" cycle: str = "cycle" + database: str = "database" dryrun: str = "dry_run" env: str = "env" envcmds: str = "envcmds" @@ -159,4 +160,5 @@ class STR: valsneeded: str = "values_needed" verbose: str = "verbose" version: str = "version" + workflow: str = "workflow" ww3: str = "ww3" From c5532da5dd8437af26a083059a8cbf492f276d72 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 18 Jul 2025 16:51:37 +0000 Subject: [PATCH 02/69] Add unit tests --- src/uwtools/tests/api/test_rocoto.py | 19 +++++++++++++++++-- src/uwtools/tests/test_cli.py | 11 +++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/uwtools/tests/api/test_rocoto.py b/src/uwtools/tests/api/test_rocoto.py index dbfbe67c1..b1aac27e7 100644 --- a/src/uwtools/tests/api/test_rocoto.py +++ b/src/uwtools/tests/api/test_rocoto.py @@ -1,17 +1,32 @@ from pathlib import Path from unittest.mock import patch +from pytest import mark + from uwtools.api import rocoto -def test_realize(): +def test_api_rocoto_realize(): path1, path2 = Path("foo"), Path("bar") with patch.object(rocoto, "_realize") as _realize: rocoto.realize(config=path1, output_file=path2) _realize.assert_called_once_with(config=path1, output_file=path2) -def test_validate(): +@mark.parametrize("f", [Path, str]) +def test_api_rocoto_run(f, utc): + cycle = utc() + database = f("/path/to/rocoto.db") + task = "foo" + workflow = f("/path/to/rocoto.xml") + with patch.object(rocoto, "_run") as _run: + rocoto.run(cycle=cycle, database=database, task=task, workflow=workflow) + _run.assert_called_once_with( + cycle=cycle, database=Path(database), task=task, workflow=Path(workflow) + ) + + +def test_api_rocoto_validate(): path = Path("foo") with patch.object(rocoto, "_validate") as _validate: rocoto.validate(xml_file=path) diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index 5bd9294b6..16145ca9e 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -456,6 +456,17 @@ def test__dispatch_rocoto_realize_no_optional(): func.assert_called_once_with(config=None, output_file=None) +def test__dispatch_rocoto_run(utc): + cycle = utc() + database = Path("/path/to/rocoto.db") + task = "foo" + workflow = Path("/path/to/rocoto.xml") + args = {STR.cycle: cycle, STR.database: database, STR.task: task, STR.workflow: workflow} + with patch.object(uwtools.api.rocoto, "_run") as _run: + cli._dispatch_rocoto_run(args) + _run.assert_called_once_with(cycle=cycle, database=database, task=task, workflow=workflow) + + def test__dispatch_rocoto_validate_xml(): args = {STR.infile: 1} with patch.object(uwtools.api.rocoto, "_validate") as _validate: From 5298587008479ea460d5146729e5703a15311208 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 18 Jul 2025 16:54:53 +0000 Subject: [PATCH 03/69] Simplify function names in uwtools.rocoto module --- src/uwtools/api/rocoto.py | 6 ++--- src/uwtools/rocoto.py | 12 ++++----- src/uwtools/tests/test_rocoto.py | 46 ++++++++++++++++---------------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/uwtools/api/rocoto.py b/src/uwtools/api/rocoto.py index 1eeb1d24e..1cf8c77aa 100644 --- a/src/uwtools/api/rocoto.py +++ b/src/uwtools/api/rocoto.py @@ -6,9 +6,9 @@ from typing import TYPE_CHECKING -from uwtools.rocoto import realize_rocoto_xml as _realize -from uwtools.rocoto import run_workflow as _run -from uwtools.rocoto import validate_rocoto_xml_file as _validate +from uwtools.rocoto import realize as _realize +from uwtools.rocoto import run as _run +from uwtools.rocoto import validate_file as _validate from uwtools.utils.api import ensure_data_source as _ensure_data_source from uwtools.utils.file import str2path as _str2path diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 43dd57359..126c06ebe 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -24,7 +24,7 @@ from datetime import datetime -def realize_rocoto_xml(config: YAMLConfig | Path | None, output_file: Path | None = None) -> str: +def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) -> str: """ Realize the Rocoto workflow defined in the given YAML as XML, validating both the YAML input and XML output. @@ -35,7 +35,7 @@ def realize_rocoto_xml(config: YAMLConfig | Path | None, output_file: Path | Non """ rxml = _RocotoXML(config) xml = str(rxml).strip() - if not validate_rocoto_xml_string(xml): + if not validate_string(xml): msg = "Internal error: Invalid Rocoto XML" raise UWError(msg) with writable(output_file) as f: @@ -43,7 +43,7 @@ def realize_rocoto_xml(config: YAMLConfig | Path | None, output_file: Path | Non return xml -def run_workflow(cycle: datetime, database: Path, task: str, workflow: Path) -> bool: +def run(cycle: datetime, database: Path, task: str, workflow: Path) -> bool: assert cycle assert database assert task @@ -51,7 +51,7 @@ def run_workflow(cycle: datetime, database: Path, task: str, workflow: Path) -> return True -def validate_rocoto_xml_file(xml_file: Path | None) -> bool: +def validate_file(xml_file: Path | None) -> bool: """ Validate purported Rocoto XML file against its schema. @@ -59,10 +59,10 @@ def validate_rocoto_xml_file(xml_file: Path | None) -> bool: :return: Did the XML conform to the schema? """ with readable(xml_file) as f: - return validate_rocoto_xml_string(xml=f.read()) + return validate_string(xml=f.read()) -def validate_rocoto_xml_string(xml: str) -> bool: +def validate_string(xml: str) -> bool: """ Validate purported Rocoto XML against its schema. diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index ec5108cac..ca090e80c 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -36,56 +36,56 @@ def validation_assets(tmp_path): def test_realize_rocoto_invalid_xml(assets): cfgfile, outfile = assets - with patch.object(rocoto, "validate_rocoto_xml_string") as vrxs: + with patch.object(rocoto, "validate_string") as vrxs: vrxs.return_value = False with raises(UWError): - rocoto.realize_rocoto_xml(config=cfgfile, output_file=outfile) + rocoto.realize(config=cfgfile, output_file=outfile) -def test_realize_rocoto_xml_cfg_to_file(assets): +def test_realize_cfg_to_file(assets): cfgfile, outfile = assets - rocoto.realize_rocoto_xml(config=YAMLConfig(cfgfile), output_file=outfile) - assert rocoto.validate_rocoto_xml_file(xml_file=outfile) + rocoto.realize(config=YAMLConfig(cfgfile), output_file=outfile) + assert rocoto.validate_file(xml_file=outfile) -def test_realize_rocoto_xml_file_to_file(assets): +def test_realize_file_to_file(assets): cfgfile, outfile = assets - rocoto.realize_rocoto_xml(config=cfgfile, output_file=outfile) - assert rocoto.validate_rocoto_xml_file(xml_file=outfile) + rocoto.realize(config=cfgfile, output_file=outfile) + assert rocoto.validate_file(xml_file=outfile) -def test_realize_rocoto_xml_cfg_to_stdout(capsys, assets): +def test_realize_cfg_to_stdout(capsys, assets): cfgfile, outfile = assets - rocoto.realize_rocoto_xml(config=YAMLConfig(cfgfile)) + rocoto.realize(config=YAMLConfig(cfgfile)) outfile.write_text(capsys.readouterr().out) - assert rocoto.validate_rocoto_xml_file(xml_file=outfile) + assert rocoto.validate_file(xml_file=outfile) -def test_realize_rocoto_xml_file_to_stdout(capsys, assets): +def test_realize_file_to_stdout(capsys, assets): cfgfile, outfile = assets - rocoto.realize_rocoto_xml(config=cfgfile) + rocoto.realize(config=cfgfile) outfile.write_text(capsys.readouterr().out) - assert rocoto.validate_rocoto_xml_file(xml_file=outfile) + assert rocoto.validate_file(xml_file=outfile) -def test_validate_rocoto_xml_file_fail(validation_assets): +def test_validate_file_fail(validation_assets): xml_file_bad, _, _, _ = validation_assets - assert rocoto.validate_rocoto_xml_file(xml_file=xml_file_bad) is False + assert rocoto.validate_file(xml_file=xml_file_bad) is False -def test_validate_rocoto_xml_file_pass(validation_assets): +def test_validate_file_pass(validation_assets): _, xml_file_good, _, _ = validation_assets - assert rocoto.validate_rocoto_xml_file(xml_file=xml_file_good) is True + assert rocoto.validate_file(xml_file=xml_file_good) is True -def test_validate_rocoto_xml_string_fail(validation_assets): +def test_validate_string_fail(validation_assets): _, _, xml_string_bad, _ = validation_assets - assert rocoto.validate_rocoto_xml_string(xml=xml_string_bad) is False + assert rocoto.validate_string(xml=xml_string_bad) is False -def test_validate_rocoto_xml_string_pass(validation_assets): +def test_validate_string_pass(validation_assets): _, _, _, xml_string_good = validation_assets - assert rocoto.validate_rocoto_xml_string(xml=xml_string_good) is True + assert rocoto.validate_string(xml=xml_string_good) is True class TestRocotoXML: @@ -551,4 +551,4 @@ def test__tag_name(self, instance): def test_dump(self, instance, tmp_path): path = tmp_path / "out.xml" instance.dump(path=path) - assert rocoto.validate_rocoto_xml_file(path) + assert rocoto.validate_file(path) From e7f254c98bd61bdccaac3f91575d7244bb7cf31d Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 18 Jul 2025 17:56:53 +0000 Subject: [PATCH 04/69] WIP --- src/uwtools/rocoto.py | 27 +++++++++++++++++++++++---- src/uwtools/utils/processing.py | 2 +- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 126c06ebe..5c242d0a7 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -5,9 +5,11 @@ from __future__ import annotations import re +import sqlite3 from dataclasses import dataclass from math import log10 from pathlib import Path +from time import sleep from typing import TYPE_CHECKING, Any from lxml import etree @@ -19,6 +21,7 @@ from uwtools.exceptions import UWConfigError, UWError from uwtools.logging import log from uwtools.utils.file import readable, resource_path, writable +from uwtools.utils.processing import run_shell_cmd if TYPE_CHECKING: from datetime import datetime @@ -44,10 +47,26 @@ def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) - def run(cycle: datetime, database: Path, task: str, workflow: Path) -> bool: - assert cycle - assert database - assert task - assert workflow + query = "select state from jobs where taskname=:taskname and cycle=:cycle" + data = {"taskname": task, "cycle": int(cycle.timestamp())} + frequency = 10 # seconds + connection = None + while True: + success, output = run_shell_cmd("rocotorun -d %s -w %s" % (database, workflow)) + if not success: + return False + if not connection: + connection = sqlite3.connect(database) + cursor = connection.cursor() + result = cursor.execute(query, data) + (state,) = result.fetchone() + log.info("Rocoto task %s at %s: %s", task, cycle, state) + if state in ["SUCCEEDED"]: + break + log.info("Sleeping %s seconds", frequency) + sleep(frequency) + if connection: + connection.close() return True diff --git a/src/uwtools/utils/processing.py b/src/uwtools/utils/processing.py index c572089e2..e5785d4e8 100644 --- a/src/uwtools/utils/processing.py +++ b/src/uwtools/utils/processing.py @@ -51,6 +51,6 @@ def run_shell_cmd( success = False if output and (log_output or not success): logfunc("%sOutput:", pre) - for line in output.split("\n"): + for line in output.strip().split("\n"): logfunc("%s%s%s", pre, INDENT, line) return success, output From 6d51fd9f76613eb2279c5865b9d71f206eafcc02 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 18 Jul 2025 21:14:13 +0000 Subject: [PATCH 05/69] WIP --- src/uwtools/rocoto.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 5c242d0a7..de0fca48f 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -7,6 +7,7 @@ import re import sqlite3 from dataclasses import dataclass +from itertools import chain from math import log10 from pathlib import Path from time import sleep @@ -49,6 +50,9 @@ def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) - def run(cycle: datetime, database: Path, task: str, workflow: Path) -> bool: query = "select state from jobs where taskname=:taskname and cycle=:cycle" data = {"taskname": task, "cycle": int(cycle.timestamp())} + active = ["QUEUED", "RUNNING"] + inactive = ["COMPLETE", "DEAD", "ERROR", "STUCK", "SUCCEEDED"] + intermediate = ["CREATED", "DYING", "STALLED", "SUBMITTING"] frequency = 10 # seconds connection = None while True: @@ -60,9 +64,12 @@ def run(cycle: datetime, database: Path, task: str, workflow: Path) -> bool: cursor = connection.cursor() result = cursor.execute(query, data) (state,) = result.fetchone() - log.info("Rocoto task %s at %s: %s", task, cycle, state) - if state in ["SUCCEEDED"]: + log.info("Rocoto task '%s' for cycle %s: %s", task, cycle, state) + assert state in chain(active, inactive, intermediate) + if state in inactive: break + if state in intermediate: + continue # iterate immediately to update status log.info("Sleeping %s seconds", frequency) sleep(frequency) if connection: From ce8b384c05580cb67d520ec9289f4bfab22e6b06 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 18 Jul 2025 21:15:47 +0000 Subject: [PATCH 06/69] WIP --- src/uwtools/rocoto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index de0fca48f..4e6637d8f 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -56,7 +56,7 @@ def run(cycle: datetime, database: Path, task: str, workflow: Path) -> bool: frequency = 10 # seconds connection = None while True: - success, output = run_shell_cmd("rocotorun -d %s -w %s" % (database, workflow)) + success, _ = run_shell_cmd("rocotorun -d %s -w %s" % (database, workflow)) if not success: return False if not connection: From 3c5392c6d95c807843a0d79749236d0491cd4c25 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 18 Jul 2025 21:32:54 +0000 Subject: [PATCH 07/69] WIP --- src/uwtools/rocoto.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 4e6637d8f..53569621b 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -28,6 +28,13 @@ from datetime import datetime +STATE = { + "active": ["QUEUED", "RUNNING"], + "inactive": ["COMPLETE", "DEAD", "ERROR", "STUCK", "SUCCEEDED"], + "intermediate": ["CREATED", "DYING", "STALLED", "SUBMITTING"], +} + + def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) -> str: """ Realize the Rocoto workflow defined in the given YAML as XML, validating both the YAML input and @@ -48,33 +55,33 @@ def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) - def run(cycle: datetime, database: Path, task: str, workflow: Path) -> bool: - query = "select state from jobs where taskname=:taskname and cycle=:cycle" - data = {"taskname": task, "cycle": int(cycle.timestamp())} - active = ["QUEUED", "RUNNING"] - inactive = ["COMPLETE", "DEAD", "ERROR", "STUCK", "SUCCEEDED"] - intermediate = ["CREATED", "DYING", "STALLED", "SUBMITTING"] - frequency = 10 # seconds + iterate = "rocotorun -d %s -w %s" % (database, workflow) + query_stmt = "select state from jobs where taskname=:taskname and cycle=:cycle" + query_data = {"taskname": task, "cycle": int(cycle.timestamp())} connection = None + ok = True # optimistically while True: - success, _ = run_shell_cmd("rocotorun -d %s -w %s" % (database, workflow)) + success, _ = run_shell_cmd(iterate) if not success: - return False + ok = False + break if not connection: connection = sqlite3.connect(database) cursor = connection.cursor() - result = cursor.execute(query, data) + result = cursor.execute(query_stmt, query_data) (state,) = result.fetchone() log.info("Rocoto task '%s' for cycle %s: %s", task, cycle, state) - assert state in chain(active, inactive, intermediate) - if state in inactive: + assert state in chain.from_iterable(STATE.values()) + if state in STATE["inactive"]: break - if state in intermediate: + if state in STATE["intermediate"]: continue # iterate immediately to update status + frequency = 10 # seconds log.info("Sleeping %s seconds", frequency) sleep(frequency) if connection: connection.close() - return True + return ok def validate_file(xml_file: Path | None) -> bool: From c7f63cf200f7fd577c0a7dc7233fd90e3376a39f Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 18 Jul 2025 21:33:41 +0000 Subject: [PATCH 08/69] WIP --- src/uwtools/rocoto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 53569621b..27cad9470 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -31,7 +31,7 @@ STATE = { "active": ["QUEUED", "RUNNING"], "inactive": ["COMPLETE", "DEAD", "ERROR", "STUCK", "SUCCEEDED"], - "intermediate": ["CREATED", "DYING", "STALLED", "SUBMITTING"], + "transient": ["CREATED", "DYING", "STALLED", "SUBMITTING"], } @@ -74,7 +74,7 @@ def run(cycle: datetime, database: Path, task: str, workflow: Path) -> bool: assert state in chain.from_iterable(STATE.values()) if state in STATE["inactive"]: break - if state in STATE["intermediate"]: + if state in STATE["transient"]: continue # iterate immediately to update status frequency = 10 # seconds log.info("Sleeping %s seconds", frequency) From 29ff65716ab21d21bae342158cbea7ad879354ed Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 18 Jul 2025 22:55:44 +0000 Subject: [PATCH 09/69] Add RocotoRunner class --- src/uwtools/rocoto.py | 106 +++++++++++++++++++++++++++++++----------- 1 file changed, 79 insertions(+), 27 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 27cad9470..0e5f42faf 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -54,34 +54,86 @@ def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) - return xml +class RocotoRunner: + def __init__(self, cycle: datetime, database: Path, task: str, workflow: Path): + self.cycle = cycle + self.database = database + self.task = task + self.workflow = workflow + self._connection: sqlite3.Connection | None = None + self._cursor = None + self._frequency = 10 # seconds + + def close(self) -> None: + if self._connection: + self._connection.close() + + @property + def connection(self): + if not self._connection: + if not self.database.is_file(): + return None + self._connection = sqlite3.connect(self.database) + return self._connection + + @property + def cursor(self): + if not self._cursor: + if not self.connection: + return None + self._cursor = self.connection.cursor() + return self._cursor + + def iterate(self) -> bool: + cmd = "rocotorun -d %s -w %s" % (self.database, self.workflow) + success, _ = run_shell_cmd(cmd) + return success + + def run(self) -> bool: + initialized = False + while True: + if initialized: + if not self.iterate(): + return False + initialized = True + if state := self.state: + if state in STATE["inactive"]: + break + if state in STATE["transient"]: + continue # iterate immediately to update status + log.info("Sleeping %s seconds", self._frequency) + sleep(self._frequency) + return True + + @property + def state(self) -> str | None: + if self.cursor: + result = self.cursor.execute(self._query_stmt, self._query_data) + state: str + (state,) = result.fetchone() + log.info(self._state_msg % state) + assert state in chain.from_iterable(STATE.values()) + return state + return None + + @property + def _query_stmt(self) -> str: + return "select state from jobs where taskname=:taskname and cycle=:cycle" + + @property + def _query_data(self) -> dict: + return {"taskname": self.task, "cycle": int(self.cycle.timestamp())} + + @property + def _state_msg(self) -> str: + return f"Rocoto task '{self.task}' for cycle {self.cycle}: %s" + + def run(cycle: datetime, database: Path, task: str, workflow: Path) -> bool: - iterate = "rocotorun -d %s -w %s" % (database, workflow) - query_stmt = "select state from jobs where taskname=:taskname and cycle=:cycle" - query_data = {"taskname": task, "cycle": int(cycle.timestamp())} - connection = None - ok = True # optimistically - while True: - success, _ = run_shell_cmd(iterate) - if not success: - ok = False - break - if not connection: - connection = sqlite3.connect(database) - cursor = connection.cursor() - result = cursor.execute(query_stmt, query_data) - (state,) = result.fetchone() - log.info("Rocoto task '%s' for cycle %s: %s", task, cycle, state) - assert state in chain.from_iterable(STATE.values()) - if state in STATE["inactive"]: - break - if state in STATE["transient"]: - continue # iterate immediately to update status - frequency = 10 # seconds - log.info("Sleeping %s seconds", frequency) - sleep(frequency) - if connection: - connection.close() - return ok + runner = RocotoRunner(cycle, database, task, workflow) + success = runner.run() + runner.close() + return success def validate_file(xml_file: Path | None) -> bool: From 415794294118cdddbfe41af004ec6bf4adf94a58 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 18 Jul 2025 22:57:49 +0000 Subject: [PATCH 10/69] WIP --- src/uwtools/rocoto.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 0e5f42faf..6b0de4e61 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -95,14 +95,15 @@ def run(self) -> bool: if initialized: if not self.iterate(): return False - initialized = True if state := self.state: if state in STATE["inactive"]: break if state in STATE["transient"]: continue # iterate immediately to update status - log.info("Sleeping %s seconds", self._frequency) - sleep(self._frequency) + if initialized: + log.info("Sleeping %s seconds", self._frequency) + sleep(self._frequency) + initialized = True return True @property From d5f033234a3bb97caaa4753abae02032b2eb01e3 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 18 Jul 2025 23:02:47 +0000 Subject: [PATCH 11/69] WIP --- src/uwtools/rocoto.py | 73 +++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 6b0de4e61..c45b28ca6 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -28,32 +28,6 @@ from datetime import datetime -STATE = { - "active": ["QUEUED", "RUNNING"], - "inactive": ["COMPLETE", "DEAD", "ERROR", "STUCK", "SUCCEEDED"], - "transient": ["CREATED", "DYING", "STALLED", "SUBMITTING"], -} - - -def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) -> str: - """ - Realize the Rocoto workflow defined in the given YAML as XML, validating both the YAML input and - XML output. - - :param config: Path to YAML input file (None => read stdin), or YAMLConfig object. - :param output_file: Path to write rendered XML file (None => write to stdout). - :return: An XML string. - """ - rxml = _RocotoXML(config) - xml = str(rxml).strip() - if not validate_string(xml): - msg = "Internal error: Invalid Rocoto XML" - raise UWError(msg) - with writable(output_file) as f: - print(xml, file=f) - return xml - - class RocotoRunner: def __init__(self, cycle: datetime, database: Path, task: str, workflow: Path): self.cycle = cycle @@ -64,10 +38,12 @@ def __init__(self, cycle: datetime, database: Path, task: str, workflow: Path): self._cursor = None self._frequency = 10 # seconds - def close(self) -> None: + def __del__(self): if self._connection: self._connection.close() + # PM FIX TYPES + @property def connection(self): if not self._connection: @@ -92,13 +68,12 @@ def iterate(self) -> bool: def run(self) -> bool: initialized = False while True: - if initialized: - if not self.iterate(): - return False + if initialized and not self.iterate(): + return False if state := self.state: - if state in STATE["inactive"]: + if state in self._state["inactive"]: break - if state in STATE["transient"]: + if state in self._state["transient"]: continue # iterate immediately to update status if initialized: log.info("Sleeping %s seconds", self._frequency) @@ -113,7 +88,7 @@ def state(self) -> str | None: state: str (state,) = result.fetchone() log.info(self._state_msg % state) - assert state in chain.from_iterable(STATE.values()) + assert state in chain.from_iterable(self._state.values()) return state return None @@ -125,16 +100,40 @@ def _query_stmt(self) -> str: def _query_data(self) -> dict: return {"taskname": self.task, "cycle": int(self.cycle.timestamp())} + @property + def _state() -> dict: + return { + "active": ["QUEUED", "RUNNING"], + "inactive": ["COMPLETE", "DEAD", "ERROR", "STUCK", "SUCCEEDED"], + "transient": ["CREATED", "DYING", "STALLED", "SUBMITTING"], + } + @property def _state_msg(self) -> str: return f"Rocoto task '{self.task}' for cycle {self.cycle}: %s" +def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) -> str: + """ + Realize the Rocoto workflow defined in the given YAML as XML, validating both the YAML input and + XML output. + + :param config: Path to YAML input file (None => read stdin), or YAMLConfig object. + :param output_file: Path to write rendered XML file (None => write to stdout). + :return: An XML string. + """ + rxml = _RocotoXML(config) + xml = str(rxml).strip() + if not validate_string(xml): + msg = "Internal error: Invalid Rocoto XML" + raise UWError(msg) + with writable(output_file) as f: + print(xml, file=f) + return xml + + def run(cycle: datetime, database: Path, task: str, workflow: Path) -> bool: - runner = RocotoRunner(cycle, database, task, workflow) - success = runner.run() - runner.close() - return success + return RocotoRunner(cycle, database, task, workflow).run() def validate_file(xml_file: Path | None) -> bool: From 0067cd2501d22f36947749592f6f2b585b95f36c Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 19 Jul 2025 14:21:10 +0000 Subject: [PATCH 12/69] WIP --- src/uwtools/rocoto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index c45b28ca6..dedb84a5a 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -101,7 +101,7 @@ def _query_data(self) -> dict: return {"taskname": self.task, "cycle": int(self.cycle.timestamp())} @property - def _state() -> dict: + def _state(self) -> dict: return { "active": ["QUEUED", "RUNNING"], "inactive": ["COMPLETE", "DEAD", "ERROR", "STUCK", "SUCCEEDED"], From 6a269c5934d1b1fe4f56a4a4fd1897dd4d1e8c50 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 19 Jul 2025 14:30:27 +0000 Subject: [PATCH 13/69] Add missing types --- src/uwtools/rocoto.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index dedb84a5a..f7a89cdf8 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -35,17 +35,15 @@ def __init__(self, cycle: datetime, database: Path, task: str, workflow: Path): self.task = task self.workflow = workflow self._connection: sqlite3.Connection | None = None - self._cursor = None + self._cursor: sqlite3.Cursor | None = None self._frequency = 10 # seconds def __del__(self): if self._connection: self._connection.close() - # PM FIX TYPES - @property - def connection(self): + def connection(self) -> sqlite3.Connection | None: if not self._connection: if not self.database.is_file(): return None @@ -53,7 +51,7 @@ def connection(self): return self._connection @property - def cursor(self): + def cursor(self) -> sqlite3.Cursor | None: if not self._cursor: if not self.connection: return None From 8143433d35624533971d4caa73aefcecbb673129 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 19 Jul 2025 14:39:45 +0000 Subject: [PATCH 14/69] Make methods private --- src/uwtools/rocoto.py | 78 +++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index f7a89cdf8..b6178a7a8 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -34,44 +34,23 @@ def __init__(self, cycle: datetime, database: Path, task: str, workflow: Path): self.database = database self.task = task self.workflow = workflow - self._connection: sqlite3.Connection | None = None - self._cursor: sqlite3.Cursor | None = None self._frequency = 10 # seconds + self.__connection: sqlite3.Connection | None = None + self.__cursor: sqlite3.Cursor | None = None def __del__(self): - if self._connection: - self._connection.close() - - @property - def connection(self) -> sqlite3.Connection | None: - if not self._connection: - if not self.database.is_file(): - return None - self._connection = sqlite3.connect(self.database) - return self._connection - - @property - def cursor(self) -> sqlite3.Cursor | None: - if not self._cursor: - if not self.connection: - return None - self._cursor = self.connection.cursor() - return self._cursor - - def iterate(self) -> bool: - cmd = "rocotorun -d %s -w %s" % (self.database, self.workflow) - success, _ = run_shell_cmd(cmd) - return success + if self.__connection: + self.__connection.close() def run(self) -> bool: initialized = False while True: - if initialized and not self.iterate(): + if initialized and not self._iterate(): return False - if state := self.state: - if state in self._state["inactive"]: + if state := self._state: + if state in self._states["inactive"]: break - if state in self._state["transient"]: + if state in self._states["transient"]: continue # iterate immediately to update status if initialized: log.info("Sleeping %s seconds", self._frequency) @@ -80,15 +59,25 @@ def run(self) -> bool: return True @property - def state(self) -> str | None: - if self.cursor: - result = self.cursor.execute(self._query_stmt, self._query_data) - state: str - (state,) = result.fetchone() - log.info(self._state_msg % state) - assert state in chain.from_iterable(self._state.values()) - return state - return None + def _connection(self) -> sqlite3.Connection | None: + if not self.__connection: + if not self.database.is_file(): + return None + self.__connection = sqlite3.connect(self.database) + return self.__connection + + @property + def _cursor(self) -> sqlite3.Cursor | None: + if not self.__cursor: + if not (connection := self._connection): + return None + self.__cursor = connection.cursor() + return self.__cursor + + def _iterate(self) -> bool: + cmd = "rocotorun -d %s -w %s" % (self.database, self.workflow) + success, _ = run_shell_cmd(cmd) + return success @property def _query_stmt(self) -> str: @@ -99,7 +88,18 @@ def _query_data(self) -> dict: return {"taskname": self.task, "cycle": int(self.cycle.timestamp())} @property - def _state(self) -> dict: + def _state(self) -> str | None: + if cursor := self._cursor: + result = cursor.execute(self._query_stmt, self._query_data) + state: str + (state,) = result.fetchone() + log.info(self._state_msg % state) + assert state in chain.from_iterable(self._states.values()) + return state + return None + + @property + def _states(self) -> dict: return { "active": ["QUEUED", "RUNNING"], "inactive": ["COMPLETE", "DEAD", "ERROR", "STUCK", "SUCCEEDED"], From cc835e8f1bc6240d3c1dfd24bae3f54f732ab1f2 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 19 Jul 2025 14:40:42 +0000 Subject: [PATCH 15/69] Reorg --- src/uwtools/rocoto.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index b6178a7a8..31501b721 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -79,14 +79,14 @@ def _iterate(self) -> bool: success, _ = run_shell_cmd(cmd) return success - @property - def _query_stmt(self) -> str: - return "select state from jobs where taskname=:taskname and cycle=:cycle" - @property def _query_data(self) -> dict: return {"taskname": self.task, "cycle": int(self.cycle.timestamp())} + @property + def _query_stmt(self) -> str: + return "select state from jobs where taskname=:taskname and cycle=:cycle" + @property def _state(self) -> str | None: if cursor := self._cursor: @@ -98,6 +98,10 @@ def _state(self) -> str | None: return state return None + @property + def _state_msg(self) -> str: + return f"Rocoto task '{self.task}' for cycle {self.cycle}: %s" + @property def _states(self) -> dict: return { @@ -106,10 +110,6 @@ def _states(self) -> dict: "transient": ["CREATED", "DYING", "STALLED", "SUBMITTING"], } - @property - def _state_msg(self) -> str: - return f"Rocoto task '{self.task}' for cycle {self.cycle}: %s" - def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) -> str: """ From 5e91cc456d058b2f6bf5f7945cc630a63280f00b Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 19 Jul 2025 14:50:56 +0000 Subject: [PATCH 16/69] WIP --- src/uwtools/rocoto.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 31501b721..4bdb02fe0 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -91,10 +91,11 @@ def _query_stmt(self) -> str: def _state(self) -> str | None: if cursor := self._cursor: result = cursor.execute(self._query_stmt, self._query_data) - state: str - (state,) = result.fetchone() - log.info(self._state_msg % state) - assert state in chain.from_iterable(self._states.values()) + state = None + if row := result.fetchone(): + (state,) = row + log.info(self._state_msg % state) + assert state in chain.from_iterable(self._states.values()) return state return None From afe88521add331a1eb10802cdeddee67febacffb Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 19 Jul 2025 14:53:41 +0000 Subject: [PATCH 17/69] WIP --- src/uwtools/rocoto.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 4bdb02fe0..51546ad6a 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -52,6 +52,8 @@ def run(self) -> bool: break if state in self._states["transient"]: continue # iterate immediately to update status + else: + log.info(self._state_msg, "State not yet known") if initialized: log.info("Sleeping %s seconds", self._frequency) sleep(self._frequency) From 10f93506f64a88d2d3321a65b8ef2352cd2c6724 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 19 Jul 2025 14:55:37 +0000 Subject: [PATCH 18/69] WIP [skip ci] --- src/uwtools/rocoto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 51546ad6a..f84ab196c 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -55,7 +55,7 @@ def run(self) -> bool: else: log.info(self._state_msg, "State not yet known") if initialized: - log.info("Sleeping %s seconds", self._frequency) + log.debug("Sleeping %s seconds", self._frequency) sleep(self._frequency) initialized = True return True From fd37eabf8460371291786a90e0010b1e0ed16713 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 19 Jul 2025 15:06:40 +0000 Subject: [PATCH 19/69] WIP [skip ci] --- src/uwtools/rocoto.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index f84ab196c..897af1c71 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -52,8 +52,7 @@ def run(self) -> bool: break if state in self._states["transient"]: continue # iterate immediately to update status - else: - log.info(self._state_msg, "State not yet known") + self._report() if initialized: log.debug("Sleeping %s seconds", self._frequency) sleep(self._frequency) @@ -89,6 +88,13 @@ def _query_data(self) -> dict: def _query_stmt(self) -> str: return "select state from jobs where taskname=:taskname and cycle=:cycle" + def _report(self) -> None: + cmd = "rocotostat -d %s -w %s" % (self.database, self.workflow) + if self.database.is_file(): + _, output = run_shell_cmd(cmd) + for line in output.split("\n"): + log.info(line) + @property def _state(self) -> str | None: if cursor := self._cursor: From f0e3b51cda02352d59f9319ab1ce67cb9212b1a4 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 19 Jul 2025 20:32:05 +0000 Subject: [PATCH 20/69] WIP [skip ci] --- src/uwtools/rocoto.py | 8 +++++--- src/uwtools/utils/processing.py | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 897af1c71..66090eb80 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -76,8 +76,9 @@ def _cursor(self) -> sqlite3.Cursor | None: return self.__cursor def _iterate(self) -> bool: + log.info("Iterating workflow") cmd = "rocotorun -d %s -w %s" % (self.database, self.workflow) - success, _ = run_shell_cmd(cmd) + success, _ = run_shell_cmd(cmd, quiet=True) return success @property @@ -91,8 +92,9 @@ def _query_stmt(self) -> str: def _report(self) -> None: cmd = "rocotostat -d %s -w %s" % (self.database, self.workflow) if self.database.is_file(): - _, output = run_shell_cmd(cmd) - for line in output.split("\n"): + log.info("Workflow status:") + _, output = run_shell_cmd(cmd, quiet=True) + for line in output.strip().split("\n"): log.info(line) @property diff --git a/src/uwtools/utils/processing.py b/src/uwtools/utils/processing.py index e5785d4e8..6247cd9b5 100644 --- a/src/uwtools/utils/processing.py +++ b/src/uwtools/utils/processing.py @@ -19,6 +19,7 @@ def run_shell_cmd( env: dict[str, str] | None = None, log_output: bool | None = False, taskname: str | None = None, + quiet: bool | None = None, ) -> tuple[bool, str]: """ Run a command in a shell. @@ -28,6 +29,7 @@ def run_shell_cmd( :param env: Environment variables to set before running cmd. :param log_output: Log output from successful cmd? (Error output is always logged.) :param taskname: Name of task executing this command, for logging. + :param quiet: Log INFO messages as DEBUG. :return: A result object providing combined stder/stdout output and success values. """ pre = f"{taskname}: " if taskname else "" @@ -37,12 +39,12 @@ def run_shell_cmd( if env: kvpairs = " ".join(f"{k}={v}" for k, v in env.items()) msg += f" with environment variables {kvpairs}" - log.info(msg, pre) + logfunc = log.debug if quiet else log.info + logfunc(msg, pre) try: output = check_output( cmd, cwd=cwd, encoding="utf=8", env=env, shell=True, stderr=STDOUT, text=True ) - logfunc = log.info success = True except CalledProcessError as e: output = e.output From 57fca6f17abd291bdab3c6f5ce89b83f4a56d2c7 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 19 Jul 2025 20:39:38 +0000 Subject: [PATCH 21/69] WIP [skip ci] --- src/uwtools/tests/utils/test_processing.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/uwtools/tests/utils/test_processing.py b/src/uwtools/tests/utils/test_processing.py index f0647ee9a..f07610fe7 100644 --- a/src/uwtools/tests/utils/test_processing.py +++ b/src/uwtools/tests/utils/test_processing.py @@ -2,6 +2,11 @@ Tests for uwtools.utils.processing module. """ +import logging + +from pytest import mark + +from uwtools.logging import log from uwtools.utils import processing @@ -16,12 +21,18 @@ def test_utils_processing_run_shell_cmd__failure(logged): assert logged(" expr: division by zero") -def test_utils_processing_run_shell_cmd__success(logged, tmp_path): +@mark.parametrize("quiet", [True, False]) +def test_utils_processing_run_shell_cmd__success(caplog, logged, quiet, tmp_path): cmd = "echo hello $FOO" + if quiet: + log.setLevel(logging.INFO) success, _ = processing.run_shell_cmd( - cmd=cmd, cwd=tmp_path, env={"FOO": "bar"}, log_output=True + cmd=cmd, cwd=tmp_path, env={"FOO": "bar"}, log_output=True, quiet=quiet ) assert success - assert logged(f"Running: {cmd} in {tmp_path} with environment variables FOO=bar") - assert logged("Output:") - assert logged(" hello bar") + if quiet: + assert not caplog.messages + else: + assert logged(f"Running: {cmd} in {tmp_path} with environment variables FOO=bar") + assert logged("Output:") + assert logged(" hello bar") From 083d8c0db9861519a5b17c0c8911e08efd23d334 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 19 Jul 2025 20:40:54 +0000 Subject: [PATCH 22/69] WIP [skip ci] --- src/uwtools/rocoto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 66090eb80..e792127e5 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -28,7 +28,7 @@ from datetime import datetime -class RocotoRunner: +class _RocotoRunner: def __init__(self, cycle: datetime, database: Path, task: str, workflow: Path): self.cycle = cycle self.database = database @@ -142,7 +142,7 @@ def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) - def run(cycle: datetime, database: Path, task: str, workflow: Path) -> bool: - return RocotoRunner(cycle, database, task, workflow).run() + return _RocotoRunner(cycle, database, task, workflow).run() def validate_file(xml_file: Path | None) -> bool: From f8109d2d5d7e26219547513483155a94b98ee80a Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 19 Jul 2025 21:47:16 +0000 Subject: [PATCH 23/69] WIP [skip ci] --- src/uwtools/api/rocoto.py | 6 ++++- src/uwtools/cli.py | 12 ++++++++++ src/uwtools/rocoto.py | 34 ++++++++++++++-------------- src/uwtools/strings.py | 1 + src/uwtools/tests/api/test_rocoto.py | 5 ++-- src/uwtools/tests/test_cli.py | 11 +++++++-- 6 files changed, 47 insertions(+), 22 deletions(-) diff --git a/src/uwtools/api/rocoto.py b/src/uwtools/api/rocoto.py index 1cf8c77aa..5125895e1 100644 --- a/src/uwtools/api/rocoto.py +++ b/src/uwtools/api/rocoto.py @@ -42,7 +42,9 @@ def realize( return True -def run(cycle: datetime, database: Path | str, task: str, workflow: Path | str) -> bool: +def run( + cycle: datetime, database: Path | str, task: str, workflow: Path | str, rate: int = 10 +) -> bool: """ Run the specified Rocoto workflow to completion (or failure). @@ -50,10 +52,12 @@ def run(cycle: datetime, database: Path | str, task: str, workflow: Path | str) :param database: Path to the Rocoto database file. :param task: The workflow task to run. :param workflow: Path to the Rocoto XML workflow document. + :param rate: Seconds between workflow iterations (deault: 10)" """ return _run( cycle=cycle, database=_ensure_data_source(_str2path(database), stdin_ok=False), + rate=rate, task=task, workflow=_ensure_data_source(_str2path(workflow), stdin_ok=False), ) diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index f8e903b83..d3300f3d7 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -493,6 +493,7 @@ def _add_subparser_rocoto_run(subparsers: Subparsers) -> ActionChecks: _add_arg_task(required) _add_arg_workflow(required) optional = _basic_setup(parser) + _add_arg_rate(optional) return _add_args_verbosity(optional) @@ -544,6 +545,7 @@ def _dispatch_rocoto_run(args: Args) -> bool: return uwtools.api.rocoto.run( cycle=args[STR.cycle], database=args[STR.database], + rate=args[STR.rate], task=args[STR.task], workflow=args[STR.workflow], ) @@ -851,6 +853,16 @@ def _add_arg_quiet(group: Group) -> None: ) +def _add_arg_rate(group: Group) -> None: + group.add_argument( + _switch(STR.rate), + "-r", + help="Seconds between workflow iterations", + required=False, + type=int, + ) + + def _add_arg_report(group: Group) -> None: group.add_argument( _switch(STR.report), diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index e792127e5..99c697d34 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -29,12 +29,12 @@ class _RocotoRunner: - def __init__(self, cycle: datetime, database: Path, task: str, workflow: Path): - self.cycle = cycle - self.database = database - self.task = task - self.workflow = workflow - self._frequency = 10 # seconds + def __init__(self, cycle: datetime, database: Path, rate: int, task: str, workflow: Path): + self._cycle = cycle + self._database = database + self._rate = rate + self._task = task + self._workflow = workflow self.__connection: sqlite3.Connection | None = None self.__cursor: sqlite3.Cursor | None = None @@ -54,17 +54,17 @@ def run(self) -> bool: continue # iterate immediately to update status self._report() if initialized: - log.debug("Sleeping %s seconds", self._frequency) - sleep(self._frequency) + log.debug("Sleeping %s seconds", self._rate) + sleep(self._rate) initialized = True return True @property def _connection(self) -> sqlite3.Connection | None: if not self.__connection: - if not self.database.is_file(): + if not self._database.is_file(): return None - self.__connection = sqlite3.connect(self.database) + self.__connection = sqlite3.connect(self._database) return self.__connection @property @@ -77,21 +77,21 @@ def _cursor(self) -> sqlite3.Cursor | None: def _iterate(self) -> bool: log.info("Iterating workflow") - cmd = "rocotorun -d %s -w %s" % (self.database, self.workflow) + cmd = "rocotorun -d %s -w %s" % (self._database, self._workflow) success, _ = run_shell_cmd(cmd, quiet=True) return success @property def _query_data(self) -> dict: - return {"taskname": self.task, "cycle": int(self.cycle.timestamp())} + return {"taskname": self._task, "cycle": int(self._cycle.timestamp())} @property def _query_stmt(self) -> str: return "select state from jobs where taskname=:taskname and cycle=:cycle" def _report(self) -> None: - cmd = "rocotostat -d %s -w %s" % (self.database, self.workflow) - if self.database.is_file(): + cmd = "rocotostat -d %s -w %s" % (self._database, self._workflow) + if self._database.is_file(): log.info("Workflow status:") _, output = run_shell_cmd(cmd, quiet=True) for line in output.strip().split("\n"): @@ -111,7 +111,7 @@ def _state(self) -> str | None: @property def _state_msg(self) -> str: - return f"Rocoto task '{self.task}' for cycle {self.cycle}: %s" + return f"Rocoto task '{self._task}' for cycle {self._cycle}: %s" @property def _states(self) -> dict: @@ -141,8 +141,8 @@ def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) - return xml -def run(cycle: datetime, database: Path, task: str, workflow: Path) -> bool: - return _RocotoRunner(cycle, database, task, workflow).run() +def run(cycle: datetime, database: Path, rate: int, task: str, workflow: Path) -> bool: + return _RocotoRunner(cycle, database, rate, task, workflow).run() def validate_file(xml_file: Path | None) -> bool: diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index 2877ae3c0..cd7d5684a 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -121,6 +121,7 @@ class STR: platform: str = "platform" properties: str = "properties" quiet: str = "quiet" + rate: str = "rate" ready: str = "ready" realize: str = "realize" render: str = "render" diff --git a/src/uwtools/tests/api/test_rocoto.py b/src/uwtools/tests/api/test_rocoto.py index b1aac27e7..dabdbd8bd 100644 --- a/src/uwtools/tests/api/test_rocoto.py +++ b/src/uwtools/tests/api/test_rocoto.py @@ -17,12 +17,13 @@ def test_api_rocoto_realize(): def test_api_rocoto_run(f, utc): cycle = utc() database = f("/path/to/rocoto.db") + rate = 11 task = "foo" workflow = f("/path/to/rocoto.xml") with patch.object(rocoto, "_run") as _run: - rocoto.run(cycle=cycle, database=database, task=task, workflow=workflow) + rocoto.run(cycle=cycle, database=database, rate=rate, task=task, workflow=workflow) _run.assert_called_once_with( - cycle=cycle, database=Path(database), task=task, workflow=Path(workflow) + cycle=cycle, database=Path(database), rate=rate, task=task, workflow=Path(workflow) ) diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index 16145ca9e..4a59fb5f1 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -459,12 +459,19 @@ def test__dispatch_rocoto_realize_no_optional(): def test__dispatch_rocoto_run(utc): cycle = utc() database = Path("/path/to/rocoto.db") + rate = 11 task = "foo" workflow = Path("/path/to/rocoto.xml") - args = {STR.cycle: cycle, STR.database: database, STR.task: task, STR.workflow: workflow} + args = { + STR.cycle: cycle, + STR.database: database, + STR.rate: rate, + STR.task: task, + STR.workflow: workflow, + } with patch.object(uwtools.api.rocoto, "_run") as _run: cli._dispatch_rocoto_run(args) - _run.assert_called_once_with(cycle=cycle, database=database, task=task, workflow=workflow) + _run.assert_called_once_with(**args) def test__dispatch_rocoto_validate_xml(): From 37208d4c9de20f2fd79092c156cf7ec544d275ce Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sun, 20 Jul 2025 16:03:36 +0000 Subject: [PATCH 24/69] Fix rate in CLI --- src/uwtools/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index d3300f3d7..98c58de10 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -857,7 +857,9 @@ def _add_arg_rate(group: Group) -> None: group.add_argument( _switch(STR.rate), "-r", - help="Seconds between workflow iterations", + default=10, + help="Delay between workflow iterations (default: 10)", + metavar="SECONDS", required=False, type=int, ) From ecac20c68004fa3b63d216a30c31003f0d318473 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 06:44:39 +0000 Subject: [PATCH 25/69] Test renaming --- src/uwtools/rocoto.py | 120 +++++++++++++++---------------- src/uwtools/tests/test_rocoto.py | 104 +++++++++++++-------------- 2 files changed, 112 insertions(+), 112 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 99c697d34..0c46ae71a 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -28,6 +28,66 @@ from datetime import datetime +def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) -> str: + """ + Realize the Rocoto workflow defined in the given YAML as XML, validating both the YAML input and + XML output. + + :param config: Path to YAML input file (None => read stdin), or YAMLConfig object. + :param output_file: Path to write rendered XML file (None => write to stdout). + :return: An XML string. + """ + rxml = _RocotoXML(config) + xml = str(rxml).strip() + if not validate_string(xml): + msg = "Internal error: Invalid Rocoto XML" + raise UWError(msg) + with writable(output_file) as f: + print(xml, file=f) + return xml + + +def run(cycle: datetime, database: Path, rate: int, task: str, workflow: Path) -> bool: + return _RocotoRunner(cycle, database, rate, task, workflow).run() + + +def validate_file(xml_file: Path | None) -> bool: + """ + Validate purported Rocoto XML file against its schema. + + :param xml_file: Path to XML file (None => read stdin). + :return: Did the XML conform to the schema? + """ + with readable(xml_file) as f: + return validate_string(xml=f.read()) + + +def validate_string(xml: str) -> bool: + """ + Validate purported Rocoto XML against its schema. + + :param xml: XML to validate. + :return: Did the XML conform to the schema? + """ + tree = etree.fromstring(xml.encode("utf-8")) + path = resource_path("rocoto/schema_with_metatasks.rng") + schema = etree.RelaxNG(etree.fromstring(path.read_text())) + valid: bool = schema.validate(tree) + if valid: + log.info("Schema validation succeeded for Rocoto XML") + else: + nerr = len(schema.error_log) + log.error("%s Rocoto XML validation error%s found", nerr, "" if nerr == 1 else "s") + for err in list(schema.error_log): + log.error(err) + log.error("Invalid Rocoto XML:") + lines = xml.strip().split("\n") + fmtstr = "%{n}d %s".format(n=int(log10(len(lines))) + 1) + for n, line in enumerate(lines): + log.error(fmtstr, n + 1, line) + return valid + + class _RocotoRunner: def __init__(self, cycle: datetime, database: Path, rate: int, task: str, workflow: Path): self._cycle = cycle @@ -122,66 +182,6 @@ def _states(self) -> dict: } -def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) -> str: - """ - Realize the Rocoto workflow defined in the given YAML as XML, validating both the YAML input and - XML output. - - :param config: Path to YAML input file (None => read stdin), or YAMLConfig object. - :param output_file: Path to write rendered XML file (None => write to stdout). - :return: An XML string. - """ - rxml = _RocotoXML(config) - xml = str(rxml).strip() - if not validate_string(xml): - msg = "Internal error: Invalid Rocoto XML" - raise UWError(msg) - with writable(output_file) as f: - print(xml, file=f) - return xml - - -def run(cycle: datetime, database: Path, rate: int, task: str, workflow: Path) -> bool: - return _RocotoRunner(cycle, database, rate, task, workflow).run() - - -def validate_file(xml_file: Path | None) -> bool: - """ - Validate purported Rocoto XML file against its schema. - - :param xml_file: Path to XML file (None => read stdin). - :return: Did the XML conform to the schema? - """ - with readable(xml_file) as f: - return validate_string(xml=f.read()) - - -def validate_string(xml: str) -> bool: - """ - Validate purported Rocoto XML against its schema. - - :param xml: XML to validate. - :return: Did the XML conform to the schema? - """ - tree = etree.fromstring(xml.encode("utf-8")) - path = resource_path("rocoto/schema_with_metatasks.rng") - schema = etree.RelaxNG(etree.fromstring(path.read_text())) - valid: bool = schema.validate(tree) - if valid: - log.info("Schema validation succeeded for Rocoto XML") - else: - nerr = len(schema.error_log) - log.error("%s Rocoto XML validation error%s found", nerr, "" if nerr == 1 else "s") - for err in list(schema.error_log): - log.error(err) - log.error("Invalid Rocoto XML:") - lines = xml.strip().split("\n") - fmtstr = "%{n}d %s".format(n=int(log10(len(lines))) + 1) - for n, line in enumerate(lines): - log.error(fmtstr, n + 1, line) - return valid - - class _RocotoXML: """ Generate a Rocoto XML document from a YAML config. diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index ca090e80c..c6bca4b3e 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -34,7 +34,7 @@ def validation_assets(tmp_path): # Tests -def test_realize_rocoto_invalid_xml(assets): +def test_rocoto_realize_rocoto_invalid_xml(assets): cfgfile, outfile = assets with patch.object(rocoto, "validate_string") as vrxs: vrxs.return_value = False @@ -42,48 +42,48 @@ def test_realize_rocoto_invalid_xml(assets): rocoto.realize(config=cfgfile, output_file=outfile) -def test_realize_cfg_to_file(assets): +def test_rocoto_realize_cfg_to_file(assets): cfgfile, outfile = assets rocoto.realize(config=YAMLConfig(cfgfile), output_file=outfile) assert rocoto.validate_file(xml_file=outfile) -def test_realize_file_to_file(assets): +def test_rocoto_realize_file_to_file(assets): cfgfile, outfile = assets rocoto.realize(config=cfgfile, output_file=outfile) assert rocoto.validate_file(xml_file=outfile) -def test_realize_cfg_to_stdout(capsys, assets): +def test_rocoto_realize_cfg_to_stdout(capsys, assets): cfgfile, outfile = assets rocoto.realize(config=YAMLConfig(cfgfile)) outfile.write_text(capsys.readouterr().out) assert rocoto.validate_file(xml_file=outfile) -def test_realize_file_to_stdout(capsys, assets): +def test_rocoto_realize_file_to_stdout(capsys, assets): cfgfile, outfile = assets rocoto.realize(config=cfgfile) outfile.write_text(capsys.readouterr().out) assert rocoto.validate_file(xml_file=outfile) -def test_validate_file_fail(validation_assets): +def test_rocoto_validate_file_fail(validation_assets): xml_file_bad, _, _, _ = validation_assets assert rocoto.validate_file(xml_file=xml_file_bad) is False -def test_validate_file_pass(validation_assets): +def test_rocoto_validate_file_pass(validation_assets): _, xml_file_good, _, _ = validation_assets assert rocoto.validate_file(xml_file=xml_file_good) is True -def test_validate_string_fail(validation_assets): +def test_rocoto_validate_string_fail(validation_assets): _, _, xml_string_bad, _ = validation_assets assert rocoto.validate_string(xml=xml_string_bad) is False -def test_validate_string_pass(validation_assets): +def test_rocoto_validate_string_pass(validation_assets): _, _, _, xml_string_good = validation_assets assert rocoto.validate_string(xml=xml_string_good) is True @@ -102,18 +102,18 @@ def instance(self, assets): def root(self): return rocoto.Element("root") - def test_instantiate_from_cfgobj(self, assets): + def test__RocotoXML_instantiate_from_cfgobj(self, assets): cfgfile, _ = assets assert rocoto._RocotoXML(config=YAMLConfig(cfgfile))._root.tag == "workflow" @mark.parametrize("config", ["bar", 42]) - def test__add_compound_time_string_basic(self, config, instance, root): + def test_rocoto__RocotoXML__add_compound_time_string_basic(self, config, instance, root): instance._add_compound_time_string(e=root, config=config, tag="foo") child = root[0] assert child.tag == "foo" assert child.text == str(config) - def test__add_compound_time_string_cyclestr(self, instance, root): + def test_rocoto__RocotoXML__add_compound_time_string_cyclestr(self, instance, root): config = {"cyclestr": {"attrs": {"offset": "00:05:00"}, "value": "qux"}} errors = schema_validator("rocoto", "$defs", "cycleString") assert not errors(config) @@ -122,7 +122,7 @@ def test__add_compound_time_string_cyclestr(self, instance, root): assert cyclestr.get("offset") == "00:05:00" assert cyclestr.text == "qux" - def test__add_compound_time_string_list(self, instance, root): + def test_rocoto__RocotoXML__add_compound_time_string_list(self, instance, root): config = [ "cycle-", {"cyclestr": {"value": "%s"}}, @@ -146,7 +146,7 @@ def test__add_compound_time_string_list(self, instance, root): instance._add_compound_time_string(e=root, config=config, tag="a") assert etree.tostring(root[0]).decode("utf-8") == xml - def test__add_metatask(self, instance, root): + def test_rocoto__RocotoXML__add_metatask(self, instance, root): config = { "attrs": {"mode": "parallel", "throttle": 42}, "var": {"baz": "3", "qux": "4"}, @@ -180,7 +180,7 @@ def test__add_metatask(self, instance, root): "nest", ) - def test__add_task(self, instance, root): + def test_rocoto__RocotoXML__add_task(self, instance, root): config = { "attrs": {"foo": "1", "bar": "2"}, "account": "baz", @@ -206,12 +206,12 @@ def test__add_task(self, instance, root): mocks["_add_task_envar"].assert_called_once_with(task, "A", "apple") @mark.parametrize("cores", [1, "1"]) - def test__add_task_cores_int_or_str(self, cores, instance, root): + def test_rocoto__RocotoXML__add_task_cores_int_or_str(self, cores, instance, root): # Ensure that either int or str "cores" values are accepted. config = {"command": "c", "cores": cores, "walltime": "00:00:01"} instance._add_task(e=root, config=config, name_attr="foo") - def test__add_task_dependency_and(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency_and(self, instance, root): config = {"and": {"or_get_obs": {"taskdep": {"attrs": {"task": "foo"}}}}} errors = schema_validator("rocoto", "$defs", "dependency") assert not errors(config) @@ -226,7 +226,7 @@ def test__add_task_dependency_and(self, instance, root): "value", ["/some/file", {"cyclestr": {"value": "@Y@m@d@H", "attrs": {"offset": "06:00:00"}}}], ) - def test__add_task_dependency_datadep(self, instance, root, value): + def test_rocoto__RocotoXML__add_task_dependency_datadep(self, instance, root, value): age = "00:00:02:00" minsize = "1K" config = {"datadep": {"attrs": {"age": age, "minsize": minsize}, "value": value}} @@ -241,17 +241,17 @@ def test__add_task_dependency_datadep(self, instance, root, value): assert child.get("minsize") == minsize assert child.text == value if isinstance(value, str) else value["cyclestr"]["value"] - def test__add_task_dependency_fail(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency_fail(self, instance, root): config = {"unrecognized": "whatever"} with raises(UWConfigError): instance._add_task_dependency(e=root, config=config) - def test__add_task_dependency_fail_bad_operand(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency_fail_bad_operand(self, instance, root): config = {"and": {"unrecognized": "whatever"}} with raises(UWConfigError): instance._add_task_dependency(e=root, config=config) - def test__add_task_dependency_metataskdep(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency_metataskdep(self, instance, root): config = {"metataskdep": {"attrs": {"metatask": "foo"}}} errors = schema_validator("rocoto", "$defs", "dependency") assert not errors(config) @@ -266,7 +266,7 @@ def test__add_task_dependency_metataskdep(self, instance, root): "tag_config", [("and", {"strneq": {"left": "&RUN_GSI;", "right": "YES"}})], ) - def test__add_task_dependency_operator(self, instance, root, tag_config): + def test_rocoto__RocotoXML__add_task_dependency_operator(self, instance, root, tag_config): tag, config = tag_config errors = schema_validator("rocoto", "$defs", "dependency") assert not errors(config) @@ -274,7 +274,7 @@ def test__add_task_dependency_operator(self, instance, root, tag_config): for tag in config: assert tag == next(iter(config)) - def test__add_task_dependency_operator_datadep_operand(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency_operator_datadep_operand(self, instance, root): value = "/some/file" config = {"value": value} errors = schema_validator("rocoto", "$defs", "dependency") @@ -284,7 +284,7 @@ def test__add_task_dependency_operator_datadep_operand(self, instance, root): assert e.tag == "datadep" assert e.text == value - def test__add_task_dependency_operator_task_operand(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency_operator_task_operand(self, instance, root): taskname = "some-task" config = {"attrs": {"task": taskname}} errors = schema_validator("rocoto", "$defs", "dependency") @@ -294,7 +294,7 @@ def test__add_task_dependency_operator_task_operand(self, instance, root): assert e.tag == "taskdep" assert e.get("task") == taskname - def test__add_task_dependency_operator_timedep_operand(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency_operator_timedep_operand(self, instance, root): value = 20230103120000 config = value errors = schema_validator("rocoto", "$defs", "compoundTimeString") @@ -304,7 +304,7 @@ def test__add_task_dependency_operator_timedep_operand(self, instance, root): assert e.tag == "timedep" assert e.text == str(value) - def test__add_task_dependency_sh__no_attrs(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency_sh__no_attrs(self, instance, root): config = {"sh_foo": {"command": "ls"}} errors = schema_validator("rocoto", "$defs", "dependency") assert not errors(config) @@ -316,7 +316,7 @@ def test__add_task_dependency_sh__no_attrs(self, instance, root): assert sh.get("name") == "foo" assert sh.text == "ls" - def test__add_task_dependency_sh__with_attrs(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency_sh__with_attrs(self, instance, root): config = {"sh_foo": {"attrs": {"runopt": "-c", "shell": "/bin/bash"}, "command": "ls"}} errors = schema_validator("rocoto", "$defs", "dependency") assert not errors(config) @@ -330,7 +330,7 @@ def test__add_task_dependency_sh__with_attrs(self, instance, root): assert sh.get("shell") == "/bin/bash" assert sh.text == "ls" - def test__add_task_dependency_streq(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency_streq(self, instance, root): config = {"streq": {"left": "&RUN_GSI;", "right": "YES"}} errors = schema_validator("rocoto", "$defs", "dependency") assert not errors(config) @@ -349,7 +349,7 @@ def test__add_task_dependency_streq(self, instance, root): ("strneq", {"left": "&RUN_GSI;", "right": "YES"}), ], ) - def test__add_task_dependency_strequality(self, config, instance, root): + def test_rocoto__RocotoXML__add_task_dependency_strequality(self, config, instance, root): errors = schema_validator("rocoto", "$defs", "dependency") tag, config = config assert not errors({tag: config}) @@ -359,7 +359,7 @@ def test__add_task_dependency_strequality(self, config, instance, root): for idx, val in enumerate(config.values()): assert element[idx].text == val - def test__add_task_dependency_taskdep(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency_taskdep(self, instance, root): config = {"taskdep": {"attrs": {"task": "foo"}}} errors = schema_validator("rocoto", "$defs", "dependency") assert not errors(config) @@ -370,7 +370,7 @@ def test__add_task_dependency_taskdep(self, instance, root): assert child.tag == "taskdep" assert child.get("task") == "foo" - def test__add_task_dependency_taskvalid(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency_taskvalid(self, instance, root): config = {"taskvalid": {"attrs": {"task": "foo"}}} errors = schema_validator("rocoto", "$defs", "dependency") assert not errors(config) @@ -389,7 +389,7 @@ def test__add_task_dependency_taskvalid(self, instance, root): {"cyclestr": {"value": "@Y@m@d@H", "attrs": {"offset": "06:00:00"}}}, ], ) - def test__add_task_dependency_timedep(self, instance, root, value): + def test_rocoto__RocotoXML__add_task_dependency_timedep(self, instance, root, value): config = {"timedep": value} errors = schema_validator("rocoto", "$defs", "dependency") assert not errors(config) @@ -403,27 +403,27 @@ def test__add_task_dependency_timedep(self, instance, root, value): else: assert child.text == str(value) - def test__config_validate_config(self, assets, instance): + def test_rocoto__RocotoXML__config_validate_config(self, assets, instance): cfgfile, _ = assets instance._config_validate(config=YAMLConfig(cfgfile)) - def test__config_validate_file(self, assets, instance): + def test_rocoto__RocotoXML__config_validate_file(self, assets, instance): cfgfile, _ = assets instance._config_validate(config=cfgfile) - def test__config_validate_config_fail(self, instance, tmp_path): + def test_rocoto__RocotoXML__config_validate_config_fail(self, instance, tmp_path): cfgfile = tmp_path / "bad.yaml" cfgfile.write_text("not: ok") with raises(UWConfigError): instance._config_validate(config=YAMLConfig(cfgfile)) - def test__config_validate_file_fail(self, instance, tmp_path): + def test_rocoto__RocotoXML__config_validate_file_fail(self, instance, tmp_path): cfgfile = tmp_path / "bad.yaml" cfgfile.write_text("not: ok") with raises(UWConfigError): instance._config_validate(config=cfgfile) - def test__add_task_envar(self, instance, root): + def test_rocoto__RocotoXML__add_task_envar(self, instance, root): instance._add_task_envar(root, "foo", "bar") envar = root[0] name, value = envar @@ -432,7 +432,7 @@ def test__add_task_envar(self, instance, root): assert value.tag == "value" assert value.text == "bar" - def test__add_task_envar_compound(self, instance, root): + def test_rocoto__RocotoXML__add_task_envar_compound(self, instance, root): instance._add_task_envar(root, "foo", {"cyclestr": {"value": "bar_@Y"}}) envar = root[0] name, value = envar @@ -443,7 +443,7 @@ def test__add_task_envar_compound(self, instance, root): assert value.text is None assert child.text == "bar_@Y" - def test__add_workflow(self, instance): + def test_rocoto__RocotoXML__add_workflow(self, instance): config = { "workflow": { "attrs": {"realtime": True, "scheduler": "slurm"}, @@ -472,7 +472,7 @@ def test__add_workflow(self, instance): mocks["_add_workflow_log"].assert_called_once_with(workflow, config["workflow"]) mocks["_add_workflow_tasks"].assert_called_once_with(workflow, config["workflow"]["tasks"]) - def test__add_workflow_cycledef(self, instance, root): + def test_rocoto__RocotoXML__add_workflow_cycledef(self, instance, root): config: list[dict] = [ {"attrs": {"group": "g1"}, "spec": "t1"}, {"attrs": {"group": "g2"}, "spec": "t2"}, @@ -485,70 +485,70 @@ def test__add_workflow_cycledef(self, instance, root): assert root[i].tag == "cycledef" assert root[i].text == item["spec"] - def test__add_workflow_log_basic(self, instance, root): + def test_rocoto__RocotoXML__add_workflow_log_basic(self, instance, root): val = "/path/to/logfile" instance._add_workflow_log(e=root, config={"log": {"value": val}}) log = root[0] assert log.tag == "log" assert log.text == val - def test__add_workflow_log_cyclestr(self, instance, root): + def test_rocoto__RocotoXML__add_workflow_log_cyclestr(self, instance, root): val = "/path/to/logfile-@Y@m@d@H" instance._add_workflow_log(e=root, config={"log": {"value": {"cyclestr": {"value": val}}}}) log = root[0] assert log.tag == "log" assert log.xpath("cyclestr")[0].text == val - def test__add_workflow_log_verbosity(self, instance, root): + def test_rocoto__RocotoXML__add_workflow_log_verbosity(self, instance, root): val = "10" config = {"log": {"attrs": {"verbosity": 10}, "value": {"cyclestr": {"value": val}}}} instance._add_workflow_log(e=root, config=config) log = root[0] assert log.attrib["verbosity"] == "10" - def test__add_workflow_tasks(self, instance, root): + def test_rocoto__RocotoXML__add_workflow_tasks(self, instance, root): config = {"metatask_foo": "1", "task_bar": "2"} with patch.multiple(instance, _add_metatask=D, _add_task=D) as mocks: instance._add_workflow_tasks(e=root, config=config) mocks["_add_metatask"].assert_called_once_with(root, "1", "foo") mocks["_add_task"].assert_called_once_with(root, "2", "bar") - def test__doctype_entities(self, instance): + def test_rocoto__RocotoXML__doctype_entities(self, instance): assert '' in instance._doctype assert '' in instance._doctype - def test__doctype_entities_none(self, instance): + def test_rocoto__RocotoXML__doctype_entities_none(self, instance): del instance._config["workflow"]["entities"] assert instance._doctype is None - def test__insert_doctype(self, instance): + def test_rocoto__RocotoXML__insert_doctype(self, instance): with patch.object(rocoto._RocotoXML, "_doctype", new_callable=PropertyMock) as _doctype: _doctype.return_value = "bar" assert instance._insert_doctype("foo\nbaz\n") == "foo\nbar\nbaz\n" - def test__insert_doctype_none(self, instance): + def test_rocoto__RocotoXML__insert_doctype_none(self, instance): with patch.object(rocoto._RocotoXML, "_doctype", new_callable=PropertyMock) as _doctype: _doctype.return_value = None assert instance._insert_doctype("foo\nbaz\n") == "foo\nbaz\n" - def test__setattrs(self, instance, root): + def test_rocoto__RocotoXML__setattrs(self, instance, root): config = {"attrs": {"foo": "1", "bar": "2"}} instance._set_attrs(e=root, config=config) assert root.get("foo") == "1" assert root.get("bar") == "2" - def test__set_and_render_jobname(self, instance): + def test_rocoto__RocotoXML__set_and_render_jobname(self, instance): config = {"join": "{{jobname}}.log"} cfg = instance._set_and_render_jobname(config, "foo") assert cfg["join"] == "foo.log" assert cfg["jobname"] == "foo" - def test__tag_name(self, instance): + def test_rocoto__RocotoXML__tag_name(self, instance): assert instance._tag_name("foo") == ("foo", "") assert instance._tag_name("foo_bar") == ("foo", "bar") assert instance._tag_name("foo_bar_baz") == ("foo", "bar_baz") - def test_dump(self, instance, tmp_path): + def test_rocoto__RocotoXML_dump(self, instance, tmp_path): path = tmp_path / "out.xml" instance.dump(path=path) assert rocoto.validate_file(path) From 8c9c0a25db2c377d2d1ecd8695eec30343231394 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 06:52:50 +0000 Subject: [PATCH 26/69] Work on tests [skip ci] --- src/uwtools/tests/test_rocoto.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index c6bca4b3e..ce149049e 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -68,6 +68,29 @@ def test_rocoto_realize_file_to_stdout(capsys, assets): assert rocoto.validate_file(xml_file=outfile) +# PM# + + +@fixture +def runargs(utc, tmp_path): + return { + "cycle": utc(), + "database": tmp_path / "rocoto.db", + "rate": 11, + "task": "foo", + "workflow": tmp_path / "rocoto.xml", + } + + +def test_rocoto_run(runargs): + with patch.object(rocoto, "_RocotoRunner") as _RocotoRunner: # noqa: N806 + rocoto.run(**runargs) + _RocotoRunner.assert_called_once_with(*runargs.values()) + + +# PM# + + def test_rocoto_validate_file_fail(validation_assets): xml_file_bad, _, _, _ = validation_assets assert rocoto.validate_file(xml_file=xml_file_bad) is False From ce0375624c920c3a3e2bd8463d57f85c3f6c801f Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 14:35:39 +0000 Subject: [PATCH 27/69] Work on tests [skip ci] --- src/uwtools/rocoto.py | 20 ++-- src/uwtools/tests/test_rocoto.py | 172 +++++++++++++++++-------------- 2 files changed, 103 insertions(+), 89 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 0c46ae71a..eb0b5053b 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -95,12 +95,12 @@ def __init__(self, cycle: datetime, database: Path, rate: int, task: str, workfl self._rate = rate self._task = task self._workflow = workflow - self.__connection: sqlite3.Connection | None = None - self.__cursor: sqlite3.Cursor | None = None + self._con: sqlite3.Connection | None = None + self._cur: sqlite3.Cursor | None = None def __del__(self): - if self.__connection: - self.__connection.close() + if self._con: + self._con.close() def run(self) -> bool: initialized = False @@ -121,19 +121,19 @@ def run(self) -> bool: @property def _connection(self) -> sqlite3.Connection | None: - if not self.__connection: + if not self._con: if not self._database.is_file(): return None - self.__connection = sqlite3.connect(self._database) - return self.__connection + self._con = sqlite3.connect(self._database) + return self._con @property def _cursor(self) -> sqlite3.Cursor | None: - if not self.__cursor: + if not self._cur: if not (connection := self._connection): return None - self.__cursor = connection.cursor() - return self.__cursor + self._cur = connection.cursor() + return self._cur def _iterate(self) -> bool: log.info("Iterating workflow") diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index ce149049e..196ed0fc3 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -3,7 +3,7 @@ """ from unittest.mock import DEFAULT as D -from unittest.mock import PropertyMock, patch +from unittest.mock import Mock, PropertyMock, patch from lxml import etree from pytest import fixture, mark, raises @@ -21,6 +21,17 @@ def assets(tmp_path): return fixture_path("hello_workflow.yaml"), tmp_path / "rocoto.xml" +@fixture +def runargs(utc, tmp_path): + return { + "cycle": utc(), + "database": tmp_path / "rocoto.db", + "rate": 11, + "task": "foo", + "workflow": tmp_path / "rocoto.xml", + } + + @fixture def validation_assets(tmp_path): xml_file_good = fixture_path("hello_workflow.xml") @@ -34,7 +45,7 @@ def validation_assets(tmp_path): # Tests -def test_rocoto_realize_rocoto_invalid_xml(assets): +def test_rocoto_realize__rocoto_invalid_xml(assets): cfgfile, outfile = assets with patch.object(rocoto, "validate_string") as vrxs: vrxs.return_value = False @@ -42,80 +53,79 @@ def test_rocoto_realize_rocoto_invalid_xml(assets): rocoto.realize(config=cfgfile, output_file=outfile) -def test_rocoto_realize_cfg_to_file(assets): +def test_rocoto_realize__cfg_to_file(assets): cfgfile, outfile = assets rocoto.realize(config=YAMLConfig(cfgfile), output_file=outfile) assert rocoto.validate_file(xml_file=outfile) -def test_rocoto_realize_file_to_file(assets): +def test_rocoto_realize__file_to_file(assets): cfgfile, outfile = assets rocoto.realize(config=cfgfile, output_file=outfile) assert rocoto.validate_file(xml_file=outfile) -def test_rocoto_realize_cfg_to_stdout(capsys, assets): +def test_rocoto_realize__cfg_to_stdout(capsys, assets): cfgfile, outfile = assets rocoto.realize(config=YAMLConfig(cfgfile)) outfile.write_text(capsys.readouterr().out) assert rocoto.validate_file(xml_file=outfile) -def test_rocoto_realize_file_to_stdout(capsys, assets): +def test_rocoto_realize__file_to_stdout(capsys, assets): cfgfile, outfile = assets rocoto.realize(config=cfgfile) outfile.write_text(capsys.readouterr().out) assert rocoto.validate_file(xml_file=outfile) -# PM# - - -@fixture -def runargs(utc, tmp_path): - return { - "cycle": utc(), - "database": tmp_path / "rocoto.db", - "rate": 11, - "task": "foo", - "workflow": tmp_path / "rocoto.xml", - } - - def test_rocoto_run(runargs): with patch.object(rocoto, "_RocotoRunner") as _RocotoRunner: # noqa: N806 rocoto.run(**runargs) _RocotoRunner.assert_called_once_with(*runargs.values()) -# PM# - - -def test_rocoto_validate_file_fail(validation_assets): +def test_rocoto_validate__file_fail(validation_assets): xml_file_bad, _, _, _ = validation_assets assert rocoto.validate_file(xml_file=xml_file_bad) is False -def test_rocoto_validate_file_pass(validation_assets): +def test_rocoto_validate__file_pass(validation_assets): _, xml_file_good, _, _ = validation_assets assert rocoto.validate_file(xml_file=xml_file_good) is True -def test_rocoto_validate_string_fail(validation_assets): +def test_rocoto_validate__string_fail(validation_assets): _, _, xml_string_bad, _ = validation_assets assert rocoto.validate_string(xml=xml_string_bad) is False -def test_rocoto_validate_string_pass(validation_assets): +def test_rocoto_validate__string_pass(validation_assets): _, _, _, xml_string_good = validation_assets assert rocoto.validate_string(xml=xml_string_good) is True +# PM# + + +def test_rocoto__RocotoRunner__init_and_del(runargs): + rr = rocoto._RocotoRunner(**runargs) + con = Mock() + rr._con = con + del rr + con.close.assert_called_once_with() + + +# PM# + + class TestRocotoXML: """ Tests for class uwtools.rocoto._RocotoXML. """ + # Fixtures + @fixture def instance(self, assets): cfgfile, _ = assets @@ -125,18 +135,25 @@ def instance(self, assets): def root(self): return rocoto.Element("root") - def test__RocotoXML_instantiate_from_cfgobj(self, assets): + # Tests + + def test__RocotoXML__instantiate_from_cfgobj(self, assets): cfgfile, _ = assets assert rocoto._RocotoXML(config=YAMLConfig(cfgfile))._root.tag == "workflow" + def test_rocoto__RocotoXML_dump(self, instance, tmp_path): + path = tmp_path / "out.xml" + instance.dump(path=path) + assert rocoto.validate_file(path) + @mark.parametrize("config", ["bar", 42]) - def test_rocoto__RocotoXML__add_compound_time_string_basic(self, config, instance, root): + def test_rocoto__RocotoXML__add_compound_time_string__basic(self, config, instance, root): instance._add_compound_time_string(e=root, config=config, tag="foo") child = root[0] assert child.tag == "foo" assert child.text == str(config) - def test_rocoto__RocotoXML__add_compound_time_string_cyclestr(self, instance, root): + def test_rocoto__RocotoXML__add_compound_time_string__cyclestr(self, instance, root): config = {"cyclestr": {"attrs": {"offset": "00:05:00"}, "value": "qux"}} errors = schema_validator("rocoto", "$defs", "cycleString") assert not errors(config) @@ -145,7 +162,7 @@ def test_rocoto__RocotoXML__add_compound_time_string_cyclestr(self, instance, ro assert cyclestr.get("offset") == "00:05:00" assert cyclestr.text == "qux" - def test_rocoto__RocotoXML__add_compound_time_string_list(self, instance, root): + def test_rocoto__RocotoXML__add_compound_time_string__list(self, instance, root): config = [ "cycle-", {"cyclestr": {"value": "%s"}}, @@ -229,12 +246,12 @@ def test_rocoto__RocotoXML__add_task(self, instance, root): mocks["_add_task_envar"].assert_called_once_with(task, "A", "apple") @mark.parametrize("cores", [1, "1"]) - def test_rocoto__RocotoXML__add_task_cores_int_or_str(self, cores, instance, root): + def test_rocoto__RocotoXML__add_task__cores_int_or_str(self, cores, instance, root): # Ensure that either int or str "cores" values are accepted. config = {"command": "c", "cores": cores, "walltime": "00:00:01"} instance._add_task(e=root, config=config, name_attr="foo") - def test_rocoto__RocotoXML__add_task_dependency_and(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency__and(self, instance, root): config = {"and": {"or_get_obs": {"taskdep": {"attrs": {"task": "foo"}}}}} errors = schema_validator("rocoto", "$defs", "dependency") assert not errors(config) @@ -264,12 +281,12 @@ def test_rocoto__RocotoXML__add_task_dependency_datadep(self, instance, root, va assert child.get("minsize") == minsize assert child.text == value if isinstance(value, str) else value["cyclestr"]["value"] - def test_rocoto__RocotoXML__add_task_dependency_fail(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency__fail(self, instance, root): config = {"unrecognized": "whatever"} with raises(UWConfigError): instance._add_task_dependency(e=root, config=config) - def test_rocoto__RocotoXML__add_task_dependency_fail_bad_operand(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency__fail_bad_operand(self, instance, root): config = {"and": {"unrecognized": "whatever"}} with raises(UWConfigError): instance._add_task_dependency(e=root, config=config) @@ -289,7 +306,7 @@ def test_rocoto__RocotoXML__add_task_dependency_metataskdep(self, instance, root "tag_config", [("and", {"strneq": {"left": "&RUN_GSI;", "right": "YES"}})], ) - def test_rocoto__RocotoXML__add_task_dependency_operator(self, instance, root, tag_config): + def test_rocoto__RocotoXML__add_task_dependency__operator(self, instance, root, tag_config): tag, config = tag_config errors = schema_validator("rocoto", "$defs", "dependency") assert not errors(config) @@ -297,7 +314,7 @@ def test_rocoto__RocotoXML__add_task_dependency_operator(self, instance, root, t for tag in config: assert tag == next(iter(config)) - def test_rocoto__RocotoXML__add_task_dependency_operator_datadep_operand(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency__operator_datadep_operand(self, instance, root): value = "/some/file" config = {"value": value} errors = schema_validator("rocoto", "$defs", "dependency") @@ -307,7 +324,7 @@ def test_rocoto__RocotoXML__add_task_dependency_operator_datadep_operand(self, i assert e.tag == "datadep" assert e.text == value - def test_rocoto__RocotoXML__add_task_dependency_operator_task_operand(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency__operator_task_operand(self, instance, root): taskname = "some-task" config = {"attrs": {"task": taskname}} errors = schema_validator("rocoto", "$defs", "dependency") @@ -317,7 +334,9 @@ def test_rocoto__RocotoXML__add_task_dependency_operator_task_operand(self, inst assert e.tag == "taskdep" assert e.get("task") == taskname - def test_rocoto__RocotoXML__add_task_dependency_operator_timedep_operand(self, instance, root): + def test_rocoto__RocotoXML__add_task__dependency__operator_timedep_operand( + self, instance, root + ): value = 20230103120000 config = value errors = schema_validator("rocoto", "$defs", "compoundTimeString") @@ -327,7 +346,7 @@ def test_rocoto__RocotoXML__add_task_dependency_operator_timedep_operand(self, i assert e.tag == "timedep" assert e.text == str(value) - def test_rocoto__RocotoXML__add_task_dependency_sh__no_attrs(self, instance, root): + def test_rocoto__RocotoXML__add_task__dependency_sh__no_attrs(self, instance, root): config = {"sh_foo": {"command": "ls"}} errors = schema_validator("rocoto", "$defs", "dependency") assert not errors(config) @@ -353,7 +372,7 @@ def test_rocoto__RocotoXML__add_task_dependency_sh__with_attrs(self, instance, r assert sh.get("shell") == "/bin/bash" assert sh.text == "ls" - def test_rocoto__RocotoXML__add_task_dependency_streq(self, instance, root): + def test_rocoto__RocotoXML__add_task_dependency__streq(self, instance, root): config = {"streq": {"left": "&RUN_GSI;", "right": "YES"}} errors = schema_validator("rocoto", "$defs", "dependency") assert not errors(config) @@ -426,26 +445,6 @@ def test_rocoto__RocotoXML__add_task_dependency_timedep(self, instance, root, va else: assert child.text == str(value) - def test_rocoto__RocotoXML__config_validate_config(self, assets, instance): - cfgfile, _ = assets - instance._config_validate(config=YAMLConfig(cfgfile)) - - def test_rocoto__RocotoXML__config_validate_file(self, assets, instance): - cfgfile, _ = assets - instance._config_validate(config=cfgfile) - - def test_rocoto__RocotoXML__config_validate_config_fail(self, instance, tmp_path): - cfgfile = tmp_path / "bad.yaml" - cfgfile.write_text("not: ok") - with raises(UWConfigError): - instance._config_validate(config=YAMLConfig(cfgfile)) - - def test_rocoto__RocotoXML__config_validate_file_fail(self, instance, tmp_path): - cfgfile = tmp_path / "bad.yaml" - cfgfile.write_text("not: ok") - with raises(UWConfigError): - instance._config_validate(config=cfgfile) - def test_rocoto__RocotoXML__add_task_envar(self, instance, root): instance._add_task_envar(root, "foo", "bar") envar = root[0] @@ -455,7 +454,7 @@ def test_rocoto__RocotoXML__add_task_envar(self, instance, root): assert value.tag == "value" assert value.text == "bar" - def test_rocoto__RocotoXML__add_task_envar_compound(self, instance, root): + def test_rocoto__RocotoXML__add_task_envar__compound(self, instance, root): instance._add_task_envar(root, "foo", {"cyclestr": {"value": "bar_@Y"}}) envar = root[0] name, value = envar @@ -508,21 +507,21 @@ def test_rocoto__RocotoXML__add_workflow_cycledef(self, instance, root): assert root[i].tag == "cycledef" assert root[i].text == item["spec"] - def test_rocoto__RocotoXML__add_workflow_log_basic(self, instance, root): + def test_rocoto__RocotoXML__add_workflow_log__basic(self, instance, root): val = "/path/to/logfile" instance._add_workflow_log(e=root, config={"log": {"value": val}}) log = root[0] assert log.tag == "log" assert log.text == val - def test_rocoto__RocotoXML__add_workflow_log_cyclestr(self, instance, root): + def test_rocoto__RocotoXML__add_workflow_log__cyclestr(self, instance, root): val = "/path/to/logfile-@Y@m@d@H" instance._add_workflow_log(e=root, config={"log": {"value": {"cyclestr": {"value": val}}}}) log = root[0] assert log.tag == "log" assert log.xpath("cyclestr")[0].text == val - def test_rocoto__RocotoXML__add_workflow_log_verbosity(self, instance, root): + def test_rocoto__RocotoXML__add_workflow_log__verbosity(self, instance, root): val = "10" config = {"log": {"attrs": {"verbosity": 10}, "value": {"cyclestr": {"value": val}}}} instance._add_workflow_log(e=root, config=config) @@ -536,11 +535,31 @@ def test_rocoto__RocotoXML__add_workflow_tasks(self, instance, root): mocks["_add_metatask"].assert_called_once_with(root, "1", "foo") mocks["_add_task"].assert_called_once_with(root, "2", "bar") - def test_rocoto__RocotoXML__doctype_entities(self, instance): + def test_rocoto__RocotoXML__config_validate__config(self, assets, instance): + cfgfile, _ = assets + instance._config_validate(config=YAMLConfig(cfgfile)) + + def test_rocoto__RocotoXML__config_validate_file(self, assets, instance): + cfgfile, _ = assets + instance._config_validate(config=cfgfile) + + def test_rocoto__RocotoXML__config_validate__config_fail(self, instance, tmp_path): + cfgfile = tmp_path / "bad.yaml" + cfgfile.write_text("not: ok") + with raises(UWConfigError): + instance._config_validate(config=YAMLConfig(cfgfile)) + + def test_rocoto__RocotoXML__config_validate__file_fail(self, instance, tmp_path): + cfgfile = tmp_path / "bad.yaml" + cfgfile.write_text("not: ok") + with raises(UWConfigError): + instance._config_validate(config=cfgfile) + + def test_rocoto__RocotoXML__doctype__entities(self, instance): assert '' in instance._doctype assert '' in instance._doctype - def test_rocoto__RocotoXML__doctype_entities_none(self, instance): + def test_rocoto__RocotoXML__doctype__entities_none(self, instance): del instance._config["workflow"]["entities"] assert instance._doctype is None @@ -549,29 +568,24 @@ def test_rocoto__RocotoXML__insert_doctype(self, instance): _doctype.return_value = "bar" assert instance._insert_doctype("foo\nbaz\n") == "foo\nbar\nbaz\n" - def test_rocoto__RocotoXML__insert_doctype_none(self, instance): + def test_rocoto__RocotoXML__insert_doctype__none(self, instance): with patch.object(rocoto._RocotoXML, "_doctype", new_callable=PropertyMock) as _doctype: _doctype.return_value = None assert instance._insert_doctype("foo\nbaz\n") == "foo\nbaz\n" - def test_rocoto__RocotoXML__setattrs(self, instance, root): - config = {"attrs": {"foo": "1", "bar": "2"}} - instance._set_attrs(e=root, config=config) - assert root.get("foo") == "1" - assert root.get("bar") == "2" - def test_rocoto__RocotoXML__set_and_render_jobname(self, instance): config = {"join": "{{jobname}}.log"} cfg = instance._set_and_render_jobname(config, "foo") assert cfg["join"] == "foo.log" assert cfg["jobname"] == "foo" + def test_rocoto__RocotoXML__set_attrs(self, instance, root): + config = {"attrs": {"foo": "1", "bar": "2"}} + instance._set_attrs(e=root, config=config) + assert root.get("foo") == "1" + assert root.get("bar") == "2" + def test_rocoto__RocotoXML__tag_name(self, instance): assert instance._tag_name("foo") == ("foo", "") assert instance._tag_name("foo_bar") == ("foo", "bar") assert instance._tag_name("foo_bar_baz") == ("foo", "bar_baz") - - def test_rocoto__RocotoXML_dump(self, instance, tmp_path): - path = tmp_path / "out.xml" - instance.dump(path=path) - assert rocoto.validate_file(path) From 5bbbff531525410c4ebe6ab2b3cbef3b0fd68096 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 14:43:09 +0000 Subject: [PATCH 28/69] Work on tests [skip ci] --- src/uwtools/tests/test_rocoto.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index 196ed0fc3..a96717d69 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -22,7 +22,7 @@ def assets(tmp_path): @fixture -def runargs(utc, tmp_path): +def rocoto_runner_args(utc, tmp_path): return { "cycle": utc(), "database": tmp_path / "rocoto.db", @@ -79,10 +79,10 @@ def test_rocoto_realize__file_to_stdout(capsys, assets): assert rocoto.validate_file(xml_file=outfile) -def test_rocoto_run(runargs): +def test_rocoto_run(rocoto_runner_args): with patch.object(rocoto, "_RocotoRunner") as _RocotoRunner: # noqa: N806 - rocoto.run(**runargs) - _RocotoRunner.assert_called_once_with(*runargs.values()) + rocoto.run(**rocoto_runner_args) + _RocotoRunner.assert_called_once_with(*rocoto_runner_args.values()) def test_rocoto_validate__file_fail(validation_assets): @@ -105,18 +105,25 @@ def test_rocoto_validate__string_pass(validation_assets): assert rocoto.validate_string(xml=xml_string_good) is True -# PM# +class TestRocotoRunner: + """ + Tests for class uwtools.rocoto._RocotoRunner. + """ + # Fixtures -def test_rocoto__RocotoRunner__init_and_del(runargs): - rr = rocoto._RocotoRunner(**runargs) - con = Mock() - rr._con = con - del rr - con.close.assert_called_once_with() + @fixture + def instance(self, rocoto_runner_args): + return rocoto._RocotoRunner(**rocoto_runner_args) + # Tests -# PM# + def test_rocoto__RocotoRunner__init_and_del(self, rocoto_runner_args): + rr = rocoto._RocotoRunner(**rocoto_runner_args) + con = Mock() + rr._con = con + del rr + con.close.assert_called_once_with() class TestRocotoXML: From 0976e6aa081f2de3a8a75f60aa95595e8abc0457 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 17:29:45 +0000 Subject: [PATCH 29/69] Work on tests [skip ci] --- src/uwtools/tests/test_rocoto.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index a96717d69..e5ddd1dac 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -125,6 +125,15 @@ def test_rocoto__RocotoRunner__init_and_del(self, rocoto_runner_args): del rr con.close.assert_called_once_with() + def test_rocoto__RocotoRunner_run__state_initially_inactive(self, instance): + with patch.multiple( + rocoto._RocotoRunner, _iterate=D, _report=D, _state=D, new_callable=PropertyMock + ) as mocks: + mocks["_state"].return_value = "COMPLETE" + assert instance.run() is True + mocks["_iterate"].assert_not_called() + mocks["_report"].assert_not_called() + class TestRocotoXML: """ From 5301eafe69db7045c7679d791528b5c409b954b7 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 17:50:51 +0000 Subject: [PATCH 30/69] Work on tests [skip ci] --- src/uwtools/tests/test_rocoto.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index e5ddd1dac..80648c2c2 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -2,6 +2,7 @@ Tests for uwtools.rocoto module. """ +from contextlib import contextmanager from unittest.mock import DEFAULT as D from unittest.mock import Mock, PropertyMock, patch @@ -116,6 +117,15 @@ class TestRocotoRunner: def instance(self, rocoto_runner_args): return rocoto._RocotoRunner(**rocoto_runner_args) + # Helpers + + @contextmanager + def rrmocks(self): + with patch.multiple( + rocoto._RocotoRunner, _iterate=D, _report=D, _state=D, new_callable=PropertyMock + ) as mocks: + yield mocks + # Tests def test_rocoto__RocotoRunner__init_and_del(self, rocoto_runner_args): @@ -126,13 +136,11 @@ def test_rocoto__RocotoRunner__init_and_del(self, rocoto_runner_args): con.close.assert_called_once_with() def test_rocoto__RocotoRunner_run__state_initially_inactive(self, instance): - with patch.multiple( - rocoto._RocotoRunner, _iterate=D, _report=D, _state=D, new_callable=PropertyMock - ) as mocks: + with self.rrmocks() as mocks: mocks["_state"].return_value = "COMPLETE" assert instance.run() is True - mocks["_iterate"].assert_not_called() - mocks["_report"].assert_not_called() + mocks["_iterate"].assert_not_called() + mocks["_report"].assert_not_called() class TestRocotoXML: From f94bf2897f7ced31690f9f5cc4a8dd086a7bdfdf Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 17:52:26 +0000 Subject: [PATCH 31/69] Work on tests [skip ci] --- src/uwtools/tests/test_rocoto.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index 80648c2c2..687b3794a 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -121,9 +121,13 @@ def instance(self, rocoto_runner_args): @contextmanager def rrmocks(self): - with patch.multiple( - rocoto._RocotoRunner, _iterate=D, _report=D, _state=D, new_callable=PropertyMock - ) as mocks: + with ( + patch.multiple( + rocoto._RocotoRunner, _iterate=D, _report=D, _state=D, new_callable=PropertyMock + ) as mocks, + patch.object(rocoto, "sleep") as sleep, + ): + mocks["sleep"] = sleep yield mocks # Tests @@ -141,6 +145,7 @@ def test_rocoto__RocotoRunner_run__state_initially_inactive(self, instance): assert instance.run() is True mocks["_iterate"].assert_not_called() mocks["_report"].assert_not_called() + mocks["sleep"].assert_not_called() class TestRocotoXML: From d7316d2e8d02143811fb90dfbe112802dd0d583c Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 18:14:48 +0000 Subject: [PATCH 32/69] Work on tests [skip ci] --- src/uwtools/rocoto.py | 8 ++++---- src/uwtools/tests/test_rocoto.py | 25 +++++++++++++++++-------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index eb0b5053b..86803bf5b 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -97,15 +97,15 @@ def __init__(self, cycle: datetime, database: Path, rate: int, task: str, workfl self._workflow = workflow self._con: sqlite3.Connection | None = None self._cur: sqlite3.Cursor | None = None + self._initialized = False def __del__(self): if self._con: self._con.close() def run(self) -> bool: - initialized = False while True: - if initialized and not self._iterate(): + if self._initialized and not self._iterate(): return False if state := self._state: if state in self._states["inactive"]: @@ -113,10 +113,10 @@ def run(self) -> bool: if state in self._states["transient"]: continue # iterate immediately to update status self._report() - if initialized: + if self._initialized: log.debug("Sleeping %s seconds", self._rate) sleep(self._rate) - initialized = True + self._initialized = True return True @property diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index 687b3794a..620f28ddd 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -120,15 +120,14 @@ def instance(self, rocoto_runner_args): # Helpers @contextmanager - def rrmocks(self): + def mocks(self): with ( - patch.multiple( - rocoto._RocotoRunner, _iterate=D, _report=D, _state=D, new_callable=PropertyMock - ) as mocks, patch.object(rocoto, "sleep") as sleep, + patch.object(rocoto._RocotoRunner, "_iterate") as _iterate, + patch.object(rocoto._RocotoRunner, "_report") as _report, + patch.object(rocoto._RocotoRunner, "_state", new_callable=PropertyMock) as _state, ): - mocks["sleep"] = sleep - yield mocks + yield dict(sleep=sleep, _iterate=_iterate, _report=_report, _state=_state) # Tests @@ -139,14 +138,24 @@ def test_rocoto__RocotoRunner__init_and_del(self, rocoto_runner_args): del rr con.close.assert_called_once_with() - def test_rocoto__RocotoRunner_run__state_initially_inactive(self, instance): - with self.rrmocks() as mocks: + def test_rocoto__RocotoRunner_run__initially_inactive(self, instance): + with self.mocks() as mocks: mocks["_state"].return_value = "COMPLETE" assert instance.run() is True mocks["_iterate"].assert_not_called() mocks["_report"].assert_not_called() mocks["sleep"].assert_not_called() + def test_rocoto__RocotoRunner_run__iterate_failure(self, instance): + instance._initialized = True + with self.mocks() as mocks: + mocks["_iterate"].return_value = False + assert instance.run() is False + mocks["_iterate"].assert_called_once_with() + mocks["_report"].assert_not_called() + mocks["_state"].assert_not_called() + mocks["sleep"].assert_not_called() + class TestRocotoXML: """ From dc208cf3fccabc03ebd72c9c2dd1d4be3efe87c3 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 20:03:02 +0000 Subject: [PATCH 33/69] Work on tests [skip ci] --- src/uwtools/tests/test_rocoto.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index 620f28ddd..f4cce06f9 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -138,23 +138,33 @@ def test_rocoto__RocotoRunner__init_and_del(self, rocoto_runner_args): del rr con.close.assert_called_once_with() + def test_rocoto__RocotoRunner_run__initially_active(self, instance): + with self.mocks() as mocks: + mocks["_iterate"].return_value = True + mocks["_state"].side_effect = ["RUNNING", "COMPLETE"] + assert instance.run() is True + assert mocks["_iterate"].call_count == 1 + assert mocks["_state"].call_count == 2 + assert mocks["_report"].call_count == 1 + assert mocks["sleep"].call_count == 0 + def test_rocoto__RocotoRunner_run__initially_inactive(self, instance): with self.mocks() as mocks: mocks["_state"].return_value = "COMPLETE" assert instance.run() is True - mocks["_iterate"].assert_not_called() - mocks["_report"].assert_not_called() - mocks["sleep"].assert_not_called() + assert mocks["_iterate"].call_count == 0 + assert mocks["_report"].call_count == 0 + assert mocks["sleep"].call_count == 0 def test_rocoto__RocotoRunner_run__iterate_failure(self, instance): instance._initialized = True with self.mocks() as mocks: mocks["_iterate"].return_value = False assert instance.run() is False - mocks["_iterate"].assert_called_once_with() - mocks["_report"].assert_not_called() - mocks["_state"].assert_not_called() - mocks["sleep"].assert_not_called() + assert mocks["_iterate"].call_count == 1 + assert mocks["_report"].call_count == 0 + assert mocks["_state"].call_count == 0 + assert mocks["sleep"].call_count == 0 class TestRocotoXML: From fcd97f1bfd92407c722b76fe2b9e9369dbe9c561 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 20:11:04 +0000 Subject: [PATCH 34/69] Work on tests [skip ci] --- src/uwtools/tests/test_rocoto.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index f4cce06f9..7126f8c5f 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -119,6 +119,12 @@ def instance(self, rocoto_runner_args): # Helpers + def check_mock_calls_counts(self, mocks, _iterate, _report, _state, sleep): + assert mocks["_iterate"].call_count == _iterate + assert mocks["_report"].call_count == _report + assert mocks["_state"].call_count == _state + assert mocks["sleep"].call_count == sleep + @contextmanager def mocks(self): with ( @@ -138,33 +144,31 @@ def test_rocoto__RocotoRunner__init_and_del(self, rocoto_runner_args): del rr con.close.assert_called_once_with() - def test_rocoto__RocotoRunner_run__initially_active(self, instance): + def test_rocoto__RocotoRunner_run__active(self, instance): with self.mocks() as mocks: mocks["_iterate"].return_value = True mocks["_state"].side_effect = ["RUNNING", "COMPLETE"] assert instance.run() is True - assert mocks["_iterate"].call_count == 1 - assert mocks["_state"].call_count == 2 - assert mocks["_report"].call_count == 1 - assert mocks["sleep"].call_count == 0 + self.check_mock_calls_counts(mocks, _iterate=1, _report=1, _state=2, sleep=0) - def test_rocoto__RocotoRunner_run__initially_inactive(self, instance): + def test_rocoto__RocotoRunner_run__inactive(self, instance): with self.mocks() as mocks: mocks["_state"].return_value = "COMPLETE" assert instance.run() is True - assert mocks["_iterate"].call_count == 0 - assert mocks["_report"].call_count == 0 - assert mocks["sleep"].call_count == 0 + self.check_mock_calls_counts(mocks, _iterate=0, _report=0, _state=1, sleep=0) + + def test_rocoto__RocotoRunner_run__transient(self, instance): + with self.mocks() as mocks: + mocks["_state"].side_effect = ["SUBMITTING", "RUNNING", "COMPLETE"] + assert instance.run() is True + self.check_mock_calls_counts(mocks, _iterate=1, _report=1, _state=3, sleep=0) def test_rocoto__RocotoRunner_run__iterate_failure(self, instance): instance._initialized = True with self.mocks() as mocks: mocks["_iterate"].return_value = False assert instance.run() is False - assert mocks["_iterate"].call_count == 1 - assert mocks["_report"].call_count == 0 - assert mocks["_state"].call_count == 0 - assert mocks["sleep"].call_count == 0 + self.check_mock_calls_counts(mocks, _iterate=1, _report=0, _state=0, sleep=0) class TestRocotoXML: From cec60eb43af68be603a65b6639dbc46d9853b382 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 20:17:58 +0000 Subject: [PATCH 35/69] Work on tests [skip ci] --- src/uwtools/tests/test_rocoto.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index 7126f8c5f..34e538303 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -133,6 +133,7 @@ def mocks(self): patch.object(rocoto._RocotoRunner, "_report") as _report, patch.object(rocoto._RocotoRunner, "_state", new_callable=PropertyMock) as _state, ): + _iterate.return_value = True yield dict(sleep=sleep, _iterate=_iterate, _report=_report, _state=_state) # Tests @@ -146,14 +147,13 @@ def test_rocoto__RocotoRunner__init_and_del(self, rocoto_runner_args): def test_rocoto__RocotoRunner_run__active(self, instance): with self.mocks() as mocks: - mocks["_iterate"].return_value = True mocks["_state"].side_effect = ["RUNNING", "COMPLETE"] assert instance.run() is True self.check_mock_calls_counts(mocks, _iterate=1, _report=1, _state=2, sleep=0) def test_rocoto__RocotoRunner_run__inactive(self, instance): with self.mocks() as mocks: - mocks["_state"].return_value = "COMPLETE" + mocks["_state"].side_effect = ["COMPLETE"] assert instance.run() is True self.check_mock_calls_counts(mocks, _iterate=0, _report=0, _state=1, sleep=0) @@ -170,6 +170,12 @@ def test_rocoto__RocotoRunner_run__iterate_failure(self, instance): assert instance.run() is False self.check_mock_calls_counts(mocks, _iterate=1, _report=0, _state=0, sleep=0) + def test_rocoto__RocotoRunner_run__sleeps(self, instance): + with self.mocks() as mocks: + mocks["_state"].side_effect = ["RUNNING", "RUNNING", "COMPLETE"] + assert instance.run() is True + self.check_mock_calls_counts(mocks, _iterate=2, _report=2, _state=3, sleep=1) + class TestRocotoXML: """ From 4b25a04d6cae97a2279781677f033f30c31d1dcb Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 20:23:07 +0000 Subject: [PATCH 36/69] Work on tests [skip ci] --- src/uwtools/tests/test_rocoto.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index 34e538303..2129e8d1b 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -2,6 +2,7 @@ Tests for uwtools.rocoto module. """ +import sqlite3 from contextlib import contextmanager from unittest.mock import DEFAULT as D from unittest.mock import Mock, PropertyMock, patch @@ -176,6 +177,20 @@ def test_rocoto__RocotoRunner_run__sleeps(self, instance): assert instance.run() is True self.check_mock_calls_counts(mocks, _iterate=2, _report=2, _state=3, sleep=1) + def test_rocoto__RocotoRunner__connection(self, instance): + instance._database.touch() + assert isinstance(instance._connection, sqlite3.Connection) + + def test_rocoto__RocotoRunner__connection__no_file(self, instance): + assert instance._connection is None + + def test_rocoto__RocotoRunner__cursor(self, instance): + instance._database.touch() + assert isinstance(instance._cursor, sqlite3.Cursor) + + def test_rocoto__RocotoRunner__cursor__no_file(self, instance): + assert instance._cursor is None + class TestRocotoXML: """ From f2b9baa25f1de56fb24d9629a26b7d0567f201ef Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 20:27:44 +0000 Subject: [PATCH 37/69] Work on tests [skip ci] --- src/uwtools/tests/test_rocoto.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index 2129e8d1b..d8c6e4762 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -191,6 +191,14 @@ def test_rocoto__RocotoRunner__cursor(self, instance): def test_rocoto__RocotoRunner__cursor__no_file(self, instance): assert instance._cursor is None + def test_rocoto__RocotoRunner__iterate(self, instance, logged): + with patch.object(rocoto, "run_shell_cmd", return_value=(True, "")) as run_shell_cmd: + assert instance._iterate() is True + run_shell_cmd.assert_called_once_with( + "rocotorun -d %s -w %s" % (instance._database, instance._workflow), quiet=True + ) + assert logged("Iterating workflow") + class TestRocotoXML: """ From e889c016b76ac0860090ec8452eac2511d2a8063 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 20:34:19 +0000 Subject: [PATCH 38/69] Work on tests [skip ci] --- src/uwtools/tests/test_rocoto.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index d8c6e4762..fe5c4260e 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -26,7 +26,7 @@ def assets(tmp_path): @fixture def rocoto_runner_args(utc, tmp_path): return { - "cycle": utc(), + "cycle": utc(2025, 7, 21, 12), "database": tmp_path / "rocoto.db", "rate": 11, "task": "foo", @@ -192,13 +192,34 @@ def test_rocoto__RocotoRunner__cursor__no_file(self, instance): assert instance._cursor is None def test_rocoto__RocotoRunner__iterate(self, instance, logged): - with patch.object(rocoto, "run_shell_cmd", return_value=(True, "")) as run_shell_cmd: + retval = (True, "") + with patch.object(rocoto, "run_shell_cmd", return_value=retval) as run_shell_cmd: assert instance._iterate() is True run_shell_cmd.assert_called_once_with( "rocotorun -d %s -w %s" % (instance._database, instance._workflow), quiet=True ) assert logged("Iterating workflow") + def test_rocoto__RocotoRunner__query_data(self, instance): + assert instance._query_data == {"taskname": "foo", "cycle": 1753099200} + + def test_rocoto__RocotoRunner__query_stmt(self, instance): + assert ( + instance._query_stmt + == "select state from jobs where taskname=:taskname and cycle=:cycle" + ) + + def test_rocoto_RocotoRunner__report(self, instance, logged): + instance._database.touch() + retval = (True, "foo\nbar\n") + with patch.object(rocoto, "run_shell_cmd", return_value=retval) as run_shell_cmd: + instance._report() + for line in ["Workflow status:", "foo", "bar"]: + assert logged(line) + run_shell_cmd.assert_called_once_with( + "rocotostat -d %s -w %s" % (instance._database, instance._workflow), quiet=True + ) + class TestRocotoXML: """ From 5acf570e1ac1ec56d1dc55a3b56d627d881e552f Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 21:12:10 +0000 Subject: [PATCH 39/69] Work on tests [skip ci] --- src/uwtools/tests/test_rocoto.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index fe5c4260e..de0da6fc6 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -126,6 +126,18 @@ def check_mock_calls_counts(self, mocks, _iterate, _report, _state, sleep): assert mocks["_state"].call_count == _state assert mocks["sleep"].call_count == sleep + def dbsetup(self, instance): + instance._database.touch() + columns = ", ".join( + [ + "id integer primary key", + "taskname varchar(64)", + "cycle datetime", + "state varchar(64)", + ] + ) + instance._cursor.execute(f"create table jobs ({columns});") + @contextmanager def mocks(self): with ( @@ -209,7 +221,7 @@ def test_rocoto__RocotoRunner__query_stmt(self, instance): == "select state from jobs where taskname=:taskname and cycle=:cycle" ) - def test_rocoto_RocotoRunner__report(self, instance, logged): + def test_rocoto__RocotoRunner__report(self, instance, logged): instance._database.touch() retval = (True, "foo\nbar\n") with patch.object(rocoto, "run_shell_cmd", return_value=retval) as run_shell_cmd: @@ -220,6 +232,18 @@ def test_rocoto_RocotoRunner__report(self, instance, logged): "rocotostat -d %s -w %s" % (instance._database, instance._workflow), quiet=True ) + def test_rocoto__RocotoRunner__state(self, instance): + self.dbsetup(instance) + instance._cursor.execute( + "insert into jobs values (:id, :taskname, :cycle, :state)", + {"id": 1, "taskname": "foo", "cycle": instance._cycle.timestamp(), "state": "COMPLETE"}, + ) + assert instance._state == "COMPLETE" + + def test_rocoto__RocotoRunner__state__none(self, instance): + self.dbsetup(instance) + assert instance._state is None + class TestRocotoXML: """ From 8e38b742169594335dbef60781dcecc6b143d772 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 21:18:32 +0000 Subject: [PATCH 40/69] Work on tests --- src/uwtools/rocoto.py | 5 ++--- src/uwtools/tests/test_rocoto.py | 9 ++++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 86803bf5b..dcff7dde2 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -159,15 +159,14 @@ def _report(self) -> None: @property def _state(self) -> str | None: + state = None if cursor := self._cursor: result = cursor.execute(self._query_stmt, self._query_data) - state = None if row := result.fetchone(): (state,) = row log.info(self._state_msg % state) assert state in chain.from_iterable(self._states.values()) - return state - return None + return state @property def _state_msg(self) -> str: diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index de0da6fc6..3e510e456 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -232,18 +232,25 @@ def test_rocoto__RocotoRunner__report(self, instance, logged): "rocotostat -d %s -w %s" % (instance._database, instance._workflow), quiet=True ) - def test_rocoto__RocotoRunner__state(self, instance): + def test_rocoto__RocotoRunner__state(self, instance, logged): self.dbsetup(instance) instance._cursor.execute( "insert into jobs values (:id, :taskname, :cycle, :state)", {"id": 1, "taskname": "foo", "cycle": instance._cycle.timestamp(), "state": "COMPLETE"}, ) assert instance._state == "COMPLETE" + assert logged(f"Rocoto task '{instance._task}' for cycle {instance._cycle}: COMPLETE") def test_rocoto__RocotoRunner__state__none(self, instance): self.dbsetup(instance) assert instance._state is None + def test_rocoto__RocotoRunner__state_msg(self, instance): + assert instance._state_msg == "Rocoto task 'foo' for cycle 2025-07-21 12:00:00: %s" + + def test_rocoto__RocotoRunner__states(self, instance): + assert list(instance._states.keys()) == ["active", "inactive", "transient"] + class TestRocotoXML: """ From 9a30879f2bc3adc6339766b166220459e79f2f67 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 22:11:39 +0000 Subject: [PATCH 41/69] Work on docs --- docs/index.rst | 43 +++++++++++-------- .../user_guide/cli/tools/execute/help.out | 2 +- docs/sections/user_guide/cli/tools/rocoto.rst | 30 +++++++++++++ .../user_guide/cli/tools/rocoto/foobar.xml | 26 +++++++++++ .../user_guide/cli/tools/rocoto/help.out | 2 + .../user_guide/cli/tools/rocoto/run-help.cmd | 1 + .../user_guide/cli/tools/rocoto/run-help.out | 27 ++++++++++++ 7 files changed, 112 insertions(+), 19 deletions(-) create mode 100644 docs/sections/user_guide/cli/tools/rocoto/foobar.xml create mode 100644 docs/sections/user_guide/cli/tools/rocoto/run-help.cmd create mode 100644 docs/sections/user_guide/cli/tools/rocoto/run-help.out diff --git a/docs/index.rst b/docs/index.rst index 13c64c129..40499ca98 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,24 +30,24 @@ Configuration Management The config tool suite helps you compare, transform, modify, and even validate your configuration. The package supports YAML, shell, Fortran namelist, and INI file formats. Configuration in any of these formats may use :jinja2:`Jinja2 syntax` to express values. These values can reference others, or compute new values by evaluating mathematical expressions, building paths, manipulating strings, etc. -Compare Mode -"""""""""""" +Compare Action +"""""""""""""" When the Linux diff tool just doesn't work for comparing unordered namelists with mixed-case keys, this is your go-to! The Fortran namelists are the *real* catalyst behind this gem, but it also works on the other configuration formats. | :any:`CLI documentation with examples` -Realize Mode -"""""""""""" +Realize Action +"""""""""""""" -This mode renders values created by :jinja2:`Jinja2 templates`, and lets you override values in one file or object with those from others, not necessarily with the same configuration format. With ``uwtools``, you can even reference the contents of other files to build up a configuration from its pieces. +To realiez a config is to render values encoded in :jinja2:`Jinja2 expressions`, potentially overriding values in one file or object with those from others, not necessarily with the same configuration format. With ``uwtools``, you can even reference the contents of other files to build up a configuration from its pieces. | :any:`CLI documentation with examples` -Validate Mode -""""""""""""" +Validate Action +""""""""""""""" -In this mode, you can provide a :json-schema:`JSON Schema<>` file alongside your configuration to validate that it meets the requirements set by the schema. We've enabled robust logging to make it easier to repair your configs when problems arise. +Provide a :json-schema:`JSON Schema<>` file alongside your configuration to validate that it meets the requirements set by the schema. We've enabled robust logging to make it easier to repair your configs when problems arise. | :any:`CLI documentation with examples` @@ -57,15 +57,15 @@ Templating | **CLI**: ``uw template -h`` | **API**: ``import uwtools.api.template`` -Render Mode -""""""""""" +Render Action +""""""""""""" -The ``render`` mode that gives you the full power of rendering a :jinja2:`Jinja2 template` in the same easy-to-use interface as your other workflow tools. +This gives you the full power of rendering a :jinja2:`Jinja2 template` in the same easy-to-use interface as your other workflow tools. | :any:`CLI documentation with examples` -Translate Mode -"""""""""""""" +Translate Action +"""""""""""""""" This tool helps transform legacy configuration files templated with the atparse tool (common at :noaa:`NOAA<>`) into :jinja2:`Jinja2 templates` for use with the ``uw config realize`` and ``uw template render`` tools, or their API equivalents. @@ -95,17 +95,24 @@ Rocoto Configurability This tool is all about creating a configurable interface to the :rocoto:`Rocoto<>` workflow manager tool that produces the Rocoto XML for a totally arbitrary set of tasks. The ``uwtools`` package defines a structured YAML interface that relies on tasks you define to run. Paired with the uw config tool suite, this interface becomes highly configurable and requires no XML syntax! -Realize Mode -"""""""""""" +Realize Action +"""""""""""""" This is where you put in your structured YAML that defines your workflow of choice, and it pops out a verified Rocoto XML. | :any:`CLI documentation with examples` -Validate Mode -""""""""""""" +Run Action +"""""""""" + +Given a Rocoto XML workflow document, invoke Rocoto in a loop, monitoring its progress, until a specified task is complete. + +| :any:`CLI documentation with examples` + +Validate Action +""""""""""""""" -Do you already have a Rocoto XML but don't want to run Rocoto to make sure it works? Use the validate mode to check to see if Rocoto will be happy. +Do you already have a Rocoto XML but don't want to run Rocoto to make sure it works? Use ``rocoto validate`` to check to see if Rocoto will be happy. | :any:`CLI documentation with examples` diff --git a/docs/sections/user_guide/cli/tools/execute/help.out b/docs/sections/user_guide/cli/tools/execute/help.out index 010a91345..2b50458da 100644 --- a/docs/sections/user_guide/cli/tools/execute/help.out +++ b/docs/sections/user_guide/cli/tools/execute/help.out @@ -12,7 +12,7 @@ Required arguments: --classname CLASSNAME Name of driver class --task TASK - Driver task to execute + Task to execute Optional arguments: -h, --help diff --git a/docs/sections/user_guide/cli/tools/rocoto.rst b/docs/sections/user_guide/cli/tools/rocoto.rst index 89ccbbdfa..9688d570a 100644 --- a/docs/sections/user_guide/cli/tools/rocoto.rst +++ b/docs/sections/user_guide/cli/tools/rocoto.rst @@ -64,6 +64,36 @@ The examples in this section use a UW YAML file ``rocoto.yaml`` with contents: .. literalinclude:: rocoto/realize-exec-stdout-verbose.out :language: xml +.. _cli_rocoto_run_examples: + +``run`` +------- + +.. literalinclude:: rocoto/run-help.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: rocoto/run-help.out + :language: text + +Examples +^^^^^^^^ + +.. note:: Use of ``uw rocoto run`` requires that the ``rocotorun`` and ``rocotostat`` executables are present on ``PATH``. on HPCs, this is typically achieved by loading a system module providing Rocoto. + +The following examples make use of a simple Rocoto XML workflow document similar to: + +.. literalinclude:: rocoto/foobar.xml + :language: xml + +* To run only the ``foo`` task, which has no dependencies: + + .. literalinclude:: rocoto/run-foo.txt + :language: text + :emphasize-lines: 1 + .. literalinclude:: rocoto/run-foo.out + :language: text + + .. _cli_rocoto_validate_examples: ``validate`` diff --git a/docs/sections/user_guide/cli/tools/rocoto/foobar.xml b/docs/sections/user_guide/cli/tools/rocoto/foobar.xml new file mode 100644 index 000000000..6842794da --- /dev/null +++ b/docs/sections/user_guide/cli/tools/rocoto/foobar.xml @@ -0,0 +1,26 @@ + + + 202507170000 202507170000 00:00:01 + /path/to/foobar.log + + account + echo foo >/path/tofoo.txt + 1 + /path/to/foo.batchout + service + batch + 00:01:00 + + + account + echo bar >/path/to/bar.txt + 1 + /path/to/bar.batchout + service + batch + 00:01:00 + + + + + diff --git a/docs/sections/user_guide/cli/tools/rocoto/help.out b/docs/sections/user_guide/cli/tools/rocoto/help.out index 7b066c847..80487672c 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/help.out +++ b/docs/sections/user_guide/cli/tools/rocoto/help.out @@ -12,5 +12,7 @@ Positional arguments: ACTION realize Realize a Rocoto XML workflow document + run + Run a Rocoto workflow validate Validate Rocoto XML diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-help.cmd b/docs/sections/user_guide/cli/tools/rocoto/run-help.cmd new file mode 100644 index 000000000..de1a19d35 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/rocoto/run-help.cmd @@ -0,0 +1 @@ +uw rocoto run --help diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-help.out b/docs/sections/user_guide/cli/tools/rocoto/run-help.out new file mode 100644 index 000000000..bc657d4ca --- /dev/null +++ b/docs/sections/user_guide/cli/tools/rocoto/run-help.out @@ -0,0 +1,27 @@ +usage: uw rocoto run [--cycle CYCLE] --database DATABASE --task TASK + --workflow WORKFLOW [-h] [--version] [--rate SECONDS] + [--quiet] [--verbose] + +Run a Rocoto workflow + +Required arguments: + --cycle CYCLE + The cycle in ISO8601 format (e.g. yyyy-mm-ddThh) + --database DATABASE, -d DATABASE + The Rocoto database file + --task TASK + Task to execute + --workflow WORKFLOW, -w WORKFLOW + The Rocoto XML file + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --rate SECONDS, -r SECONDS + Delay between workflow iterations (default: 10) + --quiet, -q + Print no logging messages + --verbose, -v + Print all logging messages From c86ad7a72177e6c135494ba0c8e7d84770158ccc Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 22:38:45 +0000 Subject: [PATCH 42/69] Work on docs --- docs/sections/user_guide/cli/Makefile.outputs | 6 ------ docs/sections/user_guide/cli/tools/rocoto.rst | 19 +++++++++++++++++-- .../user_guide/cli/tools/rocoto/foobar.xml | 4 ++-- .../user_guide/cli/tools/rocoto/run-bar.out | 10 ++++++++++ .../user_guide/cli/tools/rocoto/run-bar.txt | 1 + .../cli/tools/rocoto/run-foo-complete.out | 1 + .../user_guide/cli/tools/rocoto/run-foo.out | 4 ++++ .../user_guide/cli/tools/rocoto/run-foo.txt | 1 + 8 files changed, 36 insertions(+), 10 deletions(-) create mode 100644 docs/sections/user_guide/cli/tools/rocoto/run-bar.out create mode 100644 docs/sections/user_guide/cli/tools/rocoto/run-bar.txt create mode 100644 docs/sections/user_guide/cli/tools/rocoto/run-foo-complete.out create mode 100644 docs/sections/user_guide/cli/tools/rocoto/run-foo.out create mode 100644 docs/sections/user_guide/cli/tools/rocoto/run-foo.txt diff --git a/docs/sections/user_guide/cli/Makefile.outputs b/docs/sections/user_guide/cli/Makefile.outputs index abce0a427..587c5fdc9 100644 --- a/docs/sections/user_guide/cli/Makefile.outputs +++ b/docs/sections/user_guide/cli/Makefile.outputs @@ -9,9 +9,3 @@ all: $(OUTPUTS) $(OUTPUTS): @bash $(basename $@).cmd >$@ 2>&1 | true - -%.out: %.txt %.yaml - @bash $< >$@ 2>&1 | true - -%.out: %.txt - @bash $< >$@ 2>&1 | true diff --git a/docs/sections/user_guide/cli/tools/rocoto.rst b/docs/sections/user_guide/cli/tools/rocoto.rst index 9688d570a..6d459c65b 100644 --- a/docs/sections/user_guide/cli/tools/rocoto.rst +++ b/docs/sections/user_guide/cli/tools/rocoto.rst @@ -80,12 +80,12 @@ Examples .. note:: Use of ``uw rocoto run`` requires that the ``rocotorun`` and ``rocotostat`` executables are present on ``PATH``. on HPCs, this is typically achieved by loading a system module providing Rocoto. -The following examples make use of a simple Rocoto XML workflow document similar to: +The following examples make use of a simple Rocoto XML workflow document (which might have been created by ``uw rocoto realize``) similar to: .. literalinclude:: rocoto/foobar.xml :language: xml -* To run only the ``foo`` task, which has no dependencies: +* To run only task ``foo``, which has no dependencies, iterating the workflow at the default rate (every 10 seconds): .. literalinclude:: rocoto/run-foo.txt :language: text @@ -93,6 +93,21 @@ The following examples make use of a simple Rocoto XML workflow document similar .. literalinclude:: rocoto/run-foo.out :language: text + A second invocation of ``uw rocoto run`` immediately shows task ``foo`` in its final state: + + .. literalinclude:: rocoto/run-foo.txt + :language: text + :emphasize-lines: 1 + .. literalinclude:: rocoto/run-foo-complete.out + :language: text + +* To run task ``bar``, which depends on task ``foo``, iterating every 3 seconds (after deleting the ``foobar.db`` database file from the previous example): + + .. literalinclude:: rocoto/run-bar.txt + :language: text + :emphasize-lines: 1 + .. literalinclude:: rocoto/run-bar.out + :language: text .. _cli_rocoto_validate_examples: diff --git a/docs/sections/user_guide/cli/tools/rocoto/foobar.xml b/docs/sections/user_guide/cli/tools/rocoto/foobar.xml index 6842794da..6537ee471 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/foobar.xml +++ b/docs/sections/user_guide/cli/tools/rocoto/foobar.xml @@ -1,5 +1,5 @@ - - + + 202507170000 202507170000 00:00:01 /path/to/foobar.log diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-bar.out b/docs/sections/user_guide/cli/tools/rocoto/run-bar.out new file mode 100644 index 000000000..11aeb7d2a --- /dev/null +++ b/docs/sections/user_guide/cli/tools/rocoto/run-bar.out @@ -0,0 +1,10 @@ +[2025-07-21T22:30:40] INFO Iterating workflow +[2025-07-21T22:31:11] INFO Workflow status: +[2025-07-21T22:31:12] INFO CYCLE TASK JOBID STATE EXIT STATUS TRIES DURATION +[2025-07-21T22:31:12] INFO ================================================================================================================================ +[2025-07-21T22:31:12] INFO 202507170000 foo druby://10.178.9.5:40925 SUBMITTING - 0 0.0 +[2025-07-21T22:31:12] INFO 202507170000 bar - - - - - +[2025-07-21T22:31:15] INFO Iterating workflow +[2025-07-21T22:31:46] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUBMITTING +[2025-07-21T22:31:46] INFO Iterating workflow +[2025-07-21T22:31:48] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUCCEEDED diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-bar.txt b/docs/sections/user_guide/cli/tools/rocoto/run-bar.txt new file mode 100644 index 000000000..3676817a7 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/rocoto/run-bar.txt @@ -0,0 +1 @@ +uw rocoto run --cycle 2025-07-17T00 --database foobar.db --rate 3 --task bar --workflow foobar.xml diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-foo-complete.out b/docs/sections/user_guide/cli/tools/rocoto/run-foo-complete.out new file mode 100644 index 000000000..38360ebf2 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/rocoto/run-foo-complete.out @@ -0,0 +1 @@ +[2025-07-21T22:28:38] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUCCEEDED diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-foo.out b/docs/sections/user_guide/cli/tools/rocoto/run-foo.out new file mode 100644 index 000000000..90af78d3b --- /dev/null +++ b/docs/sections/user_guide/cli/tools/rocoto/run-foo.out @@ -0,0 +1,4 @@ +[2025-07-21T22:27:26] INFO Iterating workflow +[2025-07-21T22:27:57] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUBMITTING +[2025-07-21T22:27:57] INFO Iterating workflow +[2025-07-21T22:28:28] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUCCEEDED diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-foo.txt b/docs/sections/user_guide/cli/tools/rocoto/run-foo.txt new file mode 100644 index 000000000..ca74a4712 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/rocoto/run-foo.txt @@ -0,0 +1 @@ +uw rocoto run --cycle 2025-07-17T00 --database foobar.db --task foo --workflow foobar.xml From 974651e2d2b45e04a2bd15054be5a57d7e1e8d62 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 22:44:15 +0000 Subject: [PATCH 43/69] Work on docs --- docs/sections/user_guide/cli/tools/rocoto.rst | 14 +++++++++++--- .../cli/tools/rocoto/run-bar-complete.out | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 docs/sections/user_guide/cli/tools/rocoto/run-bar-complete.out diff --git a/docs/sections/user_guide/cli/tools/rocoto.rst b/docs/sections/user_guide/cli/tools/rocoto.rst index 6d459c65b..048e55671 100644 --- a/docs/sections/user_guide/cli/tools/rocoto.rst +++ b/docs/sections/user_guide/cli/tools/rocoto.rst @@ -78,14 +78,14 @@ The examples in this section use a UW YAML file ``rocoto.yaml`` with contents: Examples ^^^^^^^^ -.. note:: Use of ``uw rocoto run`` requires that the ``rocotorun`` and ``rocotostat`` executables are present on ``PATH``. on HPCs, this is typically achieved by loading a system module providing Rocoto. +.. note:: Use of ``uw rocoto run`` requires presence of the ``rocotorun`` and ``rocotostat`` executables on ``PATH``. On HPCs, this is typically achieved by loading a system module providing Rocoto. The following examples make use of a simple Rocoto XML workflow document (which might have been created by ``uw rocoto realize``) similar to: .. literalinclude:: rocoto/foobar.xml :language: xml -* To run only task ``foo``, which has no dependencies, iterating the workflow at the default rate (every 10 seconds): +* To run only task ``foo``, which has no dependencies: .. literalinclude:: rocoto/run-foo.txt :language: text @@ -101,7 +101,7 @@ The following examples make use of a simple Rocoto XML workflow document (which .. literalinclude:: rocoto/run-foo-complete.out :language: text -* To run task ``bar``, which depends on task ``foo``, iterating every 3 seconds (after deleting the ``foobar.db`` database file from the previous example): +* To run task ``bar``, which depends on task ``foo``, iterating every 3 seconds, and after deleting the ``foobar.db`` database file from the previous example: .. literalinclude:: rocoto/run-bar.txt :language: text @@ -109,6 +109,14 @@ The following examples make use of a simple Rocoto XML workflow document (which .. literalinclude:: rocoto/run-bar.out :language: text + A second invocation of ``uw rocoto run`` immediately shows task ``bar`` in its final state: + + .. literalinclude:: rocoto/run-bar.txt + :language: text + :emphasize-lines: 1 + .. literalinclude:: rocoto/run-bar-complete.out + :language: text + .. _cli_rocoto_validate_examples: ``validate`` diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-bar-complete.out b/docs/sections/user_guide/cli/tools/rocoto/run-bar-complete.out new file mode 100644 index 000000000..9c2bec3f4 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/rocoto/run-bar-complete.out @@ -0,0 +1 @@ +[2025-07-21T22:43:27] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUCCEEDED From 0ed6935bc9c135c52f457c29ec5471215eab4d31 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 22:50:49 +0000 Subject: [PATCH 44/69] Fix typo --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 40499ca98..5defc6209 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -40,7 +40,7 @@ When the Linux diff tool just doesn't work for comparing unordered namelists wit Realize Action """""""""""""" -To realiez a config is to render values encoded in :jinja2:`Jinja2 expressions`, potentially overriding values in one file or object with those from others, not necessarily with the same configuration format. With ``uwtools``, you can even reference the contents of other files to build up a configuration from its pieces. +To realize a config is to render values encoded in :jinja2:`Jinja2 expressions`, potentially overriding values in one file or object with those from others, not necessarily with the same configuration format. With ``uwtools``, you can even reference the contents of other files to build up a configuration from its pieces. | :any:`CLI documentation with examples` From e42f6ce0029b73991fe2aafe9e7276ddcbe52af1 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 23:12:18 +0000 Subject: [PATCH 45/69] DRY out 10-second default rate --- src/uwtools/api/rocoto.py | 9 +++++++-- src/uwtools/cli.py | 5 +++-- src/uwtools/rocoto.py | 3 +++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/uwtools/api/rocoto.py b/src/uwtools/api/rocoto.py index 5125895e1..89fd64737 100644 --- a/src/uwtools/api/rocoto.py +++ b/src/uwtools/api/rocoto.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING +from uwtools.rocoto import DEFAULT_ITERATION_RATE from uwtools.rocoto import realize as _realize from uwtools.rocoto import run as _run from uwtools.rocoto import validate_file as _validate @@ -43,7 +44,11 @@ def realize( def run( - cycle: datetime, database: Path | str, task: str, workflow: Path | str, rate: int = 10 + cycle: datetime, + database: Path | str, + task: str, + workflow: Path | str, + rate: int = DEFAULT_ITERATION_RATE, ) -> bool: """ Run the specified Rocoto workflow to completion (or failure). @@ -52,7 +57,7 @@ def run( :param database: Path to the Rocoto database file. :param task: The workflow task to run. :param workflow: Path to the Rocoto XML workflow document. - :param rate: Seconds between workflow iterations (deault: 10)" + :param rate: Seconds between workflow iterations. """ return _run( cycle=cycle, diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 98c58de10..287820cde 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -854,11 +854,12 @@ def _add_arg_quiet(group: Group) -> None: def _add_arg_rate(group: Group) -> None: + default_rate = uwtools.rocoto.DEFAULT_ITERATION_RATE group.add_argument( _switch(STR.rate), "-r", - default=10, - help="Delay between workflow iterations (default: 10)", + default=default_rate, + help="Delay between workflow iterations (default: %s)" % default_rate, metavar="SECONDS", required=False, type=int, diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index dcff7dde2..9a202b80e 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -28,6 +28,9 @@ from datetime import datetime +DEFAULT_ITERATION_RATE = 10 + + def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) -> str: """ Realize the Rocoto workflow defined in the given YAML as XML, validating both the YAML input and From d424abb3e714a896da0716022aa9e1b592cbedae Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 21 Jul 2025 23:48:49 +0000 Subject: [PATCH 46/69] Simplify run --- src/uwtools/rocoto.py | 18 +++++++----------- src/uwtools/tests/test_rocoto.py | 17 +++++------------ 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 9a202b80e..de9fb58d8 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -100,7 +100,6 @@ def __init__(self, cycle: datetime, database: Path, rate: int, task: str, workfl self._workflow = workflow self._con: sqlite3.Connection | None = None self._cur: sqlite3.Cursor | None = None - self._initialized = False def __del__(self): if self._con: @@ -108,18 +107,15 @@ def __del__(self): def run(self) -> bool: while True: - if self._initialized and not self._iterate(): + if self._state in self._states["inactive"]: + break + if not self._iterate(): return False - if state := self._state: - if state in self._states["inactive"]: - break - if state in self._states["transient"]: - continue # iterate immediately to update status + if self._state in self._states["transient"]: + continue self._report() - if self._initialized: - log.debug("Sleeping %s seconds", self._rate) - sleep(self._rate) - self._initialized = True + log.debug("Sleeping %s seconds", self._rate) + sleep(self._rate) return True @property diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index 3e510e456..337e28751 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -160,9 +160,9 @@ def test_rocoto__RocotoRunner__init_and_del(self, rocoto_runner_args): def test_rocoto__RocotoRunner_run__active(self, instance): with self.mocks() as mocks: - mocks["_state"].side_effect = ["RUNNING", "COMPLETE"] + mocks["_state"].side_effect = ["RUNNING", "RUNNING", "COMPLETE"] assert instance.run() is True - self.check_mock_calls_counts(mocks, _iterate=1, _report=1, _state=2, sleep=0) + self.check_mock_calls_counts(mocks, _iterate=1, _report=1, _state=3, sleep=1) def test_rocoto__RocotoRunner_run__inactive(self, instance): with self.mocks() as mocks: @@ -172,22 +172,15 @@ def test_rocoto__RocotoRunner_run__inactive(self, instance): def test_rocoto__RocotoRunner_run__transient(self, instance): with self.mocks() as mocks: - mocks["_state"].side_effect = ["SUBMITTING", "RUNNING", "COMPLETE"] + mocks["_state"].side_effect = [None, "SUBMITTING", "RUNNING", "RUNNING", "COMPLETE"] assert instance.run() is True - self.check_mock_calls_counts(mocks, _iterate=1, _report=1, _state=3, sleep=0) + self.check_mock_calls_counts(mocks, _iterate=2, _report=1, _state=5, sleep=1) def test_rocoto__RocotoRunner_run__iterate_failure(self, instance): - instance._initialized = True with self.mocks() as mocks: mocks["_iterate"].return_value = False assert instance.run() is False - self.check_mock_calls_counts(mocks, _iterate=1, _report=0, _state=0, sleep=0) - - def test_rocoto__RocotoRunner_run__sleeps(self, instance): - with self.mocks() as mocks: - mocks["_state"].side_effect = ["RUNNING", "RUNNING", "COMPLETE"] - assert instance.run() is True - self.check_mock_calls_counts(mocks, _iterate=2, _report=2, _state=3, sleep=1) + self.check_mock_calls_counts(mocks, _iterate=1, _report=0, _state=1, sleep=0) def test_rocoto__RocotoRunner__connection(self, instance): instance._database.touch() From f88d46941f47e0baa7532f2d30c0cd0f93e5160c Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 22 Jul 2025 01:00:48 +0000 Subject: [PATCH 47/69] Simplify run --- src/uwtools/rocoto.py | 6 ++++-- src/uwtools/tests/test_rocoto.py | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index de9fb58d8..7cc2e5362 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -106,12 +106,14 @@ def __del__(self): self._con.close() def run(self) -> bool: + state = self._state while True: - if self._state in self._states["inactive"]: + if state in self._states["inactive"]: break if not self._iterate(): return False - if self._state in self._states["transient"]: + state = self._state + if state in self._states["transient"]: continue self._report() log.debug("Sleeping %s seconds", self._rate) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index 337e28751..df8662379 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -160,9 +160,9 @@ def test_rocoto__RocotoRunner__init_and_del(self, rocoto_runner_args): def test_rocoto__RocotoRunner_run__active(self, instance): with self.mocks() as mocks: - mocks["_state"].side_effect = ["RUNNING", "RUNNING", "COMPLETE"] + mocks["_state"].side_effect = ["RUNNING", "COMPLETE"] assert instance.run() is True - self.check_mock_calls_counts(mocks, _iterate=1, _report=1, _state=3, sleep=1) + self.check_mock_calls_counts(mocks, _iterate=1, _report=1, _state=2, sleep=1) def test_rocoto__RocotoRunner_run__inactive(self, instance): with self.mocks() as mocks: @@ -172,9 +172,9 @@ def test_rocoto__RocotoRunner_run__inactive(self, instance): def test_rocoto__RocotoRunner_run__transient(self, instance): with self.mocks() as mocks: - mocks["_state"].side_effect = [None, "SUBMITTING", "RUNNING", "RUNNING", "COMPLETE"] + mocks["_state"].side_effect = [None, "SUBMITTING", "RUNNING", "COMPLETE"] assert instance.run() is True - self.check_mock_calls_counts(mocks, _iterate=2, _report=1, _state=5, sleep=1) + self.check_mock_calls_counts(mocks, _iterate=3, _report=2, _state=4, sleep=2) def test_rocoto__RocotoRunner_run__iterate_failure(self, instance): with self.mocks() as mocks: From 78364714524739fae3f09bff69907014949c08fa Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 22 Jul 2025 01:05:10 +0000 Subject: [PATCH 48/69] Simplify run --- src/uwtools/rocoto.py | 2 ++ src/uwtools/tests/test_rocoto.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 7cc2e5362..3ed549d18 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -113,6 +113,8 @@ def run(self) -> bool: if not self._iterate(): return False state = self._state + if state in self._states["inactive"]: + break if state in self._states["transient"]: continue self._report() diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index df8662379..e5c21e3d8 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -162,7 +162,7 @@ def test_rocoto__RocotoRunner_run__active(self, instance): with self.mocks() as mocks: mocks["_state"].side_effect = ["RUNNING", "COMPLETE"] assert instance.run() is True - self.check_mock_calls_counts(mocks, _iterate=1, _report=1, _state=2, sleep=1) + self.check_mock_calls_counts(mocks, _iterate=1, _report=0, _state=2, sleep=0) def test_rocoto__RocotoRunner_run__inactive(self, instance): with self.mocks() as mocks: @@ -174,7 +174,7 @@ def test_rocoto__RocotoRunner_run__transient(self, instance): with self.mocks() as mocks: mocks["_state"].side_effect = [None, "SUBMITTING", "RUNNING", "COMPLETE"] assert instance.run() is True - self.check_mock_calls_counts(mocks, _iterate=3, _report=2, _state=4, sleep=2) + self.check_mock_calls_counts(mocks, _iterate=3, _report=1, _state=4, sleep=1) def test_rocoto__RocotoRunner_run__iterate_failure(self, instance): with self.mocks() as mocks: From e927c8bb547b4d8fa41892647c580f4dbf253bf8 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 22 Jul 2025 01:09:10 +0000 Subject: [PATCH 49/69] Simplify run --- src/uwtools/rocoto.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 3ed549d18..29a679f3b 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -107,19 +107,14 @@ def __del__(self): def run(self) -> bool: state = self._state - while True: - if state in self._states["inactive"]: - break + while state not in self._states["inactive"]: if not self._iterate(): return False state = self._state - if state in self._states["inactive"]: - break - if state in self._states["transient"]: - continue - self._report() - log.debug("Sleeping %s seconds", self._rate) - sleep(self._rate) + if state not in chain(self._states["inactive"], self._states["transient"]): + self._report() + log.debug("Sleeping %s seconds", self._rate) + sleep(self._rate) return True @property From 9a1dcfb4a90e913cc5f1fcc13bbac233eec686b4 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 22 Jul 2025 01:11:57 +0000 Subject: [PATCH 50/69] Simplify run --- src/uwtools/rocoto.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 29a679f3b..85c4789f5 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -110,8 +110,7 @@ def run(self) -> bool: while state not in self._states["inactive"]: if not self._iterate(): return False - state = self._state - if state not in chain(self._states["inactive"], self._states["transient"]): + if (state := self._state) in self._states["active"]: self._report() log.debug("Sleeping %s seconds", self._rate) sleep(self._rate) From e7b97c77a919279c67fd6bd30da7101a9b7acbbf Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 22 Jul 2025 01:17:22 +0000 Subject: [PATCH 51/69] Simplify run --- src/uwtools/rocoto.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 85c4789f5..f021ecbdd 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -110,7 +110,8 @@ def run(self) -> bool: while state not in self._states["inactive"]: if not self._iterate(): return False - if (state := self._state) in self._states["active"]: + state = self._state + if state is None or state in self._states["active"]: self._report() log.debug("Sleeping %s seconds", self._rate) sleep(self._rate) From 521635a9029129bf9cda887466a9f0288ae4947e Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 22 Jul 2025 01:19:27 +0000 Subject: [PATCH 52/69] Simplify run --- .../cli/tools/rocoto/run-bar-complete.out | 2 +- .../user_guide/cli/tools/rocoto/run-bar.out | 20 +++++++++---------- .../cli/tools/rocoto/run-foo-complete.out | 2 +- .../user_guide/cli/tools/rocoto/run-foo.out | 8 ++++---- src/uwtools/rocoto.py | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-bar-complete.out b/docs/sections/user_guide/cli/tools/rocoto/run-bar-complete.out index 9c2bec3f4..8c4951c87 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/run-bar-complete.out +++ b/docs/sections/user_guide/cli/tools/rocoto/run-bar-complete.out @@ -1 +1 @@ -[2025-07-21T22:43:27] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUCCEEDED +[2025-07-22T01:18:52] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUCCEEDED diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-bar.out b/docs/sections/user_guide/cli/tools/rocoto/run-bar.out index 11aeb7d2a..768a99006 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/run-bar.out +++ b/docs/sections/user_guide/cli/tools/rocoto/run-bar.out @@ -1,10 +1,10 @@ -[2025-07-21T22:30:40] INFO Iterating workflow -[2025-07-21T22:31:11] INFO Workflow status: -[2025-07-21T22:31:12] INFO CYCLE TASK JOBID STATE EXIT STATUS TRIES DURATION -[2025-07-21T22:31:12] INFO ================================================================================================================================ -[2025-07-21T22:31:12] INFO 202507170000 foo druby://10.178.9.5:40925 SUBMITTING - 0 0.0 -[2025-07-21T22:31:12] INFO 202507170000 bar - - - - - -[2025-07-21T22:31:15] INFO Iterating workflow -[2025-07-21T22:31:46] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUBMITTING -[2025-07-21T22:31:46] INFO Iterating workflow -[2025-07-21T22:31:48] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUCCEEDED +[2025-07-22T01:17:34] INFO Iterating workflow +[2025-07-22T01:18:06] INFO Workflow status: +[2025-07-22T01:18:07] INFO CYCLE TASK JOBID STATE EXIT STATUS TRIES DURATION +[2025-07-22T01:18:07] INFO ================================================================================================================================ +[2025-07-22T01:18:07] INFO 202507170000 foo druby://10.178.9.5:45143 SUBMITTING - 0 0.0 +[2025-07-22T01:18:07] INFO 202507170000 bar - - - - - +[2025-07-22T01:18:10] INFO Iterating workflow +[2025-07-22T01:18:41] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUBMITTING +[2025-07-22T01:18:41] INFO Iterating workflow +[2025-07-22T01:18:43] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUCCEEDED diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-foo-complete.out b/docs/sections/user_guide/cli/tools/rocoto/run-foo-complete.out index 38360ebf2..fd42f2a86 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/run-foo-complete.out +++ b/docs/sections/user_guide/cli/tools/rocoto/run-foo-complete.out @@ -1 +1 @@ -[2025-07-21T22:28:38] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUCCEEDED +[2025-07-22T01:13:14] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUCCEEDED diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-foo.out b/docs/sections/user_guide/cli/tools/rocoto/run-foo.out index 90af78d3b..8968eb1ca 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/run-foo.out +++ b/docs/sections/user_guide/cli/tools/rocoto/run-foo.out @@ -1,4 +1,4 @@ -[2025-07-21T22:27:26] INFO Iterating workflow -[2025-07-21T22:27:57] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUBMITTING -[2025-07-21T22:27:57] INFO Iterating workflow -[2025-07-21T22:28:28] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUCCEEDED +[2025-07-22T01:12:07] INFO Iterating workflow +[2025-07-22T01:12:39] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUBMITTING +[2025-07-22T01:12:39] INFO Iterating workflow +[2025-07-22T01:13:10] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUCCEEDED diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index f021ecbdd..00d630284 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -111,7 +111,7 @@ def run(self) -> bool: if not self._iterate(): return False state = self._state - if state is None or state in self._states["active"]: + if not state or state in self._states["active"]: self._report() log.debug("Sleeping %s seconds", self._rate) sleep(self._rate) From 30b694061b90c4fd87e61c9ec87ec97a8bf04a3d Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 22 Jul 2025 13:56:15 +0000 Subject: [PATCH 53/69] Reorder tests --- src/uwtools/tests/test_rocoto.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index e5c21e3d8..ea73e076b 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -47,30 +47,22 @@ def validation_assets(tmp_path): # Tests -def test_rocoto_realize__rocoto_invalid_xml(assets): - cfgfile, outfile = assets - with patch.object(rocoto, "validate_string") as vrxs: - vrxs.return_value = False - with raises(UWError): - rocoto.realize(config=cfgfile, output_file=outfile) - - def test_rocoto_realize__cfg_to_file(assets): cfgfile, outfile = assets rocoto.realize(config=YAMLConfig(cfgfile), output_file=outfile) assert rocoto.validate_file(xml_file=outfile) -def test_rocoto_realize__file_to_file(assets): +def test_rocoto_realize__cfg_to_stdout(capsys, assets): cfgfile, outfile = assets - rocoto.realize(config=cfgfile, output_file=outfile) + rocoto.realize(config=YAMLConfig(cfgfile)) + outfile.write_text(capsys.readouterr().out) assert rocoto.validate_file(xml_file=outfile) -def test_rocoto_realize__cfg_to_stdout(capsys, assets): +def test_rocoto_realize__file_to_file(assets): cfgfile, outfile = assets - rocoto.realize(config=YAMLConfig(cfgfile)) - outfile.write_text(capsys.readouterr().out) + rocoto.realize(config=cfgfile, output_file=outfile) assert rocoto.validate_file(xml_file=outfile) @@ -81,6 +73,14 @@ def test_rocoto_realize__file_to_stdout(capsys, assets): assert rocoto.validate_file(xml_file=outfile) +def test_rocoto_realize__invalid_xml(assets): + cfgfile, outfile = assets + with patch.object(rocoto, "validate_string") as vrxs: + vrxs.return_value = False + with raises(UWError): + rocoto.realize(config=cfgfile, output_file=outfile) + + def test_rocoto_run(rocoto_runner_args): with patch.object(rocoto, "_RocotoRunner") as _RocotoRunner: # noqa: N806 rocoto.run(**rocoto_runner_args) From da7887b7ff68a747cb75b067036f0ada33724e08 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 22 Jul 2025 22:12:05 +0000 Subject: [PATCH 54/69] Restore Makefile.outputs --- docs/sections/user_guide/cli/Makefile.outputs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/sections/user_guide/cli/Makefile.outputs b/docs/sections/user_guide/cli/Makefile.outputs index 587c5fdc9..abce0a427 100644 --- a/docs/sections/user_guide/cli/Makefile.outputs +++ b/docs/sections/user_guide/cli/Makefile.outputs @@ -9,3 +9,9 @@ all: $(OUTPUTS) $(OUTPUTS): @bash $(basename $@).cmd >$@ 2>&1 | true + +%.out: %.txt %.yaml + @bash $< >$@ 2>&1 | true + +%.out: %.txt + @bash $< >$@ 2>&1 | true From 32d4433256017ff23eaca6bc73a83a7fc39986b1 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 22 Jul 2025 23:32:14 +0000 Subject: [PATCH 55/69] Work on docs [skip ci] --- .../user_guide/cli/tools/fs/copy-glob.out | 8 ++--- docs/sections/user_guide/cli/tools/rocoto.rst | 22 +++++++------- .../cli/tools/rocoto/foobar-realize.cmd | 1 + .../cli/tools/rocoto/foobar-realize.out | 30 +++++++++++++++++++ .../user_guide/cli/tools/rocoto/foobar.xml | 26 ---------------- .../user_guide/cli/tools/rocoto/foobar.yaml | 22 ++++++++++++++ .../cli/tools/rocoto/run-foo-complete.out | 1 - .../user_guide/cli/tools/rocoto/run-foo.txt | 6 +++- 8 files changed, 74 insertions(+), 42 deletions(-) create mode 100644 docs/sections/user_guide/cli/tools/rocoto/foobar-realize.cmd create mode 100644 docs/sections/user_guide/cli/tools/rocoto/foobar-realize.out delete mode 100644 docs/sections/user_guide/cli/tools/rocoto/foobar.xml create mode 100644 docs/sections/user_guide/cli/tools/rocoto/foobar.yaml delete mode 100644 docs/sections/user_guide/cli/tools/rocoto/run-foo-complete.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-glob.out b/docs/sections/user_guide/cli/tools/fs/copy-glob.out index a0f8558f6..cef2577e1 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-glob.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-glob.out @@ -1,16 +1,16 @@ [2025-01-02T03:04:05] INFO Validating config against internal schema: files-to-stage [2025-01-02T03:04:05] INFO Schema validation succeeded for fs config [2025-01-02T03:04:05] WARNING Ignoring directory src/20240529 -[2025-01-02T03:04:05] INFO Local src/foo -> dst/copy-glob/dst/foo: Executing -[2025-01-02T03:04:05] INFO Local src/foo -> dst/copy-glob/dst/foo: Ready [2025-01-02T03:04:05] INFO Local src/bar -> dst/copy-glob/dst/bar: Executing [2025-01-02T03:04:05] INFO Local src/bar -> dst/copy-glob/dst/bar: Ready +[2025-01-02T03:04:05] INFO Local src/foo -> dst/copy-glob/dst/foo: Executing +[2025-01-02T03:04:05] INFO Local src/foo -> dst/copy-glob/dst/foo: Ready [2025-01-02T03:04:05] INFO File copies: Ready { "not-ready": [], "ready": [ - "dst/copy-glob/dst/foo", - "dst/copy-glob/dst/bar" + "dst/copy-glob/dst/bar", + "dst/copy-glob/dst/foo" ] } diff --git a/docs/sections/user_guide/cli/tools/rocoto.rst b/docs/sections/user_guide/cli/tools/rocoto.rst index 048e55671..3ecf657b6 100644 --- a/docs/sections/user_guide/cli/tools/rocoto.rst +++ b/docs/sections/user_guide/cli/tools/rocoto.rst @@ -80,26 +80,28 @@ Examples .. note:: Use of ``uw rocoto run`` requires presence of the ``rocotorun`` and ``rocotostat`` executables on ``PATH``. On HPCs, this is typically achieved by loading a system module providing Rocoto. -The following examples make use of a simple Rocoto XML workflow document (which might have been created by ``uw rocoto realize``) similar to: +The following examples make use of this simple UW YAML for Rocoto config: -.. literalinclude:: rocoto/foobar.xml +.. literalinclude:: rocoto/foobar.yaml + :language: yaml + +It could be rendered to a Rocoto XML document like this: + +.. literalinclude:: rocoto/foobar-realize.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: rocoto/foobar-realize.out :language: xml * To run only task ``foo``, which has no dependencies: .. literalinclude:: rocoto/run-foo.txt :language: text - :emphasize-lines: 1 + :emphasize-lines: 4-5 .. literalinclude:: rocoto/run-foo.out :language: text - A second invocation of ``uw rocoto run`` immediately shows task ``foo`` in its final state: - - .. literalinclude:: rocoto/run-foo.txt - :language: text - :emphasize-lines: 1 - .. literalinclude:: rocoto/run-foo-complete.out - :language: text + Note that the second invocation of ``uw rocoto run`` immediately shows task ``foo`` in its final state. * To run task ``bar``, which depends on task ``foo``, iterating every 3 seconds, and after deleting the ``foobar.db`` database file from the previous example: diff --git a/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.cmd b/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.cmd new file mode 100644 index 000000000..ad5ebd654 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.cmd @@ -0,0 +1 @@ +ROOT=/some/path uw rocoto realize -c foobar.yaml diff --git a/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.out b/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.out new file mode 100644 index 000000000..c86f9e6dd --- /dev/null +++ b/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.out @@ -0,0 +1,30 @@ +[2025-01-02T03:04:05] INFO Schema validation succeeded for Rocoto config +[2025-01-02T03:04:05] INFO Schema validation succeeded for Rocoto XML + + + 202507170000 202507170000 00:00:01 + {{ 'ROOT' | env }}/log + + wrfruc + 1 + service + batch + 00:01:00 + echo foo >/some/path/foo + foo + /some/path/batchout + + + wrfruc + 1 + service + batch + 00:01:00 + echo bar >/some/path/bar + bar + /some/path/batchout + + + + + diff --git a/docs/sections/user_guide/cli/tools/rocoto/foobar.xml b/docs/sections/user_guide/cli/tools/rocoto/foobar.xml deleted file mode 100644 index 6537ee471..000000000 --- a/docs/sections/user_guide/cli/tools/rocoto/foobar.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - 202507170000 202507170000 00:00:01 - /path/to/foobar.log - - account - echo foo >/path/tofoo.txt - 1 - /path/to/foo.batchout - service - batch - 00:01:00 - - - account - echo bar >/path/to/bar.txt - 1 - /path/to/bar.batchout - service - batch - 00:01:00 - - - - - diff --git a/docs/sections/user_guide/cli/tools/rocoto/foobar.yaml b/docs/sections/user_guide/cli/tools/rocoto/foobar.yaml new file mode 100644 index 000000000..05698c03f --- /dev/null +++ b/docs/sections/user_guide/cli/tools/rocoto/foobar.yaml @@ -0,0 +1,22 @@ +common: &common + account: wrfruc + attrs: {cycledefs: default} + cores: 1 + join: "{{ 'ROOT' | env }}/batchout" + partition: service + queue: batch + walltime: "00:01:00" +workflow: + attrs: {realtime: false, scheduler: slurm} + cycledef: + - attrs: {group: default} + spec: 202507170000 202507170000 00:00:01 + log: {value: "{{ 'ROOT' | env }}/log" } + tasks: + task_foo: + <<: *common + command: echo foo >{{ 'ROOT' | env }}/foo + task_bar: + <<: *common + command: echo bar >{{ 'ROOT' | env }}/bar + dependency: {taskdep: {attrs: {task: foo}}} diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-foo-complete.out b/docs/sections/user_guide/cli/tools/rocoto/run-foo-complete.out deleted file mode 100644 index fd42f2a86..000000000 --- a/docs/sections/user_guide/cli/tools/rocoto/run-foo-complete.out +++ /dev/null @@ -1 +0,0 @@ -[2025-07-22T01:13:14] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUCCEEDED diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-foo.txt b/docs/sections/user_guide/cli/tools/rocoto/run-foo.txt index ca74a4712..089f0d502 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/run-foo.txt +++ b/docs/sections/user_guide/cli/tools/rocoto/run-foo.txt @@ -1 +1,5 @@ -uw rocoto run --cycle 2025-07-17T00 --database foobar.db --task foo --workflow foobar.xml +rm -rf /tmp/foobar/foo +mkdir -pv /tmp/foobar/foo +set -x +uw rocoto run --cycle 2025-07-17T00 --database /tmp/foobar/foo/foobar.db --task foo --workflow foobar.xml +uw rocoto run --cycle 2025-07-17T00 --database /tmp/foobar/foo/foobar.db --task foo --workflow foobar.xml From b81ead7cb2cf957b8615de3183978b953722867b Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 23 Jul 2025 01:25:14 +0000 Subject: [PATCH 56/69] Add commentary to Makefile.outputs --- docs/sections/user_guide/cli/Makefile.outputs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/sections/user_guide/cli/Makefile.outputs b/docs/sections/user_guide/cli/Makefile.outputs index abce0a427..63bb6abe5 100644 --- a/docs/sections/user_guide/cli/Makefile.outputs +++ b/docs/sections/user_guide/cli/Makefile.outputs @@ -10,6 +10,12 @@ all: $(OUTPUTS) $(OUTPUTS): @bash $(basename $@).cmd >$@ 2>&1 | true +# The following targets support semi-automated output generation: They will not run automatically, +# but can be manually invoked (e.g. "make foo.out" given a "foo.txt" command file) to upate .out +# files. They must be invoked in a context where all commands in the .txt file are available, e.g. +# on an HPC where the "hsi" or "rocotorun" commands are on PATH, if those are ultimately called by +# the recipe. + %.out: %.txt %.yaml @bash $< >$@ 2>&1 | true From 81715e9008f6e9c49fae093b45e5311151e076d2 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 23 Jul 2025 01:27:38 +0000 Subject: [PATCH 57/69] WIP [skip ci] --- docs/sections/user_guide/cli/tools/fs/copy-glob.out | 8 ++++---- .../user_guide/cli/tools/rocoto/foobar-realize.out | 2 +- src/uwtools/rocoto.py | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/sections/user_guide/cli/tools/fs/copy-glob.out b/docs/sections/user_guide/cli/tools/fs/copy-glob.out index cef2577e1..a0f8558f6 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-glob.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-glob.out @@ -1,16 +1,16 @@ [2025-01-02T03:04:05] INFO Validating config against internal schema: files-to-stage [2025-01-02T03:04:05] INFO Schema validation succeeded for fs config [2025-01-02T03:04:05] WARNING Ignoring directory src/20240529 -[2025-01-02T03:04:05] INFO Local src/bar -> dst/copy-glob/dst/bar: Executing -[2025-01-02T03:04:05] INFO Local src/bar -> dst/copy-glob/dst/bar: Ready [2025-01-02T03:04:05] INFO Local src/foo -> dst/copy-glob/dst/foo: Executing [2025-01-02T03:04:05] INFO Local src/foo -> dst/copy-glob/dst/foo: Ready +[2025-01-02T03:04:05] INFO Local src/bar -> dst/copy-glob/dst/bar: Executing +[2025-01-02T03:04:05] INFO Local src/bar -> dst/copy-glob/dst/bar: Ready [2025-01-02T03:04:05] INFO File copies: Ready { "not-ready": [], "ready": [ - "dst/copy-glob/dst/bar", - "dst/copy-glob/dst/foo" + "dst/copy-glob/dst/foo", + "dst/copy-glob/dst/bar" ] } diff --git a/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.out b/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.out index c86f9e6dd..5f9ff7a7e 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.out +++ b/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.out @@ -3,7 +3,7 @@ 202507170000 202507170000 00:00:01 - {{ 'ROOT' | env }}/log + /some/path/log wrfruc 1 diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 00d630284..173211d96 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -187,6 +187,7 @@ class _RocotoXML: def __init__(self, config: dict | YAMLConfig | Path | None = None) -> None: self._config_validate(config) cfgobj = config if isinstance(config, YAMLConfig) else YAMLConfig(config) + cfgobj.dereference() self._config = cfgobj.data self._add_workflow(self._config) From d53ffddfdb8e174ed2a3687fbf4508c71f59b4ff Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 23 Jul 2025 02:48:17 +0000 Subject: [PATCH 58/69] WIP [skip ci] --- .../sections/user_guide/cli/tools/fs/copy-glob.out | 8 ++++---- docs/sections/user_guide/cli/tools/rocoto.rst | 2 +- .../user_guide/cli/tools/rocoto/.gitignore | 1 + .../user_guide/cli/tools/rocoto/foobar-realize.cmd | 2 +- .../user_guide/cli/tools/rocoto/foobar.yaml | 7 +++---- .../user_guide/cli/tools/rocoto/run-foo.out | 11 +++++++---- .../user_guide/cli/tools/rocoto/run-foo.txt | 14 +++++++++----- src/uwtools/rocoto.py | 2 +- 8 files changed, 27 insertions(+), 20 deletions(-) diff --git a/docs/sections/user_guide/cli/tools/fs/copy-glob.out b/docs/sections/user_guide/cli/tools/fs/copy-glob.out index a0f8558f6..cef2577e1 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-glob.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-glob.out @@ -1,16 +1,16 @@ [2025-01-02T03:04:05] INFO Validating config against internal schema: files-to-stage [2025-01-02T03:04:05] INFO Schema validation succeeded for fs config [2025-01-02T03:04:05] WARNING Ignoring directory src/20240529 -[2025-01-02T03:04:05] INFO Local src/foo -> dst/copy-glob/dst/foo: Executing -[2025-01-02T03:04:05] INFO Local src/foo -> dst/copy-glob/dst/foo: Ready [2025-01-02T03:04:05] INFO Local src/bar -> dst/copy-glob/dst/bar: Executing [2025-01-02T03:04:05] INFO Local src/bar -> dst/copy-glob/dst/bar: Ready +[2025-01-02T03:04:05] INFO Local src/foo -> dst/copy-glob/dst/foo: Executing +[2025-01-02T03:04:05] INFO Local src/foo -> dst/copy-glob/dst/foo: Ready [2025-01-02T03:04:05] INFO File copies: Ready { "not-ready": [], "ready": [ - "dst/copy-glob/dst/foo", - "dst/copy-glob/dst/bar" + "dst/copy-glob/dst/bar", + "dst/copy-glob/dst/foo" ] } diff --git a/docs/sections/user_guide/cli/tools/rocoto.rst b/docs/sections/user_guide/cli/tools/rocoto.rst index 3ecf657b6..36ad50767 100644 --- a/docs/sections/user_guide/cli/tools/rocoto.rst +++ b/docs/sections/user_guide/cli/tools/rocoto.rst @@ -97,7 +97,7 @@ It could be rendered to a Rocoto XML document like this: .. literalinclude:: rocoto/run-foo.txt :language: text - :emphasize-lines: 4-5 + :emphasize-lines: 6 .. literalinclude:: rocoto/run-foo.out :language: text diff --git a/docs/sections/user_guide/cli/tools/rocoto/.gitignore b/docs/sections/user_guide/cli/tools/rocoto/.gitignore index 0765e6530..6b8247338 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/.gitignore +++ b/docs/sections/user_guide/cli/tools/rocoto/.gitignore @@ -1,2 +1,3 @@ rocoto.log rocoto.xml +tmp.* diff --git a/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.cmd b/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.cmd index ad5ebd654..56f669084 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.cmd +++ b/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.cmd @@ -1 +1 @@ -ROOT=/some/path uw rocoto realize -c foobar.yaml +RUNDIR=/some/path uw rocoto realize -c foobar.yaml diff --git a/docs/sections/user_guide/cli/tools/rocoto/foobar.yaml b/docs/sections/user_guide/cli/tools/rocoto/foobar.yaml index 05698c03f..483542039 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/foobar.yaml +++ b/docs/sections/user_guide/cli/tools/rocoto/foobar.yaml @@ -1,8 +1,9 @@ common: &common account: wrfruc attrs: {cycledefs: default} + command: /bin/true cores: 1 - join: "{{ 'ROOT' | env }}/batchout" + join: "{{ 'RUNDIR' | env }}/slurm" partition: service queue: batch walltime: "00:01:00" @@ -11,12 +12,10 @@ workflow: cycledef: - attrs: {group: default} spec: 202507170000 202507170000 00:00:01 - log: {value: "{{ 'ROOT' | env }}/log" } + log: {value: "{{ 'RUNDIR' | env }}/log" } tasks: task_foo: <<: *common - command: echo foo >{{ 'ROOT' | env }}/foo task_bar: <<: *common - command: echo bar >{{ 'ROOT' | env }}/bar dependency: {taskdep: {attrs: {task: foo}}} diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-foo.out b/docs/sections/user_guide/cli/tools/rocoto/run-foo.out index 8968eb1ca..621af615d 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/run-foo.out +++ b/docs/sections/user_guide/cli/tools/rocoto/run-foo.out @@ -1,4 +1,7 @@ -[2025-07-22T01:12:07] INFO Iterating workflow -[2025-07-22T01:12:39] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUBMITTING -[2025-07-22T01:12:39] INFO Iterating workflow -[2025-07-22T01:13:10] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUCCEEDED ++ uw rocoto run --cycle 2025-07-17T00 --database db --task foo --workflow xml +[2025-01-02T03:04:05] INFO Iterating workflow +[2025-01-02T03:04:05] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUBMITTING +[2025-01-02T03:04:05] INFO Iterating workflow +[2025-01-02T03:04:05] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUCCEEDED ++ uw rocoto run --cycle 2025-07-17T00 --database db --task foo --workflow xml +[2025-01-02T03:04:05] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUCCEEDED diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-foo.txt b/docs/sections/user_guide/cli/tools/rocoto/run-foo.txt index 089f0d502..259c25713 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/run-foo.txt +++ b/docs/sections/user_guide/cli/tools/rocoto/run-foo.txt @@ -1,5 +1,9 @@ -rm -rf /tmp/foobar/foo -mkdir -pv /tmp/foobar/foo -set -x -uw rocoto run --cycle 2025-07-17T00 --database /tmp/foobar/foo/foobar.db --task foo --workflow foobar.xml -uw rocoto run --cycle 2025-07-17T00 --database /tmp/foobar/foo/foobar.db --task foo --workflow foobar.xml +export RUNDIR=$(readlink -f $(mktemp -d -p $(dirname $0))) +uw rocoto realize -c foobar.yaml >$RUNDIR/xml 2>/dev/null +for invocation in 1 2; do ( + cd $RUNDIR + set -x + uw rocoto run --cycle 2025-07-17T00 --database db --task foo --workflow xml +) +done +rm -rf $RUNDIR diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 173211d96..5d34334d2 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -145,7 +145,7 @@ def _query_data(self) -> dict: @property def _query_stmt(self) -> str: - return "select state from jobs where taskname=:taskname and cycle=:cycle" + return "select state from jobs where taskname=:taskname and cycle=:cycle order by id desc" def _report(self) -> None: cmd = "rocotostat -d %s -w %s" % (self._database, self._workflow) From e02c8687a75b491c510286a31235d1f7d0714932 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 23 Jul 2025 02:53:38 +0000 Subject: [PATCH 59/69] Work on docs --- docs/sections/user_guide/cli/tools/rocoto.rst | 14 ++++------- .../cli/tools/rocoto/run-bar-complete.out | 1 - .../user_guide/cli/tools/rocoto/run-bar.out | 23 +++++++++++-------- .../user_guide/cli/tools/rocoto/run-bar.txt | 10 +++++++- 4 files changed, 26 insertions(+), 22 deletions(-) delete mode 100644 docs/sections/user_guide/cli/tools/rocoto/run-bar-complete.out diff --git a/docs/sections/user_guide/cli/tools/rocoto.rst b/docs/sections/user_guide/cli/tools/rocoto.rst index 36ad50767..37ee122a2 100644 --- a/docs/sections/user_guide/cli/tools/rocoto.rst +++ b/docs/sections/user_guide/cli/tools/rocoto.rst @@ -101,23 +101,17 @@ It could be rendered to a Rocoto XML document like this: .. literalinclude:: rocoto/run-foo.out :language: text - Note that the second invocation of ``uw rocoto run`` immediately shows task ``foo`` in its final state. + Note that the second invocation of ``uw rocoto run`` immediately shows task ``foo`` in its final state, without iterating the workflow. -* To run task ``bar``, which depends on task ``foo``, iterating every 3 seconds, and after deleting the ``foobar.db`` database file from the previous example: +* To run task ``bar``, which depends on task ``foo``, iterating every 3 seconds: .. literalinclude:: rocoto/run-bar.txt :language: text - :emphasize-lines: 1 + :emphasize-lines: 4 .. literalinclude:: rocoto/run-bar.out :language: text - A second invocation of ``uw rocoto run`` immediately shows task ``bar`` in its final state: - - .. literalinclude:: rocoto/run-bar.txt - :language: text - :emphasize-lines: 1 - .. literalinclude:: rocoto/run-bar-complete.out - :language: text + Note that the second invocation of ``uw rocoto run`` immediately shows task ``bar`` in its final state, without iterating the workflow. .. _cli_rocoto_validate_examples: diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-bar-complete.out b/docs/sections/user_guide/cli/tools/rocoto/run-bar-complete.out deleted file mode 100644 index 8c4951c87..000000000 --- a/docs/sections/user_guide/cli/tools/rocoto/run-bar-complete.out +++ /dev/null @@ -1 +0,0 @@ -[2025-07-22T01:18:52] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUCCEEDED diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-bar.out b/docs/sections/user_guide/cli/tools/rocoto/run-bar.out index 768a99006..d96dfe681 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/run-bar.out +++ b/docs/sections/user_guide/cli/tools/rocoto/run-bar.out @@ -1,10 +1,13 @@ -[2025-07-22T01:17:34] INFO Iterating workflow -[2025-07-22T01:18:06] INFO Workflow status: -[2025-07-22T01:18:07] INFO CYCLE TASK JOBID STATE EXIT STATUS TRIES DURATION -[2025-07-22T01:18:07] INFO ================================================================================================================================ -[2025-07-22T01:18:07] INFO 202507170000 foo druby://10.178.9.5:45143 SUBMITTING - 0 0.0 -[2025-07-22T01:18:07] INFO 202507170000 bar - - - - - -[2025-07-22T01:18:10] INFO Iterating workflow -[2025-07-22T01:18:41] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUBMITTING -[2025-07-22T01:18:41] INFO Iterating workflow -[2025-07-22T01:18:43] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUCCEEDED ++ uw rocoto run --cycle 2025-07-17T00 --database db --task bar --workflow xml +[2025-01-02T03:04:05] INFO Iterating workflow +[2025-01-02T03:04:05] INFO Workflow status: +[2025-01-02T03:04:05] INFO CYCLE TASK JOBID STATE EXIT STATUS TRIES DURATION +[2025-01-02T03:04:05] INFO ================================================================================================================================ +[2025-01-02T03:04:05] INFO 202507170000 foo druby://10.178.9.5:36657 SUBMITTING - 0 0.0 +[2025-01-02T03:04:05] INFO 202507170000 bar - - - - - +[2025-01-02T03:04:05] INFO Iterating workflow +[2025-01-02T03:04:05] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUBMITTING +[2025-01-02T03:04:05] INFO Iterating workflow +[2025-01-02T03:04:05] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUCCEEDED ++ uw rocoto run --cycle 2025-07-17T00 --database db --task bar --workflow xml +[2025-01-02T03:04:05] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUCCEEDED diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-bar.txt b/docs/sections/user_guide/cli/tools/rocoto/run-bar.txt index 3676817a7..8e57415b4 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/run-bar.txt +++ b/docs/sections/user_guide/cli/tools/rocoto/run-bar.txt @@ -1 +1,9 @@ -uw rocoto run --cycle 2025-07-17T00 --database foobar.db --rate 3 --task bar --workflow foobar.xml +export RUNDIR=$(readlink -f $(mktemp -d -p $(dirname $0))) +uw rocoto realize -c foobar.yaml >$RUNDIR/xml 2>/dev/null +for invocation in 1 2; do ( + cd $RUNDIR + set -x + uw rocoto run --cycle 2025-07-17T00 --database db --task bar --workflow xml +) +done +rm -rf $RUNDIR From 13d4259d9620dd1840dea0a64cc4ea7c42010d08 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 23 Jul 2025 02:55:00 +0000 Subject: [PATCH 60/69] Work on docs --- docs/sections/user_guide/cli/tools/fs/copy-glob.out | 8 ++++---- .../user_guide/cli/tools/rocoto/foobar-realize.out | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/sections/user_guide/cli/tools/fs/copy-glob.out b/docs/sections/user_guide/cli/tools/fs/copy-glob.out index cef2577e1..a0f8558f6 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-glob.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-glob.out @@ -1,16 +1,16 @@ [2025-01-02T03:04:05] INFO Validating config against internal schema: files-to-stage [2025-01-02T03:04:05] INFO Schema validation succeeded for fs config [2025-01-02T03:04:05] WARNING Ignoring directory src/20240529 -[2025-01-02T03:04:05] INFO Local src/bar -> dst/copy-glob/dst/bar: Executing -[2025-01-02T03:04:05] INFO Local src/bar -> dst/copy-glob/dst/bar: Ready [2025-01-02T03:04:05] INFO Local src/foo -> dst/copy-glob/dst/foo: Executing [2025-01-02T03:04:05] INFO Local src/foo -> dst/copy-glob/dst/foo: Ready +[2025-01-02T03:04:05] INFO Local src/bar -> dst/copy-glob/dst/bar: Executing +[2025-01-02T03:04:05] INFO Local src/bar -> dst/copy-glob/dst/bar: Ready [2025-01-02T03:04:05] INFO File copies: Ready { "not-ready": [], "ready": [ - "dst/copy-glob/dst/bar", - "dst/copy-glob/dst/foo" + "dst/copy-glob/dst/foo", + "dst/copy-glob/dst/bar" ] } diff --git a/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.out b/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.out index 5f9ff7a7e..797327b6e 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.out +++ b/docs/sections/user_guide/cli/tools/rocoto/foobar-realize.out @@ -10,9 +10,9 @@ service batch 00:01:00 - echo foo >/some/path/foo + /bin/true foo - /some/path/batchout + /some/path/slurm wrfruc @@ -20,9 +20,9 @@ service batch 00:01:00 - echo bar >/some/path/bar + /bin/true bar - /some/path/batchout + /some/path/slurm From 9be3a8f33afe2c2f9d2dc887ef96bfc03258df1a Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 23 Jul 2025 03:04:04 +0000 Subject: [PATCH 61/69] Fix test --- src/uwtools/tests/test_rocoto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index ea73e076b..662c2cff3 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -211,7 +211,7 @@ def test_rocoto__RocotoRunner__query_data(self, instance): def test_rocoto__RocotoRunner__query_stmt(self, instance): assert ( instance._query_stmt - == "select state from jobs where taskname=:taskname and cycle=:cycle" + == "select state from jobs where taskname=:taskname and cycle=:cycle order by id desc" ) def test_rocoto__RocotoRunner__report(self, instance, logged): From 56283f4b814ea54eafbd2fb685abe111fc2d211e Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 23 Jul 2025 16:45:05 +0000 Subject: [PATCH 62/69] Improve & test hardlink error logging --- src/uwtools/tests/utils/test_tasks.py | 4 +++- src/uwtools/utils/tasks.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/uwtools/tests/utils/test_tasks.py b/src/uwtools/tests/utils/test_tasks.py index 68e3af784..80b3259f0 100644 --- a/src/uwtools/tests/utils/test_tasks.py +++ b/src/uwtools/tests/utils/test_tasks.py @@ -288,10 +288,12 @@ def test_utils_tasks_hardlink__cannot_hardlink(logged, prefix, symlink_fallback, target.touch() assert not link.exists() t2, l2 = ["%s%s" % (prefix, x) if prefix else x for x in (target, link)] - with patch.object(tasks.os, "link", side_effect=OSError): + with patch.object(tasks.os, "link", side_effect=OSError("big\ntrouble\n")): tasks.hardlink(target=t2, linkname=l2, symlink_fallback=symlink_fallback) assert link.is_symlink() is symlink_fallback if not symlink_fallback: + assert logged("big") + assert logged("trouble") assert logged("Could not hardlink %s -> %s" % (link, target)) diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index bdf174c50..5ef998108 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -232,7 +232,8 @@ def hardlink( if symlink_fallback: os.symlink(src, dst) else: - log.error(str(e)) + for line in str(e).split("\n"): + log.error(line) raise UWError("Could not hardlink %s -> %s" % (dst, src)) from e From dc7213e73c89df9c221a4d8e723abbc945e4791c Mon Sep 17 00:00:00 2001 From: Paul Madden <136389411+maddenp-cu@users.noreply.github.com> Date: Wed, 23 Jul 2025 11:04:52 -0600 Subject: [PATCH 63/69] Update docs/sections/user_guide/cli/Makefile.outputs Co-authored-by: Emily Carpenter <137525341+elcarpenterNOAA@users.noreply.github.com> --- docs/sections/user_guide/cli/Makefile.outputs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sections/user_guide/cli/Makefile.outputs b/docs/sections/user_guide/cli/Makefile.outputs index 63bb6abe5..30a69c5d5 100644 --- a/docs/sections/user_guide/cli/Makefile.outputs +++ b/docs/sections/user_guide/cli/Makefile.outputs @@ -11,7 +11,7 @@ $(OUTPUTS): @bash $(basename $@).cmd >$@ 2>&1 | true # The following targets support semi-automated output generation: They will not run automatically, -# but can be manually invoked (e.g. "make foo.out" given a "foo.txt" command file) to upate .out +# but can be manually invoked (e.g. "make foo.out" given a "foo.txt" command file) to update .out # files. They must be invoked in a context where all commands in the .txt file are available, e.g. # on an HPC where the "hsi" or "rocotorun" commands are on PATH, if those are ultimately called by # the recipe. From 16867d6377cc8f88329ddbb9e2f4053c5974347d Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 23 Jul 2025 23:03:25 +0000 Subject: [PATCH 64/69] Fix comments --- src/uwtools/config/atparse_to_jinja2.py | 3 +-- src/uwtools/fs.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/uwtools/config/atparse_to_jinja2.py b/src/uwtools/config/atparse_to_jinja2.py index 2f4063eb3..77ce5e3e9 100644 --- a/src/uwtools/config/atparse_to_jinja2.py +++ b/src/uwtools/config/atparse_to_jinja2.py @@ -50,8 +50,7 @@ def _replace(atline: str) -> str: # Set maxsplits to 1 so only first @[ is captured. before_atparse = atline.split("@[", 1)[0] within_atparse = atline.split("@[")[1].split("]")[0] - # Set maxsplits to 1 so only first ] is captured, which should be the - # bracket closing @[. + # Set maxsplits to 1 so only first ] is captured, which should be the bracket closing @[. after_atparse = atline.split("@[", 1)[1].split("]", 1)[1] atline = "".join([before_atparse, "{{ ", within_atparse, " }}", after_atparse]) return atline diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 59efd4fd8..8d1920d28 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -237,7 +237,7 @@ def go(self): # If a source path is a glob pattern, the existence of the file(s) found via glob expansion # is already assured and it is unnecessary to check again for their existence. If, however, - # a source path is a full explicit path, its existence should be checked before and attempt + # a source path is a full explicit path, its existence should be checked before any attempt # is made to copy it. yield "File copies" From b1605703b34b8de1ab1258b469d0a110702bf0cd Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 24 Jul 2025 15:45:19 +0000 Subject: [PATCH 65/69] Run -> Iterate (mostly code) --- docs/index.rst | 2 +- docs/sections/user_guide/cli/tools/rocoto.rst | 2 +- src/uwtools/api/rocoto.py | 10 +- src/uwtools/cli.py | 62 ++++++------- src/uwtools/rocoto.py | 50 +++++----- src/uwtools/strings.py | 1 + src/uwtools/tests/api/test_rocoto.py | 22 ++--- src/uwtools/tests/test_cli.py | 36 ++++---- src/uwtools/tests/test_rocoto.py | 92 +++++++++---------- 9 files changed, 139 insertions(+), 138 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 52f0ac2e9..894fedaa9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -107,7 +107,7 @@ Run Action Given a Rocoto XML workflow document, invoke Rocoto in a loop, monitoring its progress, until a specified task is complete. -| :any:`CLI documentation with examples` +| :any:`CLI documentation with examples` Validate Action """"""""""""""" diff --git a/docs/sections/user_guide/cli/tools/rocoto.rst b/docs/sections/user_guide/cli/tools/rocoto.rst index 4636bc048..e6d8fb289 100644 --- a/docs/sections/user_guide/cli/tools/rocoto.rst +++ b/docs/sections/user_guide/cli/tools/rocoto.rst @@ -69,7 +69,7 @@ The examples in this section use a UW YAML file ``rocoto.yaml`` with contents: .. literalinclude:: rocoto/realize-exec-stdout-verbose.out :language: xml -.. _cli_rocoto_run_examples: +.. _cli_rocoto_iterate_examples: ``run`` ------- diff --git a/src/uwtools/api/rocoto.py b/src/uwtools/api/rocoto.py index 89fd64737..36d35a386 100644 --- a/src/uwtools/api/rocoto.py +++ b/src/uwtools/api/rocoto.py @@ -7,8 +7,8 @@ from typing import TYPE_CHECKING from uwtools.rocoto import DEFAULT_ITERATION_RATE +from uwtools.rocoto import iterate as _iterate from uwtools.rocoto import realize as _realize -from uwtools.rocoto import run as _run from uwtools.rocoto import validate_file as _validate from uwtools.utils.api import ensure_data_source as _ensure_data_source from uwtools.utils.file import str2path as _str2path @@ -43,7 +43,7 @@ def realize( return True -def run( +def iterate( cycle: datetime, database: Path | str, task: str, @@ -51,15 +51,15 @@ def run( rate: int = DEFAULT_ITERATION_RATE, ) -> bool: """ - Run the specified Rocoto workflow to completion (or failure). + Iterate the specified Rocoto workflow to completion (or failure). :param cycle: A datetime object to make available for use in the config. :param database: Path to the Rocoto database file. - :param task: The workflow task to run. + :param task: The workflow task to iterate. :param workflow: Path to the Rocoto XML workflow document. :param rate: Seconds between workflow iterations. """ - return _run( + return _iterate( cycle=cycle, database=_ensure_data_source(_str2path(database), stdin_ok=False), rate=rate, diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 796467ab7..8beb507c3 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -498,39 +498,39 @@ def _add_subparser_rocoto(subparsers: Subparsers) -> ModeChecks: _basic_setup(parser) subparsers = _add_subparsers(parser, STR.action, STR.action.upper()) return { + STR.iterate: _add_subparser_rocoto_iterate(subparsers), STR.realize: _add_subparser_rocoto_realize(subparsers), - STR.run: _add_subparser_rocoto_run(subparsers), STR.validate: _add_subparser_rocoto_validate(subparsers), } -def _add_subparser_rocoto_realize(subparsers: Subparsers) -> ActionChecks: +def _add_subparser_rocoto_iterate(subparsers: Subparsers) -> ActionChecks: """ - Add subparser for mode: rocoto realize. + Add subparser for mode: rocoto iterate. :param subparsers: Parent parser's subparsers, to add this subparser to. """ - parser = _add_subparser(subparsers, STR.realize, "Realize a Rocoto XML workflow document") + parser = _add_subparser(subparsers, STR.iterate, "Iterate a Rocoto workflow") + required = parser.add_argument_group(TITLE_REQ_ARG) + _add_arg_cycle(required) + _add_arg_database(required) + _add_arg_task(required) + _add_arg_workflow(required) optional = _basic_setup(parser) - _add_arg_config_file(optional) - _add_arg_output_file(optional) + _add_arg_rate(optional) return _add_args_verbosity(optional) -def _add_subparser_rocoto_run(subparsers: Subparsers) -> ActionChecks: +def _add_subparser_rocoto_realize(subparsers: Subparsers) -> ActionChecks: """ - Add subparser for mode: rocoto run. + Add subparser for mode: rocoto realize. :param subparsers: Parent parser's subparsers, to add this subparser to. """ - parser = _add_subparser(subparsers, STR.run, "Run a Rocoto workflow") - required = parser.add_argument_group(TITLE_REQ_ARG) - _add_arg_cycle(required) - _add_arg_database(required) - _add_arg_task(required) - _add_arg_workflow(required) + parser = _add_subparser(subparsers, STR.realize, "Realize a Rocoto XML workflow document") optional = _basic_setup(parser) - _add_arg_rate(optional) + _add_arg_config_file(optional) + _add_arg_output_file(optional) return _add_args_verbosity(optional) @@ -553,38 +553,38 @@ def _dispatch_rocoto(args: Args) -> bool: :param args: Parsed command-line args. """ actions = { + STR.iterate: _dispatch_rocoto_iterate, STR.realize: _dispatch_rocoto_realize, - STR.run: _dispatch_rocoto_run, STR.validate: _dispatch_rocoto_validate, } return actions[args[STR.action]](args) -def _dispatch_rocoto_realize(args: Args) -> bool: +def _dispatch_rocoto_iterate(args: Args) -> bool: """ - Define dispatch logic for rocoto realize action. + Define dispatch logic for rocoto iterate action. :param args: Parsed command-line args. """ - return uwtools.api.rocoto.realize( - config=args[STR.cfgfile], - output_file=args[STR.outfile], - stdin_ok=True, + return uwtools.api.rocoto.iterate( + cycle=args[STR.cycle], + database=args[STR.database], + rate=args[STR.rate], + task=args[STR.task], + workflow=args[STR.workflow], ) -def _dispatch_rocoto_run(args: Args) -> bool: +def _dispatch_rocoto_realize(args: Args) -> bool: """ - Define dispatch logic for rocoto run action. + Define dispatch logic for rocoto realize action. :param args: Parsed command-line args. """ - return uwtools.api.rocoto.run( - cycle=args[STR.cycle], - database=args[STR.database], - rate=args[STR.rate], - task=args[STR.task], - workflow=args[STR.workflow], + return uwtools.api.rocoto.realize( + config=args[STR.cfgfile], + output_file=args[STR.outfile], + stdin_ok=True, ) @@ -711,7 +711,7 @@ def _add_arg_batch(group: Group) -> None: group.add_argument( _switch(STR.batch), action="store_true", - help="Submit run to batch scheduler", + help="Submit job to batch scheduler", ) diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 5d34334d2..671d848e3 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -28,7 +28,22 @@ from datetime import datetime -DEFAULT_ITERATION_RATE = 10 +DEFAULT_ITERATION_RATE = 10 # seconds + + +def iterate(cycle: datetime, database: Path, rate: int, task: str, workflow: Path) -> bool: + return _RocotoIterator(cycle, database, rate, task, workflow).iterate() + + +def validate_file(xml_file: Path | None) -> bool: + """ + Validate purported Rocoto XML file against its schema. + + :param xml_file: Path to XML file (None => read stdin). + :return: Did the XML conform to the schema? + """ + with readable(xml_file) as f: + return validate_string(xml=f.read()) def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) -> str: @@ -50,21 +65,6 @@ def realize(config: YAMLConfig | Path | None, output_file: Path | None = None) - return xml -def run(cycle: datetime, database: Path, rate: int, task: str, workflow: Path) -> bool: - return _RocotoRunner(cycle, database, rate, task, workflow).run() - - -def validate_file(xml_file: Path | None) -> bool: - """ - Validate purported Rocoto XML file against its schema. - - :param xml_file: Path to XML file (None => read stdin). - :return: Did the XML conform to the schema? - """ - with readable(xml_file) as f: - return validate_string(xml=f.read()) - - def validate_string(xml: str) -> bool: """ Validate purported Rocoto XML against its schema. @@ -91,7 +91,7 @@ def validate_string(xml: str) -> bool: return valid -class _RocotoRunner: +class _RocotoIterator: def __init__(self, cycle: datetime, database: Path, rate: int, task: str, workflow: Path): self._cycle = cycle self._database = database @@ -105,10 +105,10 @@ def __del__(self): if self._con: self._con.close() - def run(self) -> bool: + def iterate(self) -> bool: state = self._state while state not in self._states["inactive"]: - if not self._iterate(): + if not self._run(): return False state = self._state if not state or state in self._states["active"]: @@ -133,12 +133,6 @@ def _cursor(self) -> sqlite3.Cursor | None: self._cur = connection.cursor() return self._cur - def _iterate(self) -> bool: - log.info("Iterating workflow") - cmd = "rocotorun -d %s -w %s" % (self._database, self._workflow) - success, _ = run_shell_cmd(cmd, quiet=True) - return success - @property def _query_data(self) -> dict: return {"taskname": self._task, "cycle": int(self._cycle.timestamp())} @@ -155,6 +149,12 @@ def _report(self) -> None: for line in output.strip().split("\n"): log.info(line) + def _run(self) -> bool: + log.info("Iterating workflow") + cmd = "rocotorun -d %s -w %s" % (self._database, self._workflow) + success, _ = run_shell_cmd(cmd, quiet=True) + return success + @property def _state(self) -> str | None: state = None diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index bc6baa231..035997b25 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -96,6 +96,7 @@ class STR: infile: str = "input_file" infmt: str = "input_format" ioda: str = "ioda" + iterate: str = "iterate" jedi: str = "jedi" keypath: str = "key_path" keyvalpairs: str = "key_eq_val_pairs" diff --git a/src/uwtools/tests/api/test_rocoto.py b/src/uwtools/tests/api/test_rocoto.py index dabdbd8bd..76fc66605 100644 --- a/src/uwtools/tests/api/test_rocoto.py +++ b/src/uwtools/tests/api/test_rocoto.py @@ -6,27 +6,27 @@ from uwtools.api import rocoto -def test_api_rocoto_realize(): - path1, path2 = Path("foo"), Path("bar") - with patch.object(rocoto, "_realize") as _realize: - rocoto.realize(config=path1, output_file=path2) - _realize.assert_called_once_with(config=path1, output_file=path2) - - @mark.parametrize("f", [Path, str]) -def test_api_rocoto_run(f, utc): +def test_api_rocoto_iterate(f, utc): cycle = utc() database = f("/path/to/rocoto.db") rate = 11 task = "foo" workflow = f("/path/to/rocoto.xml") - with patch.object(rocoto, "_run") as _run: - rocoto.run(cycle=cycle, database=database, rate=rate, task=task, workflow=workflow) - _run.assert_called_once_with( + with patch.object(rocoto, "_iterate") as _iterate: + rocoto.iterate(cycle=cycle, database=database, rate=rate, task=task, workflow=workflow) + _iterate.assert_called_once_with( cycle=cycle, database=Path(database), rate=rate, task=task, workflow=Path(workflow) ) +def test_api_rocoto_realize(): + path1, path2 = Path("foo"), Path("bar") + with patch.object(rocoto, "_realize") as _realize: + rocoto.realize(config=path1, output_file=path2) + _realize.assert_called_once_with(config=path1, output_file=path2) + + def test_api_rocoto_validate(): path = Path("foo") with patch.object(rocoto, "_validate") as _validate: diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index 12d7f1264..baf6d0ca5 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -450,21 +450,7 @@ def test_cli__dispatch_rocoto(params): func.assert_called_once_with(args) -def test_cli__dispatch_rocoto_realize(): - args = {STR.cfgfile: 1, STR.outfile: 2} - with patch.object(uwtools.api.rocoto, "_realize") as _realize: - cli._dispatch_rocoto_realize(args) - _realize.assert_called_once_with(config=1, output_file=2) - - -def test_cli__dispatch_rocoto_realize_no_optional(): - args = {STR.cfgfile: None, STR.outfile: None} - with patch.object(uwtools.api.rocoto, "_realize") as func: - cli._dispatch_rocoto_realize(args) - func.assert_called_once_with(config=None, output_file=None) - - -def test_cli_dispatch_rocoto_run(utc): +def test_cli_dispatch_rocoto_iterate(utc): cycle = utc() database = Path("/path/to/rocoto.db") rate = 11 @@ -477,9 +463,23 @@ def test_cli_dispatch_rocoto_run(utc): STR.task: task, STR.workflow: workflow, } - with patch.object(uwtools.api.rocoto, "_run") as _run: - cli._dispatch_rocoto_run(args) - _run.assert_called_once_with(**args) + with patch.object(uwtools.api.rocoto, "_iterate") as _iterate: + cli._dispatch_rocoto_iterate(args) + _iterate.assert_called_once_with(**args) + + +def test_cli__dispatch_rocoto_realize(): + args = {STR.cfgfile: 1, STR.outfile: 2} + with patch.object(uwtools.api.rocoto, "_realize") as _realize: + cli._dispatch_rocoto_realize(args) + _realize.assert_called_once_with(config=1, output_file=2) + + +def test_cli__dispatch_rocoto_realize_no_optional(): + args = {STR.cfgfile: None, STR.outfile: None} + with patch.object(uwtools.api.rocoto, "_realize") as func: + cli._dispatch_rocoto_realize(args) + func.assert_called_once_with(config=None, output_file=None) def test_cli__dispatch_rocoto_validate_xml(): diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index 662c2cff3..ec757bf72 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -24,7 +24,7 @@ def assets(tmp_path): @fixture -def rocoto_runner_args(utc, tmp_path): +def rocoto_iterator_args(utc, tmp_path): return { "cycle": utc(2025, 7, 21, 12), "database": tmp_path / "rocoto.db", @@ -47,6 +47,12 @@ def validation_assets(tmp_path): # Tests +def test_rocoto_iterate(rocoto_iterator_args): + with patch.object(rocoto, "_RocotoIterator") as _RocotoIterator: # noqa: N806 + rocoto.iterate(**rocoto_iterator_args) + _RocotoIterator.assert_called_once_with(*rocoto_iterator_args.values()) + + def test_rocoto_realize__cfg_to_file(assets): cfgfile, outfile = assets rocoto.realize(config=YAMLConfig(cfgfile), output_file=outfile) @@ -81,12 +87,6 @@ def test_rocoto_realize__invalid_xml(assets): rocoto.realize(config=cfgfile, output_file=outfile) -def test_rocoto_run(rocoto_runner_args): - with patch.object(rocoto, "_RocotoRunner") as _RocotoRunner: # noqa: N806 - rocoto.run(**rocoto_runner_args) - _RocotoRunner.assert_called_once_with(*rocoto_runner_args.values()) - - def test_rocoto_validate__file_fail(validation_assets): xml_file_bad, _, _, _ = validation_assets assert rocoto.validate_file(xml_file=xml_file_bad) is False @@ -107,22 +107,22 @@ def test_rocoto_validate__string_pass(validation_assets): assert rocoto.validate_string(xml=xml_string_good) is True -class TestRocotoRunner: +class TestRocotoIterator: """ - Tests for class uwtools.rocoto._RocotoRunner. + Tests for class uwtools.rocoto._RocotoIterator. """ # Fixtures @fixture - def instance(self, rocoto_runner_args): - return rocoto._RocotoRunner(**rocoto_runner_args) + def instance(self, rocoto_iterator_args): + return rocoto._RocotoIterator(**rocoto_iterator_args) # Helpers - def check_mock_calls_counts(self, mocks, _iterate, _report, _state, sleep): - assert mocks["_iterate"].call_count == _iterate + def check_mock_calls_counts(self, mocks, _report, _run, _state, sleep): assert mocks["_report"].call_count == _report + assert mocks["_run"].call_count == _run assert mocks["_state"].call_count == _state assert mocks["sleep"].call_count == sleep @@ -142,79 +142,79 @@ def dbsetup(self, instance): def mocks(self): with ( patch.object(rocoto, "sleep") as sleep, - patch.object(rocoto._RocotoRunner, "_iterate") as _iterate, - patch.object(rocoto._RocotoRunner, "_report") as _report, - patch.object(rocoto._RocotoRunner, "_state", new_callable=PropertyMock) as _state, + patch.object(rocoto._RocotoIterator, "_report") as _report, + patch.object(rocoto._RocotoIterator, "_run") as _run, + patch.object(rocoto._RocotoIterator, "_state", new_callable=PropertyMock) as _state, ): - _iterate.return_value = True - yield dict(sleep=sleep, _iterate=_iterate, _report=_report, _state=_state) + _run.return_value = True + yield dict(sleep=sleep, _report=_report, _run=_run, _state=_state) # Tests - def test_rocoto__RocotoRunner__init_and_del(self, rocoto_runner_args): - rr = rocoto._RocotoRunner(**rocoto_runner_args) + def test_rocoto__RocotoIterator__init_and_del(self, rocoto_iterator_args): + rr = rocoto._RocotoIterator(**rocoto_iterator_args) con = Mock() rr._con = con del rr con.close.assert_called_once_with() - def test_rocoto__RocotoRunner_run__active(self, instance): + def test_rocoto__RocotoIterator_iterate__active(self, instance): with self.mocks() as mocks: mocks["_state"].side_effect = ["RUNNING", "COMPLETE"] - assert instance.run() is True - self.check_mock_calls_counts(mocks, _iterate=1, _report=0, _state=2, sleep=0) + assert instance.iterate() is True + self.check_mock_calls_counts(mocks, _report=0, _run=1, _state=2, sleep=0) - def test_rocoto__RocotoRunner_run__inactive(self, instance): + def test_rocoto__RocotoIterator_iterate__inactive(self, instance): with self.mocks() as mocks: mocks["_state"].side_effect = ["COMPLETE"] - assert instance.run() is True - self.check_mock_calls_counts(mocks, _iterate=0, _report=0, _state=1, sleep=0) + assert instance.iterate() is True + self.check_mock_calls_counts(mocks, _report=0, _run=0, _state=1, sleep=0) - def test_rocoto__RocotoRunner_run__transient(self, instance): + def test_rocoto__RocotoIterator_iterate__transient(self, instance): with self.mocks() as mocks: mocks["_state"].side_effect = [None, "SUBMITTING", "RUNNING", "COMPLETE"] - assert instance.run() is True - self.check_mock_calls_counts(mocks, _iterate=3, _report=1, _state=4, sleep=1) + assert instance.iterate() is True + self.check_mock_calls_counts(mocks, _report=1, _run=3, _state=4, sleep=1) - def test_rocoto__RocotoRunner_run__iterate_failure(self, instance): + def test_rocoto__RocotoIterator_iterate__run_failure(self, instance): with self.mocks() as mocks: - mocks["_iterate"].return_value = False - assert instance.run() is False - self.check_mock_calls_counts(mocks, _iterate=1, _report=0, _state=1, sleep=0) + mocks["_run"].return_value = False + assert instance.iterate() is False + self.check_mock_calls_counts(mocks, _report=0, _run=1, _state=1, sleep=0) - def test_rocoto__RocotoRunner__connection(self, instance): + def test_rocoto__RocotoIterator__connection(self, instance): instance._database.touch() assert isinstance(instance._connection, sqlite3.Connection) - def test_rocoto__RocotoRunner__connection__no_file(self, instance): + def test_rocoto__RocotoIterator__connection__no_file(self, instance): assert instance._connection is None - def test_rocoto__RocotoRunner__cursor(self, instance): + def test_rocoto__RocotoIterator__cursor(self, instance): instance._database.touch() assert isinstance(instance._cursor, sqlite3.Cursor) - def test_rocoto__RocotoRunner__cursor__no_file(self, instance): + def test_rocoto__RocotoIterator__cursor__no_file(self, instance): assert instance._cursor is None - def test_rocoto__RocotoRunner__iterate(self, instance, logged): + def test_rocoto__RocotoIterator__iterate(self, instance, logged): retval = (True, "") with patch.object(rocoto, "run_shell_cmd", return_value=retval) as run_shell_cmd: - assert instance._iterate() is True + assert instance._run() is True run_shell_cmd.assert_called_once_with( "rocotorun -d %s -w %s" % (instance._database, instance._workflow), quiet=True ) assert logged("Iterating workflow") - def test_rocoto__RocotoRunner__query_data(self, instance): + def test_rocoto__RocotoIterator__query_data(self, instance): assert instance._query_data == {"taskname": "foo", "cycle": 1753099200} - def test_rocoto__RocotoRunner__query_stmt(self, instance): + def test_rocoto__RocotoIterator__query_stmt(self, instance): assert ( instance._query_stmt == "select state from jobs where taskname=:taskname and cycle=:cycle order by id desc" ) - def test_rocoto__RocotoRunner__report(self, instance, logged): + def test_rocoto__RocotoIterator__report(self, instance, logged): instance._database.touch() retval = (True, "foo\nbar\n") with patch.object(rocoto, "run_shell_cmd", return_value=retval) as run_shell_cmd: @@ -225,7 +225,7 @@ def test_rocoto__RocotoRunner__report(self, instance, logged): "rocotostat -d %s -w %s" % (instance._database, instance._workflow), quiet=True ) - def test_rocoto__RocotoRunner__state(self, instance, logged): + def test_rocoto__RocotoIterator__state(self, instance, logged): self.dbsetup(instance) instance._cursor.execute( "insert into jobs values (:id, :taskname, :cycle, :state)", @@ -234,14 +234,14 @@ def test_rocoto__RocotoRunner__state(self, instance, logged): assert instance._state == "COMPLETE" assert logged(f"Rocoto task '{instance._task}' for cycle {instance._cycle}: COMPLETE") - def test_rocoto__RocotoRunner__state__none(self, instance): + def test_rocoto__RocotoIterator__state__none(self, instance): self.dbsetup(instance) assert instance._state is None - def test_rocoto__RocotoRunner__state_msg(self, instance): + def test_rocoto__RocotoIterator__state_msg(self, instance): assert instance._state_msg == "Rocoto task 'foo' for cycle 2025-07-21 12:00:00: %s" - def test_rocoto__RocotoRunner__states(self, instance): + def test_rocoto__RocotoIterator__states(self, instance): assert list(instance._states.keys()) == ["active", "inactive", "transient"] From 2edafb9393ac43297b2785647ae057842ae27f2f Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 24 Jul 2025 16:07:46 +0000 Subject: [PATCH 66/69] Update docs [skip ci] --- docs/index.rst | 18 ++-- .../cli/drivers/chgres_cube/run-help.out | 2 +- .../cli/drivers/esg_grid/run-help.out | 2 +- .../cli/drivers/filter_topo/run-help.out | 2 +- .../user_guide/cli/drivers/fv3/run-help.out | 2 +- .../drivers/global_equiv_resol/run-help.out | 2 +- .../user_guide/cli/drivers/ioda/run-help.out | 2 +- .../user_guide/cli/drivers/jedi/run-help.out | 2 +- .../cli/drivers/make_hgrid/run-help.out | 2 +- .../cli/drivers/make_solo_mosaic/run-help.out | 2 +- .../user_guide/cli/drivers/mpas/run-help.out | 2 +- .../cli/drivers/mpas_init/run-help.out | 2 +- .../user_guide/cli/drivers/orog/run-help.out | 2 +- .../cli/drivers/orog_gsl/run-help.out | 2 +- .../cli/drivers/sfc_climo_gen/run-help.out | 2 +- .../user_guide/cli/drivers/shave/run-help.out | 2 +- .../cli/drivers/ungrib/run-help.out | 2 +- .../user_guide/cli/drivers/upp/run-help.out | 2 +- .../user_guide/cli/tools/execute/help.out | 2 +- docs/sections/user_guide/cli/tools/rocoto.rst | 98 +++++++++---------- .../user_guide/cli/tools/rocoto/help.out | 4 +- .../rocoto/{run-bar.out => iterate-bar.out} | 0 .../rocoto/{run-bar.txt => iterate-bar.txt} | 2 +- .../rocoto/{run-foo.out => iterate-foo.out} | 0 .../rocoto/{run-foo.txt => iterate-foo.txt} | 2 +- .../cli/tools/rocoto/iterate-help.cmd | 1 + .../rocoto/{run-help.out => iterate-help.out} | 8 +- .../user_guide/cli/tools/rocoto/run-help.cmd | 1 - 28 files changed, 85 insertions(+), 85 deletions(-) rename docs/sections/user_guide/cli/tools/rocoto/{run-bar.out => iterate-bar.out} (100%) rename docs/sections/user_guide/cli/tools/rocoto/{run-bar.txt => iterate-bar.txt} (69%) rename docs/sections/user_guide/cli/tools/rocoto/{run-foo.out => iterate-foo.out} (100%) rename docs/sections/user_guide/cli/tools/rocoto/{run-foo.txt => iterate-foo.txt} (69%) create mode 100644 docs/sections/user_guide/cli/tools/rocoto/iterate-help.cmd rename docs/sections/user_guide/cli/tools/rocoto/{run-help.out => iterate-help.out} (70%) delete mode 100644 docs/sections/user_guide/cli/tools/rocoto/run-help.cmd diff --git a/docs/index.rst b/docs/index.rst index 894fedaa9..ca5770138 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -87,27 +87,27 @@ There is a video demonstration of the use of the ``uw fs`` tool (formerly ``uw f -Rocoto Configurability -^^^^^^^^^^^^^^^^^^^^^^ +Rocoto Support +^^^^^^^^^^^^^^ | **CLI**: ``uw rocoto -h`` | **API**: ``import uwtools.api.rocoto`` This tool is all about creating a configurable interface to the :rocoto:`Rocoto<>` workflow manager tool that produces the Rocoto XML for a totally arbitrary set of tasks. The ``uwtools`` package defines a structured YAML interface that relies on tasks you define to run. Paired with the uw config tool suite, this interface becomes highly configurable and requires no XML syntax! -Realize Action +Iterate Action """""""""""""" -This is where you put in your structured YAML that defines your workflow of choice, and it pops out a verified Rocoto XML. +Given a Rocoto XML workflow document, invoke Rocoto in a loop, monitoring its progress, until a specified task is complete. -| :any:`CLI documentation with examples` +| :any:`CLI documentation with examples` -Run Action -"""""""""" +Realize Action +"""""""""""""" -Given a Rocoto XML workflow document, invoke Rocoto in a loop, monitoring its progress, until a specified task is complete. +This is where you put in your structured YAML that defines your workflow of choice, and it pops out a verified Rocoto XML. -| :any:`CLI documentation with examples` +| :any:`CLI documentation with examples` Validate Action """"""""""""""" diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube/run-help.out b/docs/sections/user_guide/cli/drivers/chgres_cube/run-help.out index 2e6cb8fa7..826f759d1 100644 --- a/docs/sections/user_guide/cli/drivers/chgres_cube/run-help.out +++ b/docs/sections/user_guide/cli/drivers/chgres_cube/run-help.out @@ -19,7 +19,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/esg_grid/run-help.out b/docs/sections/user_guide/cli/drivers/esg_grid/run-help.out index 2a993a489..cafb5c91b 100644 --- a/docs/sections/user_guide/cli/drivers/esg_grid/run-help.out +++ b/docs/sections/user_guide/cli/drivers/esg_grid/run-help.out @@ -13,7 +13,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/filter_topo/run-help.out b/docs/sections/user_guide/cli/drivers/filter_topo/run-help.out index 721b937a8..76136a649 100644 --- a/docs/sections/user_guide/cli/drivers/filter_topo/run-help.out +++ b/docs/sections/user_guide/cli/drivers/filter_topo/run-help.out @@ -13,7 +13,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/fv3/run-help.out b/docs/sections/user_guide/cli/drivers/fv3/run-help.out index e3aafbacc..a493b4ed3 100644 --- a/docs/sections/user_guide/cli/drivers/fv3/run-help.out +++ b/docs/sections/user_guide/cli/drivers/fv3/run-help.out @@ -17,7 +17,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol/run-help.out b/docs/sections/user_guide/cli/drivers/global_equiv_resol/run-help.out index d4974f5ca..5739e5514 100644 --- a/docs/sections/user_guide/cli/drivers/global_equiv_resol/run-help.out +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol/run-help.out @@ -13,7 +13,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/ioda/run-help.out b/docs/sections/user_guide/cli/drivers/ioda/run-help.out index dec231863..26d2b9a02 100644 --- a/docs/sections/user_guide/cli/drivers/ioda/run-help.out +++ b/docs/sections/user_guide/cli/drivers/ioda/run-help.out @@ -17,7 +17,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/jedi/run-help.out b/docs/sections/user_guide/cli/drivers/jedi/run-help.out index 0f80107a9..235d8cba1 100644 --- a/docs/sections/user_guide/cli/drivers/jedi/run-help.out +++ b/docs/sections/user_guide/cli/drivers/jedi/run-help.out @@ -17,7 +17,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid/run-help.out b/docs/sections/user_guide/cli/drivers/make_hgrid/run-help.out index ab39a59d2..3d655f66b 100644 --- a/docs/sections/user_guide/cli/drivers/make_hgrid/run-help.out +++ b/docs/sections/user_guide/cli/drivers/make_hgrid/run-help.out @@ -13,7 +13,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/run-help.out b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/run-help.out index e3fd0cde8..fd44fad43 100644 --- a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/run-help.out +++ b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/run-help.out @@ -13,7 +13,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/mpas/run-help.out b/docs/sections/user_guide/cli/drivers/mpas/run-help.out index cd8d82e5a..aa2b084dd 100644 --- a/docs/sections/user_guide/cli/drivers/mpas/run-help.out +++ b/docs/sections/user_guide/cli/drivers/mpas/run-help.out @@ -17,7 +17,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/mpas_init/run-help.out b/docs/sections/user_guide/cli/drivers/mpas_init/run-help.out index e918aadc3..88b5bc00a 100644 --- a/docs/sections/user_guide/cli/drivers/mpas_init/run-help.out +++ b/docs/sections/user_guide/cli/drivers/mpas_init/run-help.out @@ -17,7 +17,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/orog/run-help.out b/docs/sections/user_guide/cli/drivers/orog/run-help.out index 7d60a6504..877be3ab0 100644 --- a/docs/sections/user_guide/cli/drivers/orog/run-help.out +++ b/docs/sections/user_guide/cli/drivers/orog/run-help.out @@ -12,7 +12,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/orog_gsl/run-help.out b/docs/sections/user_guide/cli/drivers/orog_gsl/run-help.out index 3b12edfdb..447e9b3dd 100644 --- a/docs/sections/user_guide/cli/drivers/orog_gsl/run-help.out +++ b/docs/sections/user_guide/cli/drivers/orog_gsl/run-help.out @@ -13,7 +13,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/run-help.out b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/run-help.out index 8d0660ced..75de057c6 100644 --- a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/run-help.out +++ b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/run-help.out @@ -13,7 +13,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/shave/run-help.out b/docs/sections/user_guide/cli/drivers/shave/run-help.out index 81b6359e2..3a6ec4ef4 100644 --- a/docs/sections/user_guide/cli/drivers/shave/run-help.out +++ b/docs/sections/user_guide/cli/drivers/shave/run-help.out @@ -12,7 +12,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/ungrib/run-help.out b/docs/sections/user_guide/cli/drivers/ungrib/run-help.out index b0fb973af..ef63e0bdb 100644 --- a/docs/sections/user_guide/cli/drivers/ungrib/run-help.out +++ b/docs/sections/user_guide/cli/drivers/ungrib/run-help.out @@ -17,7 +17,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/drivers/upp/run-help.out b/docs/sections/user_guide/cli/drivers/upp/run-help.out index 29a888b76..3bfcc318a 100644 --- a/docs/sections/user_guide/cli/drivers/upp/run-help.out +++ b/docs/sections/user_guide/cli/drivers/upp/run-help.out @@ -19,7 +19,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/tools/execute/help.out b/docs/sections/user_guide/cli/tools/execute/help.out index 2b50458da..cc99c1284 100644 --- a/docs/sections/user_guide/cli/tools/execute/help.out +++ b/docs/sections/user_guide/cli/tools/execute/help.out @@ -28,7 +28,7 @@ Optional arguments: --leadtime LEADTIME The leadtime as hours[:minutes[:seconds]] --batch - Submit run to batch scheduler + Submit job to batch scheduler --dry-run Only log info, making no changes --graph-file PATH diff --git a/docs/sections/user_guide/cli/tools/rocoto.rst b/docs/sections/user_guide/cli/tools/rocoto.rst index e6d8fb289..46ced30a9 100644 --- a/docs/sections/user_guide/cli/tools/rocoto.rst +++ b/docs/sections/user_guide/cli/tools/rocoto.rst @@ -14,6 +14,55 @@ The ``uw`` mode for realizing and validating Rocoto XML documents. .. literalinclude:: rocoto/help.out :language: text +.. _cli_rocoto_iterate_examples: + +``iterate`` +----------- + +.. literalinclude:: rocoto/iterate-help.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: rocoto/iterate-help.out + :language: text + +Examples +^^^^^^^^ + +.. note:: Use of ``uw rocoto iterate`` requires presence of the ``rocotorun`` and ``rocotostat`` executables on ``PATH``. On HPCs, this is typically achieved by loading a system module providing Rocoto. + +The following examples make use of this simple UW YAML for Rocoto config: + +.. literalinclude:: rocoto/foobar.yaml + :language: yaml + +It could be rendered to a Rocoto XML document like this: + +.. literalinclude:: rocoto/foobar-realize.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: rocoto/foobar-realize.out + :language: xml + +* To iterate only task ``foo``, which has no dependencies: + + .. literalinclude:: rocoto/iterate-foo.txt + :language: text + :emphasize-lines: 6 + .. literalinclude:: rocoto/iterate-foo.out + :language: text + + Note that the second invocation of ``uw rocoto iterate`` immediately shows task ``foo`` in its final state, without iterating the workflow. + +* To iterate task ``bar``, which depends on task ``foo``, iterating every 3 seconds: + + .. literalinclude:: rocoto/iterate-bar.txt + :language: text + :emphasize-lines: 4 + .. literalinclude:: rocoto/iterate-bar.out + :language: text + + Note that the second invocation of ``uw rocoto iterate`` immediately shows task ``bar`` in its final state, without iterating the workflow. + .. _cli_rocoto_realize_examples: ``realize`` @@ -69,55 +118,6 @@ The examples in this section use a UW YAML file ``rocoto.yaml`` with contents: .. literalinclude:: rocoto/realize-exec-stdout-verbose.out :language: xml -.. _cli_rocoto_iterate_examples: - -``run`` -------- - -.. literalinclude:: rocoto/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: rocoto/run-help.out - :language: text - -Examples -^^^^^^^^ - -.. note:: Use of ``uw rocoto run`` requires presence of the ``rocotorun`` and ``rocotostat`` executables on ``PATH``. On HPCs, this is typically achieved by loading a system module providing Rocoto. - -The following examples make use of this simple UW YAML for Rocoto config: - -.. literalinclude:: rocoto/foobar.yaml - :language: yaml - -It could be rendered to a Rocoto XML document like this: - -.. literalinclude:: rocoto/foobar-realize.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: rocoto/foobar-realize.out - :language: xml - -* To run only task ``foo``, which has no dependencies: - - .. literalinclude:: rocoto/run-foo.txt - :language: text - :emphasize-lines: 6 - .. literalinclude:: rocoto/run-foo.out - :language: text - - Note that the second invocation of ``uw rocoto run`` immediately shows task ``foo`` in its final state, without iterating the workflow. - -* To run task ``bar``, which depends on task ``foo``, iterating every 3 seconds: - - .. literalinclude:: rocoto/run-bar.txt - :language: text - :emphasize-lines: 4 - .. literalinclude:: rocoto/run-bar.out - :language: text - - Note that the second invocation of ``uw rocoto run`` immediately shows task ``bar`` in its final state, without iterating the workflow. - .. _cli_rocoto_validate_examples: ``validate`` diff --git a/docs/sections/user_guide/cli/tools/rocoto/help.out b/docs/sections/user_guide/cli/tools/rocoto/help.out index 80487672c..e64528fa4 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/help.out +++ b/docs/sections/user_guide/cli/tools/rocoto/help.out @@ -10,9 +10,9 @@ Optional arguments: Positional arguments: ACTION + iterate + Iterate a Rocoto workflow realize Realize a Rocoto XML workflow document - run - Run a Rocoto workflow validate Validate Rocoto XML diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-bar.out b/docs/sections/user_guide/cli/tools/rocoto/iterate-bar.out similarity index 100% rename from docs/sections/user_guide/cli/tools/rocoto/run-bar.out rename to docs/sections/user_guide/cli/tools/rocoto/iterate-bar.out diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-bar.txt b/docs/sections/user_guide/cli/tools/rocoto/iterate-bar.txt similarity index 69% rename from docs/sections/user_guide/cli/tools/rocoto/run-bar.txt rename to docs/sections/user_guide/cli/tools/rocoto/iterate-bar.txt index 8e57415b4..6eeb8520f 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/run-bar.txt +++ b/docs/sections/user_guide/cli/tools/rocoto/iterate-bar.txt @@ -3,7 +3,7 @@ uw rocoto realize -c foobar.yaml >$RUNDIR/xml 2>/dev/null for invocation in 1 2; do ( cd $RUNDIR set -x - uw rocoto run --cycle 2025-07-17T00 --database db --task bar --workflow xml + uw rocoto iterate --cycle 2025-07-17T00 --database db --task bar --workflow xml ) done rm -rf $RUNDIR diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-foo.out b/docs/sections/user_guide/cli/tools/rocoto/iterate-foo.out similarity index 100% rename from docs/sections/user_guide/cli/tools/rocoto/run-foo.out rename to docs/sections/user_guide/cli/tools/rocoto/iterate-foo.out diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-foo.txt b/docs/sections/user_guide/cli/tools/rocoto/iterate-foo.txt similarity index 69% rename from docs/sections/user_guide/cli/tools/rocoto/run-foo.txt rename to docs/sections/user_guide/cli/tools/rocoto/iterate-foo.txt index 259c25713..c6f580257 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/run-foo.txt +++ b/docs/sections/user_guide/cli/tools/rocoto/iterate-foo.txt @@ -3,7 +3,7 @@ uw rocoto realize -c foobar.yaml >$RUNDIR/xml 2>/dev/null for invocation in 1 2; do ( cd $RUNDIR set -x - uw rocoto run --cycle 2025-07-17T00 --database db --task foo --workflow xml + uw rocoto iterate --cycle 2025-07-17T00 --database db --task foo --workflow xml ) done rm -rf $RUNDIR diff --git a/docs/sections/user_guide/cli/tools/rocoto/iterate-help.cmd b/docs/sections/user_guide/cli/tools/rocoto/iterate-help.cmd new file mode 100644 index 000000000..20f54d1c5 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/rocoto/iterate-help.cmd @@ -0,0 +1 @@ +uw rocoto iterate --help diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-help.out b/docs/sections/user_guide/cli/tools/rocoto/iterate-help.out similarity index 70% rename from docs/sections/user_guide/cli/tools/rocoto/run-help.out rename to docs/sections/user_guide/cli/tools/rocoto/iterate-help.out index bc657d4ca..a74921997 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/run-help.out +++ b/docs/sections/user_guide/cli/tools/rocoto/iterate-help.out @@ -1,8 +1,8 @@ -usage: uw rocoto run [--cycle CYCLE] --database DATABASE --task TASK - --workflow WORKFLOW [-h] [--version] [--rate SECONDS] - [--quiet] [--verbose] +usage: uw rocoto iterate [--cycle CYCLE] --database DATABASE --task TASK + --workflow WORKFLOW [-h] [--version] [--rate SECONDS] + [--quiet] [--verbose] -Run a Rocoto workflow +Iterate a Rocoto workflow Required arguments: --cycle CYCLE diff --git a/docs/sections/user_guide/cli/tools/rocoto/run-help.cmd b/docs/sections/user_guide/cli/tools/rocoto/run-help.cmd deleted file mode 100644 index de1a19d35..000000000 --- a/docs/sections/user_guide/cli/tools/rocoto/run-help.cmd +++ /dev/null @@ -1 +0,0 @@ -uw rocoto run --help From 0717590ff3a012003a3bce80259397808c7c5dd0 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 24 Jul 2025 16:18:21 +0000 Subject: [PATCH 67/69] Update docs --- docs/sections/user_guide/cli/tools/rocoto/iterate-bar.out | 6 +++--- docs/sections/user_guide/cli/tools/rocoto/iterate-foo.out | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/sections/user_guide/cli/tools/rocoto/iterate-bar.out b/docs/sections/user_guide/cli/tools/rocoto/iterate-bar.out index d96dfe681..b6e7cfc07 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/iterate-bar.out +++ b/docs/sections/user_guide/cli/tools/rocoto/iterate-bar.out @@ -1,13 +1,13 @@ -+ uw rocoto run --cycle 2025-07-17T00 --database db --task bar --workflow xml ++ uw rocoto iterate --cycle 2025-07-17T00 --database db --task bar --workflow xml [2025-01-02T03:04:05] INFO Iterating workflow [2025-01-02T03:04:05] INFO Workflow status: [2025-01-02T03:04:05] INFO CYCLE TASK JOBID STATE EXIT STATUS TRIES DURATION [2025-01-02T03:04:05] INFO ================================================================================================================================ -[2025-01-02T03:04:05] INFO 202507170000 foo druby://10.178.9.5:36657 SUBMITTING - 0 0.0 +[2025-01-02T03:04:05] INFO 202507170000 foo druby://10.178.9.5:42537 SUBMITTING - 0 0.0 [2025-01-02T03:04:05] INFO 202507170000 bar - - - - - [2025-01-02T03:04:05] INFO Iterating workflow [2025-01-02T03:04:05] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUBMITTING [2025-01-02T03:04:05] INFO Iterating workflow [2025-01-02T03:04:05] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUCCEEDED -+ uw rocoto run --cycle 2025-07-17T00 --database db --task bar --workflow xml ++ uw rocoto iterate --cycle 2025-07-17T00 --database db --task bar --workflow xml [2025-01-02T03:04:05] INFO Rocoto task 'bar' for cycle 2025-07-17 00:00:00: SUCCEEDED diff --git a/docs/sections/user_guide/cli/tools/rocoto/iterate-foo.out b/docs/sections/user_guide/cli/tools/rocoto/iterate-foo.out index 621af615d..29c87d404 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/iterate-foo.out +++ b/docs/sections/user_guide/cli/tools/rocoto/iterate-foo.out @@ -1,7 +1,7 @@ -+ uw rocoto run --cycle 2025-07-17T00 --database db --task foo --workflow xml ++ uw rocoto iterate --cycle 2025-07-17T00 --database db --task foo --workflow xml [2025-01-02T03:04:05] INFO Iterating workflow [2025-01-02T03:04:05] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUBMITTING [2025-01-02T03:04:05] INFO Iterating workflow [2025-01-02T03:04:05] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUCCEEDED -+ uw rocoto run --cycle 2025-07-17T00 --database db --task foo --workflow xml ++ uw rocoto iterate --cycle 2025-07-17T00 --database db --task foo --workflow xml [2025-01-02T03:04:05] INFO Rocoto task 'foo' for cycle 2025-07-17 00:00:00: SUCCEEDED From 0b6675a0a2e6e4df9280a734516e9e8c77551000 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 24 Jul 2025 16:22:39 +0000 Subject: [PATCH 68/69] Fix highlight line in docs --- docs/sections/user_guide/cli/tools/rocoto.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sections/user_guide/cli/tools/rocoto.rst b/docs/sections/user_guide/cli/tools/rocoto.rst index 46ced30a9..8b204f21c 100644 --- a/docs/sections/user_guide/cli/tools/rocoto.rst +++ b/docs/sections/user_guide/cli/tools/rocoto.rst @@ -57,7 +57,7 @@ It could be rendered to a Rocoto XML document like this: .. literalinclude:: rocoto/iterate-bar.txt :language: text - :emphasize-lines: 4 + :emphasize-lines: 6 .. literalinclude:: rocoto/iterate-bar.out :language: text From d62b4ad6d4dac8483b61e81a6c7e9241d15e8940 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 24 Jul 2025 16:31:20 +0000 Subject: [PATCH 69/69] Remove conda-verify references --- docs/sections/contributor_guide/developer_setup.rst | 2 +- docs/sections/user_guide/installation.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sections/contributor_guide/developer_setup.rst b/docs/sections/contributor_guide/developer_setup.rst index 89bf2abb9..b168113b9 100644 --- a/docs/sections/contributor_guide/developer_setup.rst +++ b/docs/sections/contributor_guide/developer_setup.rst @@ -63,7 +63,7 @@ As an alternative to installing the :anaconda-condev:`pre-built package<>`, the .. code-block:: text # Activate your conda. Optionally, activate a non-'base' environment. - conda install -y conda-build conda-verify + conda install -y conda-build git clone https://github.com/maddenp/condev.git make -C condev package conda install -y -c $CONDA_PREFIX/conda-bld -c conda-forge --override-channels condev diff --git a/docs/sections/user_guide/installation.rst b/docs/sections/user_guide/installation.rst index e42f54a13..e5be6f69e 100644 --- a/docs/sections/user_guide/installation.rst +++ b/docs/sections/user_guide/installation.rst @@ -49,7 +49,7 @@ Build the ``uwtools`` Package Locally .. code-block:: text - conda install -y -c conda-forge --override-channels conda-build conda-verify + conda install -y -c conda-forge --override-channels conda-build #. In a clone of the :uwtools:`uwtools repository<>`, build the ``uwtools`` package: