Implement --realize

maddenp-cu · maddenp-cu · commit 7ceefc9ebac8 · 2025-09-17T04:20:59.000Z
diff --git a/docs/sections/user_guide/cli/tools/config.rst b/docs/sections/user_guide/cli/tools/config.rst
@@ -115,21 +115,46 @@ Examples
      :caption: compose-update-2.yaml
      :language: yaml
 
-Compose the three together, writing to ``stdout``:
+  Compose the three together, writing to ``stdout``:
 
-.. literalinclude:: config/compose-1.cmd
-   :language: text
-   :emphasize-lines: 1
-.. literalinclude:: config/compose-1.out
-   :language: yaml
+  .. literalinclude:: config/compose-basic.cmd
+     :language: text
+     :emphasize-lines: 1
+  .. literalinclude:: config/compose-basic.out
+     :language: yaml
+
+  Values provided by update configs override or augment values provided in the base config, while unaffected values survive to the final config. Priority of values increases from left to right.
+
+  Additionally:
+
+    * Sets of configs in the ``ini``, ``nml``, and ``sh`` formats can be similarly composed.
+    * The ``--input-format`` and ``--output-format`` options can be used to specify the format of the input and output configs, respectively, for cases when ``uwtools`` cannot deduce the format of configs from their filename extensions. When the formats are neither explicitly specified or deduced, ``yaml`` is assumed.
+    * The ``--output-file`` / ``-o`` option can be added to write the final config to a file instead of to ``stdout``.
+
+* The optional ``--realize`` switch can be used to render as many Jinja2 template expressions as possible in the final config, using the config itself as a source of values. For example:
 
-Values provided by update configs override or augment values provided in the base config, while unaffected values survive to the final config. Priority of values increases from left to right.
+  .. literalinclude:: config/compose-template.yaml
+     :caption: compose-template.yaml
+     :language: yaml
+  .. literalinclude:: config/compose-values.yaml
+     :caption: compose-values.yaml
+     :language: yaml
+
+  Without the ``--realize`` switch:
+
+  .. literalinclude:: config/compose-realize-no.cmd
+     :language: text
+     :emphasize-lines: 1
+  .. literalinclude:: config/compose-realize-no.out
+     :language: yaml
 
-Additionally:
+  And with ``--realize``:
 
-  * Sets of configs in the ``ini``, ``nml``, and ``sh`` formats can be similarly composed.
-  * The ``--input-format`` and ``--output-format`` options can be used to specify the format of the input and output configs, respectively, for cases when ``uwtools`` cannot deduce the format of configs from their filename extensions. When the formats are neither explicitly specified or deduced, ``yaml`` is assumed.
-  * The ``--output-file`` / ``-o`` option can be added to write the final config to a file instead of to ``stdout``.
+  .. literalinclude:: config/compose-realize-yes.cmd
+     :language: text
+     :emphasize-lines: 1
+  .. literalinclude:: config/compose-realize-yes.out
+     :language: yaml
 
 .. _cli_config_realize_examples:
 
diff --git a/docs/sections/user_guide/cli/tools/config/compose-basic.cmd b/docs/sections/user_guide/cli/tools/config/compose-basic.cmd
diff --git a/docs/sections/user_guide/cli/tools/config/compose-basic.out b/docs/sections/user_guide/cli/tools/config/compose-basic.out
diff --git a/docs/sections/user_guide/cli/tools/config/compose-help.out b/docs/sections/user_guide/cli/tools/config/compose-help.out
@@ -1,6 +1,7 @@
-usage: uw config compose [-h] [--version] [--input-format {ini,nml,sh,yaml}]
-                         [--output-format {ini,nml,sh,yaml}]
-                         [--output-file PATH] [--quiet] [--verbose]
+usage: uw config compose [-h] [--version] [--realize] [--output-file PATH]
+                         [--input-format {ini,nml,sh,yaml}]
+                         [--output-format {ini,nml,sh,yaml}] [--quiet]
+                         [--verbose]
                          CONFIG [CONFIG ...]
 
 Compose configs
@@ -13,12 +14,14 @@ Optional arguments:
       Show help and exit
   --version
       Show version info and exit
+  --realize
+      Try to render template expressions
+  --output-file PATH, -o PATH
+      Path to output file (default: write to stdout)
   --input-format {ini,nml,sh,yaml}
       Input format (default: yaml)
   --output-format {ini,nml,sh,yaml}
       Output format (default: yaml)
-  --output-file PATH, -o PATH
-      Path to output file (default: write to stdout)
   --quiet, -q
       Print no logging messages
   --verbose, -v
diff --git a/docs/sections/user_guide/cli/tools/config/compose-realize-no.cmd b/docs/sections/user_guide/cli/tools/config/compose-realize-no.cmd
@@ -0,0 +1 @@
+uw config compose compose-template.yaml compose-values.yaml
diff --git a/docs/sections/user_guide/cli/tools/config/compose-realize-no.out b/docs/sections/user_guide/cli/tools/config/compose-realize-no.out
@@ -0,0 +1,3 @@
+radius: !float '{{ 2.0 * pi * r }}'
+pi: 3.142
+r: 1.0
diff --git a/docs/sections/user_guide/cli/tools/config/compose-realize-yes.cmd b/docs/sections/user_guide/cli/tools/config/compose-realize-yes.cmd
@@ -0,0 +1 @@
+uw config compose compose-template.yaml compose-values.yaml --realize
diff --git a/docs/sections/user_guide/cli/tools/config/compose-realize-yes.out b/docs/sections/user_guide/cli/tools/config/compose-realize-yes.out
@@ -0,0 +1,3 @@
+radius: 6.284
+pi: 3.142
+r: 1.0
diff --git a/docs/sections/user_guide/cli/tools/config/compose-template.yaml b/docs/sections/user_guide/cli/tools/config/compose-template.yaml
@@ -0,0 +1 @@
+radius: !float '{{ 2.0 * pi * r }}'
diff --git a/docs/sections/user_guide/cli/tools/config/compose-values.yaml b/docs/sections/user_guide/cli/tools/config/compose-values.yaml
@@ -0,0 +1,2 @@
+pi: 3.142
+r: 1.0
diff --git a/src/uwtools/api/config.py b/src/uwtools/api/config.py
@@ -42,6 +42,7 @@ def compare(
 
 def compose(
     configs: list[str | Path],
+    realize: bool = False,
     output_file: Path | str | None = None,
     input_format: str | None = None,
     output_format: str | None = None,
@@ -51,6 +52,7 @@ def compose(
     """
     return _compose(
         configs=list(map(Path, configs)),
+        realize=realize,
         output_file=Path(output_file) if output_file else None,
         input_format=input_format,
         output_format=output_format,
@@ -236,6 +238,7 @@ def validate(
 Recognized file extensions are: {extensions}.
 
 :param configs: Paths to configs to compose.
+:param realize: Try to render template expressions.
 :param output_file: Output config destination (default: write to ``stdout``).
 :param input_format: Format of configs to compose (choices: {choices}, default: ``{default}``)
 :param output_format: Format of output config (choices: {choices}, default: ``{default}``)
diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py
@@ -162,9 +162,10 @@ def _add_subparser_config_compose(subparsers: Subparsers) -> ActionChecks:
     """
     parser = _add_subparser(subparsers, STR.compose, "Compose configs")
     optional = _basic_setup(parser)
+    _add_arg_realize(optional)
+    _add_arg_output_file(optional)
     _add_arg_input_format(optional, choices=FORMATS)
     _add_arg_output_format(optional, choices=FORMATS)
-    _add_arg_output_file(optional)
     checks = _add_args_verbosity(optional)
     parser.add_argument("configs", metavar="CONFIG", nargs="+", type=Path)
     return checks
@@ -248,6 +249,7 @@ def _dispatch_config_compose(args: Args) -> bool:
     return uwtools.api.config.compose(
         configs=args[STR.configs],
         output_file=args[STR.outfile],
+        realize=args[STR.realize],
         input_format=args[STR.infmt],
         output_format=args[STR.outfmt],
     )
@@ -937,6 +939,14 @@ def _add_arg_rate(group: Group) -> None:
     )
 
 
+def _add_arg_realize(group: Group) -> None:
+    group.add_argument(
+        _switch(STR.realize),
+        action="store_true",
+        help="Try to render template expressions",
+    )
+
+
 def _add_arg_report(group: Group) -> None:
     group.add_argument(
         _switch(STR.report),
diff --git a/src/uwtools/config/tools.py b/src/uwtools/config/tools.py
@@ -40,6 +40,7 @@ def compare(
 
 def compose(
     configs: list[Path],
+    realize: bool,
     output_file: Path | None = None,
     input_format: str | None = None,
     output_format: str | None = None,
@@ -58,6 +59,8 @@ def compose(
     output_format = output_format or get_config_format(output_file, "output")
     output_class = format_to_config(output_format)
     output_config = output_class(config)
+    if realize:
+        output_config.dereference()
     output_config.dump(output_file)
     return True
 
diff --git a/src/uwtools/tests/api/test_config.py b/src/uwtools/tests/api/test_config.py
@@ -36,6 +36,7 @@ def test_api_config_compose(output_file, input_format, output_format):
     pathstrs = ["/path/to/c1.yaml", "/path/to/c2.yaml"]
     kwargs: dict = {
         "configs": pathstrs,
+        "realize": False,
         "output_file": output_file,
         "input_format": input_format,
         "output_format": output_format,
@@ -44,6 +45,7 @@ def test_api_config_compose(output_file, input_format, output_format):
         config.compose(**kwargs)
     _compose.assert_called_once_with(
         configs=list(map(Path, pathstrs)),
+        realize=False,
         output_file=None if output_file is None else Path(output_file),
         input_format=input_format,
         output_format=output_format,
diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py
@@ -184,16 +184,53 @@ def test_config_tools_compare__bad_format(logged):
     assert logged(msg)
 
 
+@mark.parametrize(("configclass", "fmt"), [(INIConfig, FORMAT.ini), (NMLConfig, FORMAT.nml)])
+def test_config_tools_compose__fmt_ini_nml_2x(configclass, fmt, logged, tmp_path):
+    d = {"constants": {"pi": 3.142, "e": 2.718}, "trees": {"leaf": "elm", "needle": "spruce"}}
+    u = {"trees": {"needle": "fir"}, "colors": {"red": "crimson", "green": "clover"}}
+    suffix = f".{fmt}"
+    dpath, upath = [(tmp_path / x).with_suffix(suffix) for x in ("d", "u")]
+    configclass(d).dump(dpath)
+    configclass(u).dump(upath)
+    outpath = (tmp_path / "out").with_suffix(suffix)
+    kwargs: dict = {"configs": [dpath, upath], "realize": False, "output_file": outpath}
+    assert tools.compose(**kwargs) is True
+    assert logged(f"Reading {dpath} as base '{fmt}' config")
+    assert logged(f"Composing '{fmt}' config from {upath}")
+    expected = {
+        "constants": {"pi": 3.142, "e": 2.718},
+        "trees": {"leaf": "elm", "needle": "fir"},
+        "colors": {"red": "crimson", "green": "clover"},
+    }
+    if fmt == FORMAT.ini:
+        expected["constants"] = {"pi": "3.142", "e": "2.718"}
+    assert configclass(outpath) == configclass(expected)
+
+
+def test_config_tools_compose__fmt_sh_2x(logged, tmp_path):
+    d = {"foo": 1, "bar": 2}
+    u = {"foo": 3, "baz": 4}
+    dpath, upath = [tmp_path / x for x in ("d.sh", "u.sh")]
+    SHConfig(d).dump(dpath)
+    SHConfig(u).dump(upath)
+    outpath = tmp_path / "out.sh"
+    kwargs: dict = {"configs": [dpath, upath], "realize": False, "output_file": outpath}
+    assert tools.compose(**kwargs) is True
+    assert logged(f"Reading {dpath} as base 'sh' config")
+    assert logged(f"Composing 'sh' config from {upath}")
+    assert SHConfig(outpath) == SHConfig({"foo": "3", "bar": "2", "baz": "4"})
+
+
 @mark.parametrize("tofile", [False, True])
 @mark.parametrize("suffix", ["", ".yaml", ".foo"])
-def test_config_tools_compose__yaml_single(compose_assets_yaml, logged, suffix, tmp_path, tofile):
+def test_config_tools_compose__fmt_yaml_1x(compose_assets_yaml, logged, suffix, tmp_path, tofile):
     d1, _, d2, _, d3, _ = compose_assets_yaml
     dpath = (tmp_path / "d").with_suffix(suffix)
     for d in (d1, d2, d3):
         dpath.unlink(missing_ok=True)
         assert not dpath.exists()
         dpath.write_text(yaml.dump(d))
-        kwargs: dict = {"configs": [dpath]}
+        kwargs: dict = {"configs": [dpath], "realize": False}
         if suffix and suffix != ".yaml":
             kwargs["input_format"] = FORMAT.yaml
         if tofile:
@@ -208,7 +245,7 @@ def test_config_tools_compose__yaml_single(compose_assets_yaml, logged, suffix,
 
 @mark.parametrize("tofile", [False, True])
 @mark.parametrize("suffix", ["", ".yaml", ".foo"])
-def test_config_tools_compose__yaml_double(compose_assets_yaml, logged, suffix, tmp_path, tofile):
+def test_config_tools_compose__fmt_yaml_2x(compose_assets_yaml, logged, suffix, tmp_path, tofile):
     d1, u1, d2, u2, d3, u3 = compose_assets_yaml
     dpath, upath = [(tmp_path / x).with_suffix(suffix) for x in ("d", "u")]
     for d, u in [(d1, u1), (d2, u2), (d3, u3)]:
@@ -217,7 +254,7 @@ def test_config_tools_compose__yaml_double(compose_assets_yaml, logged, suffix,
             assert not path.exists()
         dpath.write_text(yaml.dump(d))
         upath.write_text(yaml.dump(u))
-        kwargs: dict = {"configs": [dpath, upath]}
+        kwargs: dict = {"configs": [dpath, upath], "realize": False}
         if suffix and suffix != ".yaml":
             kwargs["input_format"] = FORMAT.yaml
         if tofile:
@@ -242,41 +279,23 @@ def test_config_tools_compose__yaml_double(compose_assets_yaml, logged, suffix,
             outpath.unlink()
 
 
-@mark.parametrize(("configclass", "fmt"), [(INIConfig, FORMAT.ini), (NMLConfig, FORMAT.nml)])
-def test_config_tools_compose__ini_nml_double(configclass, fmt, logged, tmp_path):
-    d = {"constants": {"pi": 3.142, "e": 2.718}, "trees": {"leaf": "elm", "needle": "spruce"}}
-    u = {"trees": {"needle": "fir"}, "colors": {"red": "crimson", "green": "clover"}}
-    suffix = f".{fmt}"
-    dpath, upath = [(tmp_path / x).with_suffix(suffix) for x in ("d", "u")]
-    configclass(d).dump(dpath)
-    configclass(u).dump(upath)
-    outpath = (tmp_path / "out").with_suffix(suffix)
-    kwargs: dict = {"configs": [dpath, upath], "output_file": outpath}
-    assert tools.compose(**kwargs) is True
-    assert logged(f"Reading {dpath} as base '{fmt}' config")
-    assert logged(f"Composing '{fmt}' config from {upath}")
-    expected = {
-        "constants": {"pi": 3.142, "e": 2.718},
-        "trees": {"leaf": "elm", "needle": "fir"},
-        "colors": {"red": "crimson", "green": "clover"},
-    }
-    if fmt == FORMAT.ini:
-        expected["constants"] = {"pi": "3.142", "e": "2.718"}
-    assert configclass(outpath) == configclass(expected)
-
-
-def test_config_tools_compose__sh_double(logged, tmp_path):
-    d = {"foo": 1, "bar": 2}
-    u = {"foo": 3, "baz": 4}
-    dpath, upath = [tmp_path / x for x in ("d.sh", "u.sh")]
-    SHConfig(d).dump(dpath)
-    SHConfig(u).dump(upath)
-    outpath = tmp_path / "out.sh"
-    kwargs: dict = {"configs": [dpath, upath], "output_file": outpath}
-    assert tools.compose(**kwargs) is True
-    assert logged(f"Reading {dpath} as base 'sh' config")
-    assert logged(f"Composing 'sh' config from {upath}")
-    assert SHConfig(outpath) == SHConfig({"foo": "3", "bar": "2", "baz": "4"})
+@mark.parametrize("realize", [False, True])
+def test_config_tools_compose__realize(realize, tmp_path):
+    dyaml = """
+    radius: !float '{{ 2.0 * pi * r }}'
+    """
+    dpath = tmp_path / "d.yaml"
+    dpath.write_text(dedent(dyaml))
+    uyaml = """
+    pi: 3.142
+    r: 1.0
+    """
+    upath = tmp_path / "u.yaml"
+    upath.write_text(dedent(uyaml))
+    outpath = tmp_path / "out.yaml"
+    assert tools.compose(configs=[dpath, upath], realize=realize, output_file=outpath) is True
+    radius = YAMLConfig(outpath)["radius"]
+    assert (radius == 6.284) if realize else (radius.tagged_string == "!float '{{ 2.0 * pi * r }}'")
 
 
 @mark.parametrize(
diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py
@@ -335,14 +335,19 @@ def test_cli__dispatch_config_compose():
     outfile = "/path/to/output"
     args = {
         STR.configs: configs,
+        STR.realize: True,
         STR.outfile: outfile,
         STR.infmt: FORMAT.yaml,
         STR.outfmt: FORMAT.yaml,
     }
     with patch.object(cli.uwtools.api.config, "compose") as compose:
         cli._dispatch_config_compose(args)
     compose.assert_called_once_with(
-        configs=configs, output_file=outfile, input_format=FORMAT.yaml, output_format=FORMAT.yaml
+        configs=configs,
+        realize=True,
+        output_file=outfile,
+        input_format=FORMAT.yaml,
+        output_format=FORMAT.yaml,
     )
 
 

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+uw config compose compose-template.yaml compose-values.yaml`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+radius: !float '{{ 2.0 * pi * r }}'`
	`2`	`+pi: 3.142`
	`3`	`+r: 1.0`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+radius: 6.284`
	`2`	`+pi: 3.142`
	`3`	`+r: 1.0`