diff --git a/docs/index.rst b/docs/index.rst index 4c23ab5e9..81d8c859e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -37,6 +37,13 @@ When the Linux diff tool just doesn't work for comparing unordered namelists wit | :any:`CLI documentation with examples` +Compose Action +"""""""""""""" + +To compose configs is to start with a base config and to update its values, structurally, from the contents of one or more other configs of the same type (YAML, Fortran namelist, etc.). Composition supports building up complex experiment configurations by, for example, combining default values appropriate to the overall application with values needed on a particular machine, and then with specific user requirements, each stored in their own config files. Such a hierarchical approach to configuration management provides flexibility and avoids repetition of common values across files. + +| :any:`CLI documentation with examples` + Realize Action """""""""""""" diff --git a/docs/sections/user_guide/cli/tools/config.rst b/docs/sections/user_guide/cli/tools/config.rst index a1ad8256f..c936adfa0 100644 --- a/docs/sections/user_guide/cli/tools/config.rst +++ b/docs/sections/user_guide/cli/tools/config.rst @@ -87,6 +87,75 @@ The examples that follow use identical namelist files ``a.nml`` and ``b.nml`` wi .. literalinclude:: config/compare-format-mismatch.out :language: text +.. _cli_config_compose_examples: + +``compose`` +----------- + +The ``compose`` action builds up a final config by repeatedly updating a base config with the contents of other configs of the same format. + +.. literalinclude:: config/compose-help.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: config/compose-help.out + :language: text + +Examples +^^^^^^^^ + +* Consider three YAML configs: + + .. literalinclude:: config/compose-base.yaml + :caption: compose-base.yaml + :language: yaml + .. literalinclude:: config/compose-update-1.yaml + :caption: compose-update-1.yaml + :language: yaml + .. literalinclude:: config/compose-update-2.yaml + :caption: compose-update-2.yaml + :language: yaml + + Compose the three together, writing to ``stdout``: + + .. literalinclude:: config/compose-basic.cmd + :language: text + :emphasize-lines: 1 + .. literalinclude:: config/compose-basic.out + :language: yaml + + Values provided by update configs override or augment values provided in the base config, while unaffected values survive to the final config. Priority of values increases from left to right. + + Additionally: + + * Sets of configs in the ``ini``, ``nml``, and ``sh`` formats can be similarly composed. + * The ``--input-format`` and ``--output-format`` options can be used to specify the format of the input and output configs, respectively, for cases when ``uwtools`` cannot deduce the format of configs from their filename extensions. When the formats are neither explicitly specified or deduced, ``yaml`` is assumed. + * The ``--output-file`` / ``-o`` option can be added to write the final config to a file instead of to ``stdout``. + +* The optional ``--realize`` switch can be used to render as many Jinja2 template expressions as possible in the final config, using the config itself as a source of values. For example: + + .. literalinclude:: config/compose-template.yaml + :caption: compose-template.yaml + :language: yaml + .. literalinclude:: config/compose-values.yaml + :caption: compose-values.yaml + :language: yaml + + Without the ``--realize`` switch: + + .. literalinclude:: config/compose-realize-no.cmd + :language: text + :emphasize-lines: 1 + .. literalinclude:: config/compose-realize-no.out + :language: yaml + + And with ``--realize``: + + .. literalinclude:: config/compose-realize-yes.cmd + :language: text + :emphasize-lines: 1 + .. literalinclude:: config/compose-realize-yes.out + :language: yaml + .. _cli_config_realize_examples: ``realize`` diff --git a/docs/sections/user_guide/cli/tools/config/compose-base.yaml b/docs/sections/user_guide/cli/tools/config/compose-base.yaml new file mode 100644 index 000000000..0e050e998 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/config/compose-base.yaml @@ -0,0 +1,4 @@ +constants: + pi: 3.142 +color: blue +flower: violet diff --git a/docs/sections/user_guide/cli/tools/config/compose-basic.cmd b/docs/sections/user_guide/cli/tools/config/compose-basic.cmd new file mode 100644 index 000000000..aa04384b2 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/config/compose-basic.cmd @@ -0,0 +1,2 @@ +uw config compose compose-base.yaml compose-update-1.yaml compose-update-2.yaml + diff --git a/docs/sections/user_guide/cli/tools/config/compose-basic.out b/docs/sections/user_guide/cli/tools/config/compose-basic.out new file mode 100644 index 000000000..acb77993a --- /dev/null +++ b/docs/sections/user_guide/cli/tools/config/compose-basic.out @@ -0,0 +1,5 @@ +constants: + pi: 3.142 + e: 2.718 +color: blue +flower: rose diff --git a/docs/sections/user_guide/cli/tools/config/compose-help.cmd b/docs/sections/user_guide/cli/tools/config/compose-help.cmd new file mode 100644 index 000000000..742b3cff0 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/config/compose-help.cmd @@ -0,0 +1 @@ +uw config compose --help diff --git a/docs/sections/user_guide/cli/tools/config/compose-help.out b/docs/sections/user_guide/cli/tools/config/compose-help.out new file mode 100644 index 000000000..59d8173d5 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/config/compose-help.out @@ -0,0 +1,28 @@ +usage: uw config compose [-h] [--version] [--realize] [--output-file PATH] + [--input-format {ini,nml,sh,yaml}] + [--output-format {ini,nml,sh,yaml}] [--quiet] + [--verbose] + CONFIG [CONFIG ...] + +Compose configs + +positional arguments: + CONFIG + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --realize + Render template expressions where possible + --output-file PATH, -o PATH + Path to output file (default: write to stdout) + --input-format {ini,nml,sh,yaml} + Input format (default: yaml) + --output-format {ini,nml,sh,yaml} + Output format (default: yaml) + --quiet, -q + Print no logging messages + --verbose, -v + Print all logging messages diff --git a/docs/sections/user_guide/cli/tools/config/compose-realize-no.cmd b/docs/sections/user_guide/cli/tools/config/compose-realize-no.cmd new file mode 100644 index 000000000..36d282c35 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/config/compose-realize-no.cmd @@ -0,0 +1 @@ +uw config compose compose-template.yaml compose-values.yaml diff --git a/docs/sections/user_guide/cli/tools/config/compose-realize-no.out b/docs/sections/user_guide/cli/tools/config/compose-realize-no.out new file mode 100644 index 000000000..58733b6c2 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/config/compose-realize-no.out @@ -0,0 +1,3 @@ +radius: !float '{{ 2.0 * pi * r }}' +pi: 3.142 +r: 1.0 diff --git a/docs/sections/user_guide/cli/tools/config/compose-realize-yes.cmd b/docs/sections/user_guide/cli/tools/config/compose-realize-yes.cmd new file mode 100644 index 000000000..58b22369a --- /dev/null +++ b/docs/sections/user_guide/cli/tools/config/compose-realize-yes.cmd @@ -0,0 +1 @@ +uw config compose compose-template.yaml compose-values.yaml --realize diff --git a/docs/sections/user_guide/cli/tools/config/compose-realize-yes.out b/docs/sections/user_guide/cli/tools/config/compose-realize-yes.out new file mode 100644 index 000000000..4024e4f8f --- /dev/null +++ b/docs/sections/user_guide/cli/tools/config/compose-realize-yes.out @@ -0,0 +1,3 @@ +radius: 6.284 +pi: 3.142 +r: 1.0 diff --git a/docs/sections/user_guide/cli/tools/config/compose-template.yaml b/docs/sections/user_guide/cli/tools/config/compose-template.yaml new file mode 100644 index 000000000..1cafba0ea --- /dev/null +++ b/docs/sections/user_guide/cli/tools/config/compose-template.yaml @@ -0,0 +1 @@ +radius: !float '{{ 2.0 * pi * r }}' diff --git a/docs/sections/user_guide/cli/tools/config/compose-update-1.yaml b/docs/sections/user_guide/cli/tools/config/compose-update-1.yaml new file mode 100644 index 000000000..47f1ad217 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/config/compose-update-1.yaml @@ -0,0 +1,2 @@ +constants: + e: 2.718 diff --git a/docs/sections/user_guide/cli/tools/config/compose-update-2.yaml b/docs/sections/user_guide/cli/tools/config/compose-update-2.yaml new file mode 100644 index 000000000..c2b9d2bb0 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/config/compose-update-2.yaml @@ -0,0 +1 @@ +flower: rose diff --git a/docs/sections/user_guide/cli/tools/config/compose-values.yaml b/docs/sections/user_guide/cli/tools/config/compose-values.yaml new file mode 100644 index 000000000..71871573e --- /dev/null +++ b/docs/sections/user_guide/cli/tools/config/compose-values.yaml @@ -0,0 +1,2 @@ +pi: 3.142 +r: 1.0 diff --git a/docs/sections/user_guide/cli/tools/config/help.out b/docs/sections/user_guide/cli/tools/config/help.out index b3e075f75..ca4bd537d 100644 --- a/docs/sections/user_guide/cli/tools/config/help.out +++ b/docs/sections/user_guide/cli/tools/config/help.out @@ -12,6 +12,8 @@ Positional arguments: ACTION compare Compare configs + compose + Compose configs realize Realize config validate diff --git a/docs/sections/user_guide/cli/tools/config/realize-help.out b/docs/sections/user_guide/cli/tools/config/realize-help.out index acc724049..9c3237c8a 100644 --- a/docs/sections/user_guide/cli/tools/config/realize-help.out +++ b/docs/sections/user_guide/cli/tools/config/realize-help.out @@ -15,17 +15,17 @@ Optional arguments: --version Show version info and exit --input-file PATH, -i PATH - Path to input file (defaults to stdin) + Path to input file (default: read from stdin) --input-format {ini,nml,sh,yaml} - Input format + Input format (default: yaml) --update-file PATH, -u PATH - Path to update file (defaults to stdin) + Path to update file (default: read from stdin) --update-format {ini,nml,sh,yaml} Update format --output-file PATH, -o PATH - Path to output file (defaults to stdout) + Path to output file (default: write to stdout) --output-format {ini,nml,sh,yaml} - Output format + Output format (default: yaml) --key-path KEY[.KEY...] Dot-separated path of keys to the block to be output --values-needed diff --git a/docs/sections/user_guide/cli/tools/config/validate-help.out b/docs/sections/user_guide/cli/tools/config/validate-help.out index bc69e8087..a260fb606 100644 --- a/docs/sections/user_guide/cli/tools/config/validate-help.out +++ b/docs/sections/user_guide/cli/tools/config/validate-help.out @@ -13,7 +13,7 @@ Optional arguments: --version Show version info and exit --input-file PATH, -i PATH - Path to input file (defaults to stdin) + Path to input file (default: read from stdin) --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/tools/rocoto/realize-help.out b/docs/sections/user_guide/cli/tools/rocoto/realize-help.out index 25a08e3af..614c1957d 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/realize-help.out +++ b/docs/sections/user_guide/cli/tools/rocoto/realize-help.out @@ -11,7 +11,7 @@ Optional arguments: --config-file PATH, -c PATH Path to UW YAML config file (default: read from stdin) --output-file PATH, -o PATH - Path to output file (defaults to stdout) + Path to output file (default: write to stdout) --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/tools/rocoto/validate-help.out b/docs/sections/user_guide/cli/tools/rocoto/validate-help.out index 23c28085e..4816b797e 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/validate-help.out +++ b/docs/sections/user_guide/cli/tools/rocoto/validate-help.out @@ -9,7 +9,7 @@ Optional arguments: --version Show version info and exit --input-file PATH, -i PATH - Path to input file (defaults to stdin) + Path to input file (default: read from stdin) --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/tools/template/render-help.out b/docs/sections/user_guide/cli/tools/template/render-help.out index c48e430bd..3aeef68fc 100644 --- a/docs/sections/user_guide/cli/tools/template/render-help.out +++ b/docs/sections/user_guide/cli/tools/template/render-help.out @@ -13,9 +13,9 @@ Optional arguments: --version Show version info and exit --input-file PATH, -i PATH - Path to input file (defaults to stdin) + Path to input file (default: read from stdin) --output-file PATH, -o PATH - Path to output file (defaults to stdout) + Path to output file (default: write to stdout) --values-file PATH Path to file providing override or interpolation values --values-format {ini,nml,sh,yaml} diff --git a/docs/sections/user_guide/cli/tools/template/translate-help.out b/docs/sections/user_guide/cli/tools/template/translate-help.out index d1ca93c39..9107720c0 100644 --- a/docs/sections/user_guide/cli/tools/template/translate-help.out +++ b/docs/sections/user_guide/cli/tools/template/translate-help.out @@ -10,9 +10,9 @@ Optional arguments: --version Show version info and exit --input-file PATH, -i PATH - Path to input file (defaults to stdin) + Path to input file (default: read from stdin) --output-file PATH, -o PATH - Path to output file (defaults to stdout) + Path to output file (default: write to stdout) --dry-run Only log info, making no changes --quiet, -q diff --git a/docs/static/custom.css b/docs/static/custom.css index cb22e007e..b29e00523 100644 --- a/docs/static/custom.css +++ b/docs/static/custom.css @@ -21,3 +21,15 @@ html.writer-html5 .rst-content table.docutils td>p { .wy-table-responsive table td, .wy-table-responsive table th { white-space: inherit; } + +/* Hide "Listing ..." text in captions */ + +.rst-content .caption-number { + display: none; +} + +/* Left align captions. */ + +.rst-content .code-block-caption { + text-align: left; +} diff --git a/src/uwtools/api/config.py b/src/uwtools/api/config.py index a821143ca..ff058972c 100644 --- a/src/uwtools/api/config.py +++ b/src/uwtools/api/config.py @@ -14,8 +14,9 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.config.formats.sh import SHConfig from uwtools.config.formats.yaml import YAMLConfig -from uwtools.config.tools import compare_configs as _compare -from uwtools.config.tools import realize_config as _realize +from uwtools.config.tools import compare as _compare +from uwtools.config.tools import compose as _compose +from uwtools.config.tools import realize as _realize from uwtools.config.validator import ConfigDataT, ConfigPathT from uwtools.config.validator import validate_check_config as _validate_check_config from uwtools.config.validator import validate_external as _validate_external @@ -31,19 +32,30 @@ def compare( - path1: Path | str, - path2: Path | str, - format1: str | None = None, - format2: str | None = None, + path1: Path | str, path2: Path | str, format1: str | None = None, format2: str | None = None ) -> bool: """ NB: This docstring is dynamically replaced: See compare.__doc__ definition below. """ - return _compare( - path1=Path(path1), - path2=Path(path2), - format1=format1, - format2=format2, + return _compare(path1=Path(path1), path2=Path(path2), format1=format1, format2=format2) + + +def compose( + configs: list[str | Path], + realize: bool = False, + output_file: Path | str | None = None, + input_format: str | None = None, + output_format: str | None = None, +) -> bool: + """ + NB: This docstring is dynamically replaced: See compose.__doc__ definition below. + """ + return _compose( + configs=list(map(Path, configs)), + realize=realize, + output_file=Path(output_file) if output_file else None, + input_format=input_format, + output_format=output_format, ) @@ -217,8 +229,25 @@ def validate( :param format1: Format of 1st config file (optional if file's extension is recognized). :param format2: Format of 2nd config file (optional if file's extension is recognized). :return: ``False`` if config files had differences, otherwise ``True``. -""".format(extensions=", ".join(_FORMAT.extensions())).strip() - +""".format(extensions=", ".join([f"``{x}``" for x in _FORMAT.extensions()])).strip() + +compose.__doc__ = """ +Compose config files. + +Specify explicit input or output formats to override default treatment based on file extension. +Recognized file extensions are: {extensions}. + +:param configs: Paths to configs to compose. +:param realize: Render template expressions where possible. +:param output_file: Output config destination (default: write to ``stdout``). +:param input_format: Format of configs to compose (choices: {choices}, default: ``{default}``) +:param output_format: Format of output config (choices: {choices}, default: ``{default}``) +:return: ``True`` if no errors were encountered. +""".format( + default=_FORMAT.yaml, + extensions=", ".join([f"``{x}``" for x in _FORMAT.extensions()]), + choices=", ".join([f"``{x}``" for x in (_FORMAT.ini, _FORMAT.nml, _FORMAT.sh, _FORMAT.yaml)]), +).strip() realize.__doc__ = """ Realize a config based on a base input config and an optional update config. @@ -258,7 +287,7 @@ def validate( :param stdin_ok: OK to read from ``stdin``? :return: The ``dict`` representation of the realized config. :raises: ``UWConfigRealizeError`` if ``total`` is ``True`` and any Jinja2 syntax was not rendered. -""".format(extensions=", ".join(_FORMAT.extensions())).strip() # noqa: E501 +""".format(extensions=", ".join([f"``{x}``" for x in _FORMAT.extensions()])).strip() # noqa: E501 __all__ = [ "Config", @@ -268,6 +297,7 @@ def validate( "SHConfig", "YAMLConfig", "compare", + "compose", "get_fieldtable_config", "get_ini_config", "get_nml_config", diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 40745795c..6b91ddc47 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -118,6 +118,7 @@ def _add_subparser_config(subparsers: Subparsers) -> ModeChecks: subparsers = _add_subparsers(parser, STR.action, STR.action.upper()) return { STR.compare: _add_subparser_config_compare(subparsers), + STR.compose: _add_subparser_config_compose(subparsers), STR.realize: _add_subparser_config_realize(subparsers), STR.validate: _add_subparser_config_validate(subparsers), } @@ -153,6 +154,23 @@ def _add_subparser_config_compare(subparsers: Subparsers) -> ActionChecks: ] +def _add_subparser_config_compose(subparsers: Subparsers) -> ActionChecks: + """ + Add subparser for mode: config compose. + + :param subparsers: Parent parser's subparsers, to add this subparser to. + """ + parser = _add_subparser(subparsers, STR.compose, "Compose configs") + optional = _basic_setup(parser) + _add_arg_realize(optional) + _add_arg_output_file(optional) + _add_arg_input_format(optional, choices=FORMATS) + _add_arg_output_format(optional, choices=FORMATS) + checks = _add_args_verbosity(optional) + parser.add_argument("configs", metavar="CONFIG", nargs="+", type=Path) + return checks + + def _add_subparser_config_realize(subparsers: Subparsers) -> ActionChecks: """ Add subparser for mode: config realize. @@ -201,6 +219,7 @@ def _dispatch_config(args: Args) -> bool: """ actions = { STR.compare: _dispatch_config_compare, + STR.compose: _dispatch_config_compose, STR.realize: _dispatch_config_realize, STR.validate: _dispatch_config_validate, } @@ -221,6 +240,21 @@ def _dispatch_config_compare(args: Args) -> bool: ) +def _dispatch_config_compose(args: Args) -> bool: + """ + Define dispatch logic for config compose action. + + :param args: Parsed command-line args. + """ + return uwtools.api.config.compose( + configs=args[STR.configs], + output_file=args[STR.outfile], + realize=args[STR.realize], + input_format=args[STR.infmt], + output_format=args[STR.outfmt], + ) + + def _dispatch_config_realize(args: Args) -> bool: """ Define dispatch logic for config realize action. @@ -808,7 +842,7 @@ def _add_arg_input_file(group: Group, required: bool = False) -> None: group.add_argument( _switch(STR.infile), "-i", - help="Path to input file (defaults to stdin)", + help="Path to input file (default: read from stdin)", metavar="PATH", required=required, type=Path, @@ -819,7 +853,7 @@ def _add_arg_input_format(group: Group, choices: list[str], required: bool = Fal group.add_argument( _switch(STR.infmt), choices=choices, - help="Input format", + help=f"Input format (default: {FORMAT.yaml})", required=required, type=str, ) @@ -866,7 +900,7 @@ def _add_arg_output_file(group: Group, required: bool = False) -> None: group.add_argument( _switch(STR.outfile), "-o", - help="Path to output file (defaults to stdout)", + help="Path to output file (default: write to stdout)", metavar="PATH", required=required, type=Path, @@ -877,7 +911,7 @@ def _add_arg_output_format(group: Group, choices: list[str], required: bool = Fa group.add_argument( _switch(STR.outfmt), choices=choices, - help="Output format", + help=f"Output format (default: {FORMAT.yaml})", required=required, type=str, ) @@ -905,6 +939,14 @@ def _add_arg_rate(group: Group) -> None: ) +def _add_arg_realize(group: Group) -> None: + group.add_argument( + _switch(STR.realize), + action="store_true", + help="Render template expressions where possible", + ) + + def _add_arg_report(group: Group) -> None: group.add_argument( _switch(STR.report), @@ -981,7 +1023,7 @@ def _add_arg_update_file(group: Group, required: bool = False) -> None: group.add_argument( _switch(STR.updatefile), "-u", - help="Path to update file (defaults to stdin)", + help="Path to update file (default: read from stdin)", metavar="PATH", required=required, type=Path, @@ -1348,6 +1390,8 @@ def _parse_args(raw_args: list[str]) -> tuple[Args, Checks]: **drivers_with_cycle_and_leadtime, } checks = {k: modes[k]() for k in sorted(modes.keys())} + # Return a dict version of the Namespace object returned by parse_args(), supporting lookups + # like args[STR.foo], which would otherwise have to be the even noisier getattr(args, STR.foo). return vars(parser.parse_args(raw_args)), checks diff --git a/src/uwtools/config/tools.py b/src/uwtools/config/tools.py index 407f7e561..ada854688 100644 --- a/src/uwtools/config/tools.py +++ b/src/uwtools/config/tools.py @@ -20,14 +20,11 @@ # Public functions -def compare_configs( - path1: Path, - path2: Path, - format1: str | None = None, - format2: str | None = None, +def compare( + path1: Path, path2: Path, format1: str | None = None, format2: str | None = None ) -> bool: """ - NB: This docstring is dynamically replaced: See compare_configs.__doc__ definition below. + NB: This docstring is dynamically replaced: See compare.__doc__ definition below. """ format1 = _ensure_format("1st config file", format1, path1) format2 = _ensure_format("2nd config file", format2, path2) @@ -41,7 +38,34 @@ def compare_configs( return cfg_1.compare_config(cfg_2.as_dict()) -def realize_config( +def compose( + configs: list[Path], + realize: bool, + output_file: Path | None = None, + input_format: str | None = None, + output_format: str | None = None, +) -> bool: + """ + NB: This docstring is dynamically replaced: See compose.__doc__ definition below. + """ + basepath = configs[0] + input_format = input_format or get_config_format(basepath, "input") + input_class = format_to_config(input_format) + log.debug("Reading %s as base '%s' config", basepath, input_format) + config = input_class(basepath) + for path in configs[1:]: + log.debug("Composing '%s' config from %s", input_format, path) + config.update_from(input_class(path)) + output_format = output_format or get_config_format(output_file, "output") + output_class = format_to_config(output_format) + output_config = output_class(config) + if realize: + output_config.dereference() + output_config.dump(output_file) + return True + + +def realize( input_config: Config | Path | dict | None = None, input_format: str | None = None, update_config: Config | Path | dict | None = None, @@ -54,12 +78,12 @@ def realize_config( dry_run: bool = False, ) -> dict: """ - NB: This docstring is dynamically replaced: See realize_config.__doc__ definition below. + NB: This docstring is dynamically replaced: See realize.__doc__ definition below. """ - input_obj = _realize_config_input_setup(input_config, input_format) - input_obj = _realize_config_update(input_obj, update_config, update_format) + input_obj = _realize_input_setup(input_config, input_format) + input_obj = _realize_update(input_obj, update_config, update_format) input_obj.dereference() - output_data, output_format = _realize_config_output_setup( + output_data, output_format = _realize_output_setup( input_obj, output_file, output_format, key_path ) if dry_run: @@ -67,7 +91,7 @@ def realize_config( log.info(line) return {} if values_needed: - _realize_config_values_needed(input_obj) + _realize_values_needed(input_obj) return {} if total and unrendered(str(input_obj)): msg = "Config could not be totally realized" @@ -138,13 +162,13 @@ def _ensure_format( return get_config_format(config, desc) -def _realize_config_input_setup( +def _realize_input_setup( input_config: Config | Path | dict | None = None, input_format: str | None = None ) -> Config: """ Set up config-realize input. - :param input_config: Input config source (None => read stdin). + :param input_config: Input config source (None => read from stdin). :param input_format: Format of the input config. :return: The input Config object. """ @@ -157,7 +181,7 @@ def _realize_config_input_setup( return config_obj -def _realize_config_output_setup( +def _realize_output_setup( input_obj: Config, output_file: Path | None = None, output_format: str | None = None, @@ -184,7 +208,7 @@ def _realize_config_output_setup( return output_data, output_format -def _realize_config_update( +def _realize_update( input_obj: Config, update_config: Config | Path | dict | None = None, update_format: str | None = None, @@ -193,7 +217,7 @@ def _realize_config_update( Set up config-realize update. :param input_obj: The input Config object. - :param update_config: Input config source (None => read stdin). + :param update_config: Input config source (None => read from stdin). :param update_format: Format of the update config. :return: The updated but unrealized Config object. """ @@ -217,7 +241,7 @@ def _realize_config_update( return input_obj -def _realize_config_values_needed(input_obj: Config) -> None: +def _realize_values_needed(input_obj: Config) -> None: """ Print a report characterizing input values as complete, empty, or template placeholders. @@ -266,7 +290,7 @@ def _validate_format(other_fmt_desc: str, other_fmt: str, input_fmt: str) -> Non # work if the docstrings are inlined in the functions. They must remain separate statements to avoid # hardcoding values into them. -compare_configs.__doc__ = """ +compare.__doc__ = """ Compare two config files. Recognized file extensions are: {extensions} @@ -275,25 +299,40 @@ def _validate_format(other_fmt_desc: str, other_fmt: str, input_fmt: str) -> Non :param path2: Path to 2nd config file :param format1: Format of 1st config file (optional if file's extension is recognized) :param format2: Format of 2nd config file (optional if file's extension is recognized) -:return: ``False`` if config files had differences, otherwise ``True`` +:return: False if config files had differences, otherwise True """.format(extensions=", ".join(FORMAT.extensions())).strip() +compose.__doc__ = """ +Compose config files. + +Recognized file extensions are: {extensions} -realize_config.__doc__ = """ +:param configs: Paths to configs to compose. +:param output_file: Output config destination (default: write to stdout). +:param input_format: Format of configs to compose (choices: {choices}, default: {default}). +:param output_format: Format of output config (choices: {choices}, default: {default}). +:return: True if no errors were encountered. +""".format( + default=FORMAT.yaml, + extensions=", ".join(FORMAT.extensions()), + choices=", ".join([FORMAT.ini, FORMAT.nml, FORMAT.sh, FORMAT.yaml]), +).strip() + +realize.__doc__ = """ Realize an output config based on an input config and optional values-providing configs. Recognized file extensions are: {extensions} -:param input_config: Input config source (None => read ``stdin``). +:param input_config: Input config source (None => read from stdin). :param input_format: Input config format. -:param update_config: Input config source (None => read ``stdin``). +:param update_config: Input config source (None => read from stdin). :param update_format: Update config format. -:param output_file: Output config destination (None => write to ``stdout``). +:param output_file: Output config destination (None => write to stdout). :param output_format: Output config format. :param key_path: Path of keys to the desired output block. :param values_needed: Report complete, missing, and template values. :param total: Require rendering of all Jinja2 variables/expressions. :param dry_run: Log output instead of writing to output. -:raises: UWConfigRealizeError if ``total`` is ``True`` and config cannot be totally realized. +:raises: UWConfigRealizeError if total is True and config cannot be totally realized. :return: The realized config (or an empty-dict for no-op modes). """.format(extensions=", ".join(FORMAT.extensions())).strip() diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index 8bbbb3432..f97de674b 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -70,7 +70,9 @@ class STR: chgrescube: str = "chgres_cube" classname: str = "classname" compare: str = "compare" + compose: str = "compose" config: str = "config" + configs: str = "configs" copy: str = "copy" cycle: str = "cycle" database: str = "database" diff --git a/src/uwtools/tests/api/test_config.py b/src/uwtools/tests/api/test_config.py index 29a5db9b6..db0377a05 100644 --- a/src/uwtools/tests/api/test_config.py +++ b/src/uwtools/tests/api/test_config.py @@ -11,7 +11,7 @@ from uwtools.utils.file import FORMAT -def test_compare(): +def test_api_config_compare(): kwargs: dict = { "path1": "path1", "format1": "fmt1", @@ -29,6 +29,29 @@ def test_compare(): ) +@mark.parametrize("output_file", [None, "/path/to/out.yaml"]) +@mark.parametrize("input_format", [None, FORMAT.yaml, FORMAT.nml]) +@mark.parametrize("output_format", [None, FORMAT.yaml, FORMAT.nml]) +def test_api_config_compose(output_file, input_format, output_format): + pathstrs = ["/path/to/c1.yaml", "/path/to/c2.yaml"] + kwargs: dict = { + "configs": pathstrs, + "realize": False, + "output_file": output_file, + "input_format": input_format, + "output_format": output_format, + } + with patch.object(config, "_compose") as _compose: + config.compose(**kwargs) + _compose.assert_called_once_with( + configs=list(map(Path, pathstrs)), + realize=False, + output_file=None if output_file is None else Path(output_file), + input_format=input_format, + output_format=output_format, + ) + + @mark.parametrize( ("classname", "f"), [ @@ -39,14 +62,14 @@ def test_compare(): ("YAMLConfig", config.get_yaml_config), ], ) -def test_get_config(classname, f): +def test_api_config__get_config(classname, f): kwargs: dict = dict(config={}) with patch.object(config, classname) as constructor: f(**kwargs) constructor.assert_called_once_with(**kwargs) -def test_realize(): +def test_api_config_realize(): kwargs: dict = { "input_config": "path1", "input_format": "fmt1", @@ -71,32 +94,13 @@ def test_realize(): ) -def test_realize_to_dict(): - kwargs: dict = { - "input_config": "path1", - "input_format": "fmt1", - "update_config": None, - "update_format": None, - "key_path": None, - "values_needed": True, - "total": False, - "dry_run": False, - "stdin_ok": False, - } - with patch.object(config, "realize") as realize: - config.realize_to_dict(**kwargs) - realize.assert_called_once_with( - **{**kwargs, "output_file": Path(os.devnull), "output_format": FORMAT.yaml} - ) - - -def test_realize_update_config_from_stdin(): +def test_api_config_realize__update_config_from_stdin(): with raises(UWError) as e: config.realize(input_config={}, output_file="output.yaml", update_format="yaml") assert str(e.value) == "Set stdin_ok=True to permit read from stdin" -def test_realize_update_config_none(): +def test_api_config_realize__update_config_none(): input_config = {"n": 42} output_file = Path("output.yaml") with patch.object(config, "_realize") as _realize: @@ -115,8 +119,27 @@ def test_realize_update_config_none(): ) +def test_api_config_realize_to_dict(): + kwargs: dict = { + "input_config": "path1", + "input_format": "fmt1", + "update_config": None, + "update_format": None, + "key_path": None, + "values_needed": True, + "total": False, + "dry_run": False, + "stdin_ok": False, + } + with patch.object(config, "realize") as realize: + config.realize_to_dict(**kwargs) + realize.assert_called_once_with( + **{**kwargs, "output_file": Path(os.devnull), "output_format": FORMAT.yaml} + ) + + @mark.parametrize("cfg", [{"foo": "bar"}, YAMLConfig(config={"foo": "bar"})]) -def test_validate_config_data(cfg): +def test_api_config_validate__config_data(cfg): kwargs: dict = {"schema_file": "schema-file", "config_data": cfg} with patch.object(config, "_validate_external") as _validate_external: assert config.validate(**kwargs) is True @@ -131,7 +154,7 @@ def test_validate_config_data(cfg): @mark.parametrize("cast", [str, Path]) -def test_validate_config_path(cast, tmp_path): +def test_api_config__validate_config_path(cast, tmp_path): cfg = tmp_path / "config.yaml" cfg.write_text(yaml.dump({})) kwargs: dict = {"schema_file": "schema-file", "config_path": cast(cfg)} diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index dc2d4d12e..e26fd8a1e 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -28,7 +28,7 @@ @fixture -def compare_configs_assets(tmp_path): +def compare_assets(tmp_path): d = {"foo": {"bar": 42}, "baz": {"qux": 43}} a = tmp_path / "a" b = tmp_path / "b" @@ -40,12 +40,23 @@ def compare_configs_assets(tmp_path): @fixture -def realize_config_testobj(realize_config_yaml_input): - return YAMLConfig(config=realize_config_yaml_input) +def compose_assets_yaml(): + d1 = {"k10": "v10", "k11": "v11"} + u1 = {"k10": "u10", "k12": "v12"} + d2 = {"k10": {"k21": "v21", "k22": "v22"}, "k11": "v11"} + u2 = {"k10": {"k21": "u21"}, "k12": "v12"} + d3 = {"k10": {"k21": {"k31": "v31", "k32": "v32"}, "k22": "v22"}, "k11": "v11"} + u3 = {"k10": {"k21": {"k31": ["v310", "v311"], "k33": "v33"}}} + return d1, u1, d2, u2, d3, u3 @fixture -def realize_config_yaml_input(tmp_path): +def realize_testobj(realize_yaml_input): + return YAMLConfig(config=realize_yaml_input) + + +@fixture +def realize_yaml_input(tmp_path): path = tmp_path / "a.yaml" d = {1: {2: {3: 42}}} # depth 3 with writable(path) as f: @@ -56,20 +67,20 @@ def realize_config_yaml_input(tmp_path): # Helpers -def help_realize_config_double_tag(config, expected, tmp_path): +def help_realize_double_tag(config, expected, tmp_path): path_in = tmp_path / "in.yaml" path_out = tmp_path / "out.yaml" path_in.write_text(dedent(config).strip()) - tools.realize_config(input_config=path_in, output_file=path_out) + tools.realize(input_config=path_in, output_file=path_out) assert path_out.read_text().strip() == dedent(expected).strip() -def help_realize_config_fmt2fmt(input_file, input_format, update_file, update_format, tmpdir): +def help_realize_fmt2fmt(input_file, input_format, update_file, update_format, tmpdir): input_file = fixture_path(input_file) update_file = fixture_path(update_file) ext = Path(input_file).suffix output_file = tmpdir / f"output_file{ext}" - tools.realize_config( + tools.realize( input_config=input_file, input_format=input_format, update_config=update_file, @@ -85,11 +96,11 @@ def help_realize_config_fmt2fmt(input_file, input_format, update_file, update_fo assert compare_files(reference, output_file) -def help_realize_config_simple(infn, infmt, tmpdir): +def help_realize_simple(infn, infmt, tmpdir): infile = fixture_path(infn) ext = Path(infile).suffix outfile = tmpdir / f"outfile{ext}" - tools.realize_config( + tools.realize( input_config=infile, input_format=infmt, output_file=outfile, @@ -104,18 +115,18 @@ def help_realize_config_simple(infn, infmt, tmpdir): # Tests -def test_config_tools_compare_configs__good(compare_configs_assets, logged): - _, a, b = compare_configs_assets - assert tools.compare_configs(path1=a, format1=FORMAT.yaml, path2=b, format2=FORMAT.yaml) +def test_config_tools_compare__good(compare_assets, logged): + _, a, b = compare_assets + assert tools.compare(path1=a, format1=FORMAT.yaml, path2=b, format2=FORMAT.yaml) assert logged(".*", regex=True) -def test_config_tools_compare_configs__changed_value(compare_configs_assets, logged): - d, a, b = compare_configs_assets +def test_config_tools_compare__changed_value(compare_assets, logged): + d, a, b = compare_assets d["baz"]["qux"] = 11 with writable(b) as f: yaml.dump(d, f) - assert not tools.compare_configs(path1=a, format1=FORMAT.yaml, path2=b, format2=FORMAT.yaml) + assert not tools.compare(path1=a, format1=FORMAT.yaml, path2=b, format2=FORMAT.yaml) expected = """ - %s + %s @@ -137,13 +148,13 @@ def test_config_tools_compare_configs__changed_value(compare_configs_assets, log assert logged(line) -def test_config_tools_compare_configs__missing_key(compare_configs_assets, logged): - d, a, b = compare_configs_assets +def test_config_tools_compare__missing_key(compare_assets, logged): + d, a, b = compare_assets del d["baz"] with writable(b) as f: yaml.dump(d, f) # Note that a and b are swapped: - assert not tools.compare_configs(path1=b, format1=FORMAT.yaml, path2=a, format2=FORMAT.yaml) + assert not tools.compare(path1=b, format1=FORMAT.yaml, path2=a, format2=FORMAT.yaml) expected = """ - %s + %s @@ -162,8 +173,8 @@ def test_config_tools_compare_configs__missing_key(compare_configs_assets, logge assert logged(line) -def test_config_tools_compare_configs__bad_format(logged): - assert not tools.compare_configs( +def test_config_tools_compare__bad_format(logged): + assert not tools.compare( path1=Path("/not/used"), format1="jpg", path2=Path("/not/used"), @@ -173,6 +184,122 @@ def test_config_tools_compare_configs__bad_format(logged): assert logged(msg) +@mark.parametrize(("configclass", "fmt"), [(INIConfig, FORMAT.ini), (NMLConfig, FORMAT.nml)]) +def test_config_tools_compose__fmt_ini_nml_2x(configclass, fmt, logged, tmp_path): + d = {"constants": {"pi": 3.142, "e": 2.718}, "trees": {"leaf": "elm", "needle": "spruce"}} + u = {"trees": {"needle": "fir"}, "colors": {"red": "crimson", "green": "clover"}} + suffix = f".{fmt}" + dpath, upath = [(tmp_path / x).with_suffix(suffix) for x in ("d", "u")] + configclass(d).dump(dpath) + configclass(u).dump(upath) + outpath = (tmp_path / "out").with_suffix(suffix) + kwargs: dict = {"configs": [dpath, upath], "realize": False, "output_file": outpath} + assert tools.compose(**kwargs) is True + assert logged(f"Reading {dpath} as base '{fmt}' config") + assert logged(f"Composing '{fmt}' config from {upath}") + constants = {"pi": 3.142, "e": 2.718} + expected = { + "constants": constants, + "trees": {"leaf": "elm", "needle": "fir"}, + "colors": {"red": "crimson", "green": "clover"}, + } + if fmt == FORMAT.ini: + # All values in INI configs are strings: + expected["constants"] = {k: str(v) for k, v in constants.items()} + assert configclass(outpath) == configclass(expected) + + +def test_config_tools_compose__fmt_sh_2x(logged, tmp_path): + d = {"foo": 1, "bar": 2} + u = {"foo": 3, "baz": 4} + dpath, upath = [tmp_path / x for x in ("d.sh", "u.sh")] + SHConfig(d).dump(dpath) + SHConfig(u).dump(upath) + outpath = tmp_path / "out.sh" + kwargs: dict = {"configs": [dpath, upath], "realize": False, "output_file": outpath} + assert tools.compose(**kwargs) is True + assert logged(f"Reading {dpath} as base 'sh' config") + assert logged(f"Composing 'sh' config from {upath}") + assert SHConfig(outpath) == SHConfig({"foo": "3", "bar": "2", "baz": "4"}) + + +@mark.parametrize("tofile", [False, True]) +@mark.parametrize("suffix", ["", ".yaml", ".foo"]) +def test_config_tools_compose__fmt_yaml_1x(compose_assets_yaml, logged, suffix, tmp_path, tofile): + d1, _, d2, _, d3, _ = compose_assets_yaml + dpath = (tmp_path / "d").with_suffix(suffix) + for d in (d1, d2, d3): + dpath.unlink(missing_ok=True) + assert not dpath.exists() + dpath.write_text(yaml.dump(d)) + kwargs: dict = {"configs": [dpath], "realize": False} + if suffix and suffix != ".yaml": + kwargs["input_format"] = FORMAT.yaml + if tofile: + outpath = (tmp_path / "out").with_suffix(suffix) + kwargs["output_file"] = outpath + assert tools.compose(**kwargs) is True + assert logged(f"Reading {dpath} as base 'yaml' config") + if tofile: + assert YAMLConfig(outpath) == d + outpath.unlink() + + +@mark.parametrize("tofile", [False, True]) +@mark.parametrize("suffix", ["", ".yaml", ".foo"]) +def test_config_tools_compose__fmt_yaml_2x(compose_assets_yaml, logged, suffix, tmp_path, tofile): + d1, u1, d2, u2, d3, u3 = compose_assets_yaml + dpath, upath = [(tmp_path / x).with_suffix(suffix) for x in ("d", "u")] + for d, u in [(d1, u1), (d2, u2), (d3, u3)]: + for path in [dpath, upath]: + path.unlink(missing_ok=True) + assert not path.exists() + dpath.write_text(yaml.dump(d)) + upath.write_text(yaml.dump(u)) + kwargs: dict = {"configs": [dpath, upath], "realize": False} + if suffix and suffix != ".yaml": + kwargs["input_format"] = FORMAT.yaml + if tofile: + outpath = (tmp_path / "out").with_suffix(suffix) + kwargs["output_file"] = outpath + assert tools.compose(**kwargs) is True + assert logged(f"Reading {dpath} as base 'yaml' config") + assert logged(f"Composing 'yaml' config from {upath}") + if tofile: + expected = { + str(u1): {"k10": "u10", "k11": "v11", "k12": "v12"}, + str(u2): {"k10": {"k21": "u21", "k22": "v22"}, "k11": "v11", "k12": "v12"}, + str(u3): { + "k10": { + "k21": {"k31": ["v310", "v311"], "k32": "v32", "k33": "v33"}, + "k22": "v22", + }, + "k11": "v11", + }, + } + assert YAMLConfig(outpath) == expected[str(u)] + outpath.unlink() + + +@mark.parametrize("realize", [False, True]) +def test_config_tools_compose__realize(realize, tmp_path): + dyaml = """ + radius: !float '{{ 2.0 * pi * r }}' + """ + dpath = tmp_path / "d.yaml" + dpath.write_text(dedent(dyaml)) + uyaml = """ + pi: 3.142 + r: 1.0 + """ + upath = tmp_path / "u.yaml" + upath.write_text(dedent(uyaml)) + outpath = tmp_path / "out.yaml" + assert tools.compose(configs=[dpath, upath], realize=realize, output_file=outpath) is True + radius = YAMLConfig(outpath)["radius"] + assert (radius == 6.284) if realize else (radius.tagged_string == "!float '{{ 2.0 * pi * r }}'") + + @mark.parametrize( ("cfgtype", "fmt"), [ @@ -192,13 +319,13 @@ def test_config_tools_config_tools_format_to_config__fail(): tools.format_to_config("no-such-config-type") -def test_config_tools_realize_config__conversion_cfg_to_yaml(tmp_path): +def test_config_tools_realize__conversion_cfg_to_yaml(tmp_path): """ Test that a .cfg file can be used to create a YAML object. """ infile = fixture_path("srw_example_yaml.cfg") outfile = tmp_path / "test_ouput.yaml" - tools.realize_config( + tools.realize( input_config=infile, input_format=FORMAT.yaml, output_file=outfile, @@ -212,25 +339,25 @@ def test_config_tools_realize_config__conversion_cfg_to_yaml(tmp_path): assert outfile.read_text()[-1] == "\n" -def test_config_tools_realize_config__depth_mismatch_to_ini(realize_config_yaml_input): +def test_config_tools_realize__depth_mismatch_to_ini(realize_yaml_input): with raises(UWConfigError): - tools.realize_config( - input_config=realize_config_yaml_input, + tools.realize( + input_config=realize_yaml_input, input_format=FORMAT.yaml, output_format=FORMAT.ini, ) -def test_config_tools_realize_config__depth_mismatch_to_sh(realize_config_yaml_input): +def test_config_tools_realize__depth_mismatch_to_sh(realize_yaml_input): with raises(UWConfigError): - tools.realize_config( - input_config=realize_config_yaml_input, + tools.realize( + input_config=realize_yaml_input, input_format=FORMAT.yaml, output_format=FORMAT.sh, ) -def test_config_tools_realize_config__double_tag_flat(tmp_path): +def test_config_tools_realize__double_tag_flat(tmp_path): config = """ a: 1 b: 2 @@ -243,10 +370,10 @@ def test_config_tools_realize_config__double_tag_flat(tmp_path): foo: 3 bar: 3 """ - help_realize_config_double_tag(config, expected, tmp_path) + help_realize_double_tag(config, expected, tmp_path) -def test_config_tools_realize_config__double_tag_nest(tmp_path): +def test_config_tools_realize__double_tag_nest(tmp_path): config = """ a: 1.0 b: 2.0 @@ -261,10 +388,10 @@ def test_config_tools_realize_config__double_tag_nest(tmp_path): foo: 3.0 bar: 3.0 """ - help_realize_config_double_tag(config, expected, tmp_path) + help_realize_double_tag(config, expected, tmp_path) -def test_config_tools_realize_config__double_tag_nest_forward_reference(tmp_path): +def test_config_tools_realize__double_tag_nest_forward_reference(tmp_path): config = """ a: true b: false @@ -279,17 +406,17 @@ def test_config_tools_realize_config__double_tag_nest_forward_reference(tmp_path qux: foo: true """ - help_realize_config_double_tag(config, expected, tmp_path) + help_realize_double_tag(config, expected, tmp_path) -def test_config_tools_realize_config__dry_run(logged): +def test_config_tools_realize__dry_run(logged): """ Test that providing a YAML base file with a dry-run flag will print an YAML config file. """ infile = fixture_path("fruit_config.yaml") yaml_config = YAMLConfig(infile) yaml_config.dereference() - tools.realize_config( + tools.realize( input_config=infile, input_format=FORMAT.yaml, output_format=FORMAT.yaml, @@ -298,13 +425,13 @@ def test_config_tools_realize_config__dry_run(logged): assert logged(str(yaml_config), multiline=True) -def test_config_tools_realize_config__field_table(tmp_path): +def test_config_tools_realize__field_table(tmp_path): """ Test reading a YAML config object and generating a field file table. """ infile = fixture_path("FV3_GFS_v16.yaml") outfile = tmp_path / "field_table_from_yaml.FV3_GFS" - tools.realize_config( + tools.realize( input_config=infile, input_format=FORMAT.yaml, output_file=outfile, @@ -319,51 +446,51 @@ def test_config_tools_realize_config__field_table(tmp_path): assert line1 in line2 -def test_config_tools_realize_config__fmt2fmt_nml2nml(tmp_path): +def test_config_tools_realize__fmt2fmt_nml2nml(tmp_path): """ Test that providing a namelist base input file and a config file will create and update namelist config file. """ - help_realize_config_fmt2fmt("simple.nml", FORMAT.nml, "simple2.nml", FORMAT.nml, tmp_path) + help_realize_fmt2fmt("simple.nml", FORMAT.nml, "simple2.nml", FORMAT.nml, tmp_path) -def test_config_tools_realize_config__fmt2fmt_ini2ini(tmp_path): +def test_config_tools_realize__fmt2fmt_ini2ini(tmp_path): """ Test that providing an INI base input file and an INI config file will create and update INI config file. """ - help_realize_config_fmt2fmt("simple.ini", FORMAT.ini, "simple2.ini", FORMAT.ini, tmp_path) + help_realize_fmt2fmt("simple.ini", FORMAT.ini, "simple2.ini", FORMAT.ini, tmp_path) -def test_config_tools_realize_config__fmt2fmt_yaml2yaml(tmp_path): +def test_config_tools_realize__fmt2fmt_yaml2yaml(tmp_path): """ Test that providing a YAML base input file and a YAML config file will create and update YAML config file. """ - help_realize_config_fmt2fmt( + help_realize_fmt2fmt( "fruit_config.yaml", FORMAT.yaml, "fruit_config_similar.yaml", FORMAT.yaml, tmp_path ) -def test_config_tools_realize_config__incompatible_file_type(): +def test_config_tools_realize__incompatible_file_type(): """ Test that providing an incompatible file type for input base file will return print statement. """ with raises(UWError): - tools.realize_config( + tools.realize( input_config=fixture_path("model_configure.sample"), input_format="sample", output_format=FORMAT.yaml, ) -def test_config_tools_realize_config__output_file_format(tmp_path): +def test_config_tools_realize__output_file_format(tmp_path): """ Test that output_format overrides bad output_file extension. """ infile = fixture_path("simple.nml") outfile = tmp_path / "test_ouput.cfg" - tools.realize_config( + tools.realize( input_config=infile, output_file=outfile, output_format=FORMAT.nml, @@ -371,7 +498,7 @@ def test_config_tools_realize_config__output_file_format(tmp_path): assert compare_files(outfile, infile) -def test_config_tools_realize_config__remove_nml_to_nml(tmp_path): +def test_config_tools_realize__remove_nml_to_nml(tmp_path): input_config = NMLConfig({"constants": {"pi": 3.141, "e": 2.718}}) s = """ constants: @@ -381,7 +508,7 @@ def test_config_tools_realize_config__remove_nml_to_nml(tmp_path): update_config.write_text(dedent(s).strip()) output_file = tmp_path / "config.nml" assert not output_file.is_file() - tools.realize_config( + tools.realize( input_config=input_config, update_config=update_config, output_file=output_file, @@ -389,7 +516,7 @@ def test_config_tools_realize_config__remove_nml_to_nml(tmp_path): assert f90nml.read(output_file) == {"constants": {"pi": 3.141}} -def test_config_tools_realize_config__remove_yaml_to_yaml_scalar(tmp_path): +def test_config_tools_realize__remove_yaml_to_yaml_scalar(tmp_path): input_config = YAMLConfig({"a": {"b": {"c": 11, "d": 22, "e": 33}}}) s = """ a: @@ -398,14 +525,14 @@ def test_config_tools_realize_config__remove_yaml_to_yaml_scalar(tmp_path): """ update_config = tmp_path / "update.yaml" update_config.write_text(dedent(s).strip()) - assert tools.realize_config( + assert tools.realize( input_config=input_config, update_config=update_config, output_format=FORMAT.yaml, ) == {"a": {"b": {"c": 11, "e": 33}}} -def test_config_tools_realize_config__remove_yaml_to_yaml_subtree(tmp_path): +def test_config_tools_realize__remove_yaml_to_yaml_subtree(tmp_path): input_config = YAMLConfig(yaml.safe_load("a: {b: {c: 11, d: 22, e: 33}}")) s = """ a: @@ -413,16 +540,16 @@ def test_config_tools_realize_config__remove_yaml_to_yaml_subtree(tmp_path): """ update_config = tmp_path / "update.yaml" update_config.write_text(dedent(s).strip()) - assert tools.realize_config( + assert tools.realize( input_config=input_config, update_config=update_config, output_format=FORMAT.yaml, ) == {"a": {}} -def test_config_tools_realize_config__scalar_value(capsys): +def test_config_tools_realize__scalar_value(capsys): stdinproxy.cache_clear() - tools.realize_config( + tools.realize( input_config=YAMLConfig(config={"foo": {"bar": "baz"}}), output_format="yaml", key_path=["foo", "bar"], @@ -430,42 +557,42 @@ def test_config_tools_realize_config__scalar_value(capsys): assert capsys.readouterr().out.strip() == "baz" -def test_config_tools_realize_config__simple_ini(tmp_path): +def test_config_tools_realize__simple_ini(tmp_path): """ Test that providing an INI file with necessary settings will create an INI config file. """ - help_realize_config_simple("simple.ini", FORMAT.ini, tmp_path) + help_realize_simple("simple.ini", FORMAT.ini, tmp_path) -def test_config_tools_realize_config__simple_namelist(tmp_path): +def test_config_tools_realize__simple_namelist(tmp_path): """ Test that providing a namelist file with necessary settings will create a namelist config file. """ - help_realize_config_simple("simple.nml", FORMAT.nml, tmp_path) + help_realize_simple("simple.nml", FORMAT.nml, tmp_path) -def test_config_tools_realize_config__simple_sh(tmp_path): +def test_config_tools_realize__simple_sh(tmp_path): """ Test that providing an sh file with necessary settings will create an sh config file. """ - help_realize_config_simple("simple.sh", FORMAT.sh, tmp_path) + help_realize_simple("simple.sh", FORMAT.sh, tmp_path) -def test_config_tools_realize_config__simple_yaml(tmp_path): +def test_config_tools_realize__simple_yaml(tmp_path): """ Test that providing a YAML base file with necessary settings will create a YAML config file. """ - help_realize_config_simple("simple2.yaml", FORMAT.yaml, tmp_path) + help_realize_simple("simple2.yaml", FORMAT.yaml, tmp_path) -def test_config_tools_realize_config__single_dereference(capsys, tmp_path): +def test_config_tools_realize__single_dereference(capsys, tmp_path): input_config = tmp_path / "a.yaml" update_config = tmp_path / "b.yaml" with writable(input_config) as f: yaml.dump({"1": "a", "2": "{{ deref }}", "3": "{{ temporalis }}"}, f) with writable(update_config) as f: yaml.dump({"2": "b", "temporalis": "c", "deref": "d"}, f) - tools.realize_config( + tools.realize( input_config=input_config, update_config=update_config, output_format=FORMAT.yaml, @@ -481,22 +608,22 @@ def test_config_tools_realize_config__single_dereference(capsys, tmp_path): assert actual == dedent(expected).strip() -def test_config_tools_realize_config__total_fail(): +def test_config_tools_realize__total_fail(): with raises(UWConfigError) as e: - tools.realize_config( + tools.realize( input_config=YAMLConfig({"foo": "{{ bar }}"}), output_format=FORMAT.yaml, total=True ) assert str(e.value) == "Config could not be totally realized" -def test_config_tools_realize_config__update_bad_format_defaults_to_yaml(capsys, tmp_path): +def test_config_tools_realize__update_bad_format_defaults_to_yaml(capsys, tmp_path): input_config = tmp_path / "a.yaml" update_config = tmp_path / "b.clj" with writable(input_config) as f: yaml.dump({"1": "a", "2": "{{ deref }}", "3": "{{ temporalis }}", "deref": "b"}, f) with writable(update_config) as f: yaml.dump({"2": "b", "temporalis": "c"}, f) - tools.realize_config( + tools.realize( input_config=input_config, update_config=update_config, output_format=FORMAT.yaml, @@ -511,11 +638,11 @@ def test_config_tools_realize_config__update_bad_format_defaults_to_yaml(capsys, assert capsys.readouterr().out.strip() == dedent(expected).strip() -def test_config_tools_realize_config__update_none(capsys, tmp_path): +def test_config_tools_realize__update_none(capsys, tmp_path): path = tmp_path / "a.yaml" with writable(path) as f: yaml.dump({"1": "a", "2": "{{ deref }}", "3": "{{ temporalis }}", "deref": "b"}, f) - tools.realize_config( + tools.realize( input_config=path, input_format=FORMAT.yaml, output_format=FORMAT.yaml, @@ -530,12 +657,12 @@ def test_config_tools_realize_config__update_none(capsys, tmp_path): assert actual == dedent(expected).strip() -def test_config_tools_realize_config__values_needed_ini(logged): +def test_config_tools_realize__values_needed_ini(logged): """ Test that the values_needed flag logs keys completed and keys containing unrendered Jinja2 variables/expressions. """ - tools.realize_config( + tools.realize( input_config=fixture_path("simple3.ini"), input_format=FORMAT.ini, output_format=FORMAT.ini, @@ -562,12 +689,12 @@ def test_config_tools_realize_config__values_needed_ini(logged): assert logged(dedent(expected), multiline=True) -def test_config_tools_realize_config__values_needed_yaml(logged): +def test_config_tools_realize__values_needed_yaml(logged): """ Test that the values_needed flag logs keys completed and keys containing unrendered Jinja2 variables/expressions. """ - tools.realize_config( + tools.realize( input_config=fixture_path("srw_example.yaml"), input_format=FORMAT.yaml, output_format=FORMAT.yaml, @@ -673,25 +800,25 @@ def test_config_tools__ensure_format__explicitly_specified_with_path(): ) -def test_config_tools__realize_config_input_setup__ini_cfgobj(): +def test_config_tools__realize_input_setup__ini_cfgobj(): data = {"section": {"foo": "bar"}} cfgobj = INIConfig(config=data) - input_obj = tools._realize_config_input_setup(input_config=cfgobj) + input_obj = tools._realize_input_setup(input_config=cfgobj) assert input_obj.data == data -def test_config_tools__realize_config_input_setup__ini_file(tmp_path): +def test_config_tools__realize_input_setup__ini_file(tmp_path): data = """ [section] foo = bar """ path = tmp_path / "config.ini" path.write_text(dedent(data).strip()) - input_obj = tools._realize_config_input_setup(input_config=path) + input_obj = tools._realize_input_setup(input_config=path) assert input_obj.data == {"section": {"foo": "bar"}} -def test_config_tools__realize_config_input_setup__ini_stdin(logged): +def test_config_tools__realize_input_setup__ini_stdin(logged): data = """ [section] foo = bar @@ -702,19 +829,19 @@ def test_config_tools__realize_config_input_setup__ini_stdin(logged): print(dedent(data).strip(), file=sio) sio.seek(0) with patch.object(sys, "stdin", new=sio): - input_obj = tools._realize_config_input_setup(input_format=FORMAT.ini) + input_obj = tools._realize_input_setup(input_format=FORMAT.ini) assert input_obj.data == {"section": {"foo": "bar", "baz": "42"}} # note: 42 is str, not int assert logged("Reading input from stdin") -def test_config_tools__realize_config_input_setup__nml_cfgobj(): +def test_config_tools__realize_input_setup__nml_cfgobj(): data = {"nl": {"pi": 3.14}} cfgobj = NMLConfig(config=data) - input_obj = tools._realize_config_input_setup(input_config=cfgobj) + input_obj = tools._realize_input_setup(input_config=cfgobj) assert input_obj.data == data -def test_config_tools__realize_config_input_setup__nml_file(tmp_path): +def test_config_tools__realize_input_setup__nml_file(tmp_path): data = """ &nl pi = 3.14 @@ -722,11 +849,11 @@ def test_config_tools__realize_config_input_setup__nml_file(tmp_path): """ path = tmp_path / "config.nml" path.write_text(dedent(data).strip()) - input_obj = tools._realize_config_input_setup(input_config=path) + input_obj = tools._realize_input_setup(input_config=path) assert input_obj["nl"]["pi"] == 3.14 -def test_config_tools__realize_config_input_setup__nml_stdin(logged): +def test_config_tools__realize_input_setup__nml_stdin(logged): data = """ &nl pi = 3.14 @@ -737,29 +864,29 @@ def test_config_tools__realize_config_input_setup__nml_stdin(logged): print(dedent(data).strip(), file=sio) sio.seek(0) with patch.object(sys, "stdin", new=sio): - input_obj = tools._realize_config_input_setup(input_format=FORMAT.nml) + input_obj = tools._realize_input_setup(input_format=FORMAT.nml) assert input_obj["nl"]["pi"] == 3.14 assert logged("Reading input from stdin") -def test_config_tools__realize_config_input_setup__sh_cfgobj(): +def test_config_tools__realize_input_setup__sh_cfgobj(): data = {"foo": "bar"} cfgobj = SHConfig(config=data) - input_obj = tools._realize_config_input_setup(input_config=cfgobj) + input_obj = tools._realize_input_setup(input_config=cfgobj) assert input_obj.data == data -def test_config_tools__realize_config_input_setup__sh_file(tmp_path): +def test_config_tools__realize_input_setup__sh_file(tmp_path): data = """ foo=bar """ path = tmp_path / "config.sh" path.write_text(dedent(data).strip()) - input_obj = tools._realize_config_input_setup(input_config=path) + input_obj = tools._realize_input_setup(input_config=path) assert input_obj.data == {"foo": "bar"} -def test_config_tools__realize_config_input_setup__sh_stdin(logged): +def test_config_tools__realize_input_setup__sh_stdin(logged): data = """ foo=bar """ @@ -768,29 +895,29 @@ def test_config_tools__realize_config_input_setup__sh_stdin(logged): print(dedent(data).strip(), file=sio) sio.seek(0) with patch.object(sys, "stdin", new=sio): - input_obj = tools._realize_config_input_setup(input_format=FORMAT.sh) + input_obj = tools._realize_input_setup(input_format=FORMAT.sh) assert input_obj.data == {"foo": "bar"} assert logged("Reading input from stdin") -def test_config_tools__realize_config_input_setup__yaml_cfgobj(): +def test_config_tools__realize_input_setup__yaml_cfgobj(): data = {"foo": "bar"} cfgobj = YAMLConfig(config=data) - input_obj = tools._realize_config_input_setup(input_config=cfgobj) + input_obj = tools._realize_input_setup(input_config=cfgobj) assert input_obj.data == data -def test_config_tools__realize_config_input_setup__yaml_file(tmp_path): +def test_config_tools__realize_input_setup__yaml_file(tmp_path): data = """ foo: bar """ path = tmp_path / "config.yaml" path.write_text(dedent(data).strip()) - input_obj = tools._realize_config_input_setup(input_config=path) + input_obj = tools._realize_input_setup(input_config=path) assert input_obj.data == {"foo": "bar"} -def test_config_tools__realize_config_input_setup__yaml_stdin(logged): +def test_config_tools__realize_input_setup__yaml_stdin(logged): data = """ foo: bar """ @@ -799,70 +926,68 @@ def test_config_tools__realize_config_input_setup__yaml_stdin(logged): print(dedent(data).strip(), file=sio) sio.seek(0) with patch.object(sys, "stdin", new=sio): - input_obj = tools._realize_config_input_setup(input_format=FORMAT.yaml) + input_obj = tools._realize_input_setup(input_format=FORMAT.yaml) assert input_obj.data == {"foo": "bar"} assert logged("Reading input from stdin") -def test_config_tools__realize_config_output_setup(logged, tmp_path): +def test_config_tools__realize_output_setup(logged, tmp_path): input_obj = YAMLConfig({"a": {"b": {"foo": "bar"}}}) output_file = tmp_path / "output.yaml" - assert tools._realize_config_output_setup( + assert tools._realize_output_setup( input_obj=input_obj, output_file=output_file, key_path=["a", "b"] ) == ({"foo": "bar"}, FORMAT.yaml) assert logged(f"Writing output to {output_file}") -def test_config_tools__realize_config_update__cfgobj(realize_config_testobj): - assert realize_config_testobj[1][2][3] == 42 +def test_config_tools__realize_update__cfgobj(realize_testobj): + assert realize_testobj[1][2][3] == 42 update_config = YAMLConfig(config={1: {2: {3: 43}}}) - o = tools._realize_config_update(input_obj=realize_config_testobj, update_config=update_config) + o = tools._realize_update(input_obj=realize_testobj, update_config=update_config) assert o[1][2][3] == 43 -def test_config_tools__realize_config_update__stdin(logged, realize_config_testobj): +def test_config_tools__realize_update__stdin(logged, realize_testobj): stdinproxy.cache_clear() - assert realize_config_testobj[1][2][3] == 42 + assert realize_testobj[1][2][3] == 42 with StringIO() as sio: print("{1: {2: {3: 43}}}", file=sio) sio.seek(0) with patch.object(sys, "stdin", new=sio): - o = tools._realize_config_update( - input_obj=realize_config_testobj, update_format=FORMAT.yaml - ) + o = tools._realize_update(input_obj=realize_testobj, update_format=FORMAT.yaml) assert o[1][2][3] == 43 assert logged("Reading update from stdin") -def test_config_tools__realize_config_update__noop(realize_config_testobj): - assert tools._realize_config_update(input_obj=realize_config_testobj) == realize_config_testobj +def test_config_tools__realize_update__noop(realize_testobj): + assert tools._realize_update(input_obj=realize_testobj) == realize_testobj -def test_config_tools__realize_config_update__file(realize_config_testobj, tmp_path): - assert realize_config_testobj[1][2][3] == 42 +def test_config_tools__realize_update__file(realize_testobj, tmp_path): + assert realize_testobj[1][2][3] == 42 values = {1: {2: {3: 43}}} update_config = tmp_path / "config.yaml" update_config.write_text(yaml.dump(values)) - o = tools._realize_config_update(input_obj=realize_config_testobj, update_config=update_config) + o = tools._realize_update(input_obj=realize_testobj, update_config=update_config) assert o[1][2][3] == 43 -def test_config_tools__realize_config_values_needed(logged, tmp_path): +def test_config_tools__realize_values_needed(logged, tmp_path): path = tmp_path / "a.yaml" with writable(path) as f: yaml.dump({1: "complete", 2: "{{ jinja2 }}", 3: ""}, f) c = YAMLConfig(config=path) - tools._realize_config_values_needed(input_obj=c) + tools._realize_values_needed(input_obj=c) assert logged("Keys that are complete:\n 1", multiline=True) assert logged("Keys with unrendered Jinja2 variables/expressions:\n 2", multiline=True) -def test_config_tools__realize_config_values_needed__negative_results(logged, tmp_path): +def test_config_tools__realize_values_needed__negative_results(logged, tmp_path): path = tmp_path / "a.yaml" with writable(path) as f: yaml.dump({}, f) c = YAMLConfig(config=path) - tools._realize_config_values_needed(input_obj=c) + tools._realize_values_needed(input_obj=c) assert logged("No keys are complete.") assert logged("No keys have unrendered Jinja2 variables/expressions.") diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index e310f2d71..8b11a8ff8 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -80,7 +80,12 @@ def test_cli__abort(capsys): def test_cli__add_subparser_config(subparsers): cli._add_subparser_config(subparsers) - assert actions(subparsers.choices[STR.config]) == [STR.compare, STR.realize, STR.validate] + assert actions(subparsers.choices[STR.config]) == [ + STR.compare, + STR.compose, + STR.realize, + STR.validate, + ] def test_cli__add_subparser_config_compare(subparsers): @@ -88,6 +93,11 @@ def test_cli__add_subparser_config_compare(subparsers): assert subparsers.choices[STR.compare] +def test_cli__add_subparser_config_compose(subparsers): + cli._add_subparser_config_compose(subparsers) + assert subparsers.choices[STR.compose] + + def test_cli__add_subparser_config_realize(subparsers): cli._add_subparser_config_realize(subparsers) assert subparsers.choices[STR.realize] @@ -295,6 +305,7 @@ def test_cli__dict_from_key_eq_val_strings(): "params", [ (STR.compare, "_dispatch_config_compare"), + (STR.compose, "_dispatch_config_compose"), (STR.realize, "_dispatch_config_realize"), (STR.validate, "_dispatch_config_validate"), ], @@ -319,6 +330,27 @@ def test_cli__dispatch_config_compare(): ) +def test_cli__dispatch_config_compose(): + configs = ["/path/to/a", "/path/to/b"] + outfile = "/path/to/output" + args = { + STR.configs: configs, + STR.realize: True, + STR.outfile: outfile, + STR.infmt: FORMAT.yaml, + STR.outfmt: FORMAT.yaml, + } + with patch.object(cli.uwtools.api.config, "compose") as compose: + cli._dispatch_config_compose(args) + compose.assert_called_once_with( + configs=configs, + realize=True, + output_file=outfile, + input_format=FORMAT.yaml, + output_format=FORMAT.yaml, + ) + + def test_cli__dispatch_config_realize(args_config_realize): with patch.object(cli.uwtools.api.config, "realize") as realize: cli._dispatch_config_realize(args_config_realize)