diff --git a/docs/sections/user_guide/cli/tools/config/compare-bad-extension.out b/docs/sections/user_guide/cli/tools/config/compare-bad-extension.out index de658e238..b1e1ad3f3 100644 --- a/docs/sections/user_guide/cli/tools/config/compare-bad-extension.out +++ b/docs/sections/user_guide/cli/tools/config/compare-bad-extension.out @@ -1 +1 @@ -Cannot deduce format of 'a.txt' from unknown extension 'txt' +[2025-01-02T03:04:05] ERROR Formats do not match: yaml vs nml diff --git a/docs/sections/user_guide/cli/tools/config/realize-extension-file-bad.out b/docs/sections/user_guide/cli/tools/config/realize-extension-file-bad.out index 32b1809c0..788934c89 100644 --- a/docs/sections/user_guide/cli/tools/config/realize-extension-file-bad.out +++ b/docs/sections/user_guide/cli/tools/config/realize-extension-file-bad.out @@ -1 +1,7 @@ -Cannot deduce format of 'config.txt' from unknown extension 'txt' +values: + date: 20240105 + empty: null + greeting: Good Night + message: Good Night Moon Good Night Moon + recipient: Moon + repeat: 2 diff --git a/docs/sections/user_guide/cli/tools/config/realize-extension-stdin-bad.out b/docs/sections/user_guide/cli/tools/config/realize-extension-stdin-bad.out index dc6d2b7cc..788934c89 100644 --- a/docs/sections/user_guide/cli/tools/config/realize-extension-stdin-bad.out +++ b/docs/sections/user_guide/cli/tools/config/realize-extension-stdin-bad.out @@ -1 +1,7 @@ -Specify --input-format when --input-file is not specified +values: + date: 20240105 + empty: null + greeting: Good Night + message: Good Night Moon Good Night Moon + recipient: Moon + repeat: 2 diff --git a/docs/sections/user_guide/cli/tools/template/render-exec-bad-extension.out b/docs/sections/user_guide/cli/tools/template/render-exec-bad-extension.out index 90d48f3ec..8ab686eaf 100644 --- a/docs/sections/user_guide/cli/tools/template/render-exec-bad-extension.out +++ b/docs/sections/user_guide/cli/tools/template/render-exec-bad-extension.out @@ -1 +1 @@ -Cannot deduce format of 'values.txt' from unknown extension 'txt' +Hello, World! diff --git a/notebooks/config.ipynb b/notebooks/config.ipynb index 0b81945f1..7a701a215 100644 --- a/notebooks/config.ipynb +++ b/notebooks/config.ipynb @@ -34,7 +34,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 48, "id": "6d02d033-0992-4990-861d-3f80d09d7083", "metadata": {}, "outputs": [], @@ -59,7 +59,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 49, "id": "ab0e21c3-a4b6-404c-bffd-e0d393d9b0a2", "metadata": {}, "outputs": [ @@ -96,7 +96,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 50, "id": "c6e049df-38f6-4879-8e0d-68356226d94b", "metadata": {}, "outputs": [ @@ -125,7 +125,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 51, "id": "cc3020a6-4eb4-4830-9263-a9fc8fac7450", "metadata": {}, "outputs": [ @@ -158,7 +158,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 52, "id": "f01ac223-4a02-40ba-822f-8e66ad39f313", "metadata": {}, "outputs": [ @@ -193,7 +193,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 53, "id": "28d23ac5-52a0-45bc-bfee-98d9ea518ca2", "metadata": {}, "outputs": [ @@ -204,7 +204,7 @@ "recipient=Mars" ] }, - "execution_count": 6, + "execution_count": 53, "metadata": {}, "output_type": "execute_result" } @@ -226,7 +226,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 54, "id": "b3a0a5bc-9d1b-4d48-a05f-be6f94fb6e1d", "metadata": {}, "outputs": [ @@ -258,7 +258,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 55, "id": "6837e75b-bd20-4c3b-bd33-650e4b4f9f23", "metadata": {}, "outputs": [ @@ -270,7 +270,7 @@ "recipient = Mars" ] }, - "execution_count": 8, + "execution_count": 55, "metadata": {}, "output_type": "execute_result" } @@ -292,7 +292,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 56, "id": "9655b36b-2d39-4fc1-b3b8-9cb3443cf4b8", "metadata": {}, "outputs": [ @@ -326,7 +326,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 57, "id": "2c3f1b75-b26f-4893-beb7-37a58c09f511", "metadata": {}, "outputs": [ @@ -362,11 +362,11 @@ " Recognized file extensions are: ini, nml, sh, yaml\n", " \n", " :param input_config: Input config file (``None`` => read ``stdin``).\n", - " :param input_format: Format of the input config (optional if file's extension is recognized).\n", + " :param input_format: Input config format (default: deduced from filename extension; ``yaml`` if that fails).\n", " :param update_config: Update config file (``None`` => read ``stdin``).\n", - " :param update_format: Format of the update config (optional if file's extension is recognized).\n", + " :param update_format: Update config format (default: deduced from filename extension; ``yaml`` if that fails).\n", " :param output_file: Output config file (``None`` => write to ``stdout``).\n", - " :param output_format: Format of the output config (optional if file's extension is recognized).\n", + " :param output_format: Output config format (default: deduced from filename extension; ``yaml`` if that fails).\n", " :param key_path: Path of keys to the desired output block.\n", " :param values_needed: Report complete, missing, and template values.\n", " :param total: Require rendering of all Jinja2 variables/expressions.\n", @@ -393,7 +393,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 58, "id": "77873e14-db3c-417d-be7a-2ba12c9a38f6", "metadata": {}, "outputs": [ @@ -403,7 +403,7 @@ "{'greeting': 'Hello', 'recipient': 'World'}" ] }, - "execution_count": 11, + "execution_count": 58, "metadata": {}, "output_type": "execute_result" } @@ -426,7 +426,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 59, "id": "4f237a73-da83-4632-990f-644632b15cd9", "metadata": {}, "outputs": [ @@ -455,7 +455,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 60, "id": "b8854dc6-9dd2-4843-99e4-278b116b9767", "metadata": {}, "outputs": [ @@ -465,7 +465,7 @@ "{'greeting': 'Hello', 'recipient': 'World'}" ] }, - "execution_count": 13, + "execution_count": 60, "metadata": {}, "output_type": "execute_result" } @@ -490,7 +490,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 61, "id": "0a59a3e8-27b5-4daa-a924-941aceaad157", "metadata": {}, "outputs": [ @@ -521,7 +521,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 62, "id": "2d72bbc1-e438-48b2-8bd7-554b598c6f24", "metadata": {}, "outputs": [ @@ -553,7 +553,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 63, "id": "7648fdd5-5752-4bf3-b366-db8da1eac601", "metadata": {}, "outputs": [ @@ -567,7 +567,7 @@ " 'recipient': 'Mars'}}" ] }, - "execution_count": 16, + "execution_count": 63, "metadata": {}, "output_type": "execute_result" } @@ -591,7 +591,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 64, "id": "f4538965-d3b0-4c0c-a878-b6852f8d8ab0", "metadata": {}, "outputs": [ @@ -627,7 +627,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 65, "id": "7ce0e917-f0d0-4302-9c8c-b136ffc5410a", "metadata": {}, "outputs": [ @@ -658,7 +658,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 66, "id": "00150efa-848c-44eb-ac0c-dab3845546b8", "metadata": {}, "outputs": [ @@ -693,7 +693,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 67, "id": "928a23d7-8ba9-4217-935d-01563bb36cb6", "metadata": {}, "outputs": [ @@ -725,7 +725,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 68, "id": "d22c692d-e98e-4f88-bdac-a369f0a1962f", "metadata": {}, "outputs": [ @@ -733,13 +733,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:28:47] INFO Keys that are complete:\n", - "[2025-05-17T15:28:47] INFO memo\n", - "[2025-05-17T15:28:47] INFO memo.sent\n", - "[2025-05-17T15:28:47] INFO \n", - "[2025-05-17T15:28:47] INFO Keys with unrendered Jinja2 variables/expressions:\n", - "[2025-05-17T15:28:47] INFO memo.sender_id: {{ id }}\n", - "[2025-05-17T15:28:47] INFO memo.message: {{ greeting }}, {{ recipient }}!\n" + "[2025-09-10T23:38:14] INFO Keys that are complete:\n", + "[2025-09-10T23:38:14] INFO memo\n", + "[2025-09-10T23:38:14] INFO memo.sent\n", + "[2025-09-10T23:38:14] INFO \n", + "[2025-09-10T23:38:14] INFO Keys with unrendered Jinja2 variables/expressions:\n", + "[2025-09-10T23:38:14] INFO memo.sender_id: {{ id }}\n", + "[2025-09-10T23:38:14] INFO memo.message: {{ greeting }}, {{ recipient }}!\n" ] }, { @@ -748,7 +748,7 @@ "{}" ] }, - "execution_count": 21, + "execution_count": 68, "metadata": {}, "output_type": "execute_result" } @@ -775,7 +775,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 69, "id": "647538d9-cd22-4f94-b15b-c34d68a324da", "metadata": {}, "outputs": [ @@ -807,7 +807,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 70, "id": "ba360102-f558-4dba-b0b6-c6f550c7d40f", "metadata": {}, "outputs": [ @@ -842,7 +842,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 71, "id": "e891a446-f699-460d-b4a8-568d9d4cf631", "metadata": {}, "outputs": [ @@ -857,7 +857,7 @@ " 'id': 321}}" ] }, - "execution_count": 24, + "execution_count": 71, "metadata": {}, "output_type": "execute_result" } @@ -882,7 +882,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 72, "id": "aeab0ec5-7e6e-4309-b484-4de5dd9324b5", "metadata": {}, "outputs": [ @@ -919,7 +919,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 73, "id": "bdc98ce1-e213-41ac-b1f2-03bd52238e30", "metadata": {}, "outputs": [ @@ -948,7 +948,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 74, "id": "ae7f648e-6586-4700-87e7-492ca3a02a06", "metadata": {}, "outputs": [ @@ -958,7 +958,7 @@ "{'id': '456', 'greeting': 'Hello', 'recipient': 'World'}" ] }, - "execution_count": 27, + "execution_count": 74, "metadata": {}, "output_type": "execute_result" } @@ -985,7 +985,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 75, "id": "249ae7a9-1ad3-4401-98b0-bc3c433f22f4", "metadata": {}, "outputs": [ @@ -1024,7 +1024,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 76, "id": "0218e5de-2c25-4d7a-a7b6-0f05ad81afb2", "metadata": {}, "outputs": [ @@ -1064,7 +1064,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 77, "id": "988b72bc-7983-4d24-9d37-c9ba32a31ac4", "metadata": {}, "outputs": [ @@ -1072,16 +1072,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:28:47] INFO - fixtures/config/base-config.nml\n", - "[2025-05-17T15:28:47] INFO + fixtures/config/alt-config.nml\n", - "[2025-05-17T15:28:47] INFO ---------------------------------------------------------------------\n", - "[2025-05-17T15:28:47] INFO ↓ ? = info | -/+ = line unique to - or + file | blank = matching line\n", - "[2025-05-17T15:28:47] INFO ---------------------------------------------------------------------\n", - "[2025-05-17T15:28:47] INFO memo:\n", - "[2025-05-17T15:28:47] INFO message: '{{ greeting }}, {{ recipient }}!'\n", - "[2025-05-17T15:28:47] INFO sender_id: '{{ id }}'\n", - "[2025-05-17T15:28:47] INFO - sent: false\n", - "[2025-05-17T15:28:47] INFO + sent: true\n" + "[2025-09-10T23:38:14] INFO - fixtures/config/base-config.nml\n", + "[2025-09-10T23:38:14] INFO + fixtures/config/alt-config.nml\n", + "[2025-09-10T23:38:14] INFO ---------------------------------------------------------------------\n", + "[2025-09-10T23:38:14] INFO ↓ ? = info | -/+ = line unique to - or + file | blank = matching line\n", + "[2025-09-10T23:38:14] INFO ---------------------------------------------------------------------\n", + "[2025-09-10T23:38:14] INFO memo:\n", + "[2025-09-10T23:38:14] INFO message: '{{ greeting }}, {{ recipient }}!'\n", + "[2025-09-10T23:38:14] INFO sender_id: '{{ id }}'\n", + "[2025-09-10T23:38:14] INFO - sent: false\n", + "[2025-09-10T23:38:14] INFO + sent: true\n" ] }, { @@ -1090,7 +1090,7 @@ "False" ] }, - "execution_count": 30, + "execution_count": 77, "metadata": {}, "output_type": "execute_result" } @@ -1115,7 +1115,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 78, "id": "ceb7f893-398b-41e4-818a-49cd036c2bfe", "metadata": {}, "outputs": [], @@ -1135,7 +1135,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 79, "id": "8b495735-2396-436e-87db-d886ac6769fa", "metadata": {}, "outputs": [ @@ -1143,8 +1143,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:28:47] INFO - fixtures/config/base-config.nml\n", - "[2025-05-17T15:28:47] INFO + tmp/config-copy.nml\n" + "[2025-09-10T23:38:14] INFO - fixtures/config/base-config.nml\n", + "[2025-09-10T23:38:14] INFO + tmp/config-copy.nml\n" ] }, { @@ -1153,7 +1153,7 @@ "True" ] }, - "execution_count": 32, + "execution_count": 79, "metadata": {}, "output_type": "execute_result" } @@ -1176,7 +1176,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 80, "id": "438ccd5b-30d5-49b4-ba12-6e6e89f46d28", "metadata": {}, "outputs": [ @@ -1184,7 +1184,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:28:47] ERROR Formats do not match: yaml vs nml\n" + "[2025-09-10T23:38:14] ERROR Formats do not match: yaml vs nml\n" ] }, { @@ -1193,7 +1193,7 @@ "False" ] }, - "execution_count": 33, + "execution_count": 80, "metadata": {}, "output_type": "execute_result" } @@ -1218,7 +1218,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 81, "id": "2eb5b2ce-bebd-449d-90a5-764484aa03aa", "metadata": {}, "outputs": [ @@ -1259,7 +1259,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 82, "id": "dc7d2b80-ea92-4301-ae85-4edbf61bf510", "metadata": {}, "outputs": [ @@ -1288,7 +1288,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 83, "id": "f9c66a24-2a7e-43be-839f-3ad5b136b646", "metadata": {}, "outputs": [ @@ -1330,7 +1330,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 84, "id": "f3873050-d857-490d-aeeb-f9217a7f808c", "metadata": {}, "outputs": [ @@ -1338,7 +1338,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:28:47] INFO Schema validation succeeded for config\n" + "[2025-09-10T23:38:14] INFO Schema validation succeeded for config\n" ] }, { @@ -1347,7 +1347,7 @@ "True" ] }, - "execution_count": 37, + "execution_count": 84, "metadata": {}, "output_type": "execute_result" } @@ -1370,7 +1370,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 85, "id": "b5664e58-8ddc-438c-8180-1e2911838744", "metadata": {}, "outputs": [ @@ -1378,9 +1378,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:28:47] ERROR 1 schema-validation error found in config\n", - "[2025-05-17T15:28:47] ERROR Error at recipient:\n", - "[2025-05-17T15:28:47] ERROR 47 is not of type 'string'\n" + "[2025-09-10T23:38:14] ERROR 1 schema-validation error found in config\n", + "[2025-09-10T23:38:14] ERROR Error at recipient:\n", + "[2025-09-10T23:38:14] ERROR 47 is not of type 'string'\n" ] }, { @@ -1389,7 +1389,7 @@ "False" ] }, - "execution_count": 38, + "execution_count": 85, "metadata": {}, "output_type": "execute_result" } @@ -1414,7 +1414,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 86, "id": "6f4df804-cda2-48c4-bf42-a5b57de5e066", "metadata": {}, "outputs": [ @@ -1544,7 +1544,7 @@ " | clear(self)\n", " | D.clear() -> None. Remove all items from D.\n", " | \n", - " | pop(self, key, default=)\n", + " | pop(self, key, default=)\n", " | D.pop(k[,d]) -> v, remove specified key and return the corresponding value.\n", " | If key is not found, d is returned if given, otherwise KeyError is raised.\n", " | \n", @@ -1603,7 +1603,7 @@ " | __class_getitem__ = GenericAlias(...) from abc.ABCMeta\n", " | Represent a PEP 585 generic type\n", " | \n", - " | E.g. for t = list[int], t.__origin__ is list and t.__args__ is (int,).\n", + " | E.g. for t = list[int], t.origin is list and t.args is (int,).\n", "\n" ] } @@ -1623,7 +1623,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 87, "id": "0e8d6c8a-cc3c-49d7-9b97-32187cd5f754", "metadata": {}, "outputs": [ @@ -1658,7 +1658,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 88, "id": "73925576-6b07-4af6-a9b0-7bd6f4235987", "metadata": {}, "outputs": [ @@ -1668,7 +1668,7 @@ "True" ] }, - "execution_count": 41, + "execution_count": 88, "metadata": {}, "output_type": "execute_result" } @@ -1691,7 +1691,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 89, "id": "9d94a325-93f6-43b4-bc56-0ce5a1830647", "metadata": {}, "outputs": [ @@ -1699,17 +1699,17 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:28:47] INFO ---------------------------------------------------------------------\n", - "[2025-05-17T15:28:47] INFO ↓ ? = info | -/+ = line unique to - or + file | blank = matching line\n", - "[2025-05-17T15:28:47] INFO ---------------------------------------------------------------------\n", - "[2025-05-17T15:28:47] INFO fruit count:\n", - "[2025-05-17T15:28:47] INFO apples: '3'\n", - "[2025-05-17T15:28:47] INFO - grapes: '{{ grape_count }}'\n", - "[2025-05-17T15:28:47] INFO + grapes: '8'\n", - "[2025-05-17T15:28:47] INFO - kiwis: '2'\n", - "[2025-05-17T15:28:47] INFO ? ^\n", - "[2025-05-17T15:28:47] INFO + kiwis: '1'\n", - "[2025-05-17T15:28:47] INFO ? ^\n" + "[2025-09-10T23:38:15] INFO ---------------------------------------------------------------------\n", + "[2025-09-10T23:38:15] INFO ↓ ? = info | -/+ = line unique to - or + file | blank = matching line\n", + "[2025-09-10T23:38:15] INFO ---------------------------------------------------------------------\n", + "[2025-09-10T23:38:15] INFO fruit count:\n", + "[2025-09-10T23:38:15] INFO apples: '3'\n", + "[2025-09-10T23:38:15] INFO - grapes: '{{ grape_count }}'\n", + "[2025-09-10T23:38:15] INFO + grapes: '8'\n", + "[2025-09-10T23:38:15] INFO - kiwis: '2'\n", + "[2025-09-10T23:38:15] INFO ? ^\n", + "[2025-09-10T23:38:15] INFO + kiwis: '1'\n", + "[2025-09-10T23:38:15] INFO ? ^\n" ] }, { @@ -1718,7 +1718,7 @@ "False" ] }, - "execution_count": 42, + "execution_count": 89, "metadata": {}, "output_type": "execute_result" } @@ -1742,7 +1742,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 90, "id": "65315885-28f7-4e34-b5e9-f07b51b85e42", "metadata": {}, "outputs": [ @@ -1777,7 +1777,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 91, "id": "7cd35285-9eb9-4796-b1d9-cf84160c0c0e", "metadata": {}, "outputs": [], @@ -1798,7 +1798,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 92, "id": "f2ce9e59-f03c-4203-b929-87b4c0eae9de", "metadata": {}, "outputs": [ @@ -1829,7 +1829,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 93, "id": "b896632e-6a5a-4756-a32b-0630bc02d504", "metadata": {}, "outputs": [ @@ -1864,7 +1864,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 94, "id": "7a3f8247-75f6-4cc6-9a9d-7e69896120e7", "metadata": {}, "outputs": [ @@ -1904,7 +1904,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.9.0" } }, "nbformat": 4, diff --git a/notebooks/exp-config-cb.ipynb b/notebooks/exp-config-cb.ipynb index 94bf01f3f..8855e5730 100644 --- a/notebooks/exp-config-cb.ipynb +++ b/notebooks/exp-config-cb.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 9, "id": "3a872d35-99f1-434c-927e-5c8fee3f0f2d", "metadata": {}, "outputs": [], @@ -39,7 +39,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 10, "id": "8b42bfca-18ae-48b1-a7ad-a1b76b9e24a8", "metadata": {}, "outputs": [ @@ -93,7 +93,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 11, "id": "e7fbe2c2-90af-446c-b398-621d91c763c9", "metadata": {}, "outputs": [ @@ -126,7 +126,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 12, "id": "d3bbf762-b4fc-49d8-90e4-e7851c9da49a", "metadata": {}, "outputs": [ @@ -161,7 +161,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 13, "id": "4bda78dc-33ee-4a23-82a8-271b40abca7b", "metadata": {}, "outputs": [ @@ -215,7 +215,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 14, "id": "e863d5ac-f727-4d91-a4bd-9bf813d35e6c", "metadata": {}, "outputs": [ @@ -282,7 +282,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 15, "id": "0fc07baf-1094-4d8c-a51e-c4e541ae4df6", "metadata": {}, "outputs": [ @@ -342,7 +342,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 16, "id": "f9f0c0df-821e-492a-9669-3ac5e43e3151", "metadata": {}, "outputs": [ @@ -350,20 +350,20 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:07] INFO Validating config against internal schema: chgres-cube\n", - "[2025-05-17T15:29:07] INFO Schema validation succeeded for chgres_cube config\n", - "[2025-05-17T15:29:07] INFO Validating config against internal schema: platform\n", - "[2025-05-17T15:29:07] INFO Schema validation succeeded for platform config\n", - "[2025-05-17T15:29:07] INFO 20250517 21:29:06 chgres_cube valid schema: Ready\n" + "[2025-09-10T23:38:38] INFO Validating config against internal schema: chgres-cube\n", + "[2025-09-10T23:38:38] INFO Schema validation succeeded for chgres_cube config\n", + "[2025-09-10T23:38:38] INFO Validating config against internal schema: platform\n", + "[2025-09-10T23:38:38] INFO Schema validation succeeded for platform config\n", + "[2025-09-10T23:38:38] INFO 20250911 05:38:38 chgres_cube valid schema: Ready\n" ] }, { "data": { "text/plain": [ - "20250517 21:29:06 chgres_cube valid schema <281472538537552>" + "20250911 05:38:38 chgres_cube valid schema <281472574893360>" ] }, - "execution_count": 8, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -395,7 +395,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.9.0" } }, "nbformat": 4, diff --git a/notebooks/fs.ipynb b/notebooks/fs.ipynb index f29e7501a..8de048ac3 100644 --- a/notebooks/fs.ipynb +++ b/notebooks/fs.ipynb @@ -31,7 +31,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 13, "id": "77864d80-e6f4-48c2-a5d5-88fc512106a9", "metadata": {}, "outputs": [], @@ -57,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 14, "id": "834654da-dfa9-4997-bcc5-846420381b18", "metadata": {}, "outputs": [ @@ -88,7 +88,7 @@ }, { "cell_type": "markdown", - "id": "0585971b-47c6-48aa-9f1f-d5890cbb2061", + "id": "a939b7ac-8064-4469-a29e-819afb5128f6", "metadata": {}, "source": [ "Files to be copied are specified by a mapping from keys destination-pathname keys to source-pathname values, either in a YAML file or a a Python ``dict``." @@ -96,7 +96,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 15, "id": "a959522f-d769-48c6-918d-d42776b3600a", "metadata": {}, "outputs": [ @@ -108,50 +108,6 @@ "data/file2-copy.txt: fixtures/fs/data/file2.txt\n", "data/file3-copy.csv: fixtures/fs/data/file3.csv\n" ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[2025-05-17T15:29:22] INFO Directory tmp/dir-target/foo: Executing\n", - "[2025-05-17T15:29:22] INFO Directory tmp/dir-target/foo: Ready\n", - "[2025-05-17T15:29:22] INFO Directory tmp/dir-target/bar/baz: Executing\n", - "[2025-05-17T15:29:22] INFO Directory tmp/dir-target/bar/baz: Ready\n", - "[2025-05-17T15:29:22] INFO Directories: Ready\n", - "[2025-05-17T15:29:22] INFO Directory tmp/dir-keys-target/foo/bar: Executing\n", - "[2025-05-17T15:29:22] INFO Directory tmp/dir-keys-target/foo/bar: Ready\n", - "[2025-05-17T15:29:22] INFO Directory tmp/dir-keys-target/baz: Executing\n", - "[2025-05-17T15:29:22] INFO Directory tmp/dir-keys-target/baz: Ready\n", - "[2025-05-17T15:29:22] INFO Directories: Ready\n", - "[2025-05-17T15:29:22] INFO Directory tmp/makedirs-target/foo: Executing\n", - "[2025-05-17T15:29:22] INFO Directory tmp/makedirs-target/foo: Ready\n", - "[2025-05-17T15:29:22] INFO Directory tmp/makedirs-target/bar/baz: Executing\n", - "[2025-05-17T15:29:22] INFO Directory tmp/makedirs-target/bar/baz: Ready\n", - "[2025-05-17T15:29:22] INFO Directories: Ready\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file3.csv -> tmp/glob-copy/file3.csv: Executing\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file3.csv -> tmp/glob-copy/file3.csv: Ready\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file1.nml -> tmp/glob-copy/file1.nml: Executing\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file1.nml -> tmp/glob-copy/file1.nml: Ready\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file2.txt -> tmp/glob-copy/file2.txt: Executing\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file2.txt -> tmp/glob-copy/file2.txt: Ready\n", - "[2025-05-17T15:29:22] INFO File copies: Ready\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file1.nml -> tmp/glob-copy-recursive/file1.nml: Executing\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file1.nml -> tmp/glob-copy-recursive/file1.nml: Ready\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/subdir1/file4.nml -> tmp/glob-copy-recursive/subdir1/file4.nml: Executing\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/subdir1/file4.nml -> tmp/glob-copy-recursive/subdir1/file4.nml: Ready\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/subdir2/file5.nml -> tmp/glob-copy-recursive/subdir2/file5.nml: Executing\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/subdir2/file5.nml -> tmp/glob-copy-recursive/subdir2/file5.nml: Ready\n", - "[2025-05-17T15:29:22] INFO File copies: Ready\n", - "[2025-05-17T15:29:22] INFO File copies: Ready\n", - "[2025-05-17T15:29:22] INFO Link tmp/glob-link-dirs/subdir1 -> fixtures/fs/data/subdir1: Executing\n", - "[2025-05-17T15:29:22] INFO Link tmp/glob-link-dirs/subdir1 -> fixtures/fs/data/subdir1: Ready\n", - "[2025-05-17T15:29:22] INFO Link tmp/glob-link-dirs/subdir2 -> fixtures/fs/data/subdir2: Executing\n", - "[2025-05-17T15:29:22] INFO Link tmp/glob-link-dirs/subdir2 -> fixtures/fs/data/subdir2: Ready\n", - "[2025-05-17T15:29:22] INFO File links: Ready\n", - "[2025-05-17T15:29:22] INFO HTTP https://raw.githubusercontent.com/ufs-community/uwtools/refs/heads/main/LICENSE -> tmp/licenses/gpl: Executing\n", - "[2025-05-17T15:29:22] INFO HTTP https://raw.githubusercontent.com/ufs-community/uwtools/refs/heads/main/LICENSE -> tmp/licenses/gpl: Ready\n", - "[2025-05-17T15:29:22] INFO File copies: Ready\n" - ] } ], "source": [ @@ -169,7 +125,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 16, "id": "a6aff6e3-815c-496e-81d7-d8756be9c232", "metadata": {}, "outputs": [ @@ -177,15 +133,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:21] INFO Validating config against internal schema: files-to-stage\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file1.nml -> tmp/copy-target/file1-copy.nml: Executing\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file1.nml -> tmp/copy-target/file1-copy.nml: Ready\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file2.txt -> tmp/copy-target/data/file2-copy.txt: Executing\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file2.txt -> tmp/copy-target/data/file2-copy.txt: Ready\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file3.csv -> tmp/copy-target/data/file3-copy.csv: Executing\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file3.csv -> tmp/copy-target/data/file3-copy.csv: Ready\n", - "[2025-05-17T15:29:22] INFO File copies: Ready\n" + "[2025-09-10T23:47:34] INFO Validating config against internal schema: files-to-stage\n", + "[2025-09-10T23:47:34] INFO Schema validation succeeded for fs config\n", + "[2025-09-10T23:47:34] INFO Local fixtures/fs/data/file1.nml -> tmp/copy-target/file1-copy.nml: Executing\n", + "[2025-09-10T23:47:34] INFO Local fixtures/fs/data/file1.nml -> tmp/copy-target/file1-copy.nml: Ready\n", + "[2025-09-10T23:47:34] INFO Local fixtures/fs/data/file2.txt -> tmp/copy-target/data/file2-copy.txt: Executing\n", + "[2025-09-10T23:47:34] INFO Local fixtures/fs/data/file2.txt -> tmp/copy-target/data/file2-copy.txt: Ready\n", + "[2025-09-10T23:47:34] INFO Local fixtures/fs/data/file3.csv -> tmp/copy-target/data/file3-copy.csv: Executing\n", + "[2025-09-10T23:47:34] INFO Local fixtures/fs/data/file3.csv -> tmp/copy-target/data/file3-copy.csv: Ready\n", + "[2025-09-10T23:47:34] INFO File copies: Ready\n" ] }, { @@ -197,7 +153,7 @@ " 'not-ready': []}" ] }, - "execution_count": 4, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -220,7 +176,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 17, "id": "e67fdb49-beef-4006-9e36-1a22829f21fc", "metadata": {}, "outputs": [ @@ -255,7 +211,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 21, "id": "b1fa6662-c4f3-4f7a-9b5d-8ee258cd6e0e", "metadata": {}, "outputs": [ @@ -263,15 +219,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: files-to-stage\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n", - "[2025-05-17T15:29:22] WARNING File fixtures/fs/data/missing-file.nml: Not ready [external asset]\n", - "[2025-05-17T15:29:22] WARNING Local fixtures/fs/data/missing-file.nml -> tmp/copy-target/missing-copy.nml: Not ready\n", - "[2025-05-17T15:29:22] WARNING Local fixtures/fs/data/missing-file.nml -> tmp/copy-target/missing-copy.nml: Requires:\n", - "[2025-05-17T15:29:22] WARNING Local fixtures/fs/data/missing-file.nml -> tmp/copy-target/missing-copy.nml: ✖ File fixtures/fs/data/missing-file.nml\n", - "[2025-05-17T15:29:22] WARNING File copies: Not ready\n", - "[2025-05-17T15:29:22] WARNING File copies: Requires:\n", - "[2025-05-17T15:29:22] WARNING File copies: ✖ Local fixtures/fs/data/missing-file.nml -> tmp/copy-target/missing-copy.nml\n" + "[2025-09-10T23:47:46] INFO Validating config against internal schema: files-to-stage\n", + "[2025-09-10T23:47:46] INFO Schema validation succeeded for fs config\n", + "[2025-09-10T23:47:46] WARNING File fixtures/fs/data/missing-file.nml: Not ready [external asset]\n", + "[2025-09-10T23:47:46] WARNING Local fixtures/fs/data/missing-file.nml -> tmp/copy-target/missing-copy.nml: Not ready\n", + "[2025-09-10T23:47:46] WARNING Local fixtures/fs/data/missing-file.nml -> tmp/copy-target/missing-copy.nml: Requires:\n", + "[2025-09-10T23:47:46] WARNING Local fixtures/fs/data/missing-file.nml -> tmp/copy-target/missing-copy.nml: ✖ File fixtures/fs/data/missing-file.nml\n", + "[2025-09-10T23:47:46] WARNING File copies: Not ready\n", + "[2025-09-10T23:47:46] WARNING File copies: Requires:\n", + "[2025-09-10T23:47:46] WARNING File copies: ✖ Local fixtures/fs/data/missing-file.nml -> tmp/copy-target/missing-copy.nml\n" ] }, { @@ -280,7 +236,7 @@ "{'ready': [], 'not-ready': ['tmp/copy-target/missing-copy.nml']}" ] }, - "execution_count": 6, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -302,7 +258,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 22, "id": "ab122bde-f483-4981-8308-fc6d4a90e50d", "metadata": {}, "outputs": [ @@ -337,7 +293,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 23, "id": "1f567844-ff8d-4e7f-87be-dffae9e15643", "metadata": {}, "outputs": [ @@ -369,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 24, "id": "dda3e407-a1a2-4b11-823a-3b6fdc39f67a", "metadata": {}, "outputs": [ @@ -377,15 +333,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: files-to-stage\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Executing\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Ready\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Executing\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Ready\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Executing\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Ready\n", - "[2025-05-17T15:29:22] INFO File copies: Ready\n" + "[2025-09-10T23:47:50] INFO Validating config against internal schema: files-to-stage\n", + "[2025-09-10T23:47:50] INFO Schema validation succeeded for fs config\n", + "[2025-09-10T23:47:50] INFO Local fixtures/fs/data/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Executing\n", + "[2025-09-10T23:47:50] INFO Local fixtures/fs/data/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Ready\n", + "[2025-09-10T23:47:50] INFO Local fixtures/fs/data/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Executing\n", + "[2025-09-10T23:47:50] INFO Local fixtures/fs/data/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Ready\n", + "[2025-09-10T23:47:50] INFO Local fixtures/fs/data/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Executing\n", + "[2025-09-10T23:47:50] INFO Local fixtures/fs/data/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Ready\n", + "[2025-09-10T23:47:50] INFO File copies: Ready\n" ] }, { @@ -397,7 +353,7 @@ " 'not-ready': []}" ] }, - "execution_count": 9, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -421,7 +377,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 25, "id": "59c67e22-fe98-4e74-8b0b-b40e24a804e8", "metadata": {}, "outputs": [ @@ -456,7 +412,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 26, "id": "7434dee7-fb52-4d9b-b2a1-d414165f3186", "metadata": {}, "outputs": [ @@ -492,8 +448,6 @@ " | Methods inherited from Stager:\n", " | \n", " | __init__(self, config: 'dict | str | Path | None' = None, target_dir: 'str | Path | None' = None, cycle: 'dt.datetime | None' = None, leadtime: 'dt.timedelta | None' = None, key_path: 'list[YAMLKey] | None' = None) -> 'None'\n", - " | Stage files and directories.\n", - " | \n", " | :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", " | :param target_dir: Path to target directory.\n", " | :param cycle: A ``datetime`` object to make available for use in the config.\n", @@ -527,7 +481,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 27, "id": "578cc091-c0eb-4293-8dbd-ee74a69a0940", "metadata": {}, "outputs": [ @@ -535,24 +489,24 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: files-to-stage\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file1.nml -> tmp/copier-target/file1-copy.nml: Executing\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file1.nml -> tmp/copier-target/file1-copy.nml: Ready\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file2.txt -> tmp/copier-target/data/file2-copy.txt: Executing\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file2.txt -> tmp/copier-target/data/file2-copy.txt: Ready\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file3.csv -> tmp/copier-target/data/file3-copy.csv: Executing\n", - "[2025-05-17T15:29:22] INFO Local fixtures/fs/data/file3.csv -> tmp/copier-target/data/file3-copy.csv: Ready\n", - "[2025-05-17T15:29:22] INFO File copies: Ready\n" + "[2025-09-10T23:47:56] INFO Validating config against internal schema: files-to-stage\n", + "[2025-09-10T23:47:56] INFO Schema validation succeeded for fs config\n", + "[2025-09-10T23:47:56] INFO Local fixtures/fs/data/file1.nml -> tmp/copier-target/file1-copy.nml: Executing\n", + "[2025-09-10T23:47:56] INFO Local fixtures/fs/data/file1.nml -> tmp/copier-target/file1-copy.nml: Ready\n", + "[2025-09-10T23:47:56] INFO Local fixtures/fs/data/file2.txt -> tmp/copier-target/data/file2-copy.txt: Executing\n", + "[2025-09-10T23:47:56] INFO Local fixtures/fs/data/file2.txt -> tmp/copier-target/data/file2-copy.txt: Ready\n", + "[2025-09-10T23:47:56] INFO Local fixtures/fs/data/file3.csv -> tmp/copier-target/data/file3-copy.csv: Executing\n", + "[2025-09-10T23:47:56] INFO Local fixtures/fs/data/file3.csv -> tmp/copier-target/data/file3-copy.csv: Ready\n", + "[2025-09-10T23:47:56] INFO File copies: Ready\n" ] }, { "data": { "text/plain": [ - "File copies <281473002154448>" + "File copies <281472361801952>" ] }, - "execution_count": 12, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -576,7 +530,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 28, "id": "c6aaac2b-bb72-433d-8ad4-349a1056cfa3", "metadata": {}, "outputs": [ @@ -611,7 +565,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 29, "id": "404d051e-18e1-4927-a24f-cbe98ab01ce9", "metadata": {}, "outputs": [ @@ -621,16 +575,23 @@ "text": [ "Help on function link in module uwtools.api.fs:\n", "\n", - "link(config: 'Path | dict | str | None' = None, target_dir: 'Path | str | None' = None, cycle: 'dt.datetime | None' = None, leadtime: 'dt.timedelta | None' = None, key_path: 'list[YAMLKey] | None' = None, dry_run: 'bool' = False, stdin_ok: 'bool' = False) -> 'dict[str, list[str]]'\n", - " Link files.\n", + "link(config: 'Path | dict | str | None' = None, target_dir: 'Path | str | None' = None, cycle: 'dt.datetime | None' = None, hardlink: 'bool | None' = False, leadtime: 'dt.timedelta | None' = None, key_path: 'list[YAMLKey] | None' = None, dry_run: 'bool' = False, stdin_ok: 'bool' = False, symlink_fallback: 'bool' = False) -> 'dict[str, list[str]]'\n", + " Create links to filesystem items.\n", + " \n", + " When ``hardlink`` is ``False`` (the default), links may target files, hardlinks, symlinks, and\n", + " directories; when ``True``, links may not be made across filesystems, or to directories. When\n", + " ``symlink_fallback`` is ``True``, a symlink will be created, if possible, if a hardlink cannot\n", + " be created.\n", " \n", " :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", " :param target_dir: Path to target directory.\n", " :param cycle: A datetime object to make available for use in the config.\n", + " :param hardlink: Create hardlinks instead of symlinks?\n", " :param leadtime: A timedelta object to make available for use in the config.\n", " :param key_path: Path of keys to config block to use.\n", " :param dry_run: Do not link files.\n", " :param stdin_ok: OK to read from ``stdin``?\n", + " :param symlink_fallback: Symlink if hardlink fails when hardlink=True?\n", " :return: A report on files linked / not linked.\n", "\n" ] @@ -650,7 +611,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 30, "id": "097b896c-aef4-48ac-aea5-eb2d463d172b", "metadata": {}, "outputs": [ @@ -679,7 +640,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 31, "id": "b6317f8a-c5fb-4114-93fa-236df3fd8805", "metadata": {}, "outputs": [ @@ -687,15 +648,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: files-to-stage\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n", - "[2025-05-17T15:29:22] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/data/file1.nml: Executing\n", - "[2025-05-17T15:29:22] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/data/file1.nml: Ready\n", - "[2025-05-17T15:29:22] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/data/file2.txt: Executing\n", - "[2025-05-17T15:29:22] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/data/file2.txt: Ready\n", - "[2025-05-17T15:29:22] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/data/file3.csv: Executing\n", - "[2025-05-17T15:29:22] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/data/file3.csv: Ready\n", - "[2025-05-17T15:29:22] INFO File links: Ready\n" + "[2025-09-10T23:48:18] INFO Validating config against internal schema: files-to-stage\n", + "[2025-09-10T23:48:18] INFO Schema validation succeeded for fs config\n", + "[2025-09-10T23:48:18] INFO Symlink tmp/link-target/file1-link.nml -> fixtures/fs/data/file1.nml: Executing\n", + "[2025-09-10T23:48:18] INFO Symlink tmp/link-target/file1-link.nml -> fixtures/fs/data/file1.nml: Ready\n", + "[2025-09-10T23:48:18] INFO Symlink tmp/link-target/file2-link.txt -> fixtures/fs/data/file2.txt: Executing\n", + "[2025-09-10T23:48:18] INFO Symlink tmp/link-target/file2-link.txt -> fixtures/fs/data/file2.txt: Ready\n", + "[2025-09-10T23:48:18] INFO Symlink tmp/link-target/data/file3-link.csv -> fixtures/fs/data/file3.csv: Executing\n", + "[2025-09-10T23:48:18] INFO Symlink tmp/link-target/data/file3-link.csv -> fixtures/fs/data/file3.csv: Ready\n", + "[2025-09-10T23:48:18] INFO File links: Ready\n" ] }, { @@ -707,7 +668,7 @@ " 'not-ready': []}" ] }, - "execution_count": 16, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -730,7 +691,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 32, "id": "b31ca50e-01c4-4665-81e0-de70a75ceb2a", "metadata": {}, "outputs": [ @@ -765,7 +726,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 34, "id": "59d93133-891d-4903-a965-23607cc72474", "metadata": {}, "outputs": [ @@ -773,15 +734,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: files-to-stage\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n", - "[2025-05-17T15:29:22] WARNING Target fixtures/fs/missing-file.nml: Not ready [external asset]\n", - "[2025-05-17T15:29:22] WARNING Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Not ready\n", - "[2025-05-17T15:29:22] WARNING Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Requires:\n", - "[2025-05-17T15:29:22] WARNING Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: ✖ Target fixtures/fs/missing-file.nml\n", - "[2025-05-17T15:29:22] WARNING File links: Not ready\n", - "[2025-05-17T15:29:22] WARNING File links: Requires:\n", - "[2025-05-17T15:29:22] WARNING File links: ✖ Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml\n" + "[2025-09-10T23:48:24] INFO Validating config against internal schema: files-to-stage\n", + "[2025-09-10T23:48:24] INFO Schema validation succeeded for fs config\n", + "[2025-09-10T23:48:24] WARNING Target fixtures/fs/missing-file.nml: Not ready [external asset]\n", + "[2025-09-10T23:48:24] WARNING Symlink tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Not ready\n", + "[2025-09-10T23:48:24] WARNING Symlink tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Requires:\n", + "[2025-09-10T23:48:24] WARNING Symlink tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: ✖ Target fixtures/fs/missing-file.nml\n", + "[2025-09-10T23:48:24] WARNING File links: Not ready\n", + "[2025-09-10T23:48:24] WARNING File links: Requires:\n", + "[2025-09-10T23:48:24] WARNING File links: ✖ Symlink tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml\n" ] }, { @@ -790,7 +751,7 @@ "{'ready': [], 'not-ready': ['tmp/link-target/missing-link.nml']}" ] }, - "execution_count": 18, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -812,7 +773,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 35, "id": "7a6e94b9-1161-4f41-9333-55736aec07b3", "metadata": {}, "outputs": [ @@ -847,7 +808,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 36, "id": "1de6cbd4-3b10-4b18-a8a5-c0cd21064bd3", "metadata": {}, "outputs": [ @@ -879,7 +840,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 38, "id": "ee4bf2a3-4101-4d95-afd5-120e95e64550", "metadata": {}, "outputs": [ @@ -887,15 +848,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: files-to-stage\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n", - "[2025-05-17T15:29:22] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/data/file1.nml: Executing\n", - "[2025-05-17T15:29:22] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/data/file1.nml: Ready\n", - "[2025-05-17T15:29:22] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/data/file2.txt: Executing\n", - "[2025-05-17T15:29:22] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/data/file2.txt: Ready\n", - "[2025-05-17T15:29:22] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/data/file3.csv: Executing\n", - "[2025-05-17T15:29:22] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/data/file3.csv: Ready\n", - "[2025-05-17T15:29:22] INFO File links: Ready\n" + "[2025-09-10T23:48:30] INFO Validating config against internal schema: files-to-stage\n", + "[2025-09-10T23:48:30] INFO Schema validation succeeded for fs config\n", + "[2025-09-10T23:48:30] INFO Symlink tmp/link-keys-target/file1-link.nml -> fixtures/fs/data/file1.nml: Executing\n", + "[2025-09-10T23:48:30] INFO Symlink tmp/link-keys-target/file1-link.nml -> fixtures/fs/data/file1.nml: Ready\n", + "[2025-09-10T23:48:30] INFO Symlink tmp/link-keys-target/file2-link.txt -> fixtures/fs/data/file2.txt: Executing\n", + "[2025-09-10T23:48:30] INFO Symlink tmp/link-keys-target/file2-link.txt -> fixtures/fs/data/file2.txt: Ready\n", + "[2025-09-10T23:48:30] INFO Symlink tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/data/file3.csv: Executing\n", + "[2025-09-10T23:48:30] INFO Symlink tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/data/file3.csv: Ready\n", + "[2025-09-10T23:48:30] INFO File links: Ready\n" ] }, { @@ -907,7 +868,7 @@ " 'not-ready': []}" ] }, - "execution_count": 21, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } @@ -931,7 +892,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 39, "id": "1393ae73-798b-49c0-9b68-e8ed28ad1df0", "metadata": {}, "outputs": [ @@ -966,7 +927,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 40, "id": "b1e5d3a2-7003-4449-9483-440236f66df7", "metadata": {}, "outputs": [ @@ -977,7 +938,7 @@ "Help on class Linker in module uwtools.fs:\n", "\n", "class Linker(FileStager)\n", - " | Linker(config: 'dict | str | Path | None' = None, target_dir: 'str | Path | None' = None, cycle: 'dt.datetime | None' = None, leadtime: 'dt.timedelta | None' = None, key_path: 'list[YAMLKey] | None' = None) -> 'None'\n", + " | Linker(config: 'dict | str | Path | None' = None, target_dir: 'str | Path | None' = None, cycle: 'dt.datetime | None' = None, hardlink: 'bool | None' = False, leadtime: 'dt.timedelta | None' = None, key_path: 'list[YAMLKey] | None' = None, symlink_fallback: 'bool' = False) -> 'None'\n", " | \n", " | Stage files by linking.\n", " | \n", @@ -990,27 +951,29 @@ " | \n", " | Methods defined here:\n", " | \n", - " | go(self)\n", - " | Link files.\n", - " | \n", - " | ----------------------------------------------------------------------\n", - " | Data and other attributes defined here:\n", - " | \n", - " | __abstractmethods__ = frozenset()\n", - " | \n", - " | ----------------------------------------------------------------------\n", - " | Methods inherited from Stager:\n", - " | \n", - " | __init__(self, config: 'dict | str | Path | None' = None, target_dir: 'str | Path | None' = None, cycle: 'dt.datetime | None' = None, leadtime: 'dt.timedelta | None' = None, key_path: 'list[YAMLKey] | None' = None) -> 'None'\n", - " | Stage files and directories.\n", - " | \n", + " | __init__(self, config: 'dict | str | Path | None' = None, target_dir: 'str | Path | None' = None, cycle: 'dt.datetime | None' = None, hardlink: 'bool | None' = False, leadtime: 'dt.timedelta | None' = None, key_path: 'list[YAMLKey] | None' = None, symlink_fallback: 'bool' = False) -> 'None'\n", " | :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", " | :param target_dir: Path to target directory.\n", " | :param cycle: A ``datetime`` object to make available for use in the config.\n", + " | :param hardlink: Create hardlinks instead of symlinks?\n", " | :param leadtime: A ``timedelta`` object to make available for use in the config.\n", " | :param key_path: Path of keys to config block to use.\n", + " | :param symlink_fallback: Symlink if hardlink fails when hardlink=True?\n", " | :raises: ``UWConfigError`` if config fails validation.\n", " | \n", + " | go(self)\n", + " | Create links to filesystem items.\n", + " | \n", + " | When ``hardlink`` is ``False`` (the default), links may target files, hardlinks, symlinks,\n", + " | and directories; when ``True``, links may not be made across filesystems, or to directories.\n", + " | When ``symlink_fallback`` is ``True``, a symlink will be created, if possible, if a hardlink\n", + " | cannot be created.\n", + " | \n", + " | ----------------------------------------------------------------------\n", + " | Data and other attributes defined here:\n", + " | \n", + " | __abstractmethods__ = frozenset()\n", + " | \n", " | ----------------------------------------------------------------------\n", " | Data descriptors inherited from Stager:\n", " | \n", @@ -1037,7 +1000,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 42, "id": "ecfa9e89-9fbd-4352-babc-dfa5b91afe6a", "metadata": {}, "outputs": [ @@ -1045,24 +1008,24 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: files-to-stage\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n", - "[2025-05-17T15:29:22] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/data/file1.nml: Executing\n", - "[2025-05-17T15:29:22] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/data/file1.nml: Ready\n", - "[2025-05-17T15:29:22] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/data/file2.txt: Executing\n", - "[2025-05-17T15:29:22] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/data/file2.txt: Ready\n", - "[2025-05-17T15:29:22] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/data/file3.csv: Executing\n", - "[2025-05-17T15:29:22] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/data/file3.csv: Ready\n", - "[2025-05-17T15:29:22] INFO File links: Ready\n" + "[2025-09-10T23:48:36] INFO Validating config against internal schema: files-to-stage\n", + "[2025-09-10T23:48:36] INFO Schema validation succeeded for fs config\n", + "[2025-09-10T23:48:36] INFO Symlink tmp/linker-target/file1-link.nml -> fixtures/fs/data/file1.nml: Executing\n", + "[2025-09-10T23:48:36] INFO Symlink tmp/linker-target/file1-link.nml -> fixtures/fs/data/file1.nml: Ready\n", + "[2025-09-10T23:48:36] INFO Symlink tmp/linker-target/file2-link.txt -> fixtures/fs/data/file2.txt: Executing\n", + "[2025-09-10T23:48:36] INFO Symlink tmp/linker-target/file2-link.txt -> fixtures/fs/data/file2.txt: Ready\n", + "[2025-09-10T23:48:36] INFO Symlink tmp/linker-target/data/file3-link.csv -> fixtures/fs/data/file3.csv: Executing\n", + "[2025-09-10T23:48:36] INFO Symlink tmp/linker-target/data/file3-link.csv -> fixtures/fs/data/file3.csv: Ready\n", + "[2025-09-10T23:48:36] INFO File links: Ready\n" ] }, { "data": { "text/plain": [ - "File links <281473001888400>" + "File links <281472361762240>" ] }, - "execution_count": 24, + "execution_count": 42, "metadata": {}, "output_type": "execute_result" } @@ -1086,7 +1049,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 43, "id": "6695f7bb-7ab7-42d1-9d2c-0bef7341147d", "metadata": {}, "outputs": [ @@ -1121,7 +1084,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 44, "id": "43b381d1-8dc2-4ea6-924c-e21149f05e7f", "metadata": {}, "outputs": [ @@ -1160,7 +1123,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 45, "id": "2f946927-509f-4cd6-a7ec-2d36f4d17318", "metadata": {}, "outputs": [ @@ -1189,7 +1152,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 47, "id": "fdd4e832-3bc5-4c7a-9b31-e387a4e7d48b", "metadata": {}, "outputs": [ @@ -1197,8 +1160,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: makedirs\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n" + "[2025-09-10T23:48:47] INFO Validating config against internal schema: makedirs\n", + "[2025-09-10T23:48:47] INFO Schema validation succeeded for fs config\n", + "[2025-09-10T23:48:47] INFO Directory tmp/dir-target/foo: Executing\n", + "[2025-09-10T23:48:47] INFO Directory tmp/dir-target/foo: Ready\n", + "[2025-09-10T23:48:47] INFO Directory tmp/dir-target/bar/baz: Executing\n", + "[2025-09-10T23:48:47] INFO Directory tmp/dir-target/bar/baz: Ready\n", + "[2025-09-10T23:48:47] INFO Directories: Ready\n" ] }, { @@ -1207,7 +1175,7 @@ "{'ready': ['tmp/dir-target/foo', 'tmp/dir-target/bar/baz'], 'not-ready': []}" ] }, - "execution_count": 28, + "execution_count": 47, "metadata": {}, "output_type": "execute_result" } @@ -1230,7 +1198,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 48, "id": "c00ec8cc-964a-498e-bd8f-a3686a468dc3", "metadata": {}, "outputs": [ @@ -1264,7 +1232,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 49, "id": "3a93956d-0acf-4c37-87bf-83c0d5287644", "metadata": {}, "outputs": [ @@ -1296,7 +1264,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 50, "id": "91549822-e85e-4d41-8860-1da05d713f75", "metadata": {}, "outputs": [ @@ -1304,8 +1272,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: makedirs\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n" + "[2025-09-10T23:48:51] INFO Validating config against internal schema: makedirs\n", + "[2025-09-10T23:48:51] INFO Schema validation succeeded for fs config\n", + "[2025-09-10T23:48:51] INFO Directory tmp/dir-keys-target/foo/bar: Executing\n", + "[2025-09-10T23:48:51] INFO Directory tmp/dir-keys-target/foo/bar: Ready\n", + "[2025-09-10T23:48:51] INFO Directory tmp/dir-keys-target/baz: Executing\n", + "[2025-09-10T23:48:51] INFO Directory tmp/dir-keys-target/baz: Ready\n", + "[2025-09-10T23:48:51] INFO Directories: Ready\n" ] }, { @@ -1315,7 +1288,7 @@ " 'not-ready': []}" ] }, - "execution_count": 31, + "execution_count": 50, "metadata": {}, "output_type": "execute_result" } @@ -1339,7 +1312,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 51, "id": "cb4ded9c-0de1-4010-af75-fbb7becd3fbc", "metadata": {}, "outputs": [ @@ -1373,7 +1346,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 52, "id": "7fe53736-d8e8-4ca9-ab2b-87729934fc19", "metadata": {}, "outputs": [ @@ -1408,8 +1381,6 @@ " | Methods inherited from Stager:\n", " | \n", " | __init__(self, config: 'dict | str | Path | None' = None, target_dir: 'str | Path | None' = None, cycle: 'dt.datetime | None' = None, leadtime: 'dt.timedelta | None' = None, key_path: 'list[YAMLKey] | None' = None) -> 'None'\n", - " | Stage files and directories.\n", - " | \n", " | :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", " | :param target_dir: Path to target directory.\n", " | :param cycle: A ``datetime`` object to make available for use in the config.\n", @@ -1443,7 +1414,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 53, "id": "950d6b43-6db7-40df-b645-beaa1369cfa4", "metadata": {}, "outputs": [ @@ -1451,17 +1422,22 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: makedirs\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n" + "[2025-09-10T23:48:55] INFO Validating config against internal schema: makedirs\n", + "[2025-09-10T23:48:55] INFO Schema validation succeeded for fs config\n", + "[2025-09-10T23:48:55] INFO Directory tmp/makedirs-target/foo: Executing\n", + "[2025-09-10T23:48:55] INFO Directory tmp/makedirs-target/foo: Ready\n", + "[2025-09-10T23:48:55] INFO Directory tmp/makedirs-target/bar/baz: Executing\n", + "[2025-09-10T23:48:55] INFO Directory tmp/makedirs-target/bar/baz: Ready\n", + "[2025-09-10T23:48:55] INFO Directories: Ready\n" ] }, { "data": { "text/plain": [ - "Directories <281473001830720>" + "Directories <281472361762192>" ] }, - "execution_count": 34, + "execution_count": 53, "metadata": {}, "output_type": "execute_result" } @@ -1485,7 +1461,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 54, "id": "a61fb9ac-df2f-4e39-9f66-bfb789c39117", "metadata": {}, "outputs": [ @@ -1519,7 +1495,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 55, "id": "329c139f-9688-47b3-ac80-a74c95a8270f", "metadata": {}, "outputs": [ @@ -1555,7 +1531,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 56, "id": "1630b959-264b-47b5-8f51-268b90599df1", "metadata": {}, "outputs": [ @@ -1574,7 +1550,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 57, "id": "13500045-c324-427d-b152-cbeb659ab00a", "metadata": {}, "outputs": [ @@ -1582,8 +1558,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: files-to-stage\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n" + "[2025-09-10T23:48:58] INFO Validating config against internal schema: files-to-stage\n", + "[2025-09-10T23:48:58] INFO Schema validation succeeded for fs config\n" ] }, { @@ -1595,7 +1571,7 @@ " 'not-ready': []}" ] }, - "execution_count": 38, + "execution_count": 57, "metadata": {}, "output_type": "execute_result" } @@ -1618,7 +1594,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 58, "id": "eda3d0a4-20e1-409b-9895-f9ad6c50daf1", "metadata": {}, "outputs": [ @@ -1652,7 +1628,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 59, "id": "8e953763-be03-416d-9331-d3dbf382078a", "metadata": {}, "outputs": [ @@ -1671,7 +1647,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 60, "id": "eb36fa38-e4fd-4b05-9462-d4b998468451", "metadata": {}, "outputs": [ @@ -1679,8 +1655,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: files-to-stage\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n" + "[2025-09-10T23:49:01] INFO Validating config against internal schema: files-to-stage\n", + "[2025-09-10T23:49:01] INFO Schema validation succeeded for fs config\n" ] }, { @@ -1692,7 +1668,7 @@ " 'not-ready': []}" ] }, - "execution_count": 41, + "execution_count": 60, "metadata": {}, "output_type": "execute_result" } @@ -1715,7 +1691,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 61, "id": "36871e13-33e0-4e55-a969-cb3557172efb", "metadata": {}, "outputs": [ @@ -1749,7 +1725,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 62, "id": "c26aada1-7b8e-4a07-982b-f1059b362243", "metadata": {}, "outputs": [ @@ -1768,7 +1744,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 63, "id": "475b1deb-497d-4079-954d-92199cc37396", "metadata": {}, "outputs": [ @@ -1776,10 +1752,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: files-to-stage\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n", - "[2025-05-17T15:29:22] WARNING Ignoring directory fixtures/fs/data/subdir1\n", - "[2025-05-17T15:29:22] WARNING Ignoring directory fixtures/fs/data/subdir2\n" + "[2025-09-10T23:49:04] INFO Validating config against internal schema: files-to-stage\n", + "[2025-09-10T23:49:04] INFO Schema validation succeeded for fs config\n", + "[2025-09-10T23:49:04] WARNING Ignoring directory fixtures/fs/data/subdir1\n", + "[2025-09-10T23:49:04] WARNING Ignoring directory fixtures/fs/data/subdir2\n" ] }, { @@ -1788,7 +1764,7 @@ "{'ready': [], 'not-ready': []}" ] }, - "execution_count": 44, + "execution_count": 63, "metadata": {}, "output_type": "execute_result" } @@ -1810,7 +1786,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 64, "id": "4ee1c195-544d-431e-98aa-e5e7184614a0", "metadata": {}, "outputs": [ @@ -1829,7 +1805,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 65, "id": "df3113d6-316f-4d08-83aa-a07d62f41d3c", "metadata": {}, "outputs": [ @@ -1837,15 +1813,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: files-to-stage\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n", - "[2025-05-17T15:29:22] INFO Link tmp/glob-link-recursive/file1.nml -> fixtures/fs/data/file1.nml: Executing\n", - "[2025-05-17T15:29:22] INFO Link tmp/glob-link-recursive/file1.nml -> fixtures/fs/data/file1.nml: Ready\n", - "[2025-05-17T15:29:22] INFO Link tmp/glob-link-recursive/subdir1/file4.nml -> fixtures/fs/data/subdir1/file4.nml: Executing\n", - "[2025-05-17T15:29:22] INFO Link tmp/glob-link-recursive/subdir1/file4.nml -> fixtures/fs/data/subdir1/file4.nml: Ready\n", - "[2025-05-17T15:29:22] INFO Link tmp/glob-link-recursive/subdir2/file5.nml -> fixtures/fs/data/subdir2/file5.nml: Executing\n", - "[2025-05-17T15:29:22] INFO Link tmp/glob-link-recursive/subdir2/file5.nml -> fixtures/fs/data/subdir2/file5.nml: Ready\n", - "[2025-05-17T15:29:22] INFO File links: Ready\n" + "[2025-09-10T23:49:05] INFO Validating config against internal schema: files-to-stage\n", + "[2025-09-10T23:49:05] INFO Schema validation succeeded for fs config\n", + "[2025-09-10T23:49:05] INFO Symlink tmp/glob-link-recursive/file1.nml -> fixtures/fs/data/file1.nml: Executing\n", + "[2025-09-10T23:49:05] INFO Symlink tmp/glob-link-recursive/file1.nml -> fixtures/fs/data/file1.nml: Ready\n", + "[2025-09-10T23:49:05] INFO Symlink tmp/glob-link-recursive/subdir1/file4.nml -> fixtures/fs/data/subdir1/file4.nml: Executing\n", + "[2025-09-10T23:49:05] INFO Symlink tmp/glob-link-recursive/subdir1/file4.nml -> fixtures/fs/data/subdir1/file4.nml: Ready\n", + "[2025-09-10T23:49:05] INFO Symlink tmp/glob-link-recursive/subdir2/file5.nml -> fixtures/fs/data/subdir2/file5.nml: Executing\n", + "[2025-09-10T23:49:05] INFO Symlink tmp/glob-link-recursive/subdir2/file5.nml -> fixtures/fs/data/subdir2/file5.nml: Ready\n", + "[2025-09-10T23:49:05] INFO File links: Ready\n" ] }, { @@ -1857,7 +1833,7 @@ " 'not-ready': []}" ] }, - "execution_count": 46, + "execution_count": 65, "metadata": {}, "output_type": "execute_result" } @@ -1872,7 +1848,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 66, "id": "e5f92299-d8b4-490b-a2b8-bfee9b4cc8e9", "metadata": {}, "outputs": [ @@ -1906,7 +1882,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 67, "id": "d79477c9-4778-4c38-9fa4-886df8628611", "metadata": {}, "outputs": [ @@ -1925,7 +1901,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 68, "id": "6852415c-1086-4db4-89b5-a2af877264f9", "metadata": {}, "outputs": [ @@ -1933,8 +1909,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: files-to-stage\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n" + "[2025-09-10T23:49:07] INFO Validating config against internal schema: files-to-stage\n", + "[2025-09-10T23:49:07] INFO Schema validation succeeded for fs config\n" ] }, { @@ -1944,7 +1920,7 @@ " 'not-ready': []}" ] }, - "execution_count": 49, + "execution_count": 68, "metadata": {}, "output_type": "execute_result" } @@ -1959,7 +1935,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 69, "id": "902c7a67-4278-427e-afb7-21b12a67f085", "metadata": {}, "outputs": [ @@ -1990,7 +1966,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 70, "id": "66056ac0-8efe-4875-95e6-9717998bbb5c", "metadata": {}, "outputs": [ @@ -2009,7 +1985,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 71, "id": "c6b0166c-190a-4af1-aa21-70cfa024a093", "metadata": {}, "outputs": [ @@ -2017,8 +1993,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:22] INFO Validating config against internal schema: files-to-stage\n", - "[2025-05-17T15:29:22] INFO Schema validation succeeded for fs config\n" + "[2025-09-10T23:49:09] INFO Validating config against internal schema: files-to-stage\n", + "[2025-09-10T23:49:09] INFO Schema validation succeeded for fs config\n", + "[2025-09-10T23:49:10] INFO HTTP https://raw.githubusercontent.com/ufs-community/uwtools/refs/heads/main/LICENSE -> tmp/licenses/gpl: Executing\n", + "[2025-09-10T23:49:10] INFO HTTP https://raw.githubusercontent.com/ufs-community/uwtools/refs/heads/main/LICENSE -> tmp/licenses/gpl: Ready\n", + "[2025-09-10T23:49:10] INFO File copies: Ready\n" ] }, { @@ -2027,7 +2006,7 @@ "{'ready': ['tmp/licenses/gpl'], 'not-ready': []}" ] }, - "execution_count": 52, + "execution_count": 71, "metadata": {}, "output_type": "execute_result" } @@ -2042,7 +2021,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 72, "id": "f1a89524-bf0d-400e-83d5-ff73fc40a9b2", "metadata": {}, "outputs": [ @@ -2079,7 +2058,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.9.0" } }, "nbformat": 4, diff --git a/notebooks/rocoto.ipynb b/notebooks/rocoto.ipynb index 3a055cdfb..ef0b17f31 100644 --- a/notebooks/rocoto.ipynb +++ b/notebooks/rocoto.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 25, "id": "7dd67340-6553-40e9-be68-d79c1979280c", "metadata": {}, "outputs": [], @@ -48,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 26, "id": "1d10d514-d918-4cbd-aa61-c2be8ee9e298", "metadata": {}, "outputs": [ @@ -90,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 27, "id": "9281ae4f-4d78-4401-bf6f-87d4b873e846", "metadata": {}, "outputs": [ @@ -104,7 +104,7 @@ " scheduler: slurm\n", " cycledef:\n", " - spec: 202410290000 202410300000 06:00:00\n", - " log:\n", + " log: \n", " value: logs/test.log\n", " tasks:\n", " task_greet:\n", @@ -132,7 +132,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 28, "id": "983636f9-7c39-4f0e-a76e-e35129d2b9fe", "metadata": {}, "outputs": [ @@ -140,8 +140,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:43] INFO Schema validation succeeded for Rocoto config\n", - "[2025-05-17T15:29:43] INFO Schema validation succeeded for Rocoto XML\n" + "[2025-09-10T23:39:41] INFO Schema validation succeeded for Rocoto config\n", + "[2025-09-10T23:39:41] INFO Schema validation succeeded for Rocoto XML\n" ] }, { @@ -150,7 +150,7 @@ "True" ] }, - "execution_count": 4, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } @@ -175,7 +175,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 29, "id": "d895e1cf-e8af-437b-9a38-2b03ec34f527", "metadata": {}, "outputs": [ @@ -213,7 +213,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 30, "id": "56d04e6b-5fa2-4c93-ac2f-c95fa23c888e", "metadata": {}, "outputs": [ @@ -249,7 +249,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 31, "id": "16704f19-cbca-4765-8c72-16512fc96e9b", "metadata": {}, "outputs": [ @@ -257,13 +257,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:43] ERROR 3 schema-validation errors found in Rocoto config\n", - "[2025-05-17T15:29:43] ERROR Error at workflow.attrs:\n", - "[2025-05-17T15:29:43] ERROR 'realtime' is a required property\n", - "[2025-05-17T15:29:43] ERROR Error at workflow.tasks.task_greet:\n", - "[2025-05-17T15:29:43] ERROR 'command' is a required property\n", - "[2025-05-17T15:29:43] ERROR Error at workflow:\n", - "[2025-05-17T15:29:43] ERROR 'log' is a required property\n" + "[2025-09-10T23:39:41] ERROR 3 schema-validation errors found in Rocoto config\n", + "[2025-09-10T23:39:41] ERROR Error at workflow.attrs:\n", + "[2025-09-10T23:39:41] ERROR 'realtime' is a required property\n", + "[2025-09-10T23:39:41] ERROR Error at workflow.tasks.task_greet:\n", + "[2025-09-10T23:39:41] ERROR 'command' is a required property\n", + "[2025-09-10T23:39:41] ERROR Error at workflow:\n", + "[2025-09-10T23:39:41] ERROR 'log' is a required property\n" ] }, { @@ -297,7 +297,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 32, "id": "074a6e7e-7c05-4037-b954-06eba8ae2241", "metadata": {}, "outputs": [ @@ -313,7 +313,7 @@ " - spec: 202410290000 202410300000 06:00:00\n", " entities:\n", " LOG: \"2024-10-29/test06:00:00.log\"\n", - " log:\n", + " log: \n", " value: logs/&LOG;\n", " tasks:\n", " task_greet:\n", @@ -339,7 +339,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 33, "id": "e4df500f-6b11-4c0e-ac44-3a5443d0ee02", "metadata": {}, "outputs": [ @@ -355,7 +355,7 @@ " - spec: 202410290000 202410300000 06:00:00\n", " entities:\n", " LOG: \"@Y-@m-@d/test@X.log\"\n", - " log:\n", + " log: \n", " value:\n", " cyclestr:\n", " value: logs/&LOG;\n", @@ -383,7 +383,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 34, "id": "5f1b9dcd-87cf-41c3-ab3c-e581e2967214", "metadata": {}, "outputs": [ @@ -391,8 +391,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:43] INFO Schema validation succeeded for Rocoto config\n", - "[2025-05-17T15:29:43] INFO Schema validation succeeded for Rocoto XML\n" + "[2025-09-10T23:39:41] INFO Schema validation succeeded for Rocoto config\n", + "[2025-09-10T23:39:41] INFO Schema validation succeeded for Rocoto XML\n" ] }, { @@ -401,7 +401,7 @@ "True" ] }, - "execution_count": 10, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -424,7 +424,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 35, "id": "1a1bea14-dbae-4e7d-96f7-0ec552b0e25a", "metadata": {}, "outputs": [ @@ -469,7 +469,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 36, "id": "33c67471-84a1-4b0f-b0bd-40f805e6615f", "metadata": {}, "outputs": [ @@ -483,7 +483,7 @@ " scheduler: slurm\n", " cycledef:\n", " - spec: 202410290000 202410300000 06:00:00\n", - " log:\n", + " log: \n", " value: logs/test.log\n", " tasks:\n", " task_bacon:\n", @@ -515,7 +515,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 37, "id": "be4d4488-9ff6-4a6d-ace0-464e21f31116", "metadata": {}, "outputs": [ @@ -529,7 +529,7 @@ " scheduler: slurm\n", " cycledef:\n", " - spec: 202410290000 202410300000 06:00:00\n", - " log:\n", + " log: \n", " value: logs/test.log\n", " tasks:\n", " task_bacon:\n", @@ -574,7 +574,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 38, "id": "001ac8ed-0f31-4012-bc63-55d63848e1d4", "metadata": {}, "outputs": [ @@ -582,8 +582,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:43] INFO Schema validation succeeded for Rocoto config\n", - "[2025-05-17T15:29:43] INFO Schema validation succeeded for Rocoto XML\n" + "[2025-09-10T23:39:41] INFO Schema validation succeeded for Rocoto config\n", + "[2025-09-10T23:39:41] INFO Schema validation succeeded for Rocoto XML\n" ] }, { @@ -592,7 +592,7 @@ "True" ] }, - "execution_count": 14, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } @@ -615,7 +615,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 39, "id": "7b9ecc04-9851-4e34-985f-d908285dc8e2", "metadata": {}, "outputs": [ @@ -676,7 +676,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 40, "id": "6c8a8b8e-62b0-47f1-b5e3-763aef2e71ea", "metadata": {}, "outputs": [ @@ -690,7 +690,7 @@ " scheduler: slurm\n", " cycledef:\n", " - spec: 202410290000 202410300000 06:00:00\n", - " log:\n", + " log: \n", " value: logs/test.log\n", " tasks:\n", " metatask_breakfast:\n", @@ -720,7 +720,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 41, "id": "112a7586-5fef-46a0-83a5-f7016257fe9b", "metadata": {}, "outputs": [ @@ -728,8 +728,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:44] INFO Schema validation succeeded for Rocoto config\n", - "[2025-05-17T15:29:44] INFO Schema validation succeeded for Rocoto XML\n" + "[2025-09-10T23:39:41] INFO Schema validation succeeded for Rocoto config\n", + "[2025-09-10T23:39:42] INFO Schema validation succeeded for Rocoto XML\n" ] }, { @@ -738,7 +738,7 @@ "True" ] }, - "execution_count": 17, + "execution_count": 41, "metadata": {}, "output_type": "execute_result" } @@ -761,7 +761,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 42, "id": "3ac7f1af-44cf-440a-8b4f-6f63e4e98fee", "metadata": {}, "outputs": [ @@ -803,7 +803,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 43, "id": "69797f73-2475-449f-b036-2529f4379440", "metadata": {}, "outputs": [ @@ -817,7 +817,7 @@ " scheduler: slurm\n", " cycledef:\n", " - spec: 202410290000 202410300000 06:00:00\n", - " log:\n", + " log: \n", " value: logs/test.log\n", " tasks:\n", " metatask_process:\n", @@ -851,7 +851,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 44, "id": "6a681e6d-800c-4d9f-87a0-270e72dcb7be", "metadata": {}, "outputs": [ @@ -886,7 +886,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 45, "id": "22a8fe77-2094-4139-9ff2-91dc897c3af3", "metadata": {}, "outputs": [ @@ -924,7 +924,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 46, "id": "fcbf5ffd-7722-4801-b6f7-5867248d471d", "metadata": {}, "outputs": [ @@ -932,7 +932,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:44] INFO Schema validation succeeded for Rocoto XML\n" + "[2025-09-10T23:39:42] INFO Schema validation succeeded for Rocoto XML\n" ] }, { @@ -941,7 +941,7 @@ "True" ] }, - "execution_count": 22, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } @@ -963,7 +963,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 47, "id": "9788207b-3c1f-4b60-bd4d-9c8a75666b24", "metadata": {}, "outputs": [ @@ -1000,7 +1000,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 48, "id": "54fcfb54-361d-47ef-9379-4b235fa54316", "metadata": {}, "outputs": [ @@ -1008,22 +1008,22 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:29:44] ERROR 4 Rocoto XML validation errors found\n", - "[2025-05-17T15:29:44] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_ATTRVALID: Element workflow failed to validate attributes\n", - "[2025-05-17T15:29:44] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_NOELEM: Expecting an element cycledef, got nothing\n", - "[2025-05-17T15:29:44] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_INTERSEQ: Invalid sequence in interleave\n", - "[2025-05-17T15:29:44] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_CONTENTVALID: Element workflow failed to validate content\n", - "[2025-05-17T15:29:44] ERROR Invalid Rocoto XML:\n", - "[2025-05-17T15:29:44] ERROR 1 \n", - "[2025-05-17T15:29:44] ERROR 2 \n", - "[2025-05-17T15:29:44] ERROR 3 logs/test.log\n", - "[2025-05-17T15:29:44] ERROR 4 \n", - "[2025-05-17T15:29:44] ERROR 5 1\n", - "[2025-05-17T15:29:44] ERROR 6 00:00:10\n", - "[2025-05-17T15:29:44] ERROR 7 echo Hello, World!\n", - "[2025-05-17T15:29:44] ERROR 8 greet\n", - "[2025-05-17T15:29:44] ERROR 9 \n", - "[2025-05-17T15:29:44] ERROR 10 \n" + "[2025-09-10T23:39:42] ERROR 4 Rocoto XML validation errors found\n", + "[2025-09-10T23:39:42] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_ATTRVALID: Element workflow failed to validate attributes\n", + "[2025-09-10T23:39:42] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_NOELEM: Expecting an element cycledef, got nothing\n", + "[2025-09-10T23:39:42] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_INTERSEQ: Invalid sequence in interleave\n", + "[2025-09-10T23:39:42] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_CONTENTVALID: Element workflow failed to validate content\n", + "[2025-09-10T23:39:42] ERROR Invalid Rocoto XML:\n", + "[2025-09-10T23:39:42] ERROR 1 \n", + "[2025-09-10T23:39:42] ERROR 2 \n", + "[2025-09-10T23:39:42] ERROR 3 logs/test.log\n", + "[2025-09-10T23:39:42] ERROR 4 \n", + "[2025-09-10T23:39:42] ERROR 5 1\n", + "[2025-09-10T23:39:42] ERROR 6 00:00:10\n", + "[2025-09-10T23:39:42] ERROR 7 echo Hello, World!\n", + "[2025-09-10T23:39:42] ERROR 8 greet\n", + "[2025-09-10T23:39:42] ERROR 9 \n", + "[2025-09-10T23:39:42] ERROR 10 \n" ] }, { @@ -1032,7 +1032,7 @@ "False" ] }, - "execution_count": 24, + "execution_count": 48, "metadata": {}, "output_type": "execute_result" } @@ -1060,7 +1060,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.9.0" } }, "nbformat": 4, diff --git a/notebooks/template.ipynb b/notebooks/template.ipynb index f8572e550..dbd07c698 100644 --- a/notebooks/template.ipynb +++ b/notebooks/template.ipynb @@ -21,7 +21,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 15, "id": "cbec4cc0-369e-41ff-a8a6-8a2699cb5759", "metadata": {}, "outputs": [], @@ -42,7 +42,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 16, "id": "54e88f1b-0b9f-4011-b070-df107f928cf9", "metadata": {}, "outputs": [ @@ -91,7 +91,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 17, "id": "91bd29fd-77ba-4ea2-946f-cd7a2d9301f1", "metadata": {}, "outputs": [ @@ -120,7 +120,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 18, "id": "040eceb1-0821-4e82-825a-5be18f06397d", "metadata": {}, "outputs": [ @@ -128,10 +128,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-05-17T15:30:01] INFO Value(s) needed to render this template are:\n", - "[2025-05-17T15:30:01] INFO first\n", - "[2025-05-17T15:30:01] INFO food\n", - "[2025-05-17T15:30:01] INFO last\n" + "[2025-09-10T23:40:02] INFO Value(s) needed to render this template are:\n", + "[2025-09-10T23:40:02] INFO first\n", + "[2025-09-10T23:40:02] INFO food\n", + "[2025-09-10T23:40:02] INFO last\n" ] }, { @@ -167,7 +167,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 19, "id": "7f794c66-8840-419a-adf5-20efddb85708", "metadata": {}, "outputs": [ @@ -196,7 +196,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 20, "id": "834b7a40-293e-4d35-81e8-121eed4cf8f2", "metadata": {}, "outputs": [ @@ -231,7 +231,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 21, "id": "55eec4f4-4f91-4618-8382-78061907bd2a", "metadata": {}, "outputs": [ @@ -267,7 +267,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 22, "id": "8601d4d9-5e53-44b7-880c-666ab810d8b8", "metadata": {}, "outputs": [ @@ -304,7 +304,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 23, "id": "ff8b80b2-590c-476f-94f7-37c4f34932f7", "metadata": {}, "outputs": [ @@ -336,7 +336,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 24, "id": "1f2cec30-0761-42f4-85fc-05593e215b23", "metadata": {}, "outputs": [ @@ -373,7 +373,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 25, "id": "2ddcefac-030d-415c-a97f-eab9e176e811", "metadata": {}, "outputs": [ @@ -413,7 +413,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 26, "id": "66fbde65-2c4e-48fa-bc49-c4faec78f944", "metadata": {}, "outputs": [ @@ -442,7 +442,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 27, "id": "bab9026c-9f5a-435d-b8a2-71fa2a325109", "metadata": {}, "outputs": [ @@ -452,7 +452,7 @@ "True" ] }, - "execution_count": 13, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -474,7 +474,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 28, "id": "5f30435c-e253-4f8a-a8e7-6bdbd8be92c9", "metadata": {}, "outputs": [ @@ -510,7 +510,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.9.0" } }, "nbformat": 4, diff --git a/src/uwtools/api/config.py b/src/uwtools/api/config.py index 3fb010ed2..a821143ca 100644 --- a/src/uwtools/api/config.py +++ b/src/uwtools/api/config.py @@ -246,11 +246,11 @@ def validate( Recognized file extensions are: {extensions} :param input_config: Input config file (``None`` => read ``stdin``). -:param input_format: Format of the input config (optional if file's extension is recognized). +:param input_format: Input config format (default: deduced from filename extension; ``yaml`` if that fails). :param update_config: Update config file (``None`` => read ``stdin``). -:param update_format: Format of the update config (optional if file's extension is recognized). +:param update_format: Update config format (default: deduced from filename extension; ``yaml`` if that fails). :param output_file: Output config file (``None`` => write to ``stdout``). -:param output_format: Format of the output config (optional if file's extension is recognized). +:param output_format: Output config format (default: deduced from filename extension; ``yaml`` if that fails). :param key_path: Path of keys to the desired output block. :param values_needed: Report complete, missing, and template values. :param total: Require rendering of all Jinja2 variables/expressions. @@ -258,7 +258,7 @@ def validate( :param stdin_ok: OK to read from ``stdin``? :return: The ``dict`` representation of the realized config. :raises: ``UWConfigRealizeError`` if ``total`` is ``True`` and any Jinja2 syntax was not rendered. -""".format(extensions=", ".join(_FORMAT.extensions())).strip() +""".format(extensions=", ".join(_FORMAT.extensions())).strip() # noqa: E501 __all__ = [ "Config", diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 3978364cd..430314260 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -29,7 +29,7 @@ from uwtools.exceptions import UWConfigRealizeError, UWError, UWTemplateRenderError from uwtools.logging import log, setup_logging from uwtools.strings import FORMAT, STR -from uwtools.utils.file import get_file_format, resource_path +from uwtools.utils.file import get_config_format, resource_path FORMATS = FORMAT.extensions() LEADTIME_DESC = "hours[:minutes[:seconds]]" @@ -1182,9 +1182,7 @@ def _basic_setup(parser: Parser) -> Group: def _check_file_vs_format(file_arg: str, format_arg: str, args: Args) -> Args: if args.get(format_arg) is None: - if args.get(file_arg) is None: - _abort("Specify %s when %s is not specified" % (_switch(format_arg), _switch(file_arg))) - args[format_arg] = get_file_format(args[file_arg]) + args[format_arg] = get_config_format(args[file_arg]) return args @@ -1194,13 +1192,13 @@ def _check_template_render_vals_args(args: Args) -> Args: # values file IS specified, its format must either be explicitly specified, or deduced from its # extension. if args.get(STR.valsfile) is not None and args.get(STR.valsfmt) is None: - args[STR.valsfmt] = get_file_format(args[STR.valsfile]) + args[STR.valsfmt] = get_config_format(args[STR.valsfile]) return args def _check_update(args: Args) -> Args: if args.get(STR.updatefile) is not None and args.get(STR.updatefmt) is None: - args[STR.updatefmt] = get_file_format(args[STR.updatefile]) + args[STR.updatefmt] = get_config_format(args[STR.updatefile]) return args diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 6d8bb5eeb..059f85674 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -22,7 +22,7 @@ uw_yaml_loader, ) from uwtools.logging import INDENT, MSGWIDTH, log -from uwtools.utils.file import get_file_format, readable, writable +from uwtools.utils.file import get_config_format, readable, writable _ConfigVal = Union[ bool, datetime, dict, float, int, list, str, timedelta, UWYAMLConvert, UWYAMLGlob, UWYAMLRemove @@ -374,7 +374,7 @@ def _supplement_values( """ values: dict if isinstance(values_src, Path): - values_format = values_format or get_file_format(values_src) + values_format = values_format or get_config_format(values_src) values_src_class = format_to_config(values_format) values = values_src_class(values_src).data log.debug("Read initial template values from %s", values_src) diff --git a/src/uwtools/config/tools.py b/src/uwtools/config/tools.py index 49a527bf7..790073d4f 100644 --- a/src/uwtools/config/tools.py +++ b/src/uwtools/config/tools.py @@ -4,8 +4,7 @@ from __future__ import annotations -from pathlib import Path -from typing import Callable, cast +from typing import TYPE_CHECKING, Callable, cast from uwtools.config.formats.base import Config from uwtools.config.jinja2 import unrendered @@ -13,7 +12,10 @@ from uwtools.exceptions import UWConfigError, UWConfigRealizeError, UWError from uwtools.logging import log from uwtools.strings import FORMAT -from uwtools.utils.file import get_file_format +from uwtools.utils.file import get_config_format + +if TYPE_CHECKING: + from pathlib import Path # Public functions @@ -144,24 +146,20 @@ def _ensure_format( desc: str, fmt: str | None = None, config: Config | Path | dict | None = None ) -> str: """ - Return the given format, or the appropriate format as deduced from the config. + Return the given format, or the deduced format. :param desc: A description of the file. :param fmt: The config format name. :param config: The input config. :return: The specified or deduced format. - :raises: UWError if the format cannot be determined. """ + if fmt: + return fmt if isinstance(config, Config): return config._get_format() # noqa: SLF001 - if isinstance(config, Path): - return fmt or get_file_format(config) if isinstance(config, dict): - return fmt or FORMAT.yaml - if fmt is None: - msg = f"Either {desc} path or format name must be specified" - raise UWError(msg) - return fmt + return FORMAT.yaml + return get_config_format(config, desc) def _realize_config_input_setup( @@ -333,11 +331,11 @@ def _validate_format(other_fmt_desc: str, other_fmt: str, input_fmt: str) -> Non Recognized file extensions are: {extensions} :param input_config: Input config source (None => read ``stdin``). -:param input_format: Format of the input config. +:param input_format: Input config format. :param update_config: Input config source (None => read ``stdin``). -:param update_format: Format of the update config. +:param update_format: Update config format. :param output_file: Output config destination (None => write to ``stdout``). -:param output_format: Format of the output config. +:param output_format: Output config format. :param key_path: Path of keys to the desired output block. :param values_needed: Report complete, missing, and template values. :param total: Require rendering of all Jinja2 variables/expressions. diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index 6f9756dbf..9c48bfa3d 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -105,13 +105,13 @@ def help_realize_config_simple(infn, infmt, tmpdir): # Tests -def test_compare_configs_good(compare_configs_assets, logged): +def test_compare_configs__good(compare_configs_assets, logged): _, a, b = compare_configs_assets assert tools.compare_configs(path1=a, format1=FORMAT.yaml, path2=b, format2=FORMAT.yaml) assert logged(".*", regex=True) -def test_compare_configs_changed_value(compare_configs_assets, logged): +def test_compare_configs__changed_value(compare_configs_assets, logged): d, a, b = compare_configs_assets d["baz"]["qux"] = 11 with writable(b) as f: @@ -138,7 +138,7 @@ def test_compare_configs_changed_value(compare_configs_assets, logged): assert logged(line) -def test_compare_configs_missing_key(compare_configs_assets, logged): +def test_compare_configs__missing_key(compare_configs_assets, logged): d, a, b = compare_configs_assets del d["baz"] with writable(b) as f: @@ -163,7 +163,7 @@ def test_compare_configs_missing_key(compare_configs_assets, logged): assert logged(line) -def test_compare_configs_bad_format(logged): +def test_compare_configs__bad_format(logged): assert not tools.compare_configs( path1=Path("/not/used"), format1="jpg", @@ -174,7 +174,7 @@ def test_compare_configs_bad_format(logged): assert logged(msg) -def test_config_check_depths_realize_fail(realize_config_testobj): +def test_config_check_depths_realize__fail(realize_config_testobj): depthin = depth(realize_config_testobj.data) with raises(UWConfigError) as e: tools.config_check_depths_realize( @@ -183,7 +183,7 @@ def test_config_check_depths_realize_fail(realize_config_testobj): assert f"Cannot realize depth-{depthin} config to type-'ini' config" in str(e.value) -def test_config_check_depths_update_fail(realize_config_testobj): +def test_config_check_depths_update__fail(realize_config_testobj): depthin = depth(realize_config_testobj.data) with raises(UWConfigError) as e: tools.config_check_depths_update( @@ -206,12 +206,12 @@ def test_config_tools_format_to_config(cfgtype, fmt): assert tools.format_to_config(fmt) is cfgtype -def test_config_tools_format_to_config_fail(): +def test_config_tools_format_to_config__fail(): with raises(UWConfigError): tools.format_to_config("no-such-config-type") -def test_realize_config_conversion_cfg_to_yaml(tmp_path): +def test_realize_config__conversion_cfg_to_yaml(tmp_path): """ Test that a .cfg file can be used to create a YAML object. """ @@ -231,7 +231,7 @@ def test_realize_config_conversion_cfg_to_yaml(tmp_path): assert outfile.read_text()[-1] == "\n" -def test_realize_config_depth_mismatch_to_ini(realize_config_yaml_input): +def test_realize_config__depth_mismatch_to_ini(realize_config_yaml_input): with raises(UWConfigError): tools.realize_config( input_config=realize_config_yaml_input, @@ -240,7 +240,7 @@ def test_realize_config_depth_mismatch_to_ini(realize_config_yaml_input): ) -def test_realize_config_depth_mismatch_to_sh(realize_config_yaml_input): +def test_realize_config__depth_mismatch_to_sh(realize_config_yaml_input): with raises(UWConfigError): tools.realize_config( input_config=realize_config_yaml_input, @@ -249,7 +249,7 @@ def test_realize_config_depth_mismatch_to_sh(realize_config_yaml_input): ) -def test_realize_config_double_tag_flat(tmp_path): +def test_realize_config__double_tag_flat(tmp_path): config = """ a: 1 b: 2 @@ -265,7 +265,7 @@ def test_realize_config_double_tag_flat(tmp_path): help_realize_config_double_tag(config, expected, tmp_path) -def test_realize_config_double_tag_nest(tmp_path): +def test_realize_config__double_tag_nest(tmp_path): config = """ a: 1.0 b: 2.0 @@ -283,7 +283,7 @@ def test_realize_config_double_tag_nest(tmp_path): help_realize_config_double_tag(config, expected, tmp_path) -def test_realize_config_double_tag_nest_forward_reference(tmp_path): +def test_realize_config__double_tag_nest_forward_reference(tmp_path): config = """ a: true b: false @@ -301,7 +301,7 @@ def test_realize_config_double_tag_nest_forward_reference(tmp_path): help_realize_config_double_tag(config, expected, tmp_path) -def test_realize_config_dry_run(logged): +def test_realize_config__dry_run(logged): """ Test that providing a YAML base file with a dry-run flag will print an YAML config file. """ @@ -317,7 +317,7 @@ def test_realize_config_dry_run(logged): assert logged(str(yaml_config), multiline=True) -def test_realize_config_field_table(tmp_path): +def test_realize_config__field_table(tmp_path): """ Test reading a YAML config object and generating a field file table. """ @@ -338,7 +338,7 @@ def test_realize_config_field_table(tmp_path): assert line1 in line2 -def test_realize_config_fmt2fmt_nml2nml(tmp_path): +def test_realize_config__fmt2fmt_nml2nml(tmp_path): """ Test that providing a namelist base input file and a config file will create and update namelist config file. @@ -346,7 +346,7 @@ def test_realize_config_fmt2fmt_nml2nml(tmp_path): help_realize_config_fmt2fmt("simple.nml", FORMAT.nml, "simple2.nml", FORMAT.nml, tmp_path) -def test_realize_config_fmt2fmt_ini2ini(tmp_path): +def test_realize_config__fmt2fmt_ini2ini(tmp_path): """ Test that providing an INI base input file and an INI config file will create and update INI config file. @@ -354,7 +354,7 @@ def test_realize_config_fmt2fmt_ini2ini(tmp_path): help_realize_config_fmt2fmt("simple.ini", FORMAT.ini, "simple2.ini", FORMAT.ini, tmp_path) -def test_realize_config_fmt2fmt_yaml2yaml(tmp_path): +def test_realize_config__fmt2fmt_yaml2yaml(tmp_path): """ Test that providing a YAML base input file and a YAML config file will create and update YAML config file. @@ -364,7 +364,7 @@ def test_realize_config_fmt2fmt_yaml2yaml(tmp_path): ) -def test_realize_config_incompatible_file_type(): +def test_realize_config__incompatible_file_type(): """ Test that providing an incompatible file type for input base file will return print statement. """ @@ -376,7 +376,7 @@ def test_realize_config_incompatible_file_type(): ) -def test_realize_config_output_file_format(tmp_path): +def test_realize_config__output_file_format(tmp_path): """ Test that output_format overrides bad output_file extension. """ @@ -390,7 +390,7 @@ def test_realize_config_output_file_format(tmp_path): assert compare_files(outfile, infile) -def test_realize_config_remove_nml_to_nml(tmp_path): +def test_realize_config__remove_nml_to_nml(tmp_path): input_config = NMLConfig({"constants": {"pi": 3.141, "e": 2.718}}) s = """ constants: @@ -408,7 +408,7 @@ def test_realize_config_remove_nml_to_nml(tmp_path): assert f90nml.read(output_file) == {"constants": {"pi": 3.141}} -def test_realize_config_remove_yaml_to_yaml_scalar(tmp_path): +def test_realize_config__remove_yaml_to_yaml_scalar(tmp_path): input_config = YAMLConfig({"a": {"b": {"c": 11, "d": 22, "e": 33}}}) s = """ a: @@ -424,7 +424,7 @@ def test_realize_config_remove_yaml_to_yaml_scalar(tmp_path): ) == {"a": {"b": {"c": 11, "e": 33}}} -def test_realize_config_remove_yaml_to_yaml_subtree(tmp_path): +def test_realize_config__remove_yaml_to_yaml_subtree(tmp_path): input_config = YAMLConfig(yaml.safe_load("a: {b: {c: 11, d: 22, e: 33}}")) s = """ a: @@ -439,7 +439,7 @@ def test_realize_config_remove_yaml_to_yaml_subtree(tmp_path): ) == {"a": {}} -def test_realize_config_scalar_value(capsys): +def test_realize_config__scalar_value(capsys): stdinproxy.cache_clear() tools.realize_config( input_config=YAMLConfig(config={"foo": {"bar": "baz"}}), @@ -449,35 +449,35 @@ def test_realize_config_scalar_value(capsys): assert capsys.readouterr().out.strip() == "baz" -def test_realize_config_simple_ini(tmp_path): +def test_realize_config__simple_ini(tmp_path): """ Test that providing an INI file with necessary settings will create an INI config file. """ help_realize_config_simple("simple.ini", FORMAT.ini, tmp_path) -def test_realize_config_simple_namelist(tmp_path): +def test_realize_config__simple_namelist(tmp_path): """ Test that providing a namelist file with necessary settings will create a namelist config file. """ help_realize_config_simple("simple.nml", FORMAT.nml, tmp_path) -def test_realize_config_simple_sh(tmp_path): +def test_realize_config__simple_sh(tmp_path): """ Test that providing an sh file with necessary settings will create an sh config file. """ help_realize_config_simple("simple.sh", FORMAT.sh, tmp_path) -def test_realize_config_simple_yaml(tmp_path): +def test_realize_config__simple_yaml(tmp_path): """ Test that providing a YAML base file with necessary settings will create a YAML config file. """ help_realize_config_simple("simple2.yaml", FORMAT.yaml, tmp_path) -def test_realize_config_single_dereference(capsys, tmp_path): +def test_realize_config__single_dereference(capsys, tmp_path): input_config = tmp_path / "a.yaml" update_config = tmp_path / "b.yaml" with writable(input_config) as f: @@ -500,25 +500,37 @@ def test_realize_config_single_dereference(capsys, tmp_path): assert actual == dedent(expected).strip() -def test_realize_config_update_bad_format(tmp_path): +def test_realize_config__total_fail(): + with raises(UWConfigError) as e: + tools.realize_config( + input_config=YAMLConfig({"foo": "{{ bar }}"}), output_format=FORMAT.yaml, total=True + ) + assert str(e.value) == "Config could not be totally realized" + + +def test_realize_config__update_bad_format_defaults_to_yaml(capsys, tmp_path): input_config = tmp_path / "a.yaml" update_config = tmp_path / "b.clj" with writable(input_config) as f: yaml.dump({"1": "a", "2": "{{ deref }}", "3": "{{ temporalis }}", "deref": "b"}, f) with writable(update_config) as f: yaml.dump({"2": "b", "temporalis": "c"}, f) - with raises(UWError) as e: - tools.realize_config( - input_config=input_config, - update_config=update_config, - output_format=FORMAT.yaml, - dry_run=True, - ) - msg = f"Cannot deduce format of '{update_config}' from unknown extension 'clj'" - assert msg in str(e.value) + tools.realize_config( + input_config=input_config, + update_config=update_config, + output_format=FORMAT.yaml, + ) + expected = """ + '1': a + '2': b + '3': c + deref: b + temporalis: c + """ + assert capsys.readouterr().out.strip() == dedent(expected).strip() -def test_realize_config_update_none(capsys, tmp_path): +def test_realize_config__update_none(capsys, tmp_path): path = tmp_path / "a.yaml" with writable(path) as f: yaml.dump({"1": "a", "2": "{{ deref }}", "3": "{{ temporalis }}", "deref": "b"}, f) @@ -537,15 +549,7 @@ def test_realize_config_update_none(capsys, tmp_path): assert actual == dedent(expected).strip() -def test_realize_config_total_fail(): - with raises(UWConfigError) as e: - tools.realize_config( - input_config=YAMLConfig({"foo": "{{ bar }}"}), output_format=FORMAT.yaml, total=True - ) - assert str(e.value) == "Config could not be totally realized" - - -def test_realize_config_values_needed_ini(logged): +def test_realize_config__values_needed_ini(logged): """ Test that the values_needed flag logs keys completed and keys containing unrendered Jinja2 variables/expressions. @@ -577,7 +581,7 @@ def test_realize_config_values_needed_ini(logged): assert logged(dedent(expected), multiline=True) -def test_realize_config_values_needed_yaml(logged): +def test_realize_config__values_needed_yaml(logged): """ Test that the values_needed flag logs keys completed and keys containing unrendered Jinja2 variables/expressions. @@ -608,65 +612,64 @@ def test_realize_config_values_needed_yaml(logged): assert logged(dedent(expected), multiline=True) -def test_walk_key_path_fail_bad_key_path(): +def test_walk_key_path(): + expected = ({"c": "cherry"}, "a.b") + assert tools.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "b"]) == expected + + +def test_walk_key_path__fail_bad_key_path(): with raises(UWError) as e: tools.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "x"]) assert str(e.value) == "Bad config path: a.x" -def test_walk_key_path_fail_bad_leaf_value(): +def test_walk_key_path__fail_bad_leaf_value(): with raises(UWError) as e: tools.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "b", "c"]) assert str(e.value) == "Value at a.b.c must be a dictionary" -def test_walk_key_path_pass(): - expected = ({"c": "cherry"}, "a.b") - assert tools.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "b"]) == expected - - -def test__ensure_format_bad_no_path_no_format(): - with raises(UWError) as e: - tools._ensure_format(desc="foo") - assert str(e.value) == "Either foo path or format name must be specified" +def test__ensure_format__no_path_no_format(logged): + assert tools._ensure_format(desc="foo") == FORMAT.yaml + assert logged(f"Treating foo config as '{FORMAT.yaml}'") -def test__ensure_format_config_obj(): +def test__ensure_format__config_obj(): config = NMLConfig({"nl": {"n": 42}}) assert tools._ensure_format(desc="foo", config=config) == FORMAT.nml -def test__ensure_format_dict_explicit(): +def test__ensure_format__dict_explicit(): assert tools._ensure_format(desc="foo", fmt=FORMAT.yaml, config={}) == FORMAT.yaml -def test__ensure_format_dict_implicit(): +def test__ensure_format__dict_implicit(): assert tools._ensure_format(desc="foo", config={}) == FORMAT.yaml -def test__ensure_format_deduced(): +def test__ensure_format__deduced(): assert tools._ensure_format(desc="foo", config=Path("/some/config.nml")) == FORMAT.nml -def test__ensure_format_explicitly_specified_no_path(): +def test__ensure_format__explicitly_specified_no_path(): assert tools._ensure_format(desc="foo", fmt=FORMAT.ini) == FORMAT.ini -def test__ensure_format_explicitly_specified_with_path(): +def test__ensure_format__explicitly_specified_with_path(): assert ( tools._ensure_format(desc="foo", fmt=FORMAT.ini, config=Path("/some/config.yaml")) == FORMAT.ini ) -def test__realize_config_input_setup_ini_cfgobj(): +def test__realize_config_input_setup__ini_cfgobj(): data = {"section": {"foo": "bar"}} cfgobj = INIConfig(config=data) input_obj = tools._realize_config_input_setup(input_config=cfgobj) assert input_obj.data == data -def test__realize_config_input_setup_ini_file(tmp_path): +def test__realize_config_input_setup__ini_file(tmp_path): data = """ [section] foo = bar @@ -677,7 +680,7 @@ def test__realize_config_input_setup_ini_file(tmp_path): assert input_obj.data == {"section": {"foo": "bar"}} -def test__realize_config_input_setup_ini_stdin(logged): +def test__realize_config_input_setup__ini_stdin(logged): data = """ [section] foo = bar @@ -693,14 +696,14 @@ def test__realize_config_input_setup_ini_stdin(logged): assert logged("Reading input from stdin") -def test__realize_config_input_setup_nml_cfgobj(): +def test__realize_config_input_setup__nml_cfgobj(): data = {"nl": {"pi": 3.14}} cfgobj = NMLConfig(config=data) input_obj = tools._realize_config_input_setup(input_config=cfgobj) assert input_obj.data == data -def test__realize_config_input_setup_nml_file(tmp_path): +def test__realize_config_input_setup__nml_file(tmp_path): data = """ &nl pi = 3.14 @@ -712,7 +715,7 @@ def test__realize_config_input_setup_nml_file(tmp_path): assert input_obj["nl"]["pi"] == 3.14 -def test__realize_config_input_setup_nml_stdin(logged): +def test__realize_config_input_setup__nml_stdin(logged): data = """ &nl pi = 3.14 @@ -728,14 +731,14 @@ def test__realize_config_input_setup_nml_stdin(logged): assert logged("Reading input from stdin") -def test__realize_config_input_setup_sh_cfgobj(): +def test__realize_config_input_setup__sh_cfgobj(): data = {"foo": "bar"} cfgobj = SHConfig(config=data) input_obj = tools._realize_config_input_setup(input_config=cfgobj) assert input_obj.data == data -def test__realize_config_input_setup_sh_file(tmp_path): +def test__realize_config_input_setup__sh_file(tmp_path): data = """ foo=bar """ @@ -745,7 +748,7 @@ def test__realize_config_input_setup_sh_file(tmp_path): assert input_obj.data == {"foo": "bar"} -def test__realize_config_input_setup_sh_stdin(logged): +def test__realize_config_input_setup__sh_stdin(logged): data = """ foo=bar """ @@ -759,14 +762,14 @@ def test__realize_config_input_setup_sh_stdin(logged): assert logged("Reading input from stdin") -def test__realize_config_input_setup_yaml_cfgobj(): +def test__realize_config_input_setup__yaml_cfgobj(): data = {"foo": "bar"} cfgobj = YAMLConfig(config=data) input_obj = tools._realize_config_input_setup(input_config=cfgobj) assert input_obj.data == data -def test__realize_config_input_setup_yaml_file(tmp_path): +def test__realize_config_input_setup__yaml_file(tmp_path): data = """ foo: bar """ @@ -776,7 +779,7 @@ def test__realize_config_input_setup_yaml_file(tmp_path): assert input_obj.data == {"foo": "bar"} -def test__realize_config_input_setup_yaml_stdin(logged): +def test__realize_config_input_setup__yaml_stdin(logged): data = """ foo: bar """ @@ -799,14 +802,14 @@ def test__realize_config_output_setup(logged, tmp_path): assert logged(f"Writing output to {output_file}") -def test__realize_config_update_cfgobj(realize_config_testobj): +def test__realize_config_update__cfgobj(realize_config_testobj): assert realize_config_testobj[1][2][3] == 42 update_config = YAMLConfig(config={1: {2: {3: 43}}}) o = tools._realize_config_update(input_obj=realize_config_testobj, update_config=update_config) assert o[1][2][3] == 43 -def test__realize_config_update_stdin(logged, realize_config_testobj): +def test__realize_config_update__stdin(logged, realize_config_testobj): stdinproxy.cache_clear() assert realize_config_testobj[1][2][3] == 42 with StringIO() as sio: @@ -820,11 +823,11 @@ def test__realize_config_update_stdin(logged, realize_config_testobj): assert logged("Reading update from stdin") -def test__realize_config_update_noop(realize_config_testobj): +def test__realize_config_update__noop(realize_config_testobj): assert tools._realize_config_update(input_obj=realize_config_testobj) == realize_config_testobj -def test__realize_config_update_file(realize_config_testobj, tmp_path): +def test__realize_config_update__file(realize_config_testobj, tmp_path): assert realize_config_testobj[1][2][3] == 42 values = {1: {2: {3: 43}}} update_config = tmp_path / "config.yaml" @@ -843,7 +846,7 @@ def test__realize_config_values_needed(logged, tmp_path): assert logged("Keys with unrendered Jinja2 variables/expressions:\n 2", multiline=True) -def test__realize_config_values_needed_negative_results(logged, tmp_path): +def test__realize_config_values_needed__negative_results(logged, tmp_path): path = tmp_path / "a.yaml" with writable(path) as f: yaml.dump({}, f) diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index baf6d0ca5..98d7efb73 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -16,7 +16,7 @@ from uwtools import cli from uwtools.cli import STR from uwtools.exceptions import UWConfigRealizeError, UWError, UWTemplateRenderError -from uwtools.utils.file import FORMAT +from uwtools.strings import FORMAT # Helpers @@ -193,7 +193,7 @@ def test_cli__dispatch_execute(utc): @mark.parametrize( - "vals", + ("file_arg", "format_arg"), [ (STR.path1, STR.fmt1), (STR.path2, STR.fmt2), @@ -202,17 +202,13 @@ def test_cli__dispatch_execute(utc): (STR.valsfile, STR.valsfmt), ], ) -def test_cli__check_file_vs_format_fail(capsys, vals): - # When reading/writing from/to stdin/stdout, the data format must be specified, since there is - # no filename to deduce it from. - file_arg, format_arg = vals - args = dict(file_arg=None, format_arg=None) - with raises(SystemExit): - cli._check_file_vs_format(file_arg=file_arg, format_arg=format_arg, args=args) - assert ( - "Specify %s when %s is not specified" % (cli._switch(format_arg), cli._switch(file_arg)) - in capsys.readouterr().err - ) +def test_cli__check_file_vs_format_fail(file_arg, format_arg): + # When reading/writing from/to stdin/stdout, the data format is assumed to be YAML when not + # otherwise specified. + args = {file_arg: None, format_arg: None} + actual = cli._check_file_vs_format(file_arg=file_arg, format_arg=format_arg, args=args) + expected = {**args, format_arg: FORMAT.yaml} + assert actual == expected def test_cli__check_file_vs_format_pass_explicit(): @@ -238,9 +234,8 @@ def test_cli__check_file_vs_format_pass_implicit(fmt): def test_cli__check_template_render_vals_args_implicit_fail(): # The values-file format cannot be deduced from the filename. args = {STR.valsfile: "a.jpg"} - with raises(UWError) as e: - cli._check_template_render_vals_args(args) - assert "Cannot deduce format" in str(e.value) + expected = {"values_file": "a.jpg", "values_format": FORMAT.yaml} + assert cli._check_template_render_vals_args(args) == expected def test_cli__check_template_render_vals_args_implicit_pass(): @@ -263,23 +258,19 @@ def test_cli__check_template_render_vals_args_noop_explicit_valsfmt(): @mark.parametrize( - ("fmt", "fn", "ok"), + ("expected", "fmt", "fn"), [ - (None, "update.txt", False), - ("yaml", "udpate.txt", True), - (None, "update.yaml", True), - ("yaml", "update.yaml", True), - ("jpg", "udpate.yaml", True), + (FORMAT.yaml, None, "a.yaml"), # yaml correctly deduced + (FORMAT.yaml, "yaml", "a.nml"), # explicit yaml overrides deduced nml + (FORMAT.yaml, "yaml", "a.txt"), # explicit yaml overrides unrecognized txt + (FORMAT.nml, None, "a.nml"), # nml correctly deduced + (FORMAT.ini, None, "a.ini"), # ini correctly deduced + (FORMAT.sh, None, "a.sh"), # sh correctly deduced ], ) -def test_cli__check_update(fmt, fn, ok): +def test_cli__check_update(expected, fmt, fn): args = {STR.updatefile: fn, STR.updatefmt: fmt} - if ok: - assert cli._check_update(args) == args - else: - with raises(UWError) as e: - cli._check_update(args) - assert "Cannot deduce format" in str(e.value) + assert cli._check_update(args) == {STR.updatefile: fn, STR.updatefmt: expected} def test_cli__check_verbosity_fail(capsys): diff --git a/src/uwtools/tests/utils/test_file.py b/src/uwtools/tests/utils/test_file.py index 5c71e133b..c912c8ca7 100644 --- a/src/uwtools/tests/utils/test_file.py +++ b/src/uwtools/tests/utils/test_file.py @@ -9,7 +9,7 @@ from pytest import fixture, mark, raises -from uwtools.exceptions import UWError +from uwtools.strings import FORMAT from uwtools.utils import file @@ -71,13 +71,12 @@ def test__stdinproxy(): "yml": "yaml", }.items(), ) -def test_get_file_format(ext, file_type): - assert file.get_file_format(Path(f"a.{ext}")) == file_type +def test_get_config_format(ext, file_type): + assert file.get_config_format(Path(f"a.{ext}")) == file_type -def test_get_file_format_unrecognized(): - with raises(UWError): - file.get_file_format(Path("a.jpg")) +def test_get_config_format_unrecognized(): + assert file.get_config_format(Path("a.jpg")) == FORMAT.yaml def test_path_if_it_exists(tmp_path): diff --git a/src/uwtools/utils/file.py b/src/uwtools/utils/file.py index 81cf34588..a8c5f1623 100644 --- a/src/uwtools/utils/file.py +++ b/src/uwtools/utils/file.py @@ -12,7 +12,7 @@ from pathlib import Path from typing import IO, TYPE_CHECKING, Any -from uwtools.exceptions import UWError +from uwtools.logging import log from uwtools.strings import FORMAT if TYPE_CHECKING: @@ -45,20 +45,22 @@ def _stdinproxy() -> StdinProxy: return StdinProxy() -def get_file_format(path: Path) -> str: +def get_config_format(path: str | Path | None, desc: str | None = None) -> str: """ - Return a standardized file format name given a path/filename. + Return a standardized config format name for a path/filename, defaulting to 'yaml'. :param path: A path or filename. + :param desc: A description of the config. :return: One of a set of supported file-format names. - :raises: ValueError if the path/filename suffix is unrecognized. """ - suffix = Path(path).suffix.replace(".", "") - try: - return FORMAT.formats()[suffix] - except KeyError: - msg = f"Cannot deduce format of '{path}' from unknown extension '{suffix}'" - raise UWError(msg) from None + default = FORMAT.yaml + if path: + path = Path(path) + suffix = path.suffix.replace(".", "") + return FORMAT.formats().get(suffix, default) + desc = f"{desc} config" if desc else str(path) if path else "config" + log.debug("Treating %s as '%s'", desc, default) + return default def path_if_it_exists(path: str) -> str: