From 99a26c503afe9a29c225d7976b075b2607ed837c Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 11 Jan 2025 19:45:18 +0100 Subject: [PATCH 01/77] changed some names --- tested/dsl/schema-strict.json | 32 ++++++++++++++++++-------------- tested/dsl/schema.json | 32 ++++++++++++++++++-------------- tested/dsl/translate_parser.py | 18 +++++++++--------- 3 files changed, 45 insertions(+), 37 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 8ec453be..8ab5f251 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -39,7 +39,7 @@ } ], "properties" : { - "files" : { + "in_files" : { "description" : "A list of files used in the test suite.", "type" : "array", "items" : { @@ -97,7 +97,7 @@ "tab" ], "properties" : { - "files" : { + "in_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -149,7 +149,7 @@ "unit" ], "properties" : { - "files" : { + "in_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -229,7 +229,7 @@ "testcases" ], "properties" : { - "files" : { + "in_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -251,7 +251,7 @@ "script" ], "properties" : { - "files" : { + "in_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -335,7 +335,7 @@ } ] }, - "files" : { + "in_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -353,7 +353,7 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, - "file": { + "out_files": { "description" : "Expected files generated by the submission.", "$ref" : "#/definitions/fileOutputChannel" }, @@ -431,7 +431,7 @@ } ] }, - "files" : { + "in_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -449,6 +449,10 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "out_files": { + "description" : "Expected files generated by the submission.", + "$ref" : "#/definitions/fileOutputChannel" + }, "exit_code" : { "type" : "integer", "description" : "Expected exit code for the run" @@ -493,14 +497,14 @@ "description" : "A file used in the test suite.", "required" : [ "name", - "url" + "path" ], "properties" : { "name" : { "type" : "string", "description" : "The filename, including the file extension." }, - "url" : { + "path" : { "type" : "string", "format" : "uri", "description" : "Relative path to the file in the `description` folder of an exercise." @@ -578,15 +582,15 @@ "type" : "object", "description" : "Built-in oracle for files.", "required" : [ - "content", - "location" + "path_expected", + "path_generated" ], "properties" : { - "content" : { + "path_expected" : { "type" : "string", "description" : "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { + "path_generated" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." }, diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 92478b6a..665c447c 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -39,7 +39,7 @@ } ], "properties" : { - "files" : { + "in_files" : { "description" : "A list of files used in the test suite.", "type" : "array", "items" : { @@ -97,7 +97,7 @@ "tab" ], "properties" : { - "files" : { + "in_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -149,7 +149,7 @@ "unit" ], "properties" : { - "files" : { + "in_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -229,7 +229,7 @@ "testcases" ], "properties" : { - "files" : { + "in_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -251,7 +251,7 @@ "script" ], "properties" : { - "files" : { + "in_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -335,7 +335,7 @@ } ] }, - "files" : { + "in_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -353,6 +353,10 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "out_files": { + "description" : "Expected files generated by the submission.", + "$ref" : "#/definitions/fileOutputChannel" + }, "exit_code" : { "type" : "integer", "description" : "Expected exit code for the run" @@ -427,7 +431,7 @@ } ] }, - "files" : { + "in_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -445,7 +449,7 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, - "file": { + "out_files": { "description" : "Expected files generated by the submission.", "$ref" : "#/definitions/fileOutputChannel" }, @@ -493,14 +497,14 @@ "description" : "A file used in the test suite.", "required" : [ "name", - "url" + "path" ], "properties" : { "name" : { "type" : "string", "description" : "The filename, including the file extension." }, - "url" : { + "path" : { "type" : "string", "format" : "uri", "description" : "Relative path to the file in the `description` folder of an exercise." @@ -578,15 +582,15 @@ "type" : "object", "description" : "Built-in oracle for files.", "required" : [ - "content", - "location" + "path_expected", + "path_generated" ], "properties" : { - "content" : { + "path_expected" : { "type" : "string", "description" : "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { + "path_generated" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." }, diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 0aa48574..4c5b78f4 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -254,9 +254,9 @@ def deepen_context(self, new_level: YamlDict | None) -> "DslContext": return self the_files = self.files - if "files" in new_level: - assert isinstance(new_level["files"], list) - additional_files = {_convert_file(f) for f in new_level["files"]} + if "in_files" in new_level: + assert isinstance(new_level["in_files"], list) + additional_files = {_convert_file(f) for f in new_level["in_files"]} the_files = list(set(self.files) | additional_files) the_config = self.config @@ -395,8 +395,8 @@ def _convert_value(value: YamlObject) -> Value: def _convert_file(link_file: YamlDict) -> FileUrl: assert isinstance(link_file["name"], str) - assert isinstance(link_file["url"], str) - return FileUrl(name=link_file["name"], url=link_file["url"]) + assert isinstance(link_file["path"], str) + return FileUrl(name=link_file["name"], url=link_file["path"]) def _convert_evaluation_function(stream: dict) -> EvaluationFunction: @@ -479,8 +479,8 @@ def _convert_file_output_channel( ) -> FileOutputChannel: assert isinstance(stream, dict) - expected = str(stream["content"]) - actual = str(stream["location"]) + expected = str(stream["path_expected"]) + actual = str(stream["path_generated"]) if "oracle" not in stream or stream["oracle"] == "builtin": config = context.merge_inheritable_with_specific_config(stream, config_name) @@ -606,8 +606,8 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: if (stdout := testcase.get("stdout")) is not None: output.stdout = _convert_text_output_channel(stdout, context, "stdout") - if (file := testcase.get("file")) is not None: - output.file = _convert_file_output_channel(file, context, "file") + if (file := testcase.get("out_files")) is not None: + output.file = _convert_file_output_channel(file, context, "out_files") if (stderr := testcase.get("stderr")) is not None: output.stderr = _convert_text_output_channel(stderr, context, "stderr") if (exception := testcase.get("exception")) is not None: From 667fd20a78a223dd69e33d1e41c19ef986814e11 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 11 Jan 2025 20:18:32 +0100 Subject: [PATCH 02/77] fixed some tests --- tested/dsl/translate_parser.py | 2 +- tested/judge/evaluation.py | 2 +- tested/languages/generation.py | 2 +- tested/testsuite.py | 2 +- tests/test_dsl_yaml.py | 16 ++++++++-------- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 4c5b78f4..68f376e8 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -396,7 +396,7 @@ def _convert_value(value: YamlObject) -> Value: def _convert_file(link_file: YamlDict) -> FileUrl: assert isinstance(link_file["name"], str) assert isinstance(link_file["path"], str) - return FileUrl(name=link_file["name"], url=link_file["path"]) + return FileUrl(name=link_file["name"], path=link_file["path"]) def _convert_evaluation_function(stream: dict) -> EvaluationFunction: diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index 44b2571a..93ba3d57 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -364,7 +364,7 @@ def evaluate_context_results( def _link_files_message(link_files: Collection[FileUrl]) -> AppendMessage: link_list = ", ".join( - f'' + f'' f'{html.escape(link_file.name)}' for link_file in link_files ) diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 7c7d883b..846a9659 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -192,7 +192,7 @@ def get_readable_input( def replace_link(match: Match) -> str: filename = match.group() the_file = url_map[filename] - the_url = urllib.parse.quote(the_file.url) + the_url = urllib.parse.quote(the_file.path) the_replacement = ( f'{filename}' ) diff --git a/tested/testsuite.py b/tested/testsuite.py index 253044dd..64635708 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -528,7 +528,7 @@ def get_functions(self) -> Iterable[FunctionCall]: @define(frozen=True) class FileUrl: - url: str + path: str name: str diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 3caef72d..3c305c29 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -750,10 +750,10 @@ def test_file_custom_check_correct(): contexts: - testcases: - statement: 'test()' - file: - content: "test/hallo.txt" + out_files: + path_expected: "test/hallo.txt" oracle: "custom_check" - location: "test.txt" + path_generated: "test.txt" name: "evaluate_test" file: "test.py" """ @@ -1174,9 +1174,9 @@ def test_files_are_propagated(): - tab: "Config ctx" files: - name: "test" - url: "test.md" + path: "test.md" - name: "two" - url: "two.md" + path: "two.md" testcases: - arguments: [ '-a', '2.125', '1.212' ] stdout: "3.34" @@ -1184,7 +1184,7 @@ def test_files_are_propagated(): stdout: "3.337" files: - name: "test" - url: "twooo.md" + path: "twooo.md" """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) @@ -1193,8 +1193,8 @@ def test_files_are_propagated(): testcases0, testcases1 = ctx0.testcases, ctx1.testcases test0, test1 = testcases0[0], testcases1[0] assert set(test0.link_files) == { - FileUrl(name="test", url="test.md"), - FileUrl(name="two", url="two.md"), + FileUrl(name="test", path="test.md"), + FileUrl(name="two", path="two.md"), } From 4a079705bed4b157ac0321868d7c9c9332b050f7 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 12 Jan 2025 11:34:16 +0100 Subject: [PATCH 03/77] fixed a test --- .../exercises/echo-function-file-output/evaluation/one.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/exercises/echo-function-file-output/evaluation/one.yaml b/tests/exercises/echo-function-file-output/evaluation/one.yaml index a428bf27..0b549773 100644 --- a/tests/exercises/echo-function-file-output/evaluation/one.yaml +++ b/tests/exercises/echo-function-file-output/evaluation/one.yaml @@ -1,7 +1,7 @@ - tab: "Test" testcases: - statement: echo_function("result.txt", "Hallo") - file: - content: "contents.txt" - location: "result.txt" + out_files: + path_expected: "contents.txt" + path_generated: "result.txt" oracle: builtin From e834e9f45d4b3aa8693424490446976549523fc4 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 12 Jan 2025 12:59:29 +0100 Subject: [PATCH 04/77] have something that works for stdin --- tested/dsl/schema-strict.json | 89 +++++++++++++++++++++++++++++----- tested/dsl/schema.json | 89 +++++++++++++++++++++++++++++----- tested/dsl/translate_parser.py | 10 ++-- 3 files changed, 161 insertions(+), 27 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 8ab5f251..41195e33 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -276,11 +276,27 @@ }, "stdin" : { "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean" + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required" : [ + "path" + ], + "properties" : { + "path": { + "type": "string", + "description": "Path to the file that contains the stdin" + } + } + } ] }, "arguments" : { @@ -372,11 +388,27 @@ }, "stdin" : { "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean" + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required" : [ + "path" + ], + "properties" : { + "path": { + "type": "string", + "description": "Path to the file that contains the stdin" + } + } + } ] }, "arguments" : { @@ -519,10 +551,23 @@ { "type" : "object", "description" : "Built-in oracle for text values.", - "required" : [ - "data" + "oneOf": [ + { + "required" : [ + "data" + ] + }, + { + "required" : [ + "path" + ] + } ], "properties" : { + "path": { + "type" : "string", + "description" : "The path to the file containing the text value." + }, "data" : { "$ref" : "#/definitions/textualType" }, @@ -537,12 +582,32 @@ { "type" : "object", "description" : "Custom oracle for text values.", + "oneOf": [ + { + "required" : [ + "oracle", + "file", + "data" + ] + }, + { + "required" : [ + "oracle", + "file", + "path" + ] + } + ], "required" : [ "oracle", "file", "data" ], "properties" : { + "path": { + "type" : "string", + "description" : "The path to the file containing the text value." + }, "data" : { "$ref" : "#/definitions/textualType" }, diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 665c447c..146db555 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -276,11 +276,27 @@ }, "stdin" : { "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean" + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required" : [ + "path" + ], + "properties" : { + "path": { + "type": "string", + "description": "Path to the file that contains the stdin" + } + } + } ] }, "arguments" : { @@ -372,11 +388,27 @@ }, "stdin" : { "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean" + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required" : [ + "path" + ], + "properties" : { + "path": { + "type": "string", + "description": "Path to the file that contains the stdin" + } + } + } ] }, "arguments" : { @@ -519,10 +551,23 @@ { "type" : "object", "description" : "Built-in oracle for text values.", - "required" : [ - "data" + "oneOf": [ + { + "required" : [ + "data" + ] + }, + { + "required" : [ + "path" + ] + } ], "properties" : { + "path": { + "type" : "string", + "description" : "The path to the file containing the text value." + }, "data" : { "$ref" : "#/definitions/textualType" }, @@ -537,12 +582,32 @@ { "type" : "object", "description" : "Custom oracle for text values.", + "oneOf": [ + { + "required" : [ + "oracle", + "file", + "data" + ] + }, + { + "required" : [ + "oracle", + "file", + "path" + ] + } + ], "required" : [ "oracle", "file", "data" ], "properties" : { + "path": { + "type" : "string", + "description" : "The path to the file containing the text value." + }, "data" : { "$ref" : "#/definitions/textualType" }, diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 68f376e8..5e08b940 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -67,7 +67,7 @@ TextBuiltin, TextData, TextOutputChannel, - ValueOutputChannel, + ValueOutputChannel, TextChannelType, ) from tested.utils import get_args, recursive_dict_merge @@ -590,8 +590,12 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: return_channel = IgnoredChannel.IGNORED if "statement" in testcase else None else: if "stdin" in testcase: - assert isinstance(testcase["stdin"], str) - stdin = TextData(data=_ensure_trailing_newline(testcase["stdin"])) + if isinstance(testcase["stdin"], dict): + path = testcase["stdin"].get("path") + stdin = TextData(data=path, type=TextChannelType.FILE) + else: + assert isinstance(testcase["stdin"], str) + stdin = TextData(data=_ensure_trailing_newline(testcase["stdin"])) else: stdin = EmptyChannel.NONE arguments = testcase.get("arguments", []) From 791836e7d14aa92d2b68f5e0c4752f4d1943c002 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 12 Jan 2025 13:38:40 +0100 Subject: [PATCH 05/77] have something working for stdout/stderr --- tested/dsl/translate_parser.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 5e08b940..49da09c4 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -445,16 +445,22 @@ def _convert_text_output_channel( ) -> TextOutputChannel: # Get the config applicable to this level. # Either attempt to get it from an object, or using the inherited options as is. + path = None + if isinstance(stream, str): config = context.config.get(config_name, dict()) raw_data = stream else: assert isinstance(stream, dict) - config = context.merge_inheritable_with_specific_config(stream, config_name) - raw_data = str(stream["data"]) + if (path := stream.get("path")) is not None: + config = context.config.get(config_name, dict()) + raw_data = path + else: + config = context.merge_inheritable_with_specific_config(stream, config_name) + raw_data = str(stream["data"]) # Normalize the data if necessary. - if config.get("normalizeTrailingNewlines", True): + if config.get("normalizeTrailingNewlines", True) and path is None: data = _ensure_trailing_newline(raw_data) else: data = raw_data @@ -464,10 +470,18 @@ def _convert_text_output_channel( else: assert isinstance(stream, dict) if "oracle" not in stream or stream["oracle"] == "builtin": + if path is not None: + return TextOutputChannel( + data=data, oracle=GenericTextOracle(options=config), type=TextChannelType.FILE + ) return TextOutputChannel( data=data, oracle=GenericTextOracle(options=config) ) elif stream["oracle"] == "custom_check": + if path is not None: + return TextOutputChannel( + data=data, oracle=_convert_custom_check_oracle(stream), type=TextChannelType.FILE + ) return TextOutputChannel( data=data, oracle=_convert_custom_check_oracle(stream) ) @@ -592,6 +606,7 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: if "stdin" in testcase: if isinstance(testcase["stdin"], dict): path = testcase["stdin"].get("path") + assert isinstance(path, str) stdin = TextData(data=path, type=TextChannelType.FILE) else: assert isinstance(testcase["stdin"], str) From ef29394778c9543ba6b64f3131f452e69295f2ba Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 12 Jan 2025 13:51:39 +0100 Subject: [PATCH 06/77] fixed all tests --- tested/dsl/schema-strict.json | 8 ++++---- tested/dsl/schema.json | 8 ++++---- tests/test_dsl_yaml.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 41195e33..7a2c29fd 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -672,19 +672,19 @@ "description" : "Custom oracle for file values.", "required" : [ "oracle", - "content", - "location", + "path_expected", + "path_generated", "file" ], "properties" : { "oracle" : { "const" : "custom_check" }, - "content" : { + "path_expected" : { "type" : "string", "description" : "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { + "path_generated" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." }, diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 146db555..8ff51f80 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -672,19 +672,19 @@ "description" : "Custom oracle for file values.", "required" : [ "oracle", - "content", - "location", + "path_expected", + "path_generated", "file" ], "properties" : { "oracle" : { "const" : "custom_check" }, - "content" : { + "path_expected" : { "type" : "string", "description" : "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { + "path_generated" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." }, diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 3c305c29..1f5041e4 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -1172,7 +1172,7 @@ def test_additional_properties_are_not_allowed(): def test_files_are_propagated(): yaml_str = """ - tab: "Config ctx" - files: + in_files: - name: "test" path: "test.md" - name: "two" @@ -1182,7 +1182,7 @@ def test_files_are_propagated(): stdout: "3.34" - arguments: [ '-a', '2.125', '1.212' ] stdout: "3.337" - files: + in_files: - name: "test" path: "twooo.md" """ From a00f09e7a12ade424830367d9dc0e193304fc9c2 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 12 Jan 2025 15:51:32 +0100 Subject: [PATCH 07/77] fixed some small issues --- tested/dsl/translate_parser.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 49da09c4..222dd7a2 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -65,9 +65,10 @@ Tab, Testcase, TextBuiltin, + TextChannelType, TextData, TextOutputChannel, - ValueOutputChannel, TextChannelType, + ValueOutputChannel, ) from tested.utils import get_args, recursive_dict_merge @@ -454,7 +455,7 @@ def _convert_text_output_channel( assert isinstance(stream, dict) if (path := stream.get("path")) is not None: config = context.config.get(config_name, dict()) - raw_data = path + raw_data = str(path) else: config = context.merge_inheritable_with_specific_config(stream, config_name) raw_data = str(stream["data"]) @@ -472,7 +473,9 @@ def _convert_text_output_channel( if "oracle" not in stream or stream["oracle"] == "builtin": if path is not None: return TextOutputChannel( - data=data, oracle=GenericTextOracle(options=config), type=TextChannelType.FILE + data=data, + oracle=GenericTextOracle(options=config), + type=TextChannelType.FILE, ) return TextOutputChannel( data=data, oracle=GenericTextOracle(options=config) @@ -480,7 +483,9 @@ def _convert_text_output_channel( elif stream["oracle"] == "custom_check": if path is not None: return TextOutputChannel( - data=data, oracle=_convert_custom_check_oracle(stream), type=TextChannelType.FILE + data=data, + oracle=_convert_custom_check_oracle(stream), + type=TextChannelType.FILE, ) return TextOutputChannel( data=data, oracle=_convert_custom_check_oracle(stream) From ee2123c84e5cc52b8318c2768c5d1a60b4bbff99 Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 31 Jan 2025 19:22:34 +0100 Subject: [PATCH 08/77] made something that can be used to generate and evaluate multiple files. --- tested/dsl/schema-strict.json | 95 +++++++++++++++++++++++++--------- tested/dsl/schema.json | 95 +++++++++++++++++++++++++--------- tested/dsl/translate_parser.py | 48 ++++++++++------- tested/oracles/text.py | 50 ++++++++++-------- tested/testsuite.py | 8 +-- 5 files changed, 201 insertions(+), 95 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 7a2c29fd..cdfc3c3d 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -39,7 +39,7 @@ } ], "properties" : { - "in_files" : { + "input_files" : { "description" : "A list of files used in the test suite.", "type" : "array", "items" : { @@ -97,7 +97,7 @@ "tab" ], "properties" : { - "in_files" : { + "input_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -149,7 +149,7 @@ "unit" ], "properties" : { - "in_files" : { + "input_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -229,7 +229,7 @@ "testcases" ], "properties" : { - "in_files" : { + "input_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -251,7 +251,7 @@ "script" ], "properties" : { - "in_files" : { + "input_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -351,7 +351,7 @@ } ] }, - "in_files" : { + "input_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -369,7 +369,7 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, - "out_files": { + "output_files": { "description" : "Expected files generated by the submission.", "$ref" : "#/definitions/fileOutputChannel" }, @@ -463,7 +463,7 @@ } ] }, - "in_files" : { + "input_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -481,7 +481,7 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, - "out_files": { + "output_files": { "description" : "Expected files generated by the submission.", "$ref" : "#/definitions/fileOutputChannel" }, @@ -643,21 +643,53 @@ }, "fileOutputChannel": { "anyOf" : [ + { + "type" : "array", + "description" : "Built-in oracle for files.", + "items" : { + "type" : "object", + "required" : [ + "path_expected", + "path_generated" + ], + "properties" : { + "path_expected" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "path_generated" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } + }, { "type" : "object", "description" : "Built-in oracle for files.", "required" : [ - "path_expected", - "path_generated" + "data" ], "properties" : { - "path_expected" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." - }, - "path_generated" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "data": { + "type": "array", + "items" : { + "type" : "object", + "required" : [ + "path_expected", + "path_generated" + ], + "properties" : { + "path_expected" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "path_generated" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } }, "oracle" : { "const" : "builtin" @@ -672,6 +704,7 @@ "description" : "Custom oracle for file values.", "required" : [ "oracle", + "data", "path_expected", "path_generated", "file" @@ -680,13 +713,25 @@ "oracle" : { "const" : "custom_check" }, - "path_expected" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." - }, - "path_generated" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "data": { + "type": "array", + "items" : { + "type" : "object", + "required" : [ + "path_expected", + "path_generated" + ], + "properties" : { + "path_expected" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "path_generated" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } }, "file" : { "type" : "string", diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 8ff51f80..c5b68e5f 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -39,7 +39,7 @@ } ], "properties" : { - "in_files" : { + "input_files" : { "description" : "A list of files used in the test suite.", "type" : "array", "items" : { @@ -97,7 +97,7 @@ "tab" ], "properties" : { - "in_files" : { + "input_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -149,7 +149,7 @@ "unit" ], "properties" : { - "in_files" : { + "input_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -229,7 +229,7 @@ "testcases" ], "properties" : { - "in_files" : { + "input_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -251,7 +251,7 @@ "script" ], "properties" : { - "in_files" : { + "input_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -351,7 +351,7 @@ } ] }, - "in_files" : { + "input_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -369,7 +369,7 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, - "out_files": { + "output_files": { "description" : "Expected files generated by the submission.", "$ref" : "#/definitions/fileOutputChannel" }, @@ -463,7 +463,7 @@ } ] }, - "in_files" : { + "input_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -481,7 +481,7 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, - "out_files": { + "output_files": { "description" : "Expected files generated by the submission.", "$ref" : "#/definitions/fileOutputChannel" }, @@ -643,21 +643,53 @@ }, "fileOutputChannel": { "anyOf" : [ + { + "type" : "array", + "description" : "Built-in oracle for files.", + "items" : { + "type" : "object", + "required" : [ + "path_expected", + "path_generated" + ], + "properties" : { + "path_expected" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "path_generated" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } + }, { "type" : "object", "description" : "Built-in oracle for files.", "required" : [ - "path_expected", - "path_generated" + "data" ], "properties" : { - "path_expected" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." - }, - "path_generated" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "data": { + "type": "array", + "items" : { + "type" : "object", + "required" : [ + "path_expected", + "path_generated" + ], + "properties" : { + "path_expected" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "path_generated" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } }, "oracle" : { "const" : "builtin" @@ -672,6 +704,7 @@ "description" : "Custom oracle for file values.", "required" : [ "oracle", + "data", "path_expected", "path_generated", "file" @@ -680,13 +713,25 @@ "oracle" : { "const" : "custom_check" }, - "path_expected" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." - }, - "path_generated" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "data": { + "type": "array", + "items" : { + "type" : "object", + "required" : [ + "path_expected", + "path_generated" + ], + "properties" : { + "path_expected" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "path_generated" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } }, "file" : { "type" : "string", diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 222dd7a2..d69c285a 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -255,9 +255,9 @@ def deepen_context(self, new_level: YamlDict | None) -> "DslContext": return self the_files = self.files - if "in_files" in new_level: - assert isinstance(new_level["in_files"], list) - additional_files = {_convert_file(f) for f in new_level["in_files"]} + if "input_files" in new_level: + assert isinstance(new_level["input_files"], list) + additional_files = {_convert_file(f) for f in new_level["input_files"]} the_files = list(set(self.files) | additional_files) the_config = self.config @@ -496,13 +496,21 @@ def _convert_text_output_channel( def _convert_file_output_channel( stream: YamlObject, context: DslContext, config_name: str ) -> FileOutputChannel: - assert isinstance(stream, dict) - - expected = str(stream["path_expected"]) - actual = str(stream["path_generated"]) - - if "oracle" not in stream or stream["oracle"] == "builtin": - config = context.merge_inheritable_with_specific_config(stream, config_name) + expected = [] + actual = [] + data = stream + if isinstance(stream, dict): + data = stream["data"] + assert isinstance(data, list) + + for item in stream: + assert isinstance(item, dict) + expected.append(str(item["path_expected"])) + actual.append(str(item["path_generated"])) + + if not isinstance(stream, dict) or "oracle" not in stream or stream["oracle"] == "builtin": + level = {} if not isinstance(stream, dict) else stream + config = context.merge_inheritable_with_specific_config(level, config_name) if "mode" not in config: config["mode"] = "full" @@ -511,16 +519,16 @@ def _convert_file_output_channel( "line", ), f"The file oracle only supports modes full and line, not {config['mode']}" return FileOutputChannel( - expected_path=expected, - actual_path=actual, - oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config), - ) + expected_path=expected, + actual_path=actual, + oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config), + ) elif stream["oracle"] == "custom_check": return FileOutputChannel( - expected_path=expected, - actual_path=actual, - oracle=_convert_custom_check_oracle(stream), - ) + expected_path=expected, + actual_path=actual, + oracle=_convert_custom_check_oracle(stream), + ) raise TypeError(f"Unknown file oracle type: {stream['oracle']}") @@ -630,8 +638,8 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: if (stdout := testcase.get("stdout")) is not None: output.stdout = _convert_text_output_channel(stdout, context, "stdout") - if (file := testcase.get("out_files")) is not None: - output.file = _convert_file_output_channel(file, context, "out_files") + if (file := testcase.get("output_files")) is not None: + output.file = _convert_file_output_channel(file, context, "output_files") if (stderr := testcase.get("stderr")) is not None: output.stderr = _convert_text_output_channel(stderr, context, "stderr") if (exception := testcase.get("exception")) is not None: diff --git a/tested/oracles/text.py b/tested/oracles/text.py index 6724e18b..a06b1375 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -134,28 +134,34 @@ def evaluate_file( messages=[message], ) - expected_path = f"{config.bundle.config.resources}/{channel.expected_path}" - - try: - with open(expected_path, "r") as file: - expected = file.read() - except FileNotFoundError: - raise ValueError(f"File {expected_path} not found in resources.") - - actual_path = config.context_dir / channel.actual_path - - try: - with open(str(actual_path), "r") as file: - actual = file.read() - except FileNotFoundError: - return OracleResult( - result=StatusMessage( - enum=Status.RUNTIME_ERROR, - human=get_i18n_string("oracles.text.file.not-found"), - ), - readable_expected=expected, - readable_actual="", - ) + expected = [] + for expected_path in channel.expected_path: + expected_path = f"{config.bundle.config.resources}/{expected_path}" + + try: + with open(expected_path, "r") as file: + expected.append(file.read()) + except FileNotFoundError: + raise ValueError(f"File {expected_path} not found in resources.") + expected = '\n'.join(expected) + + actual = [] + for actual_path in channel.actual_path: + actual_path = config.context_dir / actual_path + + try: + with open(str(actual_path), "r") as file: + actual.append(file.read()) + except FileNotFoundError: + return OracleResult( + result=StatusMessage( + enum=Status.RUNTIME_ERROR, + human=get_i18n_string("oracles.text.file.not-found"), + ), + readable_expected=expected, + readable_actual="", + ) + actual = '\n'.join(actual) if options["mode"] == "full": return compare_text(options, expected, actual) diff --git a/tested/testsuite.py b/tested/testsuite.py index 64635708..19afa31d 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -279,8 +279,8 @@ class TextOutputChannel(TextData): class FileOutputChannel(WithFeatures): """Describes the output for files.""" - expected_path: str # Path to the file to compare to. - actual_path: str # Path to the generated file (by the user code) + expected_path: list[str] # Paths to the file to compare to. + actual_path: list[str] # Paths to the generated file (by the user code) oracle: GenericTextOracle | CustomCheckOracle = field( factory=lambda: GenericTextOracle(name=TextBuiltin.FILE) ) @@ -289,9 +289,11 @@ def get_used_features(self) -> FeatureSet: return NOTHING def get_data_as_string(self, resources: Path) -> str: + file_content = [] file_path = _resolve_path(resources, self.expected_path) with open(file_path, "r") as file: - return file.read() + file_content.append(file.read()) + return '\n'.join(file_content) @fallback_field(get_converter(), {"evaluator": "oracle"}) From 1b3b892f6f081714961c53287bb5429c2c39706a Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 1 Feb 2025 16:16:52 +0100 Subject: [PATCH 09/77] Updated output files to be more consistent and do better check. Also make stdin more consistent --- tested/dsl/schema-strict.json | 84 ++++++++------------------- tested/dsl/schema.json | 84 ++++++++------------------- tested/dsl/translate_parser.py | 43 +++++++++----- tested/oracles/text.py | 103 +++++++++++++++++++-------------- tested/testsuite.py | 15 +++-- tests/test_dsl_yaml.py | 4 +- tests/test_oracles_builtin.py | 12 ++-- tests/test_suite.py | 4 +- 8 files changed, 159 insertions(+), 190 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index cdfc3c3d..53049ac8 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -276,27 +276,11 @@ }, "stdin" : { "description" : "Stdin for this context", - "oneOf": [ - { - "type" : [ - "string", - "number", - "integer", - "boolean" - ] - }, - { - "type": "object", - "required" : [ - "path" - ], - "properties" : { - "path": { - "type": "string", - "description": "Path to the file that contains the stdin" - } - } - } + "type" : [ + "string", + "number", + "integer", + "boolean" ] }, "arguments" : { @@ -388,27 +372,11 @@ }, "stdin" : { "description" : "Stdin for this context", - "oneOf": [ - { - "type" : [ - "string", - "number", - "integer", - "boolean" - ] - }, - { - "type": "object", - "required" : [ - "path" - ], - "properties" : { - "path": { - "type": "string", - "description": "Path to the file that contains the stdin" - } - } - } + "type" : [ + "string", + "number", + "integer", + "boolean" ] }, "arguments" : { @@ -649,15 +617,15 @@ "items" : { "type" : "object", "required" : [ - "path_expected", - "path_generated" + "content", + "path" ], "properties" : { - "path_expected" : { + "content" : { "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "description" : "Path or expected content for the file, relative to the evaluation directory." }, - "path_generated" : { + "path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } @@ -676,15 +644,15 @@ "items" : { "type" : "object", "required" : [ - "path_expected", - "path_generated" + "content", + "path" ], "properties" : { - "path_expected" : { + "content" : { "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "description" : "Path or expected content for the file, relative to the evaluation directory." }, - "path_generated" : { + "path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } @@ -705,8 +673,6 @@ "required" : [ "oracle", "data", - "path_expected", - "path_generated", "file" ], "properties" : { @@ -718,15 +684,15 @@ "items" : { "type" : "object", "required" : [ - "path_expected", - "path_generated" + "content", + "path" ], "properties" : { - "path_expected" : { + "content" : { "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "description" : "Path or expected content for the file, relative to the evaluation directory." }, - "path_generated" : { + "path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index c5b68e5f..eee3bbf4 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -276,27 +276,11 @@ }, "stdin" : { "description" : "Stdin for this context", - "oneOf": [ - { - "type" : [ - "string", - "number", - "integer", - "boolean" - ] - }, - { - "type": "object", - "required" : [ - "path" - ], - "properties" : { - "path": { - "type": "string", - "description": "Path to the file that contains the stdin" - } - } - } + "type" : [ + "string", + "number", + "integer", + "boolean" ] }, "arguments" : { @@ -388,27 +372,11 @@ }, "stdin" : { "description" : "Stdin for this context", - "oneOf": [ - { - "type" : [ - "string", - "number", - "integer", - "boolean" - ] - }, - { - "type": "object", - "required" : [ - "path" - ], - "properties" : { - "path": { - "type": "string", - "description": "Path to the file that contains the stdin" - } - } - } + "type" : [ + "string", + "number", + "integer", + "boolean" ] }, "arguments" : { @@ -649,15 +617,15 @@ "items" : { "type" : "object", "required" : [ - "path_expected", - "path_generated" + "content", + "path" ], "properties" : { - "path_expected" : { + "content" : { "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "description" : "Path or expected content for the file, relative to the evaluation directory." }, - "path_generated" : { + "path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } @@ -676,15 +644,15 @@ "items" : { "type" : "object", "required" : [ - "path_expected", - "path_generated" + "content", + "path" ], "properties" : { - "path_expected" : { + "content" : { "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "description" : "Path or expected content for the file, relative to the evaluation directory." }, - "path_generated" : { + "path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } @@ -705,8 +673,6 @@ "required" : [ "oracle", "data", - "path_expected", - "path_generated", "file" ], "properties" : { @@ -718,15 +684,15 @@ "items" : { "type" : "object", "required" : [ - "path_expected", - "path_generated" + "content", + "path" ], "properties" : { - "path_expected" : { + "content" : { "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "description" : "Path or expected content for the file, relative to the evaluation directory." }, - "path_generated" : { + "path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index d69c285a..af147075 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -84,6 +84,8 @@ class TestedType: class ExpressionString(str): pass +class PathString(str): + pass class ReturnOracle(dict): pass @@ -91,7 +93,7 @@ class ReturnOracle(dict): OptionDict = dict[str, int | bool] YamlObject = ( - YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle + YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle | PathString ) @@ -130,6 +132,11 @@ def _expression_string(loader: yaml.Loader, node: yaml.Node) -> ExpressionString assert isinstance(result, str), f"An expression must be a string, got {result}" return ExpressionString(result) +def _path_string(loader: yaml.Loader, node: yaml.Node) -> PathString: + result = _parse_yaml_value(loader, node) + assert isinstance(result, str), f"A path must be a string, got {result}" + return PathString(result) + def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle: result = _parse_yaml_value(loader, node) @@ -149,6 +156,7 @@ def _parse_yaml(yaml_stream: str) -> YamlObject: yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader) yaml.add_constructor("!expression", _expression_string, loader) yaml.add_constructor("!oracle", _return_oracle, loader) + yaml.add_constructor("!path", _path_string, loader) try: return yaml.load(yaml_stream, loader) @@ -187,6 +195,9 @@ def is_oracle(_checker: TypeChecker, instance: Any) -> bool: def is_expression(_checker: TypeChecker, instance: Any) -> bool: return isinstance(instance, ExpressionString) +def is_path(_checker: TypeChecker, instance: Any) -> bool: + return isinstance(instance, PathString) + def test(value: object) -> bool: if not isinstance(value, str): @@ -208,7 +219,7 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator: original_validator: Type[Validator] = validator_for(schema_object) type_checker = original_validator.TYPE_CHECKER.redefine( "oracle", is_oracle - ).redefine("expression", is_expression) + ).redefine("expression", is_expression).redefine("path", is_path) format_checker = original_validator.FORMAT_CHECKER format_checker.checks("tested-dsl-expression", SyntaxError)(test) tested_validator = extend_validator(original_validator, type_checker=type_checker) @@ -496,7 +507,8 @@ def _convert_text_output_channel( def _convert_file_output_channel( stream: YamlObject, context: DslContext, config_name: str ) -> FileOutputChannel: - expected = [] + content_type = [] + content = [] actual = [] data = stream if isinstance(stream, dict): @@ -505,8 +517,13 @@ def _convert_file_output_channel( for item in stream: assert isinstance(item, dict) - expected.append(str(item["path_expected"])) - actual.append(str(item["path_generated"])) + if isinstance(item["content"], PathString): + content_type.append(TextChannelType.FILE) + else: + content_type.append(TextChannelType.TEXT) + + content.append(str(item["content"])) + actual.append(str(item["path"])) if not isinstance(stream, dict) or "oracle" not in stream or stream["oracle"] == "builtin": level = {} if not isinstance(stream, dict) else stream @@ -519,14 +536,16 @@ def _convert_file_output_channel( "line", ), f"The file oracle only supports modes full and line, not {config['mode']}" return FileOutputChannel( - expected_path=expected, - actual_path=actual, + content_type=content_type, + content=content, + path=actual, oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config), ) elif stream["oracle"] == "custom_check": return FileOutputChannel( - expected_path=expected, - actual_path=actual, + content_type=content_type, + content=content, + path=actual, oracle=_convert_custom_check_oracle(stream), ) raise TypeError(f"Unknown file oracle type: {stream['oracle']}") @@ -617,10 +636,8 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: return_channel = IgnoredChannel.IGNORED if "statement" in testcase else None else: if "stdin" in testcase: - if isinstance(testcase["stdin"], dict): - path = testcase["stdin"].get("path") - assert isinstance(path, str) - stdin = TextData(data=path, type=TextChannelType.FILE) + if isinstance(testcase["stdin"], PathString): + stdin = TextData(data=testcase["stdin"], type=TextChannelType.FILE) else: assert isinstance(testcase["stdin"], str) stdin = TextData(data=_ensure_trailing_newline(testcase["stdin"])) diff --git a/tested/oracles/text.py b/tested/oracles/text.py index a06b1375..cffd1e4f 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -8,7 +8,8 @@ from tested.dodona import Status, StatusMessage from tested.internationalization import get_i18n_string from tested.oracles.common import OracleConfig, OracleResult -from tested.testsuite import FileOutputChannel, OutputChannel, TextOutputChannel +from tested.testsuite import FileOutputChannel, OutputChannel, TextOutputChannel, \ + TextChannelType def _is_number(string: str) -> float | None: @@ -41,8 +42,7 @@ def _file_defaults(config: OracleConfig) -> dict: raise ValueError(f"Unknown mode for file oracle: {defaults['mode']}") return defaults - -def compare_text(options: dict[str, Any], expected: str, actual: str) -> OracleResult: +def _text_comparison(options: dict[str, Any], expected: str, actual: str) -> (bool, str): # Temporary variables that may modified by the evaluation options, # Don't modify the actual values, otherwise there maybe confusion with the # solution submitted by the student @@ -61,15 +61,15 @@ def compare_text(options: dict[str, Any], expected: str, actual: str) -> OracleR expected_float = float(expected_eval.strip()) if options["applyRounding"]: numbers = int(options["roundTo"]) - # noinspection PyUnboundLocalVariable actual_float = round(actual_float, numbers) expected_float = round(expected_float, numbers) - # noinspection PyUnboundLocalVariable - result = math.isclose(actual_float, expected_float) - expected = str(expected_float) - else: - result = actual_eval == expected_eval + return math.isclose(actual_float, expected_float), str(expected_float) + + return actual_eval == expected_eval, expected +def compare_text(options: dict[str, Any], expected: str, actual: str) -> OracleResult: + + result, expected = _text_comparison(options, expected, actual) return OracleResult( result=StatusMessage(enum=Status.CORRECT if result else Status.WRONG), readable_expected=str(expected), @@ -134,48 +134,63 @@ def evaluate_file( messages=[message], ) - expected = [] - for expected_path in channel.expected_path: - expected_path = f"{config.bundle.config.resources}/{expected_path}" - - try: - with open(expected_path, "r") as file: - expected.append(file.read()) - except FileNotFoundError: - raise ValueError(f"File {expected_path} not found in resources.") - expected = '\n'.join(expected) - actual = [] - for actual_path in channel.actual_path: - actual_path = config.context_dir / actual_path + expected = [] + file_not_found = False + for i in range(len(channel.content)): + actual_path = config.context_dir / channel.path[i] + + if channel.content_type[i] == TextChannelType.FILE: + expected_path = f"{config.bundle.config.resources}/{channel.content[i]}" + try: + with open(expected_path, "r") as file: + expected.append(file.read()) + except FileNotFoundError: + raise ValueError(f"File {expected_path} not found in resources.") + else: + expected.append(channel.content[i]) try: with open(str(actual_path), "r") as file: actual.append(file.read()) except FileNotFoundError: - return OracleResult( - result=StatusMessage( - enum=Status.RUNTIME_ERROR, - human=get_i18n_string("oracles.text.file.not-found"), - ), - readable_expected=expected, - readable_actual="", - ) - actual = '\n'.join(actual) + file_not_found = True + + actual_string = '\n'.join(actual) + expected_string = '\n'.join(expected) + if file_not_found: + return OracleResult( + result=StatusMessage( + enum=Status.RUNTIME_ERROR, + human=get_i18n_string("oracles.text.file.not-found"), + ), + readable_expected=expected_string, + readable_actual=actual_string, + ) + + result = True if options["mode"] == "full": - return compare_text(options, expected, actual) + for i in range(len(expected)): + expected_value = expected[i] + actual_value = actual[i] + new_result, expected[i] = _text_comparison(options, expected_value, actual_value) + result = result and new_result else: assert options["mode"] == "line" - strip_newlines = options.get("stripNewlines", False) - expected_lines = expected.splitlines(keepends=not strip_newlines) - actual_lines = actual.splitlines(keepends=not strip_newlines) - correct = len(actual_lines) == len(expected_lines) - for expected_line, actual_line in zip(expected_lines, actual_lines): - r = compare_text(options, expected_line, actual_line) - correct = correct and r.result.enum == Status.CORRECT - return OracleResult( - result=StatusMessage(enum=Status.CORRECT if correct else Status.WRONG), - readable_expected=expected, - readable_actual=actual, - ) + for i in range(len(expected)): + expected_value = expected[i] + actual_value = actual[i] + strip_newlines = options.get("stripNewlines", False) + expected_lines = expected_value.splitlines(keepends=not strip_newlines) + actual_lines = actual_value.splitlines(keepends=not strip_newlines) + correct = len(actual_lines) == len(expected_lines) + for expected_line, actual_line in zip(expected_lines, actual_lines): + new_result, _ = _text_comparison(options, expected_line, actual_line) + correct = correct and new_result + + return OracleResult( + result=StatusMessage(enum=Status.CORRECT if result else Status.WRONG), + readable_expected='\n'.join(expected), + readable_actual=actual_string, + ) diff --git a/tested/testsuite.py b/tested/testsuite.py index 19afa31d..6f9f7159 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -279,8 +279,9 @@ class TextOutputChannel(TextData): class FileOutputChannel(WithFeatures): """Describes the output for files.""" - expected_path: list[str] # Paths to the file to compare to. - actual_path: list[str] # Paths to the generated file (by the user code) + content_type: list[TextChannelType] # True is the actual content + content: list[str] # Paths to the file to compare to. + path: list[str] # Paths to the generated file (by the user code) oracle: GenericTextOracle | CustomCheckOracle = field( factory=lambda: GenericTextOracle(name=TextBuiltin.FILE) ) @@ -290,9 +291,13 @@ def get_used_features(self) -> FeatureSet: def get_data_as_string(self, resources: Path) -> str: file_content = [] - file_path = _resolve_path(resources, self.expected_path) - with open(file_path, "r") as file: - file_content.append(file.read()) + for i in range(len(self.content)): + if self.content_type[i] == TextChannelType.FILE: + file_path = _resolve_path(resources, self.content[i]) + with open(file_path, "r") as file: + file_content.append(file.read()) + else: + file_content.append(self.content[i]) return '\n'.join(file_content) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 1f5041e4..c82078d7 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -768,8 +768,8 @@ def test_file_custom_check_correct(): assert isinstance(test.input, FunctionCall) assert isinstance(test.output.file, FileOutputChannel) assert isinstance(test.output.file.oracle, CustomCheckOracle) - assert test.output.file.actual_path == "test.txt" - assert test.output.file.expected_path == "test/hallo.txt" + assert test.output.file.path == "test.txt" + assert test.output.file.content == "test/hallo.txt" oracle = test.output.file.oracle assert oracle.function.name == "evaluate_test" assert oracle.function.file == Path("test.py") diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index 7e21ffa0..70eb36f1 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -149,7 +149,7 @@ def test_file_oracle_full_wrong( mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) channel = FileOutputChannel( - expected_path="expected.txt", actual_path="expected.txt" + content="expected.txt", path="expected.txt" ) result = evaluate_file(config, channel, "") s.assert_called_once_with(ANY, "expected\nexpected", "actual\nactual") @@ -171,7 +171,7 @@ def test_file_oracle_full_correct( mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) channel = FileOutputChannel( - expected_path="expected.txt", actual_path="expected.txt" + content="expected.txt", path="expected.txt" ) result = evaluate_file(config, channel, "") s.assert_called_once_with(ANY, "expected\nexpected", "expected\nexpected") @@ -195,7 +195,7 @@ def test_file_oracle_line_wrong( mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) channel = FileOutputChannel( - expected_path="expected.txt", actual_path="expected.txt" + content="expected.txt", path="expected.txt" ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "actual") @@ -221,7 +221,7 @@ def test_file_oracle_line_correct( mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) channel = FileOutputChannel( - expected_path="expected.txt", actual_path="expected.txt" + content="expected.txt", path="expected.txt" ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "expected") @@ -247,7 +247,7 @@ def test_file_oracle_strip_lines_correct( mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) channel = FileOutputChannel( - expected_path="expected.txt", actual_path="expected.txt" + content="expected.txt", path="expected.txt" ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "expected") @@ -273,7 +273,7 @@ def test_file_oracle_dont_strip_lines_correct( mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) channel = FileOutputChannel( - expected_path="expected.txt", actual_path="expected.txt" + content="expected.txt", path="expected.txt" ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected\n", "expected\n") diff --git a/tests/test_suite.py b/tests/test_suite.py index d194691b..3f23a965 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -113,8 +113,8 @@ def test_file_show_expected_is_accepted(): } """ result = get_converter().loads(scheme, FileOutputChannel) - assert result.expected_path == "hallo" - assert result.actual_path == "hallo" + assert result.content == "hallo" + assert result.path == "hallo" def test_value_show_expected_is_accepted(): From 32bb7d9340252b428af43c506a715082cd94c857 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 1 Feb 2025 17:51:13 +0100 Subject: [PATCH 10/77] Made inlining for stdout and stderr possible and made everything more consitent --- tested/dsl/schema-strict.json | 19 ++++++------------- tested/dsl/schema.json | 19 ++++++------------- tested/dsl/translate_parser.py | 20 +++++++++++++++++--- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 53049ac8..fdc48254 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -527,14 +527,13 @@ }, { "required" : [ - "path" + "content" ] } ], "properties" : { - "path": { - "type" : "string", - "description" : "The path to the file containing the text value." + "content": { + "$ref" : "#/definitions/textualType" }, "data" : { "$ref" : "#/definitions/textualType" @@ -562,19 +561,13 @@ "required" : [ "oracle", "file", - "path" + "content" ] } ], - "required" : [ - "oracle", - "file", - "data" - ], "properties" : { - "path": { - "type" : "string", - "description" : "The path to the file containing the text value." + "content": { + "$ref" : "#/definitions/textualType" }, "data" : { "$ref" : "#/definitions/textualType" diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index eee3bbf4..02fbaff8 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -527,14 +527,13 @@ }, { "required" : [ - "path" + "content" ] } ], "properties" : { - "path": { - "type" : "string", - "description" : "The path to the file containing the text value." + "content": { + "$ref" : "#/definitions/textualType" }, "data" : { "$ref" : "#/definitions/textualType" @@ -562,19 +561,13 @@ "required" : [ "oracle", "file", - "path" + "content" ] } ], - "required" : [ - "oracle", - "file", - "data" - ], "properties" : { - "path": { - "type" : "string", - "description" : "The path to the file containing the text value." + "content": { + "$ref" : "#/definitions/textualType" }, "data" : { "$ref" : "#/definitions/textualType" diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index af147075..7c970ec1 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -461,12 +461,18 @@ def _convert_text_output_channel( if isinstance(stream, str): config = context.config.get(config_name, dict()) + if isinstance(stream, PathString): + path = stream raw_data = stream else: assert isinstance(stream, dict) - if (path := stream.get("path")) is not None: + data = stream.get("content", stream.get("data")) + assert data is not None + + if isinstance(data, PathString): + path = data config = context.config.get(config_name, dict()) - raw_data = str(path) + raw_data = data else: config = context.merge_inheritable_with_specific_config(stream, config_name) raw_data = str(stream["data"]) @@ -478,7 +484,15 @@ def _convert_text_output_channel( data = raw_data if isinstance(stream, str): - return TextOutputChannel(data=data, oracle=GenericTextOracle(options=config)) + if path is not None: + return TextOutputChannel( + data=data, + oracle=GenericTextOracle(options=config), + type=TextChannelType.FILE, + ) + return TextOutputChannel( + data=data, oracle=GenericTextOracle(options=config) + ) else: assert isinstance(stream, dict) if "oracle" not in stream or stream["oracle"] == "builtin": From 2afdd67abeaf866f758d4d269b28eba457ecfdba Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 1 Feb 2025 18:04:12 +0100 Subject: [PATCH 11/77] Fixed some minor issues --- tested/dsl/translate_parser.py | 55 ++++++++++++++++++++++------------ tested/oracles/text.py | 52 +++++++++++++++++++------------- tested/testsuite.py | 4 +-- tests/test_oracles_builtin.py | 24 ++++----------- 4 files changed, 75 insertions(+), 60 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 7c970ec1..4cd2d29a 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -84,16 +84,27 @@ class TestedType: class ExpressionString(str): pass + class PathString(str): pass + class ReturnOracle(dict): pass OptionDict = dict[str, int | bool] YamlObject = ( - YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle | PathString + YamlDict + | list + | bool + | float + | int + | str + | None + | ExpressionString + | ReturnOracle + | PathString ) @@ -132,6 +143,7 @@ def _expression_string(loader: yaml.Loader, node: yaml.Node) -> ExpressionString assert isinstance(result, str), f"An expression must be a string, got {result}" return ExpressionString(result) + def _path_string(loader: yaml.Loader, node: yaml.Node) -> PathString: result = _parse_yaml_value(loader, node) assert isinstance(result, str), f"A path must be a string, got {result}" @@ -195,6 +207,7 @@ def is_oracle(_checker: TypeChecker, instance: Any) -> bool: def is_expression(_checker: TypeChecker, instance: Any) -> bool: return isinstance(instance, ExpressionString) + def is_path(_checker: TypeChecker, instance: Any) -> bool: return isinstance(instance, PathString) @@ -217,9 +230,11 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator: schema_object = json.load(schema_file) original_validator: Type[Validator] = validator_for(schema_object) - type_checker = original_validator.TYPE_CHECKER.redefine( - "oracle", is_oracle - ).redefine("expression", is_expression).redefine("path", is_path) + type_checker = ( + original_validator.TYPE_CHECKER.redefine("oracle", is_oracle) + .redefine("expression", is_expression) + .redefine("path", is_path) + ) format_checker = original_validator.FORMAT_CHECKER format_checker.checks("tested-dsl-expression", SyntaxError)(test) tested_validator = extend_validator(original_validator, type_checker=type_checker) @@ -490,9 +505,7 @@ def _convert_text_output_channel( oracle=GenericTextOracle(options=config), type=TextChannelType.FILE, ) - return TextOutputChannel( - data=data, oracle=GenericTextOracle(options=config) - ) + return TextOutputChannel(data=data, oracle=GenericTextOracle(options=config)) else: assert isinstance(stream, dict) if "oracle" not in stream or stream["oracle"] == "builtin": @@ -529,7 +542,7 @@ def _convert_file_output_channel( data = stream["data"] assert isinstance(data, list) - for item in stream: + for item in data: assert isinstance(item, dict) if isinstance(item["content"], PathString): content_type.append(TextChannelType.FILE) @@ -539,7 +552,11 @@ def _convert_file_output_channel( content.append(str(item["content"])) actual.append(str(item["path"])) - if not isinstance(stream, dict) or "oracle" not in stream or stream["oracle"] == "builtin": + if ( + not isinstance(stream, dict) + or "oracle" not in stream + or stream["oracle"] == "builtin" + ): level = {} if not isinstance(stream, dict) else stream config = context.merge_inheritable_with_specific_config(level, config_name) if "mode" not in config: @@ -550,18 +567,18 @@ def _convert_file_output_channel( "line", ), f"The file oracle only supports modes full and line, not {config['mode']}" return FileOutputChannel( - content_type=content_type, - content=content, - path=actual, - oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config), - ) + content_type=content_type, + content=content, + path=actual, + oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config), + ) elif stream["oracle"] == "custom_check": return FileOutputChannel( - content_type=content_type, - content=content, - path=actual, - oracle=_convert_custom_check_oracle(stream), - ) + content_type=content_type, + content=content, + path=actual, + oracle=_convert_custom_check_oracle(stream), + ) raise TypeError(f"Unknown file oracle type: {stream['oracle']}") diff --git a/tested/oracles/text.py b/tested/oracles/text.py index cffd1e4f..266a8c7d 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -8,8 +8,12 @@ from tested.dodona import Status, StatusMessage from tested.internationalization import get_i18n_string from tested.oracles.common import OracleConfig, OracleResult -from tested.testsuite import FileOutputChannel, OutputChannel, TextOutputChannel, \ - TextChannelType +from tested.testsuite import ( + FileOutputChannel, + OutputChannel, + TextChannelType, + TextOutputChannel, +) def _is_number(string: str) -> float | None: @@ -42,7 +46,10 @@ def _file_defaults(config: OracleConfig) -> dict: raise ValueError(f"Unknown mode for file oracle: {defaults['mode']}") return defaults -def _text_comparison(options: dict[str, Any], expected: str, actual: str) -> (bool, str): + +def _text_comparison( + options: dict[str, Any], expected: str, actual: str +) -> tuple[bool, str]: # Temporary variables that may modified by the evaluation options, # Don't modify the actual values, otherwise there maybe confusion with the # solution submitted by the student @@ -67,6 +74,7 @@ def _text_comparison(options: dict[str, Any], expected: str, actual: str) -> (bo return actual_eval == expected_eval, expected + def compare_text(options: dict[str, Any], expected: str, actual: str) -> OracleResult: result, expected = _text_comparison(options, expected, actual) @@ -134,8 +142,8 @@ def evaluate_file( messages=[message], ) - actual = [] - expected = [] + actual_list = [] + expected_list = [] file_not_found = False for i in range(len(channel.content)): actual_path = config.context_dir / channel.path[i] @@ -144,43 +152,45 @@ def evaluate_file( expected_path = f"{config.bundle.config.resources}/{channel.content[i]}" try: with open(expected_path, "r") as file: - expected.append(file.read()) + expected_list.append(file.read()) except FileNotFoundError: raise ValueError(f"File {expected_path} not found in resources.") else: - expected.append(channel.content[i]) + expected_list.append(channel.content[i]) try: with open(str(actual_path), "r") as file: - actual.append(file.read()) + actual_list.append(file.read()) except FileNotFoundError: file_not_found = True - actual_string = '\n'.join(actual) - expected_string = '\n'.join(expected) + actual = "\n".join(actual_list) + expected = "\n".join(expected_list) if file_not_found: return OracleResult( result=StatusMessage( enum=Status.RUNTIME_ERROR, human=get_i18n_string("oracles.text.file.not-found"), ), - readable_expected=expected_string, - readable_actual=actual_string, + readable_expected=expected, + readable_actual=actual, ) result = True if options["mode"] == "full": - for i in range(len(expected)): - expected_value = expected[i] - actual_value = actual[i] - new_result, expected[i] = _text_comparison(options, expected_value, actual_value) + for i in range(len(expected_list)): + expected_value = expected_list[i] + actual_value = actual_list[i] + new_result, expected_list[i] = _text_comparison( + options, expected_value, actual_value + ) result = result and new_result else: assert options["mode"] == "line" - for i in range(len(expected)): - expected_value = expected[i] - actual_value = actual[i] + for i in range(len(expected_list)): + expected_value = expected_list[i] + actual_value = actual_list[i] strip_newlines = options.get("stripNewlines", False) expected_lines = expected_value.splitlines(keepends=not strip_newlines) actual_lines = actual_value.splitlines(keepends=not strip_newlines) @@ -191,6 +201,6 @@ def evaluate_file( return OracleResult( result=StatusMessage(enum=Status.CORRECT if result else Status.WRONG), - readable_expected='\n'.join(expected), - readable_actual=actual_string, + readable_expected="\n".join(expected_list), + readable_actual=actual, ) diff --git a/tested/testsuite.py b/tested/testsuite.py index 6f9f7159..0b90ca64 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -279,7 +279,7 @@ class TextOutputChannel(TextData): class FileOutputChannel(WithFeatures): """Describes the output for files.""" - content_type: list[TextChannelType] # True is the actual content + content_type: list[TextChannelType] # True is the actual content content: list[str] # Paths to the file to compare to. path: list[str] # Paths to the generated file (by the user code) oracle: GenericTextOracle | CustomCheckOracle = field( @@ -298,7 +298,7 @@ def get_data_as_string(self, resources: Path) -> str: file_content.append(file.read()) else: file_content.append(self.content[i]) - return '\n'.join(file_content) + return "\n".join(file_content) @fallback_field(get_converter(), {"evaluator": "oracle"}) diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index 70eb36f1..85e1a79a 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -148,9 +148,7 @@ def test_file_oracle_full_wrong( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - content="expected.txt", path="expected.txt" - ) + channel = FileOutputChannel(content="expected.txt", path="expected.txt") result = evaluate_file(config, channel, "") s.assert_called_once_with(ANY, "expected\nexpected", "actual\nactual") assert result.result.enum == Status.WRONG @@ -170,9 +168,7 @@ def test_file_oracle_full_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - content="expected.txt", path="expected.txt" - ) + channel = FileOutputChannel(content="expected.txt", path="expected.txt") result = evaluate_file(config, channel, "") s.assert_called_once_with(ANY, "expected\nexpected", "expected\nexpected") assert result.result.enum == Status.CORRECT @@ -194,9 +190,7 @@ def test_file_oracle_line_wrong( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - content="expected.txt", path="expected.txt" - ) + channel = FileOutputChannel(content="expected.txt", path="expected.txt") result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "actual") s.assert_any_call(ANY, "expected2", "actual2") @@ -220,9 +214,7 @@ def test_file_oracle_line_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - content="expected.txt", path="expected.txt" - ) + channel = FileOutputChannel(content="expected.txt", path="expected.txt") result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "expected") s.assert_any_call(ANY, "expected2", "expected2") @@ -246,9 +238,7 @@ def test_file_oracle_strip_lines_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - content="expected.txt", path="expected.txt" - ) + channel = FileOutputChannel(content="expected.txt", path="expected.txt") result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "expected") s.assert_any_call(ANY, "expected2", "expected2") @@ -272,9 +262,7 @@ def test_file_oracle_dont_strip_lines_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - content="expected.txt", path="expected.txt" - ) + channel = FileOutputChannel(content="expected.txt", path="expected.txt") result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected\n", "expected\n") s.assert_any_call(ANY, "expected2\n", "expected2\n") From e01ebdc07abbe93b5522c312515fbc7477e4e19a Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 1 Feb 2025 18:19:33 +0100 Subject: [PATCH 12/77] Fixed the dsl yaml tests --- tested/testsuite.py | 2 +- tests/test_dsl_yaml.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/tested/testsuite.py b/tested/testsuite.py index 0b90ca64..293a81d9 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -279,7 +279,7 @@ class TextOutputChannel(TextData): class FileOutputChannel(WithFeatures): """Describes the output for files.""" - content_type: list[TextChannelType] # True is the actual content + content_type: list[TextChannelType] content: list[str] # Paths to the file to compare to. path: list[str] # Paths to the generated file (by the user code) oracle: GenericTextOracle | CustomCheckOracle = field( diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index c82078d7..88d96611 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -750,10 +750,11 @@ def test_file_custom_check_correct(): contexts: - testcases: - statement: 'test()' - out_files: - path_expected: "test/hallo.txt" + output_files: + data: + - content: !path "test/hallo.txt" + path: "test.txt" oracle: "custom_check" - path_generated: "test.txt" name: "evaluate_test" file: "test.py" """ @@ -768,8 +769,8 @@ def test_file_custom_check_correct(): assert isinstance(test.input, FunctionCall) assert isinstance(test.output.file, FileOutputChannel) assert isinstance(test.output.file.oracle, CustomCheckOracle) - assert test.output.file.path == "test.txt" - assert test.output.file.content == "test/hallo.txt" + assert test.output.file.path[0] == "test.txt" + assert test.output.file.content[0] == "test/hallo.txt" oracle = test.output.file.oracle assert oracle.function.name == "evaluate_test" assert oracle.function.file == Path("test.py") @@ -1172,7 +1173,7 @@ def test_additional_properties_are_not_allowed(): def test_files_are_propagated(): yaml_str = """ - tab: "Config ctx" - in_files: + input_files: - name: "test" path: "test.md" - name: "two" @@ -1182,7 +1183,7 @@ def test_files_are_propagated(): stdout: "3.34" - arguments: [ '-a', '2.125', '1.212' ] stdout: "3.337" - in_files: + input_files: - name: "test" path: "twooo.md" """ From 2ce7892911697439aa6693f488c775a2b4ae266d Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 1 Feb 2025 18:39:10 +0100 Subject: [PATCH 13/77] Fixed buildin oracle tests --- tested/oracles/text.py | 6 ++++-- tests/test_oracles_builtin.py | 26 +++++++++++++------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/tested/oracles/text.py b/tested/oracles/text.py index 266a8c7d..3044152d 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -194,10 +194,12 @@ def evaluate_file( strip_newlines = options.get("stripNewlines", False) expected_lines = expected_value.splitlines(keepends=not strip_newlines) actual_lines = actual_value.splitlines(keepends=not strip_newlines) - correct = len(actual_lines) == len(expected_lines) + result = len(actual_lines) == len(expected_lines) for expected_line, actual_line in zip(expected_lines, actual_lines): + print(f"{expected_line}: {actual_line}") new_result, _ = _text_comparison(options, expected_line, actual_line) - correct = correct and new_result + print(f"{new_result}") + result = result and new_result return OracleResult( result=StatusMessage(enum=Status.CORRECT if result else Status.WRONG), diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index 85e1a79a..37c82240 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -28,7 +28,7 @@ Suite, SupportedLanguage, TextOutputChannel, - ValueOutputChannel, + ValueOutputChannel, TextChannelType, ) from tests.manual_utils import configuration @@ -140,7 +140,7 @@ def test_file_oracle_full_wrong( tmp_path: Path, pytestconfig: pytest.Config, mocker: MockerFixture ): config = oracle_config(tmp_path, pytestconfig, {"mode": "full"}) - s = mocker.spy(tested.oracles.text, name="compare_text") # type: ignore[reportAttributeAccessIssue] + s = mocker.spy(tested.oracles.text, name="_text_comparison") # type: ignore[reportAttributeAccessIssue] mock_files = [ mocker.mock_open(read_data=content).return_value for content in ["expected\nexpected", "actual\nactual"] @@ -148,7 +148,7 @@ def test_file_oracle_full_wrong( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel(content="expected.txt", path="expected.txt") + channel = FileOutputChannel(content=["expected.txt"], path=["expected.txt"], content_type=[TextChannelType.FILE]) result = evaluate_file(config, channel, "") s.assert_called_once_with(ANY, "expected\nexpected", "actual\nactual") assert result.result.enum == Status.WRONG @@ -160,7 +160,7 @@ def test_file_oracle_full_correct( tmp_path: Path, pytestconfig: pytest.Config, mocker: MockerFixture ): config = oracle_config(tmp_path, pytestconfig, {"mode": "full"}) - s = mocker.spy(tested.oracles.text, name="compare_text") # type: ignore[reportAttributeAccessIssue] + s = mocker.spy(tested.oracles.text, name="_text_comparison") # type: ignore[reportAttributeAccessIssue] mock_files = [ mocker.mock_open(read_data=content).return_value for content in ["expected\nexpected", "expected\nexpected"] @@ -168,7 +168,7 @@ def test_file_oracle_full_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel(content="expected.txt", path="expected.txt") + channel = FileOutputChannel(content=["expected.txt"], path=["expected.txt"], content_type=[TextChannelType.FILE]) result = evaluate_file(config, channel, "") s.assert_called_once_with(ANY, "expected\nexpected", "expected\nexpected") assert result.result.enum == Status.CORRECT @@ -182,7 +182,7 @@ def test_file_oracle_line_wrong( config = oracle_config( tmp_path, pytestconfig, {"mode": "line", "stripNewlines": True} ) - s = mocker.spy(tested.oracles.text, name="compare_text") # type: ignore[reportAttributeAccessIssue] + s = mocker.spy(tested.oracles.text, name="_text_comparison") # type: ignore[reportAttributeAccessIssue] mock_files = [ mocker.mock_open(read_data=content).return_value for content in ["expected\nexpected2", "actual\nactual2"] @@ -190,7 +190,7 @@ def test_file_oracle_line_wrong( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel(content="expected.txt", path="expected.txt") + channel = FileOutputChannel(content=["expected.txt"], path=["expected.txt"], content_type=[TextChannelType.FILE]) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "actual") s.assert_any_call(ANY, "expected2", "actual2") @@ -206,7 +206,7 @@ def test_file_oracle_line_correct( config = oracle_config( tmp_path, pytestconfig, {"mode": "line", "stripNewlines": True} ) - s = mocker.spy(tested.oracles.text, name="compare_text") # type: ignore[reportAttributeAccessIssue] + s = mocker.spy(tested.oracles.text, name="_text_comparison") # type: ignore[reportAttributeAccessIssue] mock_files = [ mocker.mock_open(read_data=content).return_value for content in ["expected\nexpected2", "expected\nexpected2"] @@ -214,7 +214,7 @@ def test_file_oracle_line_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel(content="expected.txt", path="expected.txt") + channel = FileOutputChannel(content=["expected.txt"], path=["expected.txt"], content_type=[TextChannelType.FILE]) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "expected") s.assert_any_call(ANY, "expected2", "expected2") @@ -230,7 +230,7 @@ def test_file_oracle_strip_lines_correct( config = oracle_config( tmp_path, pytestconfig, {"mode": "line", "stripNewlines": True} ) - s = mocker.spy(tested.oracles.text, name="compare_text") # type: ignore[reportAttributeAccessIssue] + s = mocker.spy(tested.oracles.text, name="_text_comparison") # type: ignore[reportAttributeAccessIssue] mock_files = [ mocker.mock_open(read_data=content).return_value for content in ["expected\nexpected2\n", "expected\nexpected2"] @@ -238,7 +238,7 @@ def test_file_oracle_strip_lines_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel(content="expected.txt", path="expected.txt") + channel = FileOutputChannel(content=["expected.txt"], path=["expected.txt"], content_type=[TextChannelType.FILE]) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "expected") s.assert_any_call(ANY, "expected2", "expected2") @@ -254,7 +254,7 @@ def test_file_oracle_dont_strip_lines_correct( config = oracle_config( tmp_path, pytestconfig, {"mode": "line", "stripNewlines": False} ) - s = mocker.spy(tested.oracles.text, name="compare_text") # type: ignore[reportAttributeAccessIssue] + s = mocker.spy(tested.oracles.text, name="_text_comparison") # type: ignore[reportAttributeAccessIssue] mock_files = [ mocker.mock_open(read_data=content).return_value for content in ["expected\nexpected2\n", "expected\nexpected2\n"] @@ -262,7 +262,7 @@ def test_file_oracle_dont_strip_lines_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel(content="expected.txt", path="expected.txt") + channel = FileOutputChannel(content=["expected.txt"], path=["expected.txt"], content_type=[TextChannelType.FILE]) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected\n", "expected\n") s.assert_any_call(ANY, "expected2\n", "expected2\n") From 3a62f6d42221b3256ce1b483693fa018908be7e2 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 1 Feb 2025 18:51:52 +0100 Subject: [PATCH 14/77] Fixed the io function --- .../echo-function-file-output/evaluation/one.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/exercises/echo-function-file-output/evaluation/one.yaml b/tests/exercises/echo-function-file-output/evaluation/one.yaml index 0b549773..32775331 100644 --- a/tests/exercises/echo-function-file-output/evaluation/one.yaml +++ b/tests/exercises/echo-function-file-output/evaluation/one.yaml @@ -1,7 +1,8 @@ - tab: "Test" testcases: - statement: echo_function("result.txt", "Hallo") - out_files: - path_expected: "contents.txt" - path_generated: "result.txt" + output_files: + data: + - content: !path "contents.txt" + path: "result.txt" oracle: builtin From a845fb2a67ecd65406363047ad5f2589e4334589 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 2 Feb 2025 13:47:42 +0100 Subject: [PATCH 15/77] Fixed linting issue --- tests/test_oracles_builtin.py | 39 ++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index 37c82240..005c2084 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -27,8 +27,9 @@ FileOutputChannel, Suite, SupportedLanguage, + TextChannelType, TextOutputChannel, - ValueOutputChannel, TextChannelType, + ValueOutputChannel, ) from tests.manual_utils import configuration @@ -148,7 +149,11 @@ def test_file_oracle_full_wrong( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel(content=["expected.txt"], path=["expected.txt"], content_type=[TextChannelType.FILE]) + channel = FileOutputChannel( + content=["expected.txt"], + path=["expected.txt"], + content_type=[TextChannelType.FILE], + ) result = evaluate_file(config, channel, "") s.assert_called_once_with(ANY, "expected\nexpected", "actual\nactual") assert result.result.enum == Status.WRONG @@ -168,7 +173,11 @@ def test_file_oracle_full_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel(content=["expected.txt"], path=["expected.txt"], content_type=[TextChannelType.FILE]) + channel = FileOutputChannel( + content=["expected.txt"], + path=["expected.txt"], + content_type=[TextChannelType.FILE], + ) result = evaluate_file(config, channel, "") s.assert_called_once_with(ANY, "expected\nexpected", "expected\nexpected") assert result.result.enum == Status.CORRECT @@ -190,7 +199,11 @@ def test_file_oracle_line_wrong( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel(content=["expected.txt"], path=["expected.txt"], content_type=[TextChannelType.FILE]) + channel = FileOutputChannel( + content=["expected.txt"], + path=["expected.txt"], + content_type=[TextChannelType.FILE], + ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "actual") s.assert_any_call(ANY, "expected2", "actual2") @@ -214,7 +227,11 @@ def test_file_oracle_line_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel(content=["expected.txt"], path=["expected.txt"], content_type=[TextChannelType.FILE]) + channel = FileOutputChannel( + content=["expected.txt"], + path=["expected.txt"], + content_type=[TextChannelType.FILE], + ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "expected") s.assert_any_call(ANY, "expected2", "expected2") @@ -238,7 +255,11 @@ def test_file_oracle_strip_lines_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel(content=["expected.txt"], path=["expected.txt"], content_type=[TextChannelType.FILE]) + channel = FileOutputChannel( + content=["expected.txt"], + path=["expected.txt"], + content_type=[TextChannelType.FILE], + ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "expected") s.assert_any_call(ANY, "expected2", "expected2") @@ -262,7 +283,11 @@ def test_file_oracle_dont_strip_lines_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel(content=["expected.txt"], path=["expected.txt"], content_type=[TextChannelType.FILE]) + channel = FileOutputChannel( + content=["expected.txt"], + path=["expected.txt"], + content_type=[TextChannelType.FILE], + ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected\n", "expected\n") s.assert_any_call(ANY, "expected2\n", "expected2\n") From e939f30a9c18ac06d1b5e1a0f252b4f9cd13f3a2 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 18 Feb 2025 20:53:49 +0100 Subject: [PATCH 16/77] fixed last tests --- tests/test_suite.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/test_suite.py b/tests/test_suite.py index 3f23a965..2241914b 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -12,7 +12,7 @@ MainInput, TextData, TextOutputChannel, - ValueOutputChannel, + ValueOutputChannel, TextChannelType, ) @@ -33,6 +33,9 @@ def test_text_output_is_compatible_oracle(): def test_file_output_is_compatible_oracle(): old_structure = { + "content_type": [TextChannelType.TEXT], + "content": ["some content"], + "path": ["output.py"], "evaluator": { "function": {"file": "evaluate.py"}, "type": "custom_check", @@ -108,13 +111,15 @@ def test_file_show_expected_is_accepted(): scheme = """ { "show_expected": true, - "expected_path": "hallo", - "actual_path": "hallo" + "content": ["hallo"], + "path": ["hallo"], + "content_type": ["text"] } """ result = get_converter().loads(scheme, FileOutputChannel) - assert result.content == "hallo" - assert result.path == "hallo" + + assert result.content == ["hallo"] + assert result.path == ["hallo"] def test_value_show_expected_is_accepted(): From 10bd41888532cd43cb717e413c4b841453ab7fa4 Mon Sep 17 00:00:00 2001 From: breblanc Date: Wed, 19 Feb 2025 18:37:12 +0100 Subject: [PATCH 17/77] Added an extra test --- tests/test_dsl_yaml.py | 37 +++++++++++++++++++++++++++++++++++++ tests/test_suite.py | 7 ++++--- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 88d96611..8223733e 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -38,6 +38,7 @@ LanguageLiterals, LanguageSpecificOracle, SupportedLanguage, + TextChannelType, TextOutputChannel, ValueOutputChannel, parse_test_suite, @@ -69,9 +70,45 @@ def test_parse_one_tab_ctx(): tc = context.testcases[0] assert tc.is_main_testcase() assert tc.input.stdin.data == "Input string\n" + assert tc.input.stdin.type == TextChannelType.TEXT assert tc.input.arguments == ["--arg", "argument"] assert tc.output.stderr.data == "Error string\n" + assert tc.output.stderr.type == TextChannelType.TEXT assert tc.output.stdout.data == "Output string\n" + assert tc.output.stdout.type == TextChannelType.TEXT + assert tc.output.exit_code.value == 1 + + +def test_parse_one_tab_ctx_with_files(): + yaml_str = """ +namespace: "solution" +tabs: +- tab: "Ctx" + testcases: + - arguments: [ "--arg", "argument" ] + stdin: !path "input.text" + stdout: !path "output.text" + stderr: !path "error.text" + exit_code: 1 + """ + json_str = translate_to_test_suite(yaml_str) + suite = parse_test_suite(json_str) + assert suite.namespace == "solution" + assert len(suite.tabs) == 1 + tab = suite.tabs[0] + assert tab.name == "Ctx" + assert len(tab.contexts) == 1 + context = tab.contexts[0] + assert len(context.testcases) == 1 + tc = context.testcases[0] + assert tc.is_main_testcase() + assert tc.input.stdin.data == "input.text" + assert tc.input.stdin.type == TextChannelType.FILE + assert tc.input.arguments == ["--arg", "argument"] + assert tc.output.stderr.data == "error.text" + assert tc.output.stderr.type == TextChannelType.FILE + assert tc.output.stdout.data == "output.text" + assert tc.output.stdout.type == TextChannelType.FILE assert tc.output.exit_code.value == 1 diff --git a/tests/test_suite.py b/tests/test_suite.py index 2241914b..effbd889 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -10,9 +10,10 @@ ExitCodeOutputChannel, FileOutputChannel, MainInput, + TextChannelType, TextData, TextOutputChannel, - ValueOutputChannel, TextChannelType, + ValueOutputChannel, ) @@ -34,8 +35,8 @@ def test_text_output_is_compatible_oracle(): def test_file_output_is_compatible_oracle(): old_structure = { "content_type": [TextChannelType.TEXT], - "content": ["some content"], - "path": ["output.py"], + "content": ["some content"], + "path": ["output.py"], "evaluator": { "function": {"file": "evaluate.py"}, "type": "custom_check", From 1d3aebddd4fc55949171dccfac5e812aba157c23 Mon Sep 17 00:00:00 2001 From: breblanc Date: Wed, 19 Feb 2025 19:39:19 +0100 Subject: [PATCH 18/77] Added some more tests --- tests/test_dsl_yaml.py | 3 ++- tests/test_oracles_builtin.py | 25 ++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 8223733e..a6fbfa23 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -88,7 +88,8 @@ def test_parse_one_tab_ctx_with_files(): - arguments: [ "--arg", "argument" ] stdin: !path "input.text" stdout: !path "output.text" - stderr: !path "error.text" + stderr: + content: !path "error.text" exit_code: 1 """ json_str = translate_to_test_suite(yaml_str) diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index 005c2084..58b7fe46 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -160,7 +160,6 @@ def test_file_oracle_full_wrong( assert result.readable_expected == "expected\nexpected" assert result.readable_actual == "actual\nactual" - def test_file_oracle_full_correct( tmp_path: Path, pytestconfig: pytest.Config, mocker: MockerFixture ): @@ -185,6 +184,30 @@ def test_file_oracle_full_correct( assert result.readable_actual == "expected\nexpected" +def test_file_oracle_full_correct_with_mixed_content( + tmp_path: Path, pytestconfig: pytest.Config, mocker: MockerFixture +): + config = oracle_config(tmp_path, pytestconfig, {"mode": "full"}) + s = mocker.spy(tested.oracles.text, name="_text_comparison") # type: ignore[reportAttributeAccessIssue] + mock_files = [ + mocker.mock_open(read_data=content).return_value + for content in ["expected\nexpected", "expected\nexpected", "expected\nexpected", "expected\nexpected"] + ] + mock_opener = mocker.mock_open() + mock_opener.side_effect = mock_files + mocker.patch("builtins.open", mock_opener) + channel = FileOutputChannel( + content=["expected.txt", "expected\nexpected"], + path=["expected.txt", "expected.txt"], + content_type=[TextChannelType.FILE, TextChannelType.TEXT], + ) + result = evaluate_file(config, channel, "") + s.assert_called_with(ANY, "expected\nexpected", "expected\nexpected") + assert result.result.enum == Status.CORRECT + assert result.readable_expected == "expected\nexpected\nexpected\nexpected" + assert result.readable_actual == "expected\nexpected\nexpected\nexpected" + + def test_file_oracle_line_wrong( tmp_path: Path, pytestconfig: pytest.Config, mocker: MockerFixture ): From 2cb31f5129da8d0929bbf0e59bfcb39579688fc0 Mon Sep 17 00:00:00 2001 From: breblanc Date: Wed, 19 Feb 2025 19:40:10 +0100 Subject: [PATCH 19/77] lint test --- tests/test_oracles_builtin.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index 58b7fe46..ac71e5e7 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -160,6 +160,7 @@ def test_file_oracle_full_wrong( assert result.readable_expected == "expected\nexpected" assert result.readable_actual == "actual\nactual" + def test_file_oracle_full_correct( tmp_path: Path, pytestconfig: pytest.Config, mocker: MockerFixture ): @@ -191,7 +192,12 @@ def test_file_oracle_full_correct_with_mixed_content( s = mocker.spy(tested.oracles.text, name="_text_comparison") # type: ignore[reportAttributeAccessIssue] mock_files = [ mocker.mock_open(read_data=content).return_value - for content in ["expected\nexpected", "expected\nexpected", "expected\nexpected", "expected\nexpected"] + for content in [ + "expected\nexpected", + "expected\nexpected", + "expected\nexpected", + "expected\nexpected", + ] ] mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files From 2ce6701d63d253dabe48b51c8cef5b6a2057b733 Mon Sep 17 00:00:00 2001 From: breblanc Date: Wed, 19 Feb 2025 20:02:04 +0100 Subject: [PATCH 20/77] covered an extra case --- tests/test_dsl_yaml.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index a6fbfa23..51936350 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -782,7 +782,7 @@ def test_value_built_in_checks_implied(): ) -def test_file_custom_check_correct(): +def test_output_files_custom_check_correct(): yaml_str = f""" - tab: 'Test' contexts: @@ -792,6 +792,8 @@ def test_file_custom_check_correct(): data: - content: !path "test/hallo.txt" path: "test.txt" + - content: "Hallo world!" + path: "test2.txt" oracle: "custom_check" name: "evaluate_test" file: "test.py" @@ -809,6 +811,10 @@ def test_file_custom_check_correct(): assert isinstance(test.output.file.oracle, CustomCheckOracle) assert test.output.file.path[0] == "test.txt" assert test.output.file.content[0] == "test/hallo.txt" + assert test.output.file.content_type[0] == TextChannelType.FILE + assert test.output.file.path[1] == "test2.txt" + assert test.output.file.content[1] == "Hallo world!" + assert test.output.file.content_type[1] == TextChannelType.TEXT oracle = test.output.file.oracle assert oracle.function.name == "evaluate_test" assert oracle.function.file == Path("test.py") From 70e3f219f51c33f07e53aef6a97eedcbe5ddb1d5 Mon Sep 17 00:00:00 2001 From: breblanc Date: Thu, 20 Feb 2025 09:56:00 +0100 Subject: [PATCH 21/77] added an extra case for invalid input --- tests/test_oracles_builtin.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index ac71e5e7..848405a1 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -325,6 +325,19 @@ def test_file_oracle_dont_strip_lines_correct( assert result.readable_expected == "expected\nexpected2\n" assert result.readable_actual == "expected\nexpected2\n" +def test_correct_error_actual_not_found(tmp_path: Path, pytestconfig: pytest.Config): + config = oracle_config( + tmp_path, pytestconfig, {"mode": "line", "stripNewlines": False} + ) + channel = FileOutputChannel( + content=["Hallo world!"], + path=["expected.txt"], + content_type=[TextChannelType.TEXT], + ) + result = evaluate_file(config, channel, "") + assert result.result.enum == Status.RUNTIME_ERROR + assert result.result.human == "File not found." + def test_exception_oracle_only_messages_correct( tmp_path: Path, pytestconfig: pytest.Config From 441fc8b7cfef2245fd3c6f0a7c80a0ca84c7b1ab Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 24 Feb 2025 19:49:53 +0100 Subject: [PATCH 22/77] fixed test --- tests/test_oracles_builtin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index 848405a1..f49ae011 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -336,7 +336,7 @@ def test_correct_error_actual_not_found(tmp_path: Path, pytestconfig: pytest.Con ) result = evaluate_file(config, channel, "") assert result.result.enum == Status.RUNTIME_ERROR - assert result.result.human == "File not found." + assert result.result.human == "File not found." or result.result.human == "Bestand niet gevonden." def test_exception_oracle_only_messages_correct( From 9254c14a2403b42051e5a12412068e3ee36dff47 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 24 Feb 2025 19:51:01 +0100 Subject: [PATCH 23/77] fixed linting --- tests/test_oracles_builtin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index f49ae011..1b01adcc 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -325,6 +325,7 @@ def test_file_oracle_dont_strip_lines_correct( assert result.readable_expected == "expected\nexpected2\n" assert result.readable_actual == "expected\nexpected2\n" + def test_correct_error_actual_not_found(tmp_path: Path, pytestconfig: pytest.Config): config = oracle_config( tmp_path, pytestconfig, {"mode": "line", "stripNewlines": False} @@ -336,7 +337,10 @@ def test_correct_error_actual_not_found(tmp_path: Path, pytestconfig: pytest.Con ) result = evaluate_file(config, channel, "") assert result.result.enum == Status.RUNTIME_ERROR - assert result.result.human == "File not found." or result.result.human == "Bestand niet gevonden." + assert ( + result.result.human == "File not found." + or result.result.human == "Bestand niet gevonden." + ) def test_exception_oracle_only_messages_correct( From 3003cfcd846ade615739872625138e01a3386c61 Mon Sep 17 00:00:00 2001 From: breblanc Date: Wed, 26 Feb 2025 11:50:25 +0100 Subject: [PATCH 24/77] cleaned up code --- tested/dsl/translate_parser.py | 26 ++++++++------------------ tested/oracles/text.py | 2 -- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 4cd2d29a..a03da0ca 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -498,33 +498,23 @@ def _convert_text_output_channel( else: data = raw_data + if path is not None: + return TextOutputChannel( + data=data, + oracle=GenericTextOracle(options=config), + type=TextChannelType.FILE, + ) + if isinstance(stream, str): - if path is not None: - return TextOutputChannel( - data=data, - oracle=GenericTextOracle(options=config), - type=TextChannelType.FILE, - ) return TextOutputChannel(data=data, oracle=GenericTextOracle(options=config)) else: assert isinstance(stream, dict) if "oracle" not in stream or stream["oracle"] == "builtin": - if path is not None: - return TextOutputChannel( - data=data, - oracle=GenericTextOracle(options=config), - type=TextChannelType.FILE, - ) + return TextOutputChannel( data=data, oracle=GenericTextOracle(options=config) ) elif stream["oracle"] == "custom_check": - if path is not None: - return TextOutputChannel( - data=data, - oracle=_convert_custom_check_oracle(stream), - type=TextChannelType.FILE, - ) return TextOutputChannel( data=data, oracle=_convert_custom_check_oracle(stream) ) diff --git a/tested/oracles/text.py b/tested/oracles/text.py index 3044152d..5072bec4 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -196,9 +196,7 @@ def evaluate_file( actual_lines = actual_value.splitlines(keepends=not strip_newlines) result = len(actual_lines) == len(expected_lines) for expected_line, actual_line in zip(expected_lines, actual_lines): - print(f"{expected_line}: {actual_line}") new_result, _ = _text_comparison(options, expected_line, actual_line) - print(f"{new_result}") result = result and new_result return OracleResult( From 723c1a6308adbff741f8c00ec4ac2c41738e3540 Mon Sep 17 00:00:00 2001 From: breblanc Date: Wed, 5 Mar 2025 15:25:13 +0100 Subject: [PATCH 25/77] fixed a small error --- tested/dsl/translate_parser.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index a03da0ca..33220bd8 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -498,26 +498,22 @@ def _convert_text_output_channel( else: data = raw_data + text_output = TextOutputChannel(data=data) if path is not None: - return TextOutputChannel( - data=data, - oracle=GenericTextOracle(options=config), - type=TextChannelType.FILE, - ) + text_output.type = TextChannelType.FILE if isinstance(stream, str): - return TextOutputChannel(data=data, oracle=GenericTextOracle(options=config)) + text_output.oracle = GenericTextOracle(options=config) + return text_output else: assert isinstance(stream, dict) if "oracle" not in stream or stream["oracle"] == "builtin": - return TextOutputChannel( - data=data, oracle=GenericTextOracle(options=config) - ) + text_output.oracle = GenericTextOracle(options=config) + return text_output elif stream["oracle"] == "custom_check": - return TextOutputChannel( - data=data, oracle=_convert_custom_check_oracle(stream) - ) + text_output.oracle = _convert_custom_check_oracle(stream) + return text_output raise TypeError(f"Unknown text oracle type: {stream['oracle']}") From af868cfb22a05bf3ecb49fd5d0824c19a30f883d Mon Sep 17 00:00:00 2001 From: breblanc Date: Wed, 5 Mar 2025 18:00:56 +0100 Subject: [PATCH 26/77] Added a better temporary better seperator --- tested/oracles/text.py | 6 +++--- tested/testsuite.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tested/oracles/text.py b/tested/oracles/text.py index 5072bec4..f1a7dd49 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -152,15 +152,15 @@ def evaluate_file( expected_path = f"{config.bundle.config.resources}/{channel.content[i]}" try: with open(expected_path, "r") as file: - expected_list.append(file.read()) + expected_list.append(f"--- <{channel.path[i]}> ---\n{file.read()}") except FileNotFoundError: raise ValueError(f"File {expected_path} not found in resources.") else: - expected_list.append(channel.content[i]) + expected_list.append(f"--- <{channel.path[i]}> ---\n{channel.content[i]}") try: with open(str(actual_path), "r") as file: - actual_list.append(file.read()) + actual_list.append(f"--- <{channel.path[i]}> ---\n{file.read()}") except FileNotFoundError: file_not_found = True diff --git a/tested/testsuite.py b/tested/testsuite.py index 293a81d9..3a2e79c4 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -295,9 +295,9 @@ def get_data_as_string(self, resources: Path) -> str: if self.content_type[i] == TextChannelType.FILE: file_path = _resolve_path(resources, self.content[i]) with open(file_path, "r") as file: - file_content.append(file.read()) + file_content.append(f"--- <{self.path[i]}> ---\n{file.read()}") else: - file_content.append(self.content[i]) + file_content.append(f"--- <{self.path[i]}> ---\n{self.content[i]}") return "\n".join(file_content) From 6d59d54318266947947b1e1f2da7836c85689527 Mon Sep 17 00:00:00 2001 From: breblanc Date: Wed, 5 Mar 2025 19:42:04 +0100 Subject: [PATCH 27/77] Changed FileOutputChannel to work with list of objects instead of 3 lists. Also fixed the tests --- tested/dsl/translate_parser.py | 27 ++++---- tested/oracles/text.py | 29 +++++--- tested/testsuite.py | 24 ++++--- tests/test_dsl_yaml.py | 12 ++-- tests/test_oracles_builtin.py | 120 ++++++++++++++++++++++----------- tests/test_suite.py | 14 ++-- 6 files changed, 148 insertions(+), 78 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 33220bd8..ca0361c1 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -60,6 +60,7 @@ LanguageSpecificOracle, MainInput, Output, + OutputFileData, Suite, SupportedLanguage, Tab, @@ -520,9 +521,8 @@ def _convert_text_output_channel( def _convert_file_output_channel( stream: YamlObject, context: DslContext, config_name: str ) -> FileOutputChannel: - content_type = [] - content = [] - actual = [] + + file_data = [] data = stream if isinstance(stream, dict): data = stream["data"] @@ -531,12 +531,17 @@ def _convert_file_output_channel( for item in data: assert isinstance(item, dict) if isinstance(item["content"], PathString): - content_type.append(TextChannelType.FILE) + content_type = TextChannelType.FILE else: - content_type.append(TextChannelType.TEXT) + content_type = TextChannelType.TEXT - content.append(str(item["content"])) - actual.append(str(item["path"])) + file_data.append( + OutputFileData( + content_type=content_type, + content=str(item["content"]), + path=str(item["path"]), + ) + ) if ( not isinstance(stream, dict) @@ -553,16 +558,12 @@ def _convert_file_output_channel( "line", ), f"The file oracle only supports modes full and line, not {config['mode']}" return FileOutputChannel( - content_type=content_type, - content=content, - path=actual, + output_data=file_data, oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config), ) elif stream["oracle"] == "custom_check": return FileOutputChannel( - content_type=content_type, - content=content, - path=actual, + output_data=file_data, oracle=_convert_custom_check_oracle(stream), ) raise TypeError(f"Unknown file oracle type: {stream['oracle']}") diff --git a/tested/oracles/text.py b/tested/oracles/text.py index f1a7dd49..76ec1386 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -145,22 +145,23 @@ def evaluate_file( actual_list = [] expected_list = [] file_not_found = False - for i in range(len(channel.content)): - actual_path = config.context_dir / channel.path[i] + for i in range(len(channel.output_data)): + output_data = channel.output_data[i] + actual_path = config.context_dir / output_data.path - if channel.content_type[i] == TextChannelType.FILE: - expected_path = f"{config.bundle.config.resources}/{channel.content[i]}" + if output_data.content_type == TextChannelType.FILE: + expected_path = f"{config.bundle.config.resources}/{output_data.content}" try: with open(expected_path, "r") as file: - expected_list.append(f"--- <{channel.path[i]}> ---\n{file.read()}") + expected_list.append(file.read()) except FileNotFoundError: raise ValueError(f"File {expected_path} not found in resources.") else: - expected_list.append(f"--- <{channel.path[i]}> ---\n{channel.content[i]}") + expected_list.append(output_data.content) try: with open(str(actual_path), "r") as file: - actual_list.append(f"--- <{channel.path[i]}> ---\n{file.read()}") + actual_list.append(file.read()) except FileNotFoundError: file_not_found = True @@ -185,6 +186,12 @@ def evaluate_file( new_result, expected_list[i] = _text_comparison( options, expected_value, actual_value ) + expected_list[i] = ( + f"--- <{channel.output_data[i].path}> ---\n{expected_list[i]}" + ) + actual_list[i] = ( + f"--- <{channel.output_data[i].path}> ---\n{actual_list[i]}" + ) result = result and new_result else: assert options["mode"] == "line" @@ -198,9 +205,15 @@ def evaluate_file( for expected_line, actual_line in zip(expected_lines, actual_lines): new_result, _ = _text_comparison(options, expected_line, actual_line) result = result and new_result + expected_list[i] = ( + f"--- <{channel.output_data[i].path}> ---\n{expected_list[i]}" + ) + actual_list[i] = ( + f"--- <{channel.output_data[i].path}> ---\n{actual_list[i]}" + ) return OracleResult( result=StatusMessage(enum=Status.CORRECT if result else Status.WRONG), readable_expected="\n".join(expected_list), - readable_actual=actual, + readable_actual="\n".join(actual_list), ) diff --git a/tested/testsuite.py b/tested/testsuite.py index 3a2e79c4..286dbf40 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -273,15 +273,20 @@ class TextOutputChannel(TextData): oracle: GenericTextOracle | CustomCheckOracle = field(factory=GenericTextOracle) +@define(frozen=True) +class OutputFileData: + content_type: TextChannelType + content: str + path: str + + @fallback_field(get_converter(), {"evaluator": "oracle"}) @ignore_field(get_converter(), "show_expected") @define class FileOutputChannel(WithFeatures): """Describes the output for files.""" - content_type: list[TextChannelType] - content: list[str] # Paths to the file to compare to. - path: list[str] # Paths to the generated file (by the user code) + output_data: list[OutputFileData] oracle: GenericTextOracle | CustomCheckOracle = field( factory=lambda: GenericTextOracle(name=TextBuiltin.FILE) ) @@ -291,13 +296,16 @@ def get_used_features(self) -> FeatureSet: def get_data_as_string(self, resources: Path) -> str: file_content = [] - for i in range(len(self.content)): - if self.content_type[i] == TextChannelType.FILE: - file_path = _resolve_path(resources, self.content[i]) + for i in range(len(self.output_data)): + output_data = self.output_data[i] + if output_data.content_type == TextChannelType.FILE: + file_path = _resolve_path(resources, output_data.content) with open(file_path, "r") as file: - file_content.append(f"--- <{self.path[i]}> ---\n{file.read()}") + file_content.append(f"--- <{output_data.path}> ---\n{file.read()}") else: - file_content.append(f"--- <{self.path[i]}> ---\n{self.content[i]}") + file_content.append( + f"--- <{output_data.path}> ---\n{output_data.content[i]}" + ) return "\n".join(file_content) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 51936350..06ca9edb 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -809,12 +809,12 @@ def test_output_files_custom_check_correct(): assert isinstance(test.input, FunctionCall) assert isinstance(test.output.file, FileOutputChannel) assert isinstance(test.output.file.oracle, CustomCheckOracle) - assert test.output.file.path[0] == "test.txt" - assert test.output.file.content[0] == "test/hallo.txt" - assert test.output.file.content_type[0] == TextChannelType.FILE - assert test.output.file.path[1] == "test2.txt" - assert test.output.file.content[1] == "Hallo world!" - assert test.output.file.content_type[1] == TextChannelType.TEXT + assert test.output.file.output_data[0].path == "test.txt" + assert test.output.file.output_data[0].content == "test/hallo.txt" + assert test.output.file.output_data[0].content_type == TextChannelType.FILE + assert test.output.file.output_data[1].path == "test2.txt" + assert test.output.file.output_data[1].content == "Hallo world!" + assert test.output.file.output_data[1].content_type == TextChannelType.TEXT oracle = test.output.file.oracle assert oracle.function.name == "evaluate_test" assert oracle.function.file == Path("test.py") diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index 1b01adcc..852aa117 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -25,6 +25,7 @@ ExceptionOutputChannel, ExpectedException, FileOutputChannel, + OutputFileData, Suite, SupportedLanguage, TextChannelType, @@ -150,15 +151,19 @@ def test_file_oracle_full_wrong( mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) channel = FileOutputChannel( - content=["expected.txt"], - path=["expected.txt"], - content_type=[TextChannelType.FILE], + output_data=[ + OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, + ) + ] ) result = evaluate_file(config, channel, "") s.assert_called_once_with(ANY, "expected\nexpected", "actual\nactual") assert result.result.enum == Status.WRONG - assert result.readable_expected == "expected\nexpected" - assert result.readable_actual == "actual\nactual" + assert result.readable_expected == "--- ---\nexpected\nexpected" + assert result.readable_actual == "--- ---\nactual\nactual" def test_file_oracle_full_correct( @@ -174,15 +179,19 @@ def test_file_oracle_full_correct( mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) channel = FileOutputChannel( - content=["expected.txt"], - path=["expected.txt"], - content_type=[TextChannelType.FILE], + output_data=[ + OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, + ) + ] ) result = evaluate_file(config, channel, "") s.assert_called_once_with(ANY, "expected\nexpected", "expected\nexpected") assert result.result.enum == Status.CORRECT - assert result.readable_expected == "expected\nexpected" - assert result.readable_actual == "expected\nexpected" + assert result.readable_expected == "--- ---\nexpected\nexpected" + assert result.readable_actual == "--- ---\nexpected\nexpected" def test_file_oracle_full_correct_with_mixed_content( @@ -203,15 +212,30 @@ def test_file_oracle_full_correct_with_mixed_content( mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) channel = FileOutputChannel( - content=["expected.txt", "expected\nexpected"], - path=["expected.txt", "expected.txt"], - content_type=[TextChannelType.FILE, TextChannelType.TEXT], + output_data=[ + OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, + ), + OutputFileData( + content="expected\nexpected", + path="expected.txt", + content_type=TextChannelType.TEXT, + ), + ] ) result = evaluate_file(config, channel, "") s.assert_called_with(ANY, "expected\nexpected", "expected\nexpected") assert result.result.enum == Status.CORRECT - assert result.readable_expected == "expected\nexpected\nexpected\nexpected" - assert result.readable_actual == "expected\nexpected\nexpected\nexpected" + assert ( + result.readable_expected + == "--- ---\nexpected\nexpected\n--- ---\nexpected\nexpected" + ) + assert ( + result.readable_actual + == "--- ---\nexpected\nexpected\n--- ---\nexpected\nexpected" + ) def test_file_oracle_line_wrong( @@ -229,17 +253,21 @@ def test_file_oracle_line_wrong( mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) channel = FileOutputChannel( - content=["expected.txt"], - path=["expected.txt"], - content_type=[TextChannelType.FILE], + output_data=[ + OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, + ) + ] ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "actual") s.assert_any_call(ANY, "expected2", "actual2") assert s.call_count == 2 assert result.result.enum == Status.WRONG - assert result.readable_expected == "expected\nexpected2" - assert result.readable_actual == "actual\nactual2" + assert result.readable_expected == "--- ---\nexpected\nexpected2" + assert result.readable_actual == "--- ---\nactual\nactual2" def test_file_oracle_line_correct( @@ -257,17 +285,21 @@ def test_file_oracle_line_correct( mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) channel = FileOutputChannel( - content=["expected.txt"], - path=["expected.txt"], - content_type=[TextChannelType.FILE], + output_data=[ + OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, + ) + ] ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "expected") s.assert_any_call(ANY, "expected2", "expected2") assert s.call_count == 2 assert result.result.enum == Status.CORRECT - assert result.readable_expected == "expected\nexpected2" - assert result.readable_actual == "expected\nexpected2" + assert result.readable_expected == "--- ---\nexpected\nexpected2" + assert result.readable_actual == "--- ---\nexpected\nexpected2" def test_file_oracle_strip_lines_correct( @@ -285,17 +317,21 @@ def test_file_oracle_strip_lines_correct( mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) channel = FileOutputChannel( - content=["expected.txt"], - path=["expected.txt"], - content_type=[TextChannelType.FILE], + output_data=[ + OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, + ) + ] ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "expected") s.assert_any_call(ANY, "expected2", "expected2") assert s.call_count == 2 assert result.result.enum == Status.CORRECT - assert result.readable_expected == "expected\nexpected2\n" - assert result.readable_actual == "expected\nexpected2" + assert result.readable_expected == "--- ---\nexpected\nexpected2\n" + assert result.readable_actual == "--- ---\nexpected\nexpected2" def test_file_oracle_dont_strip_lines_correct( @@ -313,17 +349,21 @@ def test_file_oracle_dont_strip_lines_correct( mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) channel = FileOutputChannel( - content=["expected.txt"], - path=["expected.txt"], - content_type=[TextChannelType.FILE], + output_data=[ + OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, + ) + ] ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected\n", "expected\n") s.assert_any_call(ANY, "expected2\n", "expected2\n") assert s.call_count == 2 assert result.result.enum == Status.CORRECT - assert result.readable_expected == "expected\nexpected2\n" - assert result.readable_actual == "expected\nexpected2\n" + assert result.readable_expected == "--- ---\nexpected\nexpected2\n" + assert result.readable_actual == "--- ---\nexpected\nexpected2\n" def test_correct_error_actual_not_found(tmp_path: Path, pytestconfig: pytest.Config): @@ -331,9 +371,13 @@ def test_correct_error_actual_not_found(tmp_path: Path, pytestconfig: pytest.Con tmp_path, pytestconfig, {"mode": "line", "stripNewlines": False} ) channel = FileOutputChannel( - content=["Hallo world!"], - path=["expected.txt"], - content_type=[TextChannelType.TEXT], + output_data=[ + OutputFileData( + content="Hallo world!", + path="expected.txt", + content_type=TextChannelType.TEXT, + ) + ] ) result = evaluate_file(config, channel, "") assert result.result.enum == Status.RUNTIME_ERROR diff --git a/tests/test_suite.py b/tests/test_suite.py index effbd889..e94561e9 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -112,15 +112,19 @@ def test_file_show_expected_is_accepted(): scheme = """ { "show_expected": true, - "content": ["hallo"], - "path": ["hallo"], - "content_type": ["text"] + "output_data": [ + { + "content_type": "text", + "content": "hallo", + "path": "hallo.txt" + } + ] } """ result = get_converter().loads(scheme, FileOutputChannel) - assert result.content == ["hallo"] - assert result.path == ["hallo"] + assert result.output_data[0].content == "hallo" + assert result.output_data[0].path == "hallo.txt" def test_value_show_expected_is_accepted(): From e841287fd61fe42d3fc43a50244353c5991cdf85 Mon Sep 17 00:00:00 2001 From: breblanc Date: Thu, 6 Mar 2025 09:24:41 +0100 Subject: [PATCH 28/77] fix tests --- tests/test_suite.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_suite.py b/tests/test_suite.py index e94561e9..4eff78c2 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -34,9 +34,13 @@ def test_text_output_is_compatible_oracle(): def test_file_output_is_compatible_oracle(): old_structure = { - "content_type": [TextChannelType.TEXT], - "content": ["some content"], - "path": ["output.py"], + "output_data": [ + { + "content_type": TextChannelType.TEXT, + "content": "some content", + "path": "output.py", + } + ], "evaluator": { "function": {"file": "evaluate.py"}, "type": "custom_check", From d3e7d46cdc374244e0577858d6209cdde4c151f9 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 15 Mar 2025 15:40:52 +0100 Subject: [PATCH 29/77] changed the input_files --- tested/dsl/schema-strict.json | 9 ++-- tested/dsl/schema.json | 9 ++-- tested/dsl/translate_parser.py | 96 +++++++++++++++++++++++++--------- tested/judge/evaluation.py | 2 +- tested/languages/generation.py | 2 +- tested/main.py | 2 +- tested/testsuite.py | 2 +- tests/test_dsl_yaml.py | 4 +- tests/test_suite.py | 2 +- 9 files changed, 90 insertions(+), 38 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index fdc48254..998ceed5 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -496,15 +496,18 @@ "type" : "object", "description" : "A file used in the test suite.", "required" : [ - "name", - "path" + "name" ], "properties" : { "name" : { "type" : "string", "description" : "The filename, including the file extension." }, - "path" : { + "content" : { + "type" : "string", + "description" : "The actual content of the file." + }, + "url" : { "type" : "string", "format" : "uri", "description" : "Relative path to the file in the `description` folder of an exercise." diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 02fbaff8..f87f484c 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -496,15 +496,18 @@ "type" : "object", "description" : "A file used in the test suite.", "required" : [ - "name", - "path" + "name" ], "properties" : { "name" : { "type" : "string", "description" : "The filename, including the file extension." }, - "path" : { + "content" : { + "type" : "string", + "description" : "The actual content of the file." + }, + "url" : { "type" : "string", "format" : "uri", "description" : "Relative path to the file in the `description` folder of an exercise." diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index ca0361c1..54fa282c 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -1,4 +1,6 @@ +import base64 import json +import os import sys import textwrap from collections.abc import Callable @@ -269,12 +271,13 @@ class DslContext: config: dict[str, dict] = field(factory=dict) language: SupportedLanguage | Literal["tested"] = "tested" - def deepen_context(self, new_level: YamlDict | None) -> "DslContext": + def deepen_context(self, new_level: YamlDict | None, workdir: Path) -> "DslContext": """ Merge certain fields of the new object with the current context, resulting in a new context for the new level. :param new_level: The new object from the DSL to get information from. + :param workdir: The working directory where all files are located. :return: A new context. """ @@ -284,7 +287,9 @@ def deepen_context(self, new_level: YamlDict | None) -> "DslContext": the_files = self.files if "input_files" in new_level: assert isinstance(new_level["input_files"], list) - additional_files = {_convert_file(f) for f in new_level["input_files"]} + additional_files = { + _convert_file(f, workdir=workdir) for f in new_level["input_files"] + } the_files = list(set(self.files) | additional_files) the_config = self.config @@ -421,10 +426,34 @@ def _convert_value(value: YamlObject) -> Value: return _tested_type_to_value(tested_type) -def _convert_file(link_file: YamlDict) -> FileUrl: +def base64_encode(content: str) -> str: + sample_string_bytes = content.encode("ascii") + + base64_bytes = base64.b64encode(sample_string_bytes) + return base64_bytes.decode("ascii") + + +def _convert_file(link_file: YamlDict, workdir: Path) -> FileUrl: assert isinstance(link_file["name"], str) - assert isinstance(link_file["path"], str) - return FileUrl(name=link_file["name"], path=link_file["path"]) + if "content" in link_file: + assert isinstance(link_file["content"], str) + full_path = workdir / link_file["name"] + os.makedirs(os.path.dirname(full_path), exist_ok=True) + with open(full_path, "w", encoding="utf-8") as f: + f.write(link_file["content"]) + + if "url" in link_file: + assert isinstance(link_file["url"], str) + url = link_file["url"] + else: + url = base64_encode(link_file["content"]) + else: + # Assumed the specified files are already in the working directory. + assert "url" in link_file + assert isinstance(link_file["url"], str) + url = link_file["url"] + + return FileUrl(name=link_file["name"], url=url) def _convert_evaluation_function(stream: dict) -> EvaluationFunction: @@ -623,8 +652,10 @@ def _validate_testcase_combinations(testcase: YamlDict): raise ValueError("A statement cannot have an expected return value.") -def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: - context = context.deepen_context(testcase) +def _convert_testcase( + testcase: YamlDict, context: DslContext, workdir: Path +) -> Testcase: + context = context.deepen_context(testcase, workdir) # This is backwards compatability to some extend. # TODO: remove this at some point. @@ -727,15 +758,19 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: ) -def _convert_context(context: YamlDict, dsl_context: DslContext) -> Context: - dsl_context = dsl_context.deepen_context(context) +def _convert_context( + context: YamlDict, dsl_context: DslContext, workdir: Path +) -> Context: + dsl_context = dsl_context.deepen_context(context, workdir) raw_testcases = context.get("script", context.get("testcases")) assert isinstance(raw_testcases, list) - testcases = _convert_dsl_list(raw_testcases, dsl_context, _convert_testcase) + testcases = _convert_dsl_list( + raw_testcases, dsl_context, workdir, _convert_testcase + ) return Context(testcases=testcases) -def _convert_tab(tab: YamlDict, context: DslContext) -> Tab: +def _convert_tab(tab: YamlDict, context: DslContext, workdir: Path) -> Tab: """ Translate a DSL tab to a full test suite tab. @@ -743,29 +778,35 @@ def _convert_tab(tab: YamlDict, context: DslContext) -> Tab: :param context: The context with config for the parent level. :return: A full tab. """ - context = context.deepen_context(tab) + context = context.deepen_context(tab, workdir) name = tab.get("unit", tab.get("tab")) assert isinstance(name, str) # The tab can have testcases or contexts. if "contexts" in tab: assert isinstance(tab["contexts"], list) - contexts = _convert_dsl_list(tab["contexts"], context, _convert_context) + contexts = _convert_dsl_list( + tab["contexts"], context, workdir, _convert_context + ) elif "cases" in tab: assert "unit" in tab # We have testcases N.S. / contexts O.S. assert isinstance(tab["cases"], list) - contexts = _convert_dsl_list(tab["cases"], context, _convert_context) + contexts = _convert_dsl_list(tab["cases"], context, workdir, _convert_context) elif "testcases" in tab: # We have scripts N.S. / testcases O.S. assert "tab" in tab assert isinstance(tab["testcases"], list) - testcases = _convert_dsl_list(tab["testcases"], context, _convert_testcase) + testcases = _convert_dsl_list( + tab["testcases"], context, workdir, _convert_testcase + ) contexts = [Context(testcases=[t]) for t in testcases] else: assert "scripts" in tab assert isinstance(tab["scripts"], list) - testcases = _convert_dsl_list(tab["scripts"], context, _convert_testcase) + testcases = _convert_dsl_list( + tab["scripts"], context, workdir, _convert_testcase + ) contexts = [Context(testcases=[t]) for t in testcases] return Tab(name=name, contexts=contexts) @@ -775,7 +816,10 @@ def _convert_tab(tab: YamlDict, context: DslContext) -> Tab: def _convert_dsl_list( - dsl_list: list, context: DslContext, converter: Callable[[YamlDict, DslContext], T] + dsl_list: list, + context: DslContext, + workdir: Path, + converter: Callable[[YamlDict, DslContext, Path], T], ) -> list[T]: """ Convert a list of YAML objects into a test suite object. @@ -783,11 +827,11 @@ def _convert_dsl_list( objects = [] for dsl_object in dsl_list: assert isinstance(dsl_object, dict) - objects.append(converter(dsl_object, context)) + objects.append(converter(dsl_object, context, workdir)) return objects -def _convert_dsl(dsl_object: YamlObject) -> Suite: +def _convert_dsl(dsl_object: YamlObject, workdir: Path) -> Suite: """ Translate a DSL test suite into a full test suite. @@ -804,13 +848,13 @@ def _convert_dsl(dsl_object: YamlObject) -> Suite: else: assert isinstance(dsl_object, dict) namespace = dsl_object.get("namespace") - context = context.deepen_context(dsl_object) + context = context.deepen_context(dsl_object, workdir) tab_list = dsl_object.get("units", dsl_object.get("tabs")) assert isinstance(tab_list, list) if (language := dsl_object.get("language", "tested")) != "tested": language = SupportedLanguage(language) context = evolve(context, language=language) - tabs = _convert_dsl_list(tab_list, context, _convert_tab) + tabs = _convert_dsl_list(tab_list, context, workdir, _convert_tab) if namespace: assert isinstance(namespace, str) @@ -819,25 +863,27 @@ def _convert_dsl(dsl_object: YamlObject) -> Suite: return Suite(tabs=tabs) -def parse_dsl(dsl_string: str) -> Suite: +def parse_dsl(dsl_string: str, workdir: Path) -> Suite: """ Parse a string containing a DSL test suite into our representation, a test suite. :param dsl_string: The string containing a DSL. + :param workdir: The working directory for the test suite. :return: The parsed and converted test suite. """ dsl_object = _parse_yaml(dsl_string) _validate_dsl(dsl_object) - return _convert_dsl(dsl_object) + return _convert_dsl(dsl_object, workdir) -def translate_to_test_suite(dsl_string: str) -> str: +def translate_to_test_suite(dsl_string: str, workdir: Path) -> str: """ Convert a DSL to a test suite. :param dsl_string: The DSL. + :param workdir: The working directory for the test suite. :return: The test suite. """ - suite = parse_dsl(dsl_string) + suite = parse_dsl(dsl_string, workdir) return suite_to_json(suite) diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index 93ba3d57..44b2571a 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -364,7 +364,7 @@ def evaluate_context_results( def _link_files_message(link_files: Collection[FileUrl]) -> AppendMessage: link_list = ", ".join( - f'' + f'' f'{html.escape(link_file.name)}' for link_file in link_files ) diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 846a9659..7c7d883b 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -192,7 +192,7 @@ def get_readable_input( def replace_link(match: Match) -> str: filename = match.group() the_file = url_map[filename] - the_url = urllib.parse.quote(the_file.path) + the_url = urllib.parse.quote(the_file.url) the_replacement = ( f'{filename}' ) diff --git a/tested/main.py b/tested/main.py index b4549028..2c5313c8 100644 --- a/tested/main.py +++ b/tested/main.py @@ -30,7 +30,7 @@ def run(config: DodonaConfig, judge_output: IO): _, ext = os.path.splitext(config.test_suite) is_yaml = ext.lower() in (".yaml", ".yml") if is_yaml: - suite = parse_dsl(textual_suite) + suite = parse_dsl(textual_suite, config.workdir) else: suite = parse_test_suite(textual_suite) pack = create_bundle(config, judge_output, suite) diff --git a/tested/testsuite.py b/tested/testsuite.py index 286dbf40..ee9d74d9 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -543,7 +543,7 @@ def get_functions(self) -> Iterable[FunctionCall]: @define(frozen=True) class FileUrl: - path: str + url: str name: str diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 06ca9edb..10d7e4e2 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -1238,8 +1238,8 @@ def test_files_are_propagated(): testcases0, testcases1 = ctx0.testcases, ctx1.testcases test0, test1 = testcases0[0], testcases1[0] assert set(test0.link_files) == { - FileUrl(name="test", path="test.md"), - FileUrl(name="two", path="two.md"), + FileUrl(name="test", url="test.md"), + FileUrl(name="two", url="two.md"), } diff --git a/tests/test_suite.py b/tests/test_suite.py index 4eff78c2..20bf190f 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -128,7 +128,7 @@ def test_file_show_expected_is_accepted(): result = get_converter().loads(scheme, FileOutputChannel) assert result.output_data[0].content == "hallo" - assert result.output_data[0].path == "hallo.txt" + assert result.output_data[0].url == "hallo.txt" def test_value_show_expected_is_accepted(): From 8e929289aa8dba7175e74ef096ec3a8f301d99b3 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 15 Mar 2025 16:29:39 +0100 Subject: [PATCH 30/77] Fixed some of the issues --- tested/dsl/translate_parser.py | 31 ++++++++++++++++--------------- tests/test_suite.py | 2 +- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 54fa282c..84672a86 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -271,7 +271,7 @@ class DslContext: config: dict[str, dict] = field(factory=dict) language: SupportedLanguage | Literal["tested"] = "tested" - def deepen_context(self, new_level: YamlDict | None, workdir: Path) -> "DslContext": + def deepen_context(self, new_level: YamlDict | None, workdir: Path|None) -> "DslContext": """ Merge certain fields of the new object with the current context, resulting in a new context for the new level. @@ -433,14 +433,15 @@ def base64_encode(content: str) -> str: return base64_bytes.decode("ascii") -def _convert_file(link_file: YamlDict, workdir: Path) -> FileUrl: +def _convert_file(link_file: YamlDict, workdir: Path|None) -> FileUrl: assert isinstance(link_file["name"], str) if "content" in link_file: assert isinstance(link_file["content"], str) - full_path = workdir / link_file["name"] - os.makedirs(os.path.dirname(full_path), exist_ok=True) - with open(full_path, "w", encoding="utf-8") as f: - f.write(link_file["content"]) + if workdir is not None: + full_path = workdir / link_file["name"] + os.makedirs(os.path.dirname(full_path), exist_ok=True) + with open(full_path, "w", encoding="utf-8") as f: + f.write(link_file["content"]) if "url" in link_file: assert isinstance(link_file["url"], str) @@ -653,7 +654,7 @@ def _validate_testcase_combinations(testcase: YamlDict): def _convert_testcase( - testcase: YamlDict, context: DslContext, workdir: Path + testcase: YamlDict, context: DslContext, workdir: Path|None ) -> Testcase: context = context.deepen_context(testcase, workdir) @@ -759,7 +760,7 @@ def _convert_testcase( def _convert_context( - context: YamlDict, dsl_context: DslContext, workdir: Path + context: YamlDict, dsl_context: DslContext, workdir: Path|None ) -> Context: dsl_context = dsl_context.deepen_context(context, workdir) raw_testcases = context.get("script", context.get("testcases")) @@ -770,7 +771,7 @@ def _convert_context( return Context(testcases=testcases) -def _convert_tab(tab: YamlDict, context: DslContext, workdir: Path) -> Tab: +def _convert_tab(tab: YamlDict, context: DslContext, workdir: Path|None) -> Tab: """ Translate a DSL tab to a full test suite tab. @@ -818,8 +819,8 @@ def _convert_tab(tab: YamlDict, context: DslContext, workdir: Path) -> Tab: def _convert_dsl_list( dsl_list: list, context: DslContext, - workdir: Path, - converter: Callable[[YamlDict, DslContext, Path], T], + workdir: Path|None, + converter: Callable[[YamlDict, DslContext, Path|None], T], ) -> list[T]: """ Convert a list of YAML objects into a test suite object. @@ -831,7 +832,7 @@ def _convert_dsl_list( return objects -def _convert_dsl(dsl_object: YamlObject, workdir: Path) -> Suite: +def _convert_dsl(dsl_object: YamlObject, workdir: Path|None) -> Suite: """ Translate a DSL test suite into a full test suite. @@ -863,7 +864,7 @@ def _convert_dsl(dsl_object: YamlObject, workdir: Path) -> Suite: return Suite(tabs=tabs) -def parse_dsl(dsl_string: str, workdir: Path) -> Suite: +def parse_dsl(dsl_string: str, workdir: Path|None=None) -> Suite: """ Parse a string containing a DSL test suite into our representation, a test suite. @@ -877,7 +878,7 @@ def parse_dsl(dsl_string: str, workdir: Path) -> Suite: return _convert_dsl(dsl_object, workdir) -def translate_to_test_suite(dsl_string: str, workdir: Path) -> str: +def translate_to_test_suite(dsl_string: str) -> str: """ Convert a DSL to a test suite. @@ -885,5 +886,5 @@ def translate_to_test_suite(dsl_string: str, workdir: Path) -> str: :param workdir: The working directory for the test suite. :return: The test suite. """ - suite = parse_dsl(dsl_string, workdir) + suite = parse_dsl(dsl_string, Path(".")) return suite_to_json(suite) diff --git a/tests/test_suite.py b/tests/test_suite.py index 20bf190f..4eff78c2 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -128,7 +128,7 @@ def test_file_show_expected_is_accepted(): result = get_converter().loads(scheme, FileOutputChannel) assert result.output_data[0].content == "hallo" - assert result.output_data[0].url == "hallo.txt" + assert result.output_data[0].path == "hallo.txt" def test_value_show_expected_is_accepted(): From 9e3e0a68af98c9db96bf730daa141c98ca572c47 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 15 Mar 2025 16:32:33 +0100 Subject: [PATCH 31/77] Fixed linting --- tested/dsl/translate_parser.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 84672a86..16d15363 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -271,7 +271,9 @@ class DslContext: config: dict[str, dict] = field(factory=dict) language: SupportedLanguage | Literal["tested"] = "tested" - def deepen_context(self, new_level: YamlDict | None, workdir: Path|None) -> "DslContext": + def deepen_context( + self, new_level: YamlDict | None, workdir: Path | None + ) -> "DslContext": """ Merge certain fields of the new object with the current context, resulting in a new context for the new level. @@ -433,7 +435,7 @@ def base64_encode(content: str) -> str: return base64_bytes.decode("ascii") -def _convert_file(link_file: YamlDict, workdir: Path|None) -> FileUrl: +def _convert_file(link_file: YamlDict, workdir: Path | None) -> FileUrl: assert isinstance(link_file["name"], str) if "content" in link_file: assert isinstance(link_file["content"], str) @@ -654,7 +656,7 @@ def _validate_testcase_combinations(testcase: YamlDict): def _convert_testcase( - testcase: YamlDict, context: DslContext, workdir: Path|None + testcase: YamlDict, context: DslContext, workdir: Path | None ) -> Testcase: context = context.deepen_context(testcase, workdir) @@ -760,7 +762,7 @@ def _convert_testcase( def _convert_context( - context: YamlDict, dsl_context: DslContext, workdir: Path|None + context: YamlDict, dsl_context: DslContext, workdir: Path | None ) -> Context: dsl_context = dsl_context.deepen_context(context, workdir) raw_testcases = context.get("script", context.get("testcases")) @@ -771,7 +773,7 @@ def _convert_context( return Context(testcases=testcases) -def _convert_tab(tab: YamlDict, context: DslContext, workdir: Path|None) -> Tab: +def _convert_tab(tab: YamlDict, context: DslContext, workdir: Path | None) -> Tab: """ Translate a DSL tab to a full test suite tab. @@ -819,8 +821,8 @@ def _convert_tab(tab: YamlDict, context: DslContext, workdir: Path|None) -> Tab: def _convert_dsl_list( dsl_list: list, context: DslContext, - workdir: Path|None, - converter: Callable[[YamlDict, DslContext, Path|None], T], + workdir: Path | None, + converter: Callable[[YamlDict, DslContext, Path | None], T], ) -> list[T]: """ Convert a list of YAML objects into a test suite object. @@ -832,7 +834,7 @@ def _convert_dsl_list( return objects -def _convert_dsl(dsl_object: YamlObject, workdir: Path|None) -> Suite: +def _convert_dsl(dsl_object: YamlObject, workdir: Path | None) -> Suite: """ Translate a DSL test suite into a full test suite. @@ -864,7 +866,7 @@ def _convert_dsl(dsl_object: YamlObject, workdir: Path|None) -> Suite: return Suite(tabs=tabs) -def parse_dsl(dsl_string: str, workdir: Path|None=None) -> Suite: +def parse_dsl(dsl_string: str, workdir: Path | None = None) -> Suite: """ Parse a string containing a DSL test suite into our representation, a test suite. From 7bec9509e6e4d97f38ceaa86bf8aa867e12d9928 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 16 Mar 2025 11:15:03 +0100 Subject: [PATCH 32/77] Fixed test --- tests/test_dsl_yaml.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 10d7e4e2..b45bdb48 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -1219,9 +1219,9 @@ def test_files_are_propagated(): - tab: "Config ctx" input_files: - name: "test" - path: "test.md" + url: "test.md" - name: "two" - path: "two.md" + url: "two.md" testcases: - arguments: [ '-a', '2.125', '1.212' ] stdout: "3.34" @@ -1229,7 +1229,7 @@ def test_files_are_propagated(): stdout: "3.337" input_files: - name: "test" - path: "twooo.md" + url: "twooo.md" """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) From b6fe8e14f5cd1e997cf52207f070c52f32f6522c Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 16 Mar 2025 15:37:42 +0100 Subject: [PATCH 33/77] Added link support for stdin --- tested/dsl/translate_parser.py | 3 ++- tested/languages/generation.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 16d15363..98a68385 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -3,6 +3,7 @@ import os import sys import textwrap +import zlib from collections.abc import Callable from decimal import Decimal from pathlib import Path @@ -431,7 +432,7 @@ def _convert_value(value: YamlObject) -> Value: def base64_encode(content: str) -> str: sample_string_bytes = content.encode("ascii") - base64_bytes = base64.b64encode(sample_string_bytes) + base64_bytes = base64.b64encode(zlib.compress(sample_string_bytes)) return base64_bytes.decode("ascii") diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 7c7d883b..945ceae5 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -128,7 +128,7 @@ def get_readable_input( args = f"$ {command}" # Determine the stdin if isinstance(case.input.stdin, TextData): - stdin = case.input.stdin.get_data_as_string(bundle.config.resources) + stdin = case.input.stdin.data else: stdin = "" From 621f4a786b59eafd82a9dafe85768422abe3a4f4 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 16 Mar 2025 15:56:00 +0100 Subject: [PATCH 34/77] Added a few more checks --- tested/dsl/schema-strict.json | 6 ++++-- tested/languages/generation.py | 17 ++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 998ceed5..5074d1ee 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -280,7 +280,8 @@ "string", "number", "integer", - "boolean" + "boolean", + "path" ] }, "arguments" : { @@ -376,7 +377,8 @@ "string", "number", "integer", - "boolean" + "boolean", + "path" ] }, "arguments" : { diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 945ceae5..ace1c919 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -111,6 +111,14 @@ def get_readable_input( 3. If it is a context testcase: a. The stdin and the arguments. """ + # We have potential files. + # Check if the file names are present in the string. + # If not, we can also stop before doing ugly things. + # We construct a regex, since that can be faster than checking everything. + simple_regex = re.compile( + "|".join(map(lambda x: re.escape(x.name), case.link_files)) + ) + format_ = "text" # By default, we use text as input. if case.description: if isinstance(case.description, ExtendedMessage): @@ -129,6 +137,8 @@ def get_readable_input( # Determine the stdin if isinstance(case.input.stdin, TextData): stdin = case.input.stdin.data + if not case.link_files and not simple_regex.search(stdin): + stdin = case.input.stdin.get_data_as_string(bundle.config.resources) else: stdin = "" @@ -161,13 +171,6 @@ def get_readable_input( if not case.link_files: return ExtendedMessage(description=text, format=format_), set() - # We have potential files. - # Check if the file names are present in the string. - # If not, we can also stop before doing ugly things. - # We construct a regex, since that can be faster than checking everything. - simple_regex = re.compile( - "|".join(map(lambda x: re.escape(x.name), case.link_files)) - ) if not simple_regex.search(text): # There is no match, so bail now. From 317505a90362f4a5f5517d55f40ddaae8aa9b80d Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 16 Mar 2025 18:01:44 +0100 Subject: [PATCH 35/77] Have a current version to work with multiple output files --- tested/dsl/schema-strict.json | 36 +++++++++++++++++++++------------- tested/dsl/schema.json | 33 +++++++++++++++++++------------ tested/dsl/translate_parser.py | 19 +++++++----------- tested/judge/utils.py | 10 +++++++++- tested/oracles/text.py | 33 ++++++++++++++++++++----------- tested/testsuite.py | 6 +++--- tests/test_dsl_yaml.py | 4 ++-- tests/test_oracles_builtin.py | 18 ++++++++--------- tests/test_suite.py | 2 +- 9 files changed, 96 insertions(+), 65 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 5074d1ee..f60a7758 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -377,8 +377,7 @@ "string", "number", "integer", - "boolean", - "path" + "boolean" ] }, "arguments" : { @@ -615,15 +614,18 @@ "items" : { "type" : "object", "required" : [ - "content", - "path" + "student_path" ], "properties" : { "content" : { "type" : "string", - "description" : "Path or expected content for the file, relative to the evaluation directory." + "description" : "Expected content for the file." + }, + "url" : { + "type" : "string", + "description" : "Path to file with the expected content for the file, relative to the evaluation directory." }, - "path" : { + "student_path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } @@ -642,15 +644,18 @@ "items" : { "type" : "object", "required" : [ - "content", - "path" + "student_path" ], "properties" : { "content" : { "type" : "string", - "description" : "Path or expected content for the file, relative to the evaluation directory." + "description" : "Expected content for the file." }, - "path" : { + "url" : { + "type" : "string", + "description" : "Path to file with the expected content for the file, relative to the evaluation directory." + }, + "student_path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } @@ -682,15 +687,18 @@ "items" : { "type" : "object", "required" : [ - "content", - "path" + "student_path" ], "properties" : { "content" : { "type" : "string", - "description" : "Path or expected content for the file, relative to the evaluation directory." + "description" : "Expected content for the file." + }, + "url" : { + "type" : "string", + "description" : "Path to file with the expected content for the file, relative to the evaluation directory." }, - "path" : { + "student_path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index f87f484c..af48b57d 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -613,15 +613,18 @@ "items" : { "type" : "object", "required" : [ - "content", - "path" + "student_path" ], "properties" : { "content" : { "type" : "string", - "description" : "Path or expected content for the file, relative to the evaluation directory." + "description" : "Expected content for the file." }, - "path" : { + "url" : { + "type" : "string", + "description" : "Path to file with the expected content for the file, relative to the evaluation directory." + }, + "student_path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } @@ -640,15 +643,18 @@ "items" : { "type" : "object", "required" : [ - "content", - "path" + "student_path" ], "properties" : { "content" : { "type" : "string", - "description" : "Path or expected content for the file, relative to the evaluation directory." + "description" : "Expected content for the file." + }, + "url" : { + "type" : "string", + "description" : "Path to file with the expected content for the file, relative to the evaluation directory." }, - "path" : { + "student_path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } @@ -680,15 +686,18 @@ "items" : { "type" : "object", "required" : [ - "content", - "path" + "student_path" ], "properties" : { "content" : { "type" : "string", - "description" : "Path or expected content for the file, relative to the evaluation directory." + "description" : "Expected content for the file." + }, + "url" : { + "type" : "string", + "description" : "Path to file with the expected content for the file, relative to the evaluation directory." }, - "path" : { + "student_path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 98a68385..1f056168 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -1,9 +1,7 @@ -import base64 import json import os import sys import textwrap -import zlib from collections.abc import Callable from decimal import Decimal from pathlib import Path @@ -36,6 +34,7 @@ ) from tested.dodona import ExtendedMessage from tested.dsl.ast_translator import InvalidDslError, extract_comment, parse_string +from tested.judge.utils import base64_encode from tested.parsing import get_converter, suite_to_json from tested.serialisation import ( BooleanType, @@ -429,13 +428,6 @@ def _convert_value(value: YamlObject) -> Value: return _tested_type_to_value(tested_type) -def base64_encode(content: str) -> str: - sample_string_bytes = content.encode("ascii") - - base64_bytes = base64.b64encode(zlib.compress(sample_string_bytes)) - return base64_bytes.decode("ascii") - - def _convert_file(link_file: YamlDict, workdir: Path | None) -> FileUrl: assert isinstance(link_file["name"], str) if "content" in link_file: @@ -563,16 +555,19 @@ def _convert_file_output_channel( for item in data: assert isinstance(item, dict) - if isinstance(item["content"], PathString): + if "url" in item: content_type = TextChannelType.FILE + content = str(item["url"]) else: + assert "content" in item content_type = TextChannelType.TEXT + content = str(item["content"]) file_data.append( OutputFileData( content_type=content_type, - content=str(item["content"]), - path=str(item["path"]), + content=content, + student_path=str(item["student_path"]), ) ) diff --git a/tested/judge/utils.py b/tested/judge/utils.py index c50a00b2..8475e841 100644 --- a/tested/judge/utils.py +++ b/tested/judge/utils.py @@ -1,10 +1,11 @@ """ Common utilities for the judge. """ - +import base64 import logging import shutil import subprocess +import zlib from pathlib import Path from attrs import define @@ -145,3 +146,10 @@ def filter_files(files: list[str] | FileFilter, directory: Path) -> list[Path]: ) else: return [Path(file) for file in files] + + +def base64_encode(content: str) -> str: + sample_string_bytes = content.encode("ascii") + + base64_bytes = base64.b64encode(zlib.compress(sample_string_bytes)) + return base64_bytes.decode("ascii") diff --git a/tested/oracles/text.py b/tested/oracles/text.py index 76ec1386..c7f6ae62 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -7,6 +7,7 @@ from tested.dodona import Status, StatusMessage from tested.internationalization import get_i18n_string +from tested.judge.utils import base64_encode from tested.oracles.common import OracleConfig, OracleResult from tested.testsuite import ( FileOutputChannel, @@ -108,6 +109,16 @@ def evaluate_text( result = compare_text(options, expected, actual) return result +def make_expected_and_actual_file_output(output_data: OutputChannel, expected: str, actual: str) -> tuple[str, str]: + content_type = output_data.content_type + expected_str = output_data.content + if content_type == TextChannelType.TEXT: + expected_str = base64_encode(expected) + + return ( + f"--- <{output_data.student_path}|{content_type}> ---\n{expected_str}", + f"--- <{output_data.student_path}|text> ---\n{base64_encode(actual)}" + ) def evaluate_file( config: OracleConfig, channel: OutputChannel, actual: str @@ -147,7 +158,7 @@ def evaluate_file( file_not_found = False for i in range(len(channel.output_data)): output_data = channel.output_data[i] - actual_path = config.context_dir / output_data.path + actual_path = config.context_dir / output_data.student_path if output_data.content_type == TextChannelType.FILE: expected_path = f"{config.bundle.config.resources}/{output_data.content}" @@ -186,12 +197,12 @@ def evaluate_file( new_result, expected_list[i] = _text_comparison( options, expected_value, actual_value ) - expected_list[i] = ( - f"--- <{channel.output_data[i].path}> ---\n{expected_list[i]}" - ) - actual_list[i] = ( - f"--- <{channel.output_data[i].path}> ---\n{actual_list[i]}" + expected_list[i], actual_list[i] = make_expected_and_actual_file_output( + channel.output_data[i], + expected_list[i], + actual_list[i] ) + result = result and new_result else: assert options["mode"] == "line" @@ -205,11 +216,11 @@ def evaluate_file( for expected_line, actual_line in zip(expected_lines, actual_lines): new_result, _ = _text_comparison(options, expected_line, actual_line) result = result and new_result - expected_list[i] = ( - f"--- <{channel.output_data[i].path}> ---\n{expected_list[i]}" - ) - actual_list[i] = ( - f"--- <{channel.output_data[i].path}> ---\n{actual_list[i]}" + + expected_list[i], actual_list[i] = make_expected_and_actual_file_output( + channel.output_data[i], + expected_list[i], + actual_list[i] ) return OracleResult( diff --git a/tested/testsuite.py b/tested/testsuite.py index ee9d74d9..28ffaf0f 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -277,7 +277,7 @@ class TextOutputChannel(TextData): class OutputFileData: content_type: TextChannelType content: str - path: str + student_path: str @fallback_field(get_converter(), {"evaluator": "oracle"}) @@ -301,10 +301,10 @@ def get_data_as_string(self, resources: Path) -> str: if output_data.content_type == TextChannelType.FILE: file_path = _resolve_path(resources, output_data.content) with open(file_path, "r") as file: - file_content.append(f"--- <{output_data.path}> ---\n{file.read()}") + file_content.append(f"--- <{output_data.student_path}> ---\n{file.read()}") else: file_content.append( - f"--- <{output_data.path}> ---\n{output_data.content[i]}" + f"--- <{output_data.student_path}> ---\n{output_data.content[i]}" ) return "\n".join(file_content) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index b45bdb48..44abe28d 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -809,10 +809,10 @@ def test_output_files_custom_check_correct(): assert isinstance(test.input, FunctionCall) assert isinstance(test.output.file, FileOutputChannel) assert isinstance(test.output.file.oracle, CustomCheckOracle) - assert test.output.file.output_data[0].path == "test.txt" + assert test.output.file.output_data[0].student_path == "test.txt" assert test.output.file.output_data[0].content == "test/hallo.txt" assert test.output.file.output_data[0].content_type == TextChannelType.FILE - assert test.output.file.output_data[1].path == "test2.txt" + assert test.output.file.output_data[1].student_path == "test2.txt" assert test.output.file.output_data[1].content == "Hallo world!" assert test.output.file.output_data[1].content_type == TextChannelType.TEXT oracle = test.output.file.oracle diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index 852aa117..45a8d5e9 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -154,7 +154,7 @@ def test_file_oracle_full_wrong( output_data=[ OutputFileData( content="expected.txt", - path="expected.txt", + student_path="expected.txt", content_type=TextChannelType.FILE, ) ] @@ -182,7 +182,7 @@ def test_file_oracle_full_correct( output_data=[ OutputFileData( content="expected.txt", - path="expected.txt", + student_path="expected.txt", content_type=TextChannelType.FILE, ) ] @@ -215,12 +215,12 @@ def test_file_oracle_full_correct_with_mixed_content( output_data=[ OutputFileData( content="expected.txt", - path="expected.txt", + student_path="expected.txt", content_type=TextChannelType.FILE, ), OutputFileData( content="expected\nexpected", - path="expected.txt", + student_path="expected.txt", content_type=TextChannelType.TEXT, ), ] @@ -256,7 +256,7 @@ def test_file_oracle_line_wrong( output_data=[ OutputFileData( content="expected.txt", - path="expected.txt", + student_path="expected.txt", content_type=TextChannelType.FILE, ) ] @@ -288,7 +288,7 @@ def test_file_oracle_line_correct( output_data=[ OutputFileData( content="expected.txt", - path="expected.txt", + student_path="expected.txt", content_type=TextChannelType.FILE, ) ] @@ -320,7 +320,7 @@ def test_file_oracle_strip_lines_correct( output_data=[ OutputFileData( content="expected.txt", - path="expected.txt", + student_path="expected.txt", content_type=TextChannelType.FILE, ) ] @@ -352,7 +352,7 @@ def test_file_oracle_dont_strip_lines_correct( output_data=[ OutputFileData( content="expected.txt", - path="expected.txt", + student_path="expected.txt", content_type=TextChannelType.FILE, ) ] @@ -374,7 +374,7 @@ def test_correct_error_actual_not_found(tmp_path: Path, pytestconfig: pytest.Con output_data=[ OutputFileData( content="Hallo world!", - path="expected.txt", + student_path="expected.txt", content_type=TextChannelType.TEXT, ) ] diff --git a/tests/test_suite.py b/tests/test_suite.py index 4eff78c2..789a40f2 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -128,7 +128,7 @@ def test_file_show_expected_is_accepted(): result = get_converter().loads(scheme, FileOutputChannel) assert result.output_data[0].content == "hallo" - assert result.output_data[0].path == "hallo.txt" + assert result.output_data[0].student_path == "hallo.txt" def test_value_show_expected_is_accepted(): From aa295600939be51b528cb068dfa6fd6fcbc0b888 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 17 Mar 2025 16:08:34 +0100 Subject: [PATCH 36/77] cleaned up some code and used urls in stdin too --- tested/dsl/schema-strict.json | 14 ++++++++++++-- tested/dsl/schema.json | 15 +++++++++++++-- tested/dsl/translate_parser.py | 7 +++++-- tested/judge/utils.py | 1 + tested/languages/generation.py | 1 - tested/oracles/text.py | 17 +++++++++-------- tested/testsuite.py | 4 +++- 7 files changed, 43 insertions(+), 16 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index f60a7758..71edd736 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -281,8 +281,18 @@ "number", "integer", "boolean", - "path" - ] + "object" + ], + "required" : [ + "url" + ], + "properties" : { + "url" : { + "type" : "string", + "format" : "uri", + "description" : "Relative path to the file in the `description` folder of an exercise." + } + } }, "arguments" : { "type" : "array", diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index af48b57d..3eb9c950 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -280,8 +280,19 @@ "string", "number", "integer", - "boolean" - ] + "boolean", + "object" + ], + "required" : [ + "url" + ], + "properties" : { + "url" : { + "type" : "string", + "format" : "uri", + "description" : "Relative path to the file in the `description` folder of an exercise." + } + } }, "arguments" : { "type" : "array", diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 1f056168..0ee955ea 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -684,8 +684,11 @@ def _convert_testcase( return_channel = IgnoredChannel.IGNORED if "statement" in testcase else None else: if "stdin" in testcase: - if isinstance(testcase["stdin"], PathString): - stdin = TextData(data=testcase["stdin"], type=TextChannelType.FILE) + if isinstance(testcase["stdin"], dict): + assert "url" in testcase["stdin"] + stdin = TextData( + data=str(testcase["stdin"]["url"]), type=TextChannelType.FILE + ) else: assert isinstance(testcase["stdin"], str) stdin = TextData(data=_ensure_trailing_newline(testcase["stdin"])) diff --git a/tested/judge/utils.py b/tested/judge/utils.py index 8475e841..cc37aac0 100644 --- a/tested/judge/utils.py +++ b/tested/judge/utils.py @@ -1,6 +1,7 @@ """ Common utilities for the judge. """ + import base64 import logging import shutil diff --git a/tested/languages/generation.py b/tested/languages/generation.py index ace1c919..a68c376c 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -171,7 +171,6 @@ def get_readable_input( if not case.link_files: return ExtendedMessage(description=text, format=format_), set() - if not simple_regex.search(text): # There is no match, so bail now. return ExtendedMessage(description=text, format=format_), set() diff --git a/tested/oracles/text.py b/tested/oracles/text.py index c7f6ae62..3cc0877f 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -12,6 +12,7 @@ from tested.testsuite import ( FileOutputChannel, OutputChannel, + OutputFileData, TextChannelType, TextOutputChannel, ) @@ -109,7 +110,10 @@ def evaluate_text( result = compare_text(options, expected, actual) return result -def make_expected_and_actual_file_output(output_data: OutputChannel, expected: str, actual: str) -> tuple[str, str]: + +def make_expected_and_actual_file_output( + output_data: OutputFileData, expected: str, actual: str +) -> tuple[str, str]: content_type = output_data.content_type expected_str = output_data.content if content_type == TextChannelType.TEXT: @@ -117,9 +121,10 @@ def make_expected_and_actual_file_output(output_data: OutputChannel, expected: s return ( f"--- <{output_data.student_path}|{content_type}> ---\n{expected_str}", - f"--- <{output_data.student_path}|text> ---\n{base64_encode(actual)}" + f"--- <{output_data.student_path}|text> ---\n{base64_encode(actual)}", ) + def evaluate_file( config: OracleConfig, channel: OutputChannel, actual: str ) -> OracleResult: @@ -198,9 +203,7 @@ def evaluate_file( options, expected_value, actual_value ) expected_list[i], actual_list[i] = make_expected_and_actual_file_output( - channel.output_data[i], - expected_list[i], - actual_list[i] + channel.output_data[i], expected_list[i], actual_list[i] ) result = result and new_result @@ -218,9 +221,7 @@ def evaluate_file( result = result and new_result expected_list[i], actual_list[i] = make_expected_and_actual_file_output( - channel.output_data[i], - expected_list[i], - actual_list[i] + channel.output_data[i], expected_list[i], actual_list[i] ) return OracleResult( diff --git a/tested/testsuite.py b/tested/testsuite.py index 28ffaf0f..9be15370 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -301,7 +301,9 @@ def get_data_as_string(self, resources: Path) -> str: if output_data.content_type == TextChannelType.FILE: file_path = _resolve_path(resources, output_data.content) with open(file_path, "r") as file: - file_content.append(f"--- <{output_data.student_path}> ---\n{file.read()}") + file_content.append( + f"--- <{output_data.student_path}> ---\n{file.read()}" + ) else: file_content.append( f"--- <{output_data.student_path}> ---\n{output_data.content[i]}" From 2d1e56868b3fc235b94ed783a2c91d1d02bcfb3a Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 17 Mar 2025 19:18:31 +0100 Subject: [PATCH 37/77] Made attempt for stdout/stderr --- tested/dsl/schema-strict.json | 39 +++++++++++++++++++++++++++++++--- tested/dsl/schema.json | 39 +++++++++++++++++++++++++++++++--- tested/dsl/translate_parser.py | 14 +++++------- tested/oracles/text.py | 17 +++++++++++++-- 4 files changed, 92 insertions(+), 17 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 71edd736..370c5678 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -290,7 +290,7 @@ "url" : { "type" : "string", "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." + "description" : "Relative path to the file in the `evaluation` folder of an exercise." } } }, @@ -387,8 +387,19 @@ "string", "number", "integer", - "boolean" - ] + "boolean", + "object" + ], + "required" : [ + "url" + ], + "properties" : { + "url" : { + "type" : "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } }, "arguments" : { "type" : "array", @@ -539,6 +550,11 @@ "data" ] }, + { + "required" : [ + "url" + ] + }, { "required" : [ "content" @@ -552,6 +568,11 @@ "data" : { "$ref" : "#/definitions/textualType" }, + "url" : { + "type" : "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + }, "oracle" : { "const" : "builtin" }, @@ -571,6 +592,13 @@ "data" ] }, + { + "required" : [ + "oracle", + "file", + "url" + ] + }, { "required" : [ "oracle", @@ -586,6 +614,11 @@ "data" : { "$ref" : "#/definitions/textualType" }, + "url" : { + "type" : "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + }, "oracle" : { "const" : "custom_check" }, diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 3eb9c950..26d0f15a 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -290,7 +290,7 @@ "url" : { "type" : "string", "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." + "description" : "Relative path to the file in the `evaluation` folder of an exercise." } } }, @@ -387,8 +387,19 @@ "string", "number", "integer", - "boolean" - ] + "boolean", + "object" + ], + "required" : [ + "url" + ], + "properties" : { + "url" : { + "type" : "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } }, "arguments" : { "type" : "array", @@ -539,6 +550,11 @@ "data" ] }, + { + "required" : [ + "url" + ] + }, { "required" : [ "content" @@ -552,6 +568,11 @@ "data" : { "$ref" : "#/definitions/textualType" }, + "url" : { + "type" : "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + }, "oracle" : { "const" : "builtin" }, @@ -571,6 +592,13 @@ "data" ] }, + { + "required" : [ + "oracle", + "file", + "url" + ] + }, { "required" : [ "oracle", @@ -586,6 +614,11 @@ "data" : { "$ref" : "#/definitions/textualType" }, + "url" : { + "type" : "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + }, "oracle" : { "const" : "custom_check" }, diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 0ee955ea..09f0d09a 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -502,21 +502,17 @@ def _convert_text_output_channel( if isinstance(stream, str): config = context.config.get(config_name, dict()) - if isinstance(stream, PathString): - path = stream raw_data = stream else: assert isinstance(stream, dict) - data = stream.get("content", stream.get("data")) - assert data is not None - - if isinstance(data, PathString): - path = data + if "url" in stream: config = context.config.get(config_name, dict()) - raw_data = data + raw_data = str(stream["url"]) + path = raw_data else: + raw_data = str(stream.get("content", stream.get("data"))) config = context.merge_inheritable_with_specific_config(stream, config_name) - raw_data = str(stream["data"]) + assert raw_data is not None # Normalize the data if necessary. if config.get("normalizeTrailingNewlines", True) and path is None: diff --git a/tested/oracles/text.py b/tested/oracles/text.py index 3cc0877f..b8fdc55d 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -3,6 +3,7 @@ """ import math +import os from typing import Any from tested.dodona import Status, StatusMessage @@ -77,9 +78,16 @@ def _text_comparison( return actual_eval == expected_eval, expected -def compare_text(options: dict[str, Any], expected: str, actual: str) -> OracleResult: +def compare_text( + options: dict[str, Any], expected: str, actual: str, expected_path: str = "" +) -> OracleResult: result, expected = _text_comparison(options, expected, actual) + if expected_path: + expected = f"--- <{os.path.basename(expected_path)}|file> ---\n{expected_path}" + actual = ( + f"--- <{os.path.basename(expected_path)}|text> ---\n{base64_encode(actual)}" + ) return OracleResult( result=StatusMessage(enum=Status.CORRECT if result else Status.WRONG), readable_expected=str(expected), @@ -107,7 +115,12 @@ def evaluate_text( options = _text_options(config) expected = channel.get_data_as_string(config.bundle.config.resources) - result = compare_text(options, expected, actual) + result = compare_text( + options, + expected, + actual, + channel.data if channel.type == TextChannelType.FILE else "", + ) return result From 497f4527956f83918e94136d72a73c155fddc6f1 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 23 Mar 2025 13:59:54 +0100 Subject: [PATCH 38/77] changed names again and splitted output files over multiple expected/generated pairs --- tested/dsl/schema-strict.json | 115 +++++++++++++-------------------- tested/dsl/schema.json | 80 +++++------------------ tested/dsl/translate_parser.py | 35 +++++----- tested/judge/evaluation.py | 71 +++++++++++--------- tested/languages/generation.py | 6 +- tested/oracles/text.py | 101 +++++++++-------------------- tested/testsuite.py | 11 +++- tests/test_dsl_yaml.py | 4 +- 8 files changed, 162 insertions(+), 261 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 370c5678..a7142aab 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -278,21 +278,12 @@ "description" : "Stdin for this context", "type" : [ "string", + "path", "number", "integer", "boolean", "object" - ], - "required" : [ - "url" - ], - "properties" : { - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." - } - } + ] }, "arguments" : { "type" : "array", @@ -385,21 +376,12 @@ "description" : "Stdin for this context", "type" : [ "string", + "path", "number", "integer", "boolean", "object" - ], - "required" : [ - "url" - ], - "properties" : { - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." - } - } + ] }, "arguments" : { "type" : "array", @@ -518,10 +500,10 @@ "type" : "object", "description" : "A file used in the test suite.", "required" : [ - "name" + "path" ], "properties" : { - "name" : { + "path" : { "type" : "string", "description" : "The filename, including the file extension." }, @@ -550,11 +532,6 @@ "data" ] }, - { - "required" : [ - "url" - ] - }, { "required" : [ "content" @@ -563,16 +540,12 @@ ], "properties" : { "content": { - "$ref" : "#/definitions/textualType" + "$ref" : "#/definitions/textualType", + "description" : "Content or relative path to the file in the `evaluation` folder of an exercise." }, "data" : { "$ref" : "#/definitions/textualType" }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." - }, "oracle" : { "const" : "builtin" }, @@ -592,13 +565,6 @@ "data" ] }, - { - "required" : [ - "oracle", - "file", - "url" - ] - }, { "required" : [ "oracle", @@ -614,11 +580,6 @@ "data" : { "$ref" : "#/definitions/textualType" }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." - }, "oracle" : { "const" : "custom_check" }, @@ -657,16 +618,20 @@ "items" : { "type" : "object", "required" : [ - "student_path" + "student_path", + "content" ], "properties" : { "content" : { - "type" : "string", - "description" : "Expected content for the file." - }, - "url" : { - "type" : "string", - "description" : "Path to file with the expected content for the file, relative to the evaluation directory." + "anyOf": [ + { + "type": "string" + }, + { + "type": "path" + } + ], + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." }, "student_path" : { "type" : "string", @@ -687,16 +652,20 @@ "items" : { "type" : "object", "required" : [ - "student_path" + "student_path", + "content" ], "properties" : { "content" : { - "type" : "string", - "description" : "Expected content for the file." - }, - "url" : { - "type" : "string", - "description" : "Path to file with the expected content for the file, relative to the evaluation directory." + "anyOf": [ + { + "type": "string" + }, + { + "type": "path" + } + ], + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." }, "student_path" : { "type" : "string", @@ -730,16 +699,20 @@ "items" : { "type" : "object", "required" : [ - "student_path" + "student_path", + "content" ], "properties" : { "content" : { - "type" : "string", - "description" : "Expected content for the file." - }, - "url" : { - "type" : "string", - "description" : "Path to file with the expected content for the file, relative to the evaluation directory." + "anyOf": [ + { + "type": "string" + }, + { + "type": "path" + } + ], + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." }, "student_path" : { "type" : "string", @@ -993,7 +966,8 @@ "string", "number", "integer", - "boolean" + "boolean", + "path" ] }, "yamlValue" : { @@ -1001,7 +975,8 @@ "not" : { "type" : [ "oracle", - "expression" + "expression", + "path" ] } }, diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 26d0f15a..32120231 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -282,17 +282,7 @@ "integer", "boolean", "object" - ], - "required" : [ - "url" - ], - "properties" : { - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." - } - } + ] }, "arguments" : { "type" : "array", @@ -389,17 +379,7 @@ "integer", "boolean", "object" - ], - "required" : [ - "url" - ], - "properties" : { - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." - } - } + ] }, "arguments" : { "type" : "array", @@ -518,10 +498,10 @@ "type" : "object", "description" : "A file used in the test suite.", "required" : [ - "name" + "path" ], "properties" : { - "name" : { + "path" : { "type" : "string", "description" : "The filename, including the file extension." }, @@ -550,11 +530,6 @@ "data" ] }, - { - "required" : [ - "url" - ] - }, { "required" : [ "content" @@ -563,16 +538,12 @@ ], "properties" : { "content": { - "$ref" : "#/definitions/textualType" + "$ref" : "#/definitions/textualType", + "description" : "Content or relative path to the file in the `evaluation` folder of an exercise." }, "data" : { "$ref" : "#/definitions/textualType" }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." - }, "oracle" : { "const" : "builtin" }, @@ -592,13 +563,6 @@ "data" ] }, - { - "required" : [ - "oracle", - "file", - "url" - ] - }, { "required" : [ "oracle", @@ -614,11 +578,6 @@ "data" : { "$ref" : "#/definitions/textualType" }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." - }, "oracle" : { "const" : "custom_check" }, @@ -657,16 +616,13 @@ "items" : { "type" : "object", "required" : [ - "student_path" + "student_path", + "content" ], "properties" : { "content" : { "type" : "string", - "description" : "Expected content for the file." - }, - "url" : { - "type" : "string", - "description" : "Path to file with the expected content for the file, relative to the evaluation directory." + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." }, "student_path" : { "type" : "string", @@ -687,16 +643,13 @@ "items" : { "type" : "object", "required" : [ - "student_path" + "student_path", + "content" ], "properties" : { "content" : { "type" : "string", - "description" : "Expected content for the file." - }, - "url" : { - "type" : "string", - "description" : "Path to file with the expected content for the file, relative to the evaluation directory." + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." }, "student_path" : { "type" : "string", @@ -730,16 +683,13 @@ "items" : { "type" : "object", "required" : [ - "student_path" + "student_path", + "content" ], "properties" : { "content" : { "type" : "string", - "description" : "Expected content for the file." - }, - "url" : { - "type" : "string", - "description" : "Path to file with the expected content for the file, relative to the evaluation directory." + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." }, "student_path" : { "type" : "string", diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 09f0d09a..9e0578c7 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -429,11 +429,11 @@ def _convert_value(value: YamlObject) -> Value: def _convert_file(link_file: YamlDict, workdir: Path | None) -> FileUrl: - assert isinstance(link_file["name"], str) + assert isinstance(link_file["path"], str) if "content" in link_file: assert isinstance(link_file["content"], str) if workdir is not None: - full_path = workdir / link_file["name"] + full_path = workdir / link_file["path"] os.makedirs(os.path.dirname(full_path), exist_ok=True) with open(full_path, "w", encoding="utf-8") as f: f.write(link_file["content"]) @@ -449,7 +449,7 @@ def _convert_file(link_file: YamlDict, workdir: Path | None) -> FileUrl: assert isinstance(link_file["url"], str) url = link_file["url"] - return FileUrl(name=link_file["name"], url=url) + return FileUrl(path=link_file["path"], url=url) def _convert_evaluation_function(stream: dict) -> EvaluationFunction: @@ -503,22 +503,23 @@ def _convert_text_output_channel( if isinstance(stream, str): config = context.config.get(config_name, dict()) raw_data = stream + if isinstance(raw_data, PathString): + path = raw_data else: assert isinstance(stream, dict) - if "url" in stream: + raw_data = stream.get("content", stream.get("data")) + if not isinstance(raw_data, PathString): + config = context.merge_inheritable_with_specific_config(stream, config_name) + else: config = context.config.get(config_name, dict()) - raw_data = str(stream["url"]) path = raw_data - else: - raw_data = str(stream.get("content", stream.get("data"))) - config = context.merge_inheritable_with_specific_config(stream, config_name) assert raw_data is not None # Normalize the data if necessary. if config.get("normalizeTrailingNewlines", True) and path is None: - data = _ensure_trailing_newline(raw_data) + data = _ensure_trailing_newline(str(raw_data)) else: - data = raw_data + data = str(raw_data) text_output = TextOutputChannel(data=data) if path is not None: @@ -551,13 +552,12 @@ def _convert_file_output_channel( for item in data: assert isinstance(item, dict) - if "url" in item: + content = item["content"] + if isinstance(content, PathString): content_type = TextChannelType.FILE - content = str(item["url"]) else: - assert "content" in item + content = str(content) content_type = TextChannelType.TEXT - content = str(item["content"]) file_data.append( OutputFileData( @@ -680,11 +680,8 @@ def _convert_testcase( return_channel = IgnoredChannel.IGNORED if "statement" in testcase else None else: if "stdin" in testcase: - if isinstance(testcase["stdin"], dict): - assert "url" in testcase["stdin"] - stdin = TextData( - data=str(testcase["stdin"]["url"]), type=TextChannelType.FILE - ) + if isinstance(testcase["stdin"], PathString): + stdin = TextData(data=str(testcase["stdin"]), type=TextChannelType.FILE) else: assert isinstance(testcase["stdin"], str) stdin = TextData(data=_ensure_trailing_newline(testcase["stdin"])) diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index 44b2571a..629e37c3 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -110,39 +110,52 @@ def _evaluate_channel( bundle, context_directory, output, testcase, unexpected_status=unexpected_status ) # Run the oracle. - evaluation_result = evaluator(output, actual if actual else "") - status = evaluation_result.result + print("-------------") + print(channel) + print(output) + print("-------------") + if channel == Channel.FILE: + assert isinstance(output, FileOutputChannel) + new_output = output.output_data + else: + new_output = [output] - # Decide if we should show this channel or not. - is_correct = status.enum == Status.CORRECT - should_report_case = should_show(output, channel, evaluation_result) + missing = False + if actual is None: + missing = True - if not should_report_case and is_correct: - # We do report that a test is correct, to set the status. - return False + for output_element in new_output: + evaluation_result = evaluator(output_element, actual if actual else "") + status = evaluation_result.result - expected = evaluation_result.readable_expected - out.add(StartTest(expected=expected, channel=channel)) + # Decide if we should show this channel or not. + is_correct = status.enum == Status.CORRECT + should_report_case = should_show(output, channel, evaluation_result) - # Report any messages we received. - for message in evaluation_result.messages: - out.add(AppendMessage(message=message)) + if not should_report_case and is_correct: + # We do report that a test is correct, to set the status. + return False - missing = False - if actual is None: - out.add(AppendMessage(message=get_i18n_string("judge.evaluation.missing"))) - missing = True - elif should_report_case and timeout and not is_correct: - status.human = get_i18n_string("judge.evaluation.time-limit") - status.enum = Status.TIME_LIMIT_EXCEEDED - out.add(AppendMessage(message=status.human)) - elif should_report_case and memory and not is_correct: - status.human = get_i18n_string("judge.evaluation.memory-limit") - status.enum = Status.TIME_LIMIT_EXCEEDED - out.add(AppendMessage(message=status.human)) - - # Close the test. - out.add(CloseTest(generated=evaluation_result.readable_actual, status=status)) + expected = evaluation_result.readable_expected + out.add(StartTest(expected=expected, channel=channel)) + + # Report any messages we received. + for message in evaluation_result.messages: + out.add(AppendMessage(message=message)) + + if actual is None: + out.add(AppendMessage(message=get_i18n_string("judge.evaluation.missing"))) + elif should_report_case and timeout and not is_correct: + status.human = get_i18n_string("judge.evaluation.time-limit") + status.enum = Status.TIME_LIMIT_EXCEEDED + out.add(AppendMessage(message=status.human)) + elif should_report_case and memory and not is_correct: + status.human = get_i18n_string("judge.evaluation.memory-limit") + status.enum = Status.TIME_LIMIT_EXCEEDED + out.add(AppendMessage(message=status.human)) + + # Close the test. + out.add(CloseTest(generated=evaluation_result.readable_actual, status=status)) return missing @@ -365,7 +378,7 @@ def evaluate_context_results( def _link_files_message(link_files: Collection[FileUrl]) -> AppendMessage: link_list = ", ".join( f'' - f'{html.escape(link_file.name)}' + f'{html.escape(link_file.path)}' for link_file in link_files ) file_list_str = get_i18n_string( diff --git a/tested/languages/generation.py b/tested/languages/generation.py index a68c376c..97474da7 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -79,7 +79,7 @@ def generate_execution_unit( def _handle_link_files(link_files: Iterable[FileUrl], language: str) -> tuple[str, str]: dict_links = dict( - (link_file.name, get_converter().unstructure(link_file)) + (link_file.path, get_converter().unstructure(link_file)) for link_file in link_files ) files = json.dumps(dict_links) @@ -116,7 +116,7 @@ def get_readable_input( # If not, we can also stop before doing ugly things. # We construct a regex, since that can be faster than checking everything. simple_regex = re.compile( - "|".join(map(lambda x: re.escape(x.name), case.link_files)) + "|".join(map(lambda x: re.escape(x.path), case.link_files)) ) format_ = "text" # By default, we use text as input. @@ -185,7 +185,7 @@ def get_readable_input( generated_html = highlight_code(text, bundle.config.programming_language) # Map of file URLs. - url_map = {html.escape(x.name): x for x in case.link_files} + url_map = {html.escape(x.path): x for x in case.link_files} seen = set() escaped_regex = re.compile("|".join(url_map.keys())) diff --git a/tested/oracles/text.py b/tested/oracles/text.py index b8fdc55d..bd735471 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -78,16 +78,10 @@ def _text_comparison( return actual_eval == expected_eval, expected -def compare_text( - options: dict[str, Any], expected: str, actual: str, expected_path: str = "" -) -> OracleResult: +def compare_text(options: dict[str, Any], expected: str, actual: str) -> OracleResult: result, expected = _text_comparison(options, expected, actual) - if expected_path: - expected = f"--- <{os.path.basename(expected_path)}|file> ---\n{expected_path}" - actual = ( - f"--- <{os.path.basename(expected_path)}|text> ---\n{base64_encode(actual)}" - ) + return OracleResult( result=StatusMessage(enum=Status.CORRECT if result else Status.WRONG), readable_expected=str(expected), @@ -115,12 +109,7 @@ def evaluate_text( options = _text_options(config) expected = channel.get_data_as_string(config.bundle.config.resources) - result = compare_text( - options, - expected, - actual, - channel.data if channel.type == TextChannelType.FILE else "", - ) + result = compare_text(options, expected, actual) return result @@ -155,7 +144,7 @@ def evaluate_file( When no mode is passed, the oracle will default to ``full``. """ - assert isinstance(channel, FileOutputChannel) + assert isinstance(channel, OutputFileData) options = _text_options(config) # There must be nothing as output. @@ -171,74 +160,44 @@ def evaluate_file( messages=[message], ) - actual_list = [] - expected_list = [] - file_not_found = False - for i in range(len(channel.output_data)): - output_data = channel.output_data[i] - actual_path = config.context_dir / output_data.student_path - - if output_data.content_type == TextChannelType.FILE: - expected_path = f"{config.bundle.config.resources}/{output_data.content}" - try: - with open(expected_path, "r") as file: - expected_list.append(file.read()) - except FileNotFoundError: - raise ValueError(f"File {expected_path} not found in resources.") - else: - expected_list.append(output_data.content) + expected = channel.content + if channel.content_type == TextChannelType.FILE: + expected_path = f"{config.bundle.config.resources}/{expected}" try: - with open(str(actual_path), "r") as file: - actual_list.append(file.read()) + with open(expected_path, "r") as file: + expected = file.read() except FileNotFoundError: - file_not_found = True + raise ValueError(f"File {expected_path} not found in resources.") - actual = "\n".join(actual_list) - expected = "\n".join(expected_list) - if file_not_found: + actual_path = config.context_dir / channel.student_path + + try: + with open(str(actual_path), "r") as file: + actual = file.read() + except FileNotFoundError: return OracleResult( result=StatusMessage( enum=Status.RUNTIME_ERROR, human=get_i18n_string("oracles.text.file.not-found"), ), readable_expected=expected, - readable_actual=actual, + readable_actual="", ) - result = True - if options["mode"] == "full": - for i in range(len(expected_list)): - expected_value = expected_list[i] - actual_value = actual_list[i] - new_result, expected_list[i] = _text_comparison( - options, expected_value, actual_value - ) - expected_list[i], actual_list[i] = make_expected_and_actual_file_output( - channel.output_data[i], expected_list[i], actual_list[i] - ) - - result = result and new_result + return compare_text(options, expected, actual) else: assert options["mode"] == "line" - for i in range(len(expected_list)): - expected_value = expected_list[i] - actual_value = actual_list[i] - strip_newlines = options.get("stripNewlines", False) - expected_lines = expected_value.splitlines(keepends=not strip_newlines) - actual_lines = actual_value.splitlines(keepends=not strip_newlines) - result = len(actual_lines) == len(expected_lines) - for expected_line, actual_line in zip(expected_lines, actual_lines): - new_result, _ = _text_comparison(options, expected_line, actual_line) - result = result and new_result - - expected_list[i], actual_list[i] = make_expected_and_actual_file_output( - channel.output_data[i], expected_list[i], actual_list[i] - ) - - return OracleResult( - result=StatusMessage(enum=Status.CORRECT if result else Status.WRONG), - readable_expected="\n".join(expected_list), - readable_actual="\n".join(actual_list), - ) + strip_newlines = options.get("stripNewlines", False) + expected_lines = expected.splitlines(keepends=not strip_newlines) + actual_lines = actual.splitlines(keepends=not strip_newlines) + correct = len(actual_lines) == len(expected_lines) + for expected_line, actual_line in zip(expected_lines, actual_lines): + r = compare_text(options, expected_line, actual_line) + correct = correct and r.result.enum == Status.CORRECT + return OracleResult( + result=StatusMessage(enum=Status.CORRECT if correct else Status.WRONG), + readable_expected=expected, + readable_actual=actual, + ) diff --git a/tested/testsuite.py b/tested/testsuite.py index 9be15370..6e841af0 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -278,6 +278,9 @@ class OutputFileData: content_type: TextChannelType content: str student_path: str + oracle: GenericTextOracle | CustomCheckOracle = field( + factory=lambda: GenericTextOracle(name=TextBuiltin.FILE) + ) @fallback_field(get_converter(), {"evaluator": "oracle"}) @@ -428,7 +431,11 @@ def get_used_features(self) -> FeatureSet: SpecialOutputChannel = EmptyChannel | IgnoredChannel OracleOutputChannel = Union[ - TextOutputChannel, FileOutputChannel, ValueOutputChannel, ExceptionOutputChannel + TextOutputChannel, + FileOutputChannel, + OutputFileData, + ValueOutputChannel, + ExceptionOutputChannel, ] NormalOutputChannel = OracleOutputChannel | ExitCodeOutputChannel @@ -546,7 +553,7 @@ def get_functions(self) -> Iterable[FunctionCall]: @define(frozen=True) class FileUrl: url: str - name: str + path: str @ignore_field(get_converter(), "essential") diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 44abe28d..ced79a6c 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -1238,8 +1238,8 @@ def test_files_are_propagated(): testcases0, testcases1 = ctx0.testcases, ctx1.testcases test0, test1 = testcases0[0], testcases1[0] assert set(test0.link_files) == { - FileUrl(name="test", url="test.md"), - FileUrl(name="two", url="two.md"), + FileUrl(path="test", url="test.md"), + FileUrl(path="two", url="two.md"), } From eb7dcbe59358bfc79fb23155afce814365d9d824 Mon Sep 17 00:00:00 2001 From: breblanc Date: Wed, 26 Mar 2025 20:07:19 +0100 Subject: [PATCH 39/77] Fixed a bug and removed a few prints --- tested/judge/evaluation.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index 629e37c3..99c2a2ab 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -45,6 +45,7 @@ FileUrl, IgnoredChannel, OutputChannel, + OutputFileData, SpecialOutputChannel, Testcase, TextOutput, @@ -110,11 +111,7 @@ def _evaluate_channel( bundle, context_directory, output, testcase, unexpected_status=unexpected_status ) # Run the oracle. - print("-------------") - print(channel) - print(output) - print("-------------") - if channel == Channel.FILE: + if channel == Channel.FILE and output != "ignored": assert isinstance(output, FileOutputChannel) new_output = output.output_data else: @@ -137,7 +134,10 @@ def _evaluate_channel( return False expected = evaluation_result.readable_expected - out.add(StartTest(expected=expected, channel=channel)) + expected_channel = channel + if isinstance(output_element, OutputFileData): + expected_channel = output_element.student_path + out.add(StartTest(expected=expected, channel=expected_channel)) # Report any messages we received. for message in evaluation_result.messages: From e8602ac10916a228ad87163c7ae026a3d075385f Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 29 Mar 2025 17:10:36 +0100 Subject: [PATCH 40/77] Added input file data and removed a lot of html generation --- tested/dodona.py | 1 + tested/dsl/translate_parser.py | 12 ++++++------ tested/judge/core.py | 13 +++++++++++++ tested/judge/evaluation.py | 2 +- tested/judge/utils.py | 7 ------- tested/languages/generation.py | 26 +++----------------------- tested/oracles/text.py | 18 ------------------ tested/testsuite.py | 3 ++- 8 files changed, 26 insertions(+), 56 deletions(-) diff --git a/tested/dodona.py b/tested/dodona.py index 90e5f768..862d01e3 100644 --- a/tested/dodona.py +++ b/tested/dodona.py @@ -40,6 +40,7 @@ class Metadata: statements: str | None stdin: str | None + input_files: list[dict[str, str]] | None Message = ExtendedMessage | str diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 9e0578c7..b162b47c 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -34,7 +34,6 @@ ) from tested.dodona import ExtendedMessage from tested.dsl.ast_translator import InvalidDslError, extract_comment, parse_string -from tested.judge.utils import base64_encode from tested.parsing import get_converter, suite_to_json from tested.serialisation import ( BooleanType, @@ -430,26 +429,27 @@ def _convert_value(value: YamlObject) -> Value: def _convert_file(link_file: YamlDict, workdir: Path | None) -> FileUrl: assert isinstance(link_file["path"], str) + content = "" + url = "" if "content" in link_file: - assert isinstance(link_file["content"], str) + content = link_file["content"] + assert isinstance(content, str) if workdir is not None: full_path = workdir / link_file["path"] os.makedirs(os.path.dirname(full_path), exist_ok=True) with open(full_path, "w", encoding="utf-8") as f: - f.write(link_file["content"]) + f.write(content) if "url" in link_file: assert isinstance(link_file["url"], str) url = link_file["url"] - else: - url = base64_encode(link_file["content"]) else: # Assumed the specified files are already in the working directory. assert "url" in link_file assert isinstance(link_file["url"], str) url = link_file["url"] - return FileUrl(path=link_file["path"], url=url) + return FileUrl(path=link_file["path"], url=url, content=content) def _convert_evaluation_function(stream: dict) -> EvaluationFunction: diff --git a/tested/judge/core.py b/tested/judge/core.py index 24bdd3b3..85a13bc3 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -345,7 +345,16 @@ def _process_results( # since it only differs a bit. meta_statements = [] meta_stdin = None + input_files = [] for case in planned.context.testcases: + for file in case.link_files: + file_data = {"path": file.path} + if file.url != "": + file_data["url"] = file.url + + if file.content != "": + file_data["content"] = file.content + input_files.append(file_data) if case.is_main_testcase(): assert isinstance(case.input, MainInput) if isinstance(case.input.stdin, TextData): @@ -367,11 +376,15 @@ def _process_results( # Don't add empty statements meta_statements = None + if not input_files: + input_files = None + collector.add( CloseContext( data=Metadata( statements=meta_statements, stdin=meta_stdin, + input_files=input_files ) ), planned.context_index, diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index 99c2a2ab..d53b1cac 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -377,7 +377,7 @@ def evaluate_context_results( def _link_files_message(link_files: Collection[FileUrl]) -> AppendMessage: link_list = ", ".join( - f'' + f'' f'{html.escape(link_file.path)}' for link_file in link_files ) diff --git a/tested/judge/utils.py b/tested/judge/utils.py index cc37aac0..4b491637 100644 --- a/tested/judge/utils.py +++ b/tested/judge/utils.py @@ -147,10 +147,3 @@ def filter_files(files: list[str] | FileFilter, directory: Path) -> list[Path]: ) else: return [Path(file) for file in files] - - -def base64_encode(content: str) -> str: - sample_string_bytes = content.encode("ascii") - - base64_bytes = base64.b64encode(zlib.compress(sample_string_bytes)) - return base64_bytes.decode("ascii") diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 97474da7..0f693f41 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -77,20 +77,6 @@ def generate_execution_unit( return bundle.language.generate_execution_unit(prepared_execution) -def _handle_link_files(link_files: Iterable[FileUrl], language: str) -> tuple[str, str]: - dict_links = dict( - (link_file.path, get_converter().unstructure(link_file)) - for link_file in link_files - ) - files = json.dumps(dict_links) - return ( - f"
",
-        "
", - ) - - def _get_heredoc_token(stdin: str) -> str: delimiter = "STDIN" while delimiter in stdin: @@ -177,12 +163,9 @@ def get_readable_input( # Now we need to do ugly stuff. # Begin by compiling the HTML that will be displayed. - if format_ == "text": - generated_html = html.escape(text) - elif format_ == "console": + generated_html = html.escape(text) + if format_ == "console": generated_html = highlight_code(text) - else: - generated_html = highlight_code(text, bundle.config.programming_language) # Map of file URLs. url_map = {html.escape(x.path): x for x in case.link_files} @@ -194,16 +177,13 @@ def get_readable_input( def replace_link(match: Match) -> str: filename = match.group() the_file = url_map[filename] - the_url = urllib.parse.quote(the_file.url) the_replacement = ( - f'{filename}' + f'{filename}' ) seen.add(the_file) return the_replacement generated_html = escaped_regex.sub(replace_link, generated_html) - prefix, suffix = _handle_link_files(seen, format_) - generated_html = f"{prefix}{generated_html}{suffix}" return ExtendedMessage(description=generated_html, format="html"), seen diff --git a/tested/oracles/text.py b/tested/oracles/text.py index bd735471..3d4fdde9 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -3,15 +3,12 @@ """ import math -import os from typing import Any from tested.dodona import Status, StatusMessage from tested.internationalization import get_i18n_string -from tested.judge.utils import base64_encode from tested.oracles.common import OracleConfig, OracleResult from tested.testsuite import ( - FileOutputChannel, OutputChannel, OutputFileData, TextChannelType, @@ -112,21 +109,6 @@ def evaluate_text( result = compare_text(options, expected, actual) return result - -def make_expected_and_actual_file_output( - output_data: OutputFileData, expected: str, actual: str -) -> tuple[str, str]: - content_type = output_data.content_type - expected_str = output_data.content - if content_type == TextChannelType.TEXT: - expected_str = base64_encode(expected) - - return ( - f"--- <{output_data.student_path}|{content_type}> ---\n{expected_str}", - f"--- <{output_data.student_path}|text> ---\n{base64_encode(actual)}", - ) - - def evaluate_file( config: OracleConfig, channel: OutputChannel, actual: str ) -> OracleResult: diff --git a/tested/testsuite.py b/tested/testsuite.py index 6e841af0..b51da2d5 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -552,8 +552,9 @@ def get_functions(self) -> Iterable[FunctionCall]: @define(frozen=True) class FileUrl: - url: str path: str + content: str = "" + url: str = "" @ignore_field(get_converter(), "essential") From 83e69e1e5c1b96bf15480b08961cca03e9e953e6 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 29 Mar 2025 17:14:10 +0100 Subject: [PATCH 41/77] fixed a few linting issues --- tested/judge/core.py | 2 +- tested/judge/utils.py | 2 -- tested/languages/generation.py | 4 +--- tested/oracles/text.py | 1 + 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tested/judge/core.py b/tested/judge/core.py index 85a13bc3..8f63a9e7 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -384,7 +384,7 @@ def _process_results( data=Metadata( statements=meta_statements, stdin=meta_stdin, - input_files=input_files + input_files=input_files, ) ), planned.context_index, diff --git a/tested/judge/utils.py b/tested/judge/utils.py index 4b491637..c50a00b2 100644 --- a/tested/judge/utils.py +++ b/tested/judge/utils.py @@ -2,11 +2,9 @@ Common utilities for the judge. """ -import base64 import logging import shutil import subprocess -import zlib from pathlib import Path from attrs import define diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 0f693f41..9204b279 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -177,9 +177,7 @@ def get_readable_input( def replace_link(match: Match) -> str: filename = match.group() the_file = url_map[filename] - the_replacement = ( - f'{filename}' - ) + the_replacement = f'{filename}' seen.add(the_file) return the_replacement diff --git a/tested/oracles/text.py b/tested/oracles/text.py index 3d4fdde9..3d501e46 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -109,6 +109,7 @@ def evaluate_text( result = compare_text(options, expected, actual) return result + def evaluate_file( config: OracleConfig, channel: OutputChannel, actual: str ) -> OracleResult: From 425e82ddfe36f455737a9f7068981a5b43d0d172 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 7 Apr 2025 19:13:38 +0200 Subject: [PATCH 42/77] made urls mandatory again --- tested/dsl/schema-strict.json | 7 ++++--- tested/dsl/schema.json | 7 ++++--- tested/dsl/translate_parser.py | 13 ++----------- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index a7142aab..c08d6309 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -500,7 +500,8 @@ "type" : "object", "description" : "A file used in the test suite.", "required" : [ - "path" + "path", + "url" ], "properties" : { "path" : { @@ -618,7 +619,7 @@ "items" : { "type" : "object", "required" : [ - "student_path", + "path", "content" ], "properties" : { @@ -633,7 +634,7 @@ ], "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." }, - "student_path" : { + "path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 32120231..02228779 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -498,7 +498,8 @@ "type" : "object", "description" : "A file used in the test suite.", "required" : [ - "path" + "path", + "url" ], "properties" : { "path" : { @@ -616,7 +617,7 @@ "items" : { "type" : "object", "required" : [ - "student_path", + "path", "content" ], "properties" : { @@ -624,7 +625,7 @@ "type" : "string", "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." }, - "student_path" : { + "path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index b162b47c..c0205288 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -429,8 +429,8 @@ def _convert_value(value: YamlObject) -> Value: def _convert_file(link_file: YamlDict, workdir: Path | None) -> FileUrl: assert isinstance(link_file["path"], str) + assert isinstance(link_file["url"], str) content = "" - url = "" if "content" in link_file: content = link_file["content"] assert isinstance(content, str) @@ -440,16 +440,7 @@ def _convert_file(link_file: YamlDict, workdir: Path | None) -> FileUrl: with open(full_path, "w", encoding="utf-8") as f: f.write(content) - if "url" in link_file: - assert isinstance(link_file["url"], str) - url = link_file["url"] - else: - # Assumed the specified files are already in the working directory. - assert "url" in link_file - assert isinstance(link_file["url"], str) - url = link_file["url"] - - return FileUrl(path=link_file["path"], url=url, content=content) + return FileUrl(path=link_file["path"], url=link_file["url"], content=content) def _convert_evaluation_function(stream: dict) -> EvaluationFunction: From 06b68adaab9773d25d1f9025c6cdec9847ea572c Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 8 Apr 2025 16:33:26 +0200 Subject: [PATCH 43/77] Fixed stdin and and gave correct data for stdin --- tested/judge/core.py | 4 +--- tested/languages/generation.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/tested/judge/core.py b/tested/judge/core.py index 8f63a9e7..32ba8fa1 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -358,9 +358,7 @@ def _process_results( if case.is_main_testcase(): assert isinstance(case.input, MainInput) if isinstance(case.input.stdin, TextData): - meta_stdin = case.input.stdin.get_data_as_string( - bundle.config.resources - ) + meta_stdin = case.input.stdin.data elif isinstance(case.input, Statement): stmt = generate_statement(bundle, case.input) meta_statements.append(stmt) diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 9204b279..3d7cca69 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -37,6 +37,7 @@ LanguageLiterals, MainInput, Testcase, + TextChannelType, TextData, ) from tested.utils import is_statement_strict @@ -121,18 +122,20 @@ def get_readable_input( command = shlex.join([submission] + case.input.arguments) args = f"$ {command}" # Determine the stdin - if isinstance(case.input.stdin, TextData): - stdin = case.input.stdin.data - if not case.link_files and not simple_regex.search(stdin): - stdin = case.input.stdin.get_data_as_string(bundle.config.resources) + stdin_data = case.input.stdin + if isinstance(stdin_data, TextData): + stdin = stdin_data.data else: stdin = "" # If we have both stdin and arguments, we use a here-document. if case.input.arguments and stdin: - assert stdin[-1] == "\n", "stdin must end with a newline" - delimiter = _get_heredoc_token(stdin) - text = f"{args} << '{delimiter}'\n{stdin}{delimiter}" + if isinstance(stdin_data, TextData) and stdin_data.type == "file": + text = f"{args} << {stdin}" + else: + assert stdin[-1] == "\n", "stdin must end with a newline" + delimiter = _get_heredoc_token(stdin) + text = f"{args} << '{delimiter}'\n{stdin}{delimiter}" elif stdin: assert not case.input.arguments text = stdin @@ -164,8 +167,6 @@ def get_readable_input( # Now we need to do ugly stuff. # Begin by compiling the HTML that will be displayed. generated_html = html.escape(text) - if format_ == "console": - generated_html = highlight_code(text) # Map of file URLs. url_map = {html.escape(x.path): x for x in case.link_files} From da56d9c884cdc106303e9aa4acd89d2a629ae972 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 8 Apr 2025 16:42:26 +0200 Subject: [PATCH 44/77] changed student_path back to path for output files --- tested/dsl/schema-strict.json | 8 ++++---- tested/dsl/schema.json | 8 ++++---- tested/dsl/translate_parser.py | 2 +- tested/judge/evaluation.py | 2 +- tested/oracles/text.py | 2 +- tested/testsuite.py | 6 +++--- tests/test_dsl_yaml.py | 4 ++-- tests/test_oracles_builtin.py | 18 +++++++++--------- tests/test_suite.py | 2 +- 9 files changed, 26 insertions(+), 26 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index c08d6309..dd5a4aac 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -653,7 +653,7 @@ "items" : { "type" : "object", "required" : [ - "student_path", + "path", "content" ], "properties" : { @@ -668,7 +668,7 @@ ], "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." }, - "student_path" : { + "path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } @@ -700,7 +700,7 @@ "items" : { "type" : "object", "required" : [ - "student_path", + "path", "content" ], "properties" : { @@ -715,7 +715,7 @@ ], "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." }, - "student_path" : { + "path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 02228779..776a4b22 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -644,7 +644,7 @@ "items" : { "type" : "object", "required" : [ - "student_path", + "path", "content" ], "properties" : { @@ -652,7 +652,7 @@ "type" : "string", "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." }, - "student_path" : { + "path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } @@ -684,7 +684,7 @@ "items" : { "type" : "object", "required" : [ - "student_path", + "path", "content" ], "properties" : { @@ -692,7 +692,7 @@ "type" : "string", "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." }, - "student_path" : { + "path" : { "type" : "string", "description" : "Path to where the file generated by the submission should go." } diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index c0205288..040fece7 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -554,7 +554,7 @@ def _convert_file_output_channel( OutputFileData( content_type=content_type, content=content, - student_path=str(item["student_path"]), + path=str(item["path"]), ) ) diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index d53b1cac..6471cfae 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -136,7 +136,7 @@ def _evaluate_channel( expected = evaluation_result.readable_expected expected_channel = channel if isinstance(output_element, OutputFileData): - expected_channel = output_element.student_path + expected_channel = output_element.path out.add(StartTest(expected=expected, channel=expected_channel)) # Report any messages we received. diff --git a/tested/oracles/text.py b/tested/oracles/text.py index 3d501e46..fe324e41 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -153,7 +153,7 @@ def evaluate_file( except FileNotFoundError: raise ValueError(f"File {expected_path} not found in resources.") - actual_path = config.context_dir / channel.student_path + actual_path = config.context_dir / channel.path try: with open(str(actual_path), "r") as file: diff --git a/tested/testsuite.py b/tested/testsuite.py index b51da2d5..52ea5f5f 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -277,7 +277,7 @@ class TextOutputChannel(TextData): class OutputFileData: content_type: TextChannelType content: str - student_path: str + path: str oracle: GenericTextOracle | CustomCheckOracle = field( factory=lambda: GenericTextOracle(name=TextBuiltin.FILE) ) @@ -305,11 +305,11 @@ def get_data_as_string(self, resources: Path) -> str: file_path = _resolve_path(resources, output_data.content) with open(file_path, "r") as file: file_content.append( - f"--- <{output_data.student_path}> ---\n{file.read()}" + f"--- <{output_data.path}> ---\n{file.read()}" ) else: file_content.append( - f"--- <{output_data.student_path}> ---\n{output_data.content[i]}" + f"--- <{output_data.path}> ---\n{output_data.content[i]}" ) return "\n".join(file_content) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index ced79a6c..14dd0578 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -809,10 +809,10 @@ def test_output_files_custom_check_correct(): assert isinstance(test.input, FunctionCall) assert isinstance(test.output.file, FileOutputChannel) assert isinstance(test.output.file.oracle, CustomCheckOracle) - assert test.output.file.output_data[0].student_path == "test.txt" + assert test.output.file.output_data[0].path == "test.txt" assert test.output.file.output_data[0].content == "test/hallo.txt" assert test.output.file.output_data[0].content_type == TextChannelType.FILE - assert test.output.file.output_data[1].student_path == "test2.txt" + assert test.output.file.output_data[1].path == "test2.txt" assert test.output.file.output_data[1].content == "Hallo world!" assert test.output.file.output_data[1].content_type == TextChannelType.TEXT oracle = test.output.file.oracle diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index 45a8d5e9..852aa117 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -154,7 +154,7 @@ def test_file_oracle_full_wrong( output_data=[ OutputFileData( content="expected.txt", - student_path="expected.txt", + path="expected.txt", content_type=TextChannelType.FILE, ) ] @@ -182,7 +182,7 @@ def test_file_oracle_full_correct( output_data=[ OutputFileData( content="expected.txt", - student_path="expected.txt", + path="expected.txt", content_type=TextChannelType.FILE, ) ] @@ -215,12 +215,12 @@ def test_file_oracle_full_correct_with_mixed_content( output_data=[ OutputFileData( content="expected.txt", - student_path="expected.txt", + path="expected.txt", content_type=TextChannelType.FILE, ), OutputFileData( content="expected\nexpected", - student_path="expected.txt", + path="expected.txt", content_type=TextChannelType.TEXT, ), ] @@ -256,7 +256,7 @@ def test_file_oracle_line_wrong( output_data=[ OutputFileData( content="expected.txt", - student_path="expected.txt", + path="expected.txt", content_type=TextChannelType.FILE, ) ] @@ -288,7 +288,7 @@ def test_file_oracle_line_correct( output_data=[ OutputFileData( content="expected.txt", - student_path="expected.txt", + path="expected.txt", content_type=TextChannelType.FILE, ) ] @@ -320,7 +320,7 @@ def test_file_oracle_strip_lines_correct( output_data=[ OutputFileData( content="expected.txt", - student_path="expected.txt", + path="expected.txt", content_type=TextChannelType.FILE, ) ] @@ -352,7 +352,7 @@ def test_file_oracle_dont_strip_lines_correct( output_data=[ OutputFileData( content="expected.txt", - student_path="expected.txt", + path="expected.txt", content_type=TextChannelType.FILE, ) ] @@ -374,7 +374,7 @@ def test_correct_error_actual_not_found(tmp_path: Path, pytestconfig: pytest.Con output_data=[ OutputFileData( content="Hallo world!", - student_path="expected.txt", + path="expected.txt", content_type=TextChannelType.TEXT, ) ] diff --git a/tests/test_suite.py b/tests/test_suite.py index 789a40f2..4eff78c2 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -128,7 +128,7 @@ def test_file_show_expected_is_accepted(): result = get_converter().loads(scheme, FileOutputChannel) assert result.output_data[0].content == "hallo" - assert result.output_data[0].student_path == "hallo.txt" + assert result.output_data[0].path == "hallo.txt" def test_value_show_expected_is_accepted(): From 5713981e6d638973eccc7d19ad09295f24ffc7d7 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 13 Apr 2025 13:50:44 +0200 Subject: [PATCH 45/77] Made some new changes to stdin --- tested/dsl/schema-strict.json | 114 +++++++++++++++++++++++++++++---- tested/dsl/schema.json | 114 +++++++++++++++++++++++++++++---- tested/dsl/translate_parser.py | 32 +++++++-- tested/judge/execution.py | 2 +- tested/languages/generation.py | 39 +++++++---- tested/testsuite.py | 11 +++- 6 files changed, 263 insertions(+), 49 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index dd5a4aac..fb26c35b 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -276,13 +276,57 @@ }, "stdin" : { "description" : "Stdin for this context", - "type" : [ - "string", - "path", - "number", - "integer", - "boolean", - "object" + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties" : false, + "required": ["content"], + "properties": { + "content": { + "type" : [ + "string", + "number", + "integer", + "boolean", + "object" + ], + "description": "The actual content that will be used for stdin." + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "url": { + "type": "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } + }, + { + "type": "object", + "additionalProperties" : false, + "required": ["path", "url"], + "properties": { + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "url": { + "type": "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } + } ] }, "arguments" : { @@ -374,13 +418,55 @@ }, "stdin" : { "description" : "Stdin for this context", - "type" : [ - "string", - "path", - "number", - "integer", - "boolean", - "object" + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required": ["content"], + "properties": { + "content": { + "type" : [ + "string", + "number", + "integer", + "boolean", + "object" + ], + "description": "The actual content that will be used for stdin." + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "url": { + "type": "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } + }, + { + "type": "object", + "required": ["path", "url"], + "properties": { + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "url": { + "type": "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } + } ] }, "arguments" : { diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 776a4b22..4af6cbf5 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -276,12 +276,57 @@ }, "stdin" : { "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean", - "object" + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties" : false, + "required": ["content"], + "properties": { + "content": { + "type" : [ + "string", + "number", + "integer", + "boolean", + "object" + ], + "description": "The actual content that will be used for stdin." + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "url": { + "type": "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } + }, + { + "type": "object", + "additionalProperties" : false, + "required": ["path", "url"], + "properties": { + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "url": { + "type": "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } + } ] }, "arguments" : { @@ -373,12 +418,57 @@ }, "stdin" : { "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean", - "object" + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties" : false, + "required": ["content"], + "properties": { + "content": { + "type" : [ + "string", + "number", + "integer", + "boolean", + "object" + ], + "description": "The actual content that will be used for stdin." + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "url": { + "type": "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } + }, + { + "type": "object", + "additionalProperties" : false, + "required": ["path", "url"], + "properties": { + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "url": { + "type": "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } + } ] }, "arguments" : { diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 040fece7..10753b6b 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -512,9 +512,11 @@ def _convert_text_output_channel( else: data = str(raw_data) - text_output = TextOutputChannel(data=data) + if path is not None: - text_output.type = TextChannelType.FILE + text_output = TextOutputChannel(data=None, path=data, type=TextChannelType.FILE) + else: + text_output = TextOutputChannel(data=data) if isinstance(stream, str): text_output.oracle = GenericTextOracle(options=config) @@ -671,11 +673,29 @@ def _convert_testcase( return_channel = IgnoredChannel.IGNORED if "statement" in testcase else None else: if "stdin" in testcase: - if isinstance(testcase["stdin"], PathString): - stdin = TextData(data=str(testcase["stdin"]), type=TextChannelType.FILE) + stdin_data = testcase["stdin"] + data = None + path = "" + url = "" + if isinstance(stdin_data, str): + data = _ensure_trailing_newline(stdin_data) + else: + assert isinstance(stdin_data, dict) + if "content" in stdin_data: + content = stdin_data["content"] + assert isinstance(content, str) + data = _ensure_trailing_newline(content) + + if "path" in stdin_data: + assert "url" in stdin_data + path = stdin_data["path"] + url = stdin_data["url"] + assert isinstance(path, str) and isinstance(url, str) + + if path: + stdin = TextData(data=data, path=path, url=url, type=TextChannelType.FILE) else: - assert isinstance(testcase["stdin"], str) - stdin = TextData(data=_ensure_trailing_newline(testcase["stdin"])) + stdin = TextData(data=data) else: stdin = EmptyChannel.NONE arguments = testcase.get("arguments", []) diff --git a/tested/judge/execution.py b/tested/judge/execution.py index c5ee6f42..bc1e903e 100644 --- a/tested/judge/execution.py +++ b/tested/judge/execution.py @@ -241,7 +241,7 @@ def execute_unit( executable = executable_or_status files.remove(executable) - stdin = unit.get_stdin(bundle.config.resources) + stdin = unit.get_stdin(bundle.config.workdir) # Do the execution. base_result = execute_file( diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 3d7cca69..524166b3 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -98,13 +98,8 @@ def get_readable_input( 3. If it is a context testcase: a. The stdin and the arguments. """ - # We have potential files. - # Check if the file names are present in the string. - # If not, we can also stop before doing ugly things. - # We construct a regex, since that can be faster than checking everything. - simple_regex = re.compile( - "|".join(map(lambda x: re.escape(x.path), case.link_files)) - ) + + link_files = case.link_files format_ = "text" # By default, we use text as input. if case.description: @@ -124,18 +119,28 @@ def get_readable_input( # Determine the stdin stdin_data = case.input.stdin if isinstance(stdin_data, TextData): - stdin = stdin_data.data + if stdin_data.type == "file": + stdin = stdin_data.path + if stdin_data.data is not None: + link_files.append(FileUrl(path=stdin, content=stdin_data.data, url=stdin_data.url)) + else: + link_files.append(FileUrl(path=stdin, url=stdin_data.url)) + else: + stdin = stdin_data.data else: stdin = "" # If we have both stdin and arguments, we use a here-document. if case.input.arguments and stdin: if isinstance(stdin_data, TextData) and stdin_data.type == "file": - text = f"{args} << {stdin}" + text = f"{args} < {stdin}" else: assert stdin[-1] == "\n", "stdin must end with a newline" - delimiter = _get_heredoc_token(stdin) - text = f"{args} << '{delimiter}'\n{stdin}{delimiter}" + if stdin.count("\n") > 1: + delimiter = _get_heredoc_token(stdin) + text = f"{args} << '{delimiter}'\n{stdin}{delimiter}" + else: + text = f"{args} <<< {stdin.strip()}" elif stdin: assert not case.input.arguments text = stdin @@ -156,8 +161,16 @@ def get_readable_input( if case.line_comment: text = f"{text} {bundle.language.comment(case.line_comment)}" + # We have potential files. + # Check if the file names are present in the string. + # If not, we can also stop before doing ugly things. + # We construct a regex, since that can be faster than checking everything. + simple_regex = re.compile( + "|".join(map(lambda x: re.escape(x.path), link_files)) + ) + # If there are no files, return now. This means we don't need to do ugly stuff. - if not case.link_files: + if not link_files: return ExtendedMessage(description=text, format=format_), set() if not simple_regex.search(text): @@ -169,7 +182,7 @@ def get_readable_input( generated_html = html.escape(text) # Map of file URLs. - url_map = {html.escape(x.path): x for x in case.link_files} + url_map = {html.escape(x.path): x for x in link_files} seen = set() escaped_regex = re.compile("|".join(url_map.keys())) diff --git a/tested/testsuite.py b/tested/testsuite.py index 52ea5f5f..0115bf58 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -246,15 +246,20 @@ def _resolve_path(working_directory, file_path): class TextData(WithFeatures): """Describes textual data: either directly or in a file.""" - data: str + data: str | None + path: str = "" + url: str = "" type: TextChannelType = TextChannelType.TEXT def get_data_as_string(self, working_directory: Path) -> str: """Get the data as a string, reading the file if necessary.""" - if self.type == TextChannelType.TEXT: + if self.data is not None: return self.data + + if self.type == TextChannelType.TEXT: + return "" elif self.type == TextChannelType.FILE: - file_path = _resolve_path(working_directory, self.data) + file_path = _resolve_path(working_directory, self.path) with open(file_path, "r") as file: return file.read() else: From 2d1fa4c35fc7325a0fcdc8eda7d77babec7a4e47 Mon Sep 17 00:00:00 2001 From: breblanc Date: Thu, 17 Apr 2025 15:44:53 +0200 Subject: [PATCH 46/77] Made a few more updates to stdin --- tested/dodona.py | 1 - tested/judge/core.py | 12 ++---------- tested/languages/generation.py | 33 ++++++++++++++++----------------- 3 files changed, 18 insertions(+), 28 deletions(-) diff --git a/tested/dodona.py b/tested/dodona.py index 862d01e3..15dcbb96 100644 --- a/tested/dodona.py +++ b/tested/dodona.py @@ -39,7 +39,6 @@ class Metadata: """Currently only used for the Python tutor""" statements: str | None - stdin: str | None input_files: list[dict[str, str]] | None diff --git a/tested/judge/core.py b/tested/judge/core.py index 32ba8fa1..5af95ec3 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -344,7 +344,6 @@ def _process_results( # TODO: we could probably re-use the "readable_input" function here, # since it only differs a bit. meta_statements = [] - meta_stdin = None input_files = [] for case in planned.context.testcases: for file in case.link_files: @@ -352,20 +351,14 @@ def _process_results( if file.url != "": file_data["url"] = file.url - if file.content != "": - file_data["content"] = file.content input_files.append(file_data) - if case.is_main_testcase(): - assert isinstance(case.input, MainInput) - if isinstance(case.input.stdin, TextData): - meta_stdin = case.input.stdin.data - elif isinstance(case.input, Statement): + if isinstance(case.input, Statement): stmt = generate_statement(bundle, case.input) meta_statements.append(stmt) elif isinstance(case.input, LanguageLiterals): stmt = case.input.get_for(bundle.config.programming_language) meta_statements.append(stmt) - else: + elif not case.is_main_testcase(): raise AssertionError(f"Found unknown case input type: {case.input}") if meta_statements: @@ -381,7 +374,6 @@ def _process_results( CloseContext( data=Metadata( statements=meta_statements, - stdin=meta_stdin, input_files=input_files, ) ), diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 524166b3..1f10c262 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -99,7 +99,18 @@ def get_readable_input( a. The stdin and the arguments. """ + stdin = "" link_files = case.link_files + if case.is_main_testcase(): + assert isinstance(case.input, MainInput) + stdin_data = case.input.stdin + if isinstance(stdin_data, TextData): + if stdin_data.type == "file": + stdin = stdin_data.path + link_files.append(FileUrl(path=stdin, url=stdin_data.url)) + else: + stdin = stdin_data.data + format_ = "text" # By default, we use text as input. if case.description: @@ -109,7 +120,6 @@ def get_readable_input( else: text = case.description elif case.is_main_testcase(): - assert isinstance(case.input, MainInput) # See https://rouge-ruby.github.io/docs/Rouge/Lexers/ConsoleLexer.html format_ = "console" # Determine the command (with arguments) @@ -118,32 +128,21 @@ def get_readable_input( args = f"$ {command}" # Determine the stdin stdin_data = case.input.stdin - if isinstance(stdin_data, TextData): - if stdin_data.type == "file": - stdin = stdin_data.path - if stdin_data.data is not None: - link_files.append(FileUrl(path=stdin, content=stdin_data.data, url=stdin_data.url)) - else: - link_files.append(FileUrl(path=stdin, url=stdin_data.url)) - else: - stdin = stdin_data.data - else: - stdin = "" # If we have both stdin and arguments, we use a here-document. - if case.input.arguments and stdin: + if stdin: if isinstance(stdin_data, TextData) and stdin_data.type == "file": text = f"{args} < {stdin}" - else: + elif case.input.arguments: assert stdin[-1] == "\n", "stdin must end with a newline" if stdin.count("\n") > 1: delimiter = _get_heredoc_token(stdin) text = f"{args} << '{delimiter}'\n{stdin}{delimiter}" else: text = f"{args} <<< {stdin.strip()}" - elif stdin: - assert not case.input.arguments - text = stdin + else: + assert not case.input.arguments + text = stdin else: text = args elif isinstance(case.input, Statement): From d11fd5e6cfa28874a874595f6cf6c26a61948d4c Mon Sep 17 00:00:00 2001 From: breblanc Date: Thu, 17 Apr 2025 16:05:25 +0200 Subject: [PATCH 47/77] Changed channel to "File: " when given output file content in feedback --- tested/dsl/translate_parser.py | 5 +++-- tested/judge/evaluation.py | 2 +- tested/languages/generation.py | 6 ++---- tested/testsuite.py | 4 +--- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 10753b6b..b1c9f8a0 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -512,7 +512,6 @@ def _convert_text_output_channel( else: data = str(raw_data) - if path is not None: text_output = TextOutputChannel(data=None, path=data, type=TextChannelType.FILE) else: @@ -693,7 +692,9 @@ def _convert_testcase( assert isinstance(path, str) and isinstance(url, str) if path: - stdin = TextData(data=data, path=path, url=url, type=TextChannelType.FILE) + stdin = TextData( + data=data, path=path, url=url, type=TextChannelType.FILE + ) else: stdin = TextData(data=data) else: diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index 6471cfae..fdba3059 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -136,7 +136,7 @@ def _evaluate_channel( expected = evaluation_result.readable_expected expected_channel = channel if isinstance(output_element, OutputFileData): - expected_channel = output_element.path + expected_channel = f"File: {output_element.path}" out.add(StartTest(expected=expected, channel=expected_channel)) # Report any messages we received. diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 1f10c262..13948cfa 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -111,7 +111,6 @@ def get_readable_input( else: stdin = stdin_data.data - format_ = "text" # By default, we use text as input. if case.description: if isinstance(case.description, ExtendedMessage): @@ -120,6 +119,7 @@ def get_readable_input( else: text = case.description elif case.is_main_testcase(): + assert isinstance(case.input, MainInput) # See https://rouge-ruby.github.io/docs/Rouge/Lexers/ConsoleLexer.html format_ = "console" # Determine the command (with arguments) @@ -164,9 +164,7 @@ def get_readable_input( # Check if the file names are present in the string. # If not, we can also stop before doing ugly things. # We construct a regex, since that can be faster than checking everything. - simple_regex = re.compile( - "|".join(map(lambda x: re.escape(x.path), link_files)) - ) + simple_regex = re.compile("|".join(map(lambda x: re.escape(x.path), link_files))) # If there are no files, return now. This means we don't need to do ugly stuff. if not link_files: diff --git a/tested/testsuite.py b/tested/testsuite.py index 0115bf58..50940d60 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -309,9 +309,7 @@ def get_data_as_string(self, resources: Path) -> str: if output_data.content_type == TextChannelType.FILE: file_path = _resolve_path(resources, output_data.content) with open(file_path, "r") as file: - file_content.append( - f"--- <{output_data.path}> ---\n{file.read()}" - ) + file_content.append(f"--- <{output_data.path}> ---\n{file.read()}") else: file_content.append( f"--- <{output_data.path}> ---\n{output_data.content[i]}" From 78029c31ebad31ccd1779e0cab6ea00653f46dad Mon Sep 17 00:00:00 2001 From: breblanc Date: Thu, 17 Apr 2025 20:53:29 +0200 Subject: [PATCH 48/77] Made backward compatible and added warnings --- tested/dsl/schema-strict.json | 148 +++++++++++++++++++++++++++++++ tested/dsl/schema.json | 156 +++++++++++++++++++++++++++++++++ tested/dsl/translate_parser.py | 114 +++++++++++++++++------- tested/judge/core.py | 26 +++++- tested/testsuite.py | 5 ++ 5 files changed, 415 insertions(+), 34 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index fb26c35b..336991da 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -39,6 +39,13 @@ } ], "properties" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, "input_files" : { "description" : "A list of files used in the test suite.", "type" : "array", @@ -97,7 +104,15 @@ "tab" ], "properties" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -149,7 +164,15 @@ "unit" ], "properties" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -229,7 +252,15 @@ "testcases" ], "properties" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -251,7 +282,15 @@ "script" ], "properties" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -381,7 +420,15 @@ } ] }, + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -399,6 +446,10 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "file": { + "description" : "Expected file generated by the submission (DEPRECATED).", + "$ref" : "#/definitions/deprecatedFileOutputChannel" + }, "output_files": { "description" : "Expected files generated by the submission.", "$ref" : "#/definitions/fileOutputChannel" @@ -539,6 +590,10 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "file": { + "description" : "Expected file generated by the submission (DEPRECATED).", + "$ref" : "#/definitions/deprecatedFileOutputChannel" + }, "output_files": { "description" : "Expected files generated by the submission.", "$ref" : "#/definitions/fileOutputChannel" @@ -582,6 +637,25 @@ } ] }, + "deprecatedFile": { + "type" : "object", + "description" : "A file used in the test suite (DEPRECATED).", + "required" : [ + "name", + "url" + ], + "properties" : { + "name" : { + "type" : "string", + "description" : "The filename, including the file extension." + }, + "url" : { + "type" : "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } + }, "file" : { "type" : "object", "description" : "A file used in the test suite.", @@ -697,6 +771,80 @@ } ] }, + "deprecatedFileOutputChannel": { + "anyOf" : [ + { + "type" : "object", + "description" : "Built-in oracle for files.", + "required" : [ + "content", + "location" + ], + "properties" : { + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/fileConfigurationOptions" + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for file values.", + "required" : [ + "oracle", + "content", + "location", + "file" + ], + "properties" : { + "oracle" : { + "const" : "custom_check" + }, + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + } + ] + }, "fileOutputChannel": { "anyOf" : [ { diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 4af6cbf5..694ff642 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -39,6 +39,13 @@ } ], "properties" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, "input_files" : { "description" : "A list of files used in the test suite.", "type" : "array", @@ -97,7 +104,15 @@ "tab" ], "properties" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -149,7 +164,15 @@ "unit" ], "properties" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -229,7 +252,15 @@ "testcases" ], "properties" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -251,7 +282,15 @@ "script" ], "properties" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -381,7 +420,15 @@ } ] }, + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -399,6 +446,10 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "file": { + "description" : "Expected file generated by the submission (DEPRECATED).", + "$ref" : "#/definitions/deprecatedFileOutputChannel" + }, "output_files": { "description" : "Expected files generated by the submission.", "$ref" : "#/definitions/fileOutputChannel" @@ -523,7 +574,15 @@ } ] }, + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -541,6 +600,10 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "file": { + "description" : "Expected file generated by the submission (DEPRECATED).", + "$ref" : "#/definitions/deprecatedFileOutputChannel" + }, "output_files": { "description" : "Expected files generated by the submission.", "$ref" : "#/definitions/fileOutputChannel" @@ -584,6 +647,25 @@ } ] }, + "deprecatedFile": { + "type" : "object", + "description" : "A file used in the test suite.", + "required" : [ + "name", + "url" + ], + "properties" : { + "name" : { + "type" : "string", + "description" : "The filename, including the file extension." + }, + "url" : { + "type" : "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } + }, "file" : { "type" : "object", "description" : "A file used in the test suite.", @@ -699,6 +781,80 @@ } ] }, + "deprecatedFileOutputChannel": { + "anyOf" : [ + { + "type" : "object", + "description" : "Built-in oracle for files.", + "required" : [ + "content", + "location" + ], + "properties" : { + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/fileConfigurationOptions" + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for file values.", + "required" : [ + "oracle", + "content", + "location", + "file" + ], + "properties" : { + "oracle" : { + "const" : "custom_check" + }, + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + } + ] + }, "fileOutputChannel": { "anyOf" : [ { diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index b1c9f8a0..2c9f6273 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -14,6 +14,7 @@ from jsonschema.protocols import Validator from jsonschema.validators import extend as extend_validator from jsonschema.validators import validator_for +from typing_extensions import deprecated from tested.datatypes import ( AdvancedNumericTypes, @@ -70,7 +71,7 @@ TextChannelType, TextData, TextOutputChannel, - ValueOutputChannel, + ValueOutputChannel, DeprecatedUsage, ) from tested.utils import get_args, recursive_dict_merge @@ -272,7 +273,7 @@ class DslContext: def deepen_context( self, new_level: YamlDict | None, workdir: Path | None - ) -> "DslContext": + ) -> tuple["DslContext", set[DeprecatedUsage]]: """ Merge certain fields of the new object with the current context, resulting in a new context for the new level. @@ -285,11 +286,16 @@ def deepen_context( if new_level is None: return self + deprecated_usage = set() the_files = self.files - if "input_files" in new_level: - assert isinstance(new_level["input_files"], list) + if "input_files" in new_level or "files" in new_level: + key = "input_files" + if "files" in new_level: + key = "files" + deprecated_usage.add(DeprecatedUsage.INPUT_FILES) + assert isinstance(new_level[key], list) additional_files = { - _convert_file(f, workdir=workdir) for f in new_level["input_files"] + _convert_file(f, workdir=workdir, deprecated=len(deprecated_usage) == 0) for f in new_level[key] } the_files = list(set(self.files) | additional_files) @@ -298,7 +304,7 @@ def deepen_context( assert isinstance(new_level["config"], dict) the_config = recursive_dict_merge(the_config, new_level["config"]) - return evolve(self, files=the_files, config=the_config) + return evolve(self, files=the_files, config=the_config), deprecated_usage def merge_inheritable_with_specific_config( self, level: YamlDict, config_name: str @@ -427,20 +433,22 @@ def _convert_value(value: YamlObject) -> Value: return _tested_type_to_value(tested_type) -def _convert_file(link_file: YamlDict, workdir: Path | None) -> FileUrl: - assert isinstance(link_file["path"], str) +def _convert_file(link_file: YamlDict, workdir: Path | None, deprecated: bool) -> FileUrl: + path_key = "path" if deprecated else "name" + + assert isinstance(link_file[path_key], str) assert isinstance(link_file["url"], str) content = "" if "content" in link_file: content = link_file["content"] assert isinstance(content, str) if workdir is not None: - full_path = workdir / link_file["path"] + full_path = workdir / link_file[path_key] os.makedirs(os.path.dirname(full_path), exist_ok=True) with open(full_path, "w", encoding="utf-8") as f: f.write(content) - return FileUrl(path=link_file["path"], url=link_file["url"], content=content) + return FileUrl(path=link_file[path_key], url=link_file["url"], content=content) def _convert_evaluation_function(stream: dict) -> EvaluationFunction: @@ -531,6 +539,33 @@ def _convert_text_output_channel( return text_output raise TypeError(f"Unknown text oracle type: {stream['oracle']}") +def _convert_file_output_channel_deprecated( + stream: YamlObject, context: DslContext, config_name: str +) -> FileOutputChannel: + assert isinstance(stream, dict) + + data = OutputFileData(content_type=TextChannelType.TEXT, content=str(stream["content"]), path=str(stream["location"])) + + if "oracle" not in stream or stream["oracle"] == "builtin": + config = context.merge_inheritable_with_specific_config(stream, config_name) + if "mode" not in config: + config["mode"] = "full" + + assert config["mode"] in ( + "full", + "line", + ), f"The file oracle only supports modes full and line, not {config['mode']}" + return FileOutputChannel( + output_data=[data], + oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config), + ) + elif stream["oracle"] == "custom_check": + return FileOutputChannel( + output_data=[data], + oracle=_convert_custom_check_oracle(stream), + ) + raise TypeError(f"Unknown file oracle type: {stream['oracle']}") + def _convert_file_output_channel( stream: YamlObject, context: DslContext, config_name: str @@ -641,8 +676,8 @@ def _validate_testcase_combinations(testcase: YamlDict): def _convert_testcase( testcase: YamlDict, context: DslContext, workdir: Path | None -) -> Testcase: - context = context.deepen_context(testcase, workdir) +) -> tuple[Testcase, set[DeprecatedUsage]]: + context, deprecated_usage = context.deepen_context(testcase, workdir) # This is backwards compatability to some extend. # TODO: remove this at some point. @@ -711,6 +746,9 @@ def _convert_testcase( if (stdout := testcase.get("stdout")) is not None: output.stdout = _convert_text_output_channel(stdout, context, "stdout") + if (file := testcase.get("file")) is not None: + output.file = _convert_file_output_channel_deprecated(file, context, "file") + deprecated_usage.add(DeprecatedUsage.OUTPUT_FILES) if (file := testcase.get("output_files")) is not None: output.file = _convert_file_output_channel(file, context, "output_files") if (stderr := testcase.get("stderr")) is not None: @@ -762,22 +800,23 @@ def _convert_testcase( output=output, link_files=context.files, line_comment=line_comment, - ) + ), deprecated_usage def _convert_context( context: YamlDict, dsl_context: DslContext, workdir: Path | None -) -> Context: - dsl_context = dsl_context.deepen_context(context, workdir) +) -> tuple[Context, set[DeprecatedUsage]]: + dsl_context, deprecated_usage = dsl_context.deepen_context(context, workdir) raw_testcases = context.get("script", context.get("testcases")) assert isinstance(raw_testcases, list) - testcases = _convert_dsl_list( + testcases, du = _convert_dsl_list( raw_testcases, dsl_context, workdir, _convert_testcase ) - return Context(testcases=testcases) + deprecated_usage.update(du) + return Context(testcases=testcases), deprecated_usage -def _convert_tab(tab: YamlDict, context: DslContext, workdir: Path | None) -> Tab: +def _convert_tab(tab: YamlDict, context: DslContext, workdir: Path | None) -> tuple[Tab, set[DeprecatedUsage]]: """ Translate a DSL tab to a full test suite tab. @@ -785,38 +824,42 @@ def _convert_tab(tab: YamlDict, context: DslContext, workdir: Path | None) -> Ta :param context: The context with config for the parent level. :return: A full tab. """ - context = context.deepen_context(tab, workdir) + context, deprecated_usage = context.deepen_context(tab, workdir) name = tab.get("unit", tab.get("tab")) assert isinstance(name, str) # The tab can have testcases or contexts. if "contexts" in tab: assert isinstance(tab["contexts"], list) - contexts = _convert_dsl_list( + contexts, du = _convert_dsl_list( tab["contexts"], context, workdir, _convert_context ) + deprecated_usage.update(du) elif "cases" in tab: assert "unit" in tab # We have testcases N.S. / contexts O.S. assert isinstance(tab["cases"], list) - contexts = _convert_dsl_list(tab["cases"], context, workdir, _convert_context) + contexts, du = _convert_dsl_list(tab["cases"], context, workdir, _convert_context) + deprecated_usage.update(du) elif "testcases" in tab: # We have scripts N.S. / testcases O.S. assert "tab" in tab assert isinstance(tab["testcases"], list) - testcases = _convert_dsl_list( + testcases, du = _convert_dsl_list( tab["testcases"], context, workdir, _convert_testcase ) + deprecated_usage.update(du) contexts = [Context(testcases=[t]) for t in testcases] else: assert "scripts" in tab assert isinstance(tab["scripts"], list) - testcases = _convert_dsl_list( + testcases, du = _convert_dsl_list( tab["scripts"], context, workdir, _convert_testcase ) + deprecated_usage.update(du) contexts = [Context(testcases=[t]) for t in testcases] - return Tab(name=name, contexts=contexts) + return Tab(name=name, contexts=contexts), deprecated_usage T = TypeVar("T") @@ -826,16 +869,19 @@ def _convert_dsl_list( dsl_list: list, context: DslContext, workdir: Path | None, - converter: Callable[[YamlDict, DslContext, Path | None], T], -) -> list[T]: + converter: Callable[[YamlDict, DslContext, Path | None], tuple[T, set[DeprecatedUsage]]], +) -> tuple[list[T], set[DeprecatedUsage]]: """ Convert a list of YAML objects into a test suite object. """ objects = [] + deprecated_usage = set() for dsl_object in dsl_list: assert isinstance(dsl_object, dict) - objects.append(converter(dsl_object, context, workdir)) - return objects + ob, du = converter(dsl_object, context, workdir) + deprecated_usage.update(du) + objects.append(ob) + return objects, deprecated_usage def _convert_dsl(dsl_object: YamlObject, workdir: Path | None) -> Suite: @@ -849,25 +895,29 @@ def _convert_dsl(dsl_object: YamlObject, workdir: Path | None) -> Suite: :return: A full test suite. """ context = DslContext() + deprecated_usage = set() + if isinstance(dsl_object, list): namespace = None tab_list = dsl_object else: assert isinstance(dsl_object, dict) namespace = dsl_object.get("namespace") - context = context.deepen_context(dsl_object, workdir) + context, du = context.deepen_context(dsl_object, workdir) + deprecated_usage.update(du) tab_list = dsl_object.get("units", dsl_object.get("tabs")) assert isinstance(tab_list, list) if (language := dsl_object.get("language", "tested")) != "tested": language = SupportedLanguage(language) context = evolve(context, language=language) - tabs = _convert_dsl_list(tab_list, context, workdir, _convert_tab) + tabs, du = _convert_dsl_list(tab_list, context, workdir, _convert_tab) + deprecated_usage.update(du) if namespace: assert isinstance(namespace, str) - return Suite(tabs=tabs, namespace=namespace) + return Suite(tabs=tabs, deprecated=list(deprecated_usage), namespace=namespace) else: - return Suite(tabs=tabs) + return Suite(tabs=tabs, deprecated=list(deprecated_usage)) def parse_dsl(dsl_string: str, workdir: Path | None = None) -> Suite: diff --git a/tested/judge/core.py b/tested/judge/core.py index 5af95ec3..ad1b59f8 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -16,7 +16,7 @@ StartTab, Status, StatusMessage, - report_update, + report_update, ExtendedMessage, Permission, ) from tested.features import is_supported from tested.internationalization import get_i18n_string, set_locale @@ -46,7 +46,7 @@ generate_statement, ) from tested.serialisation import Statement -from tested.testsuite import LanguageLiterals, MainInput, TextData +from tested.testsuite import LanguageLiterals, MainInput, TextData, DeprecatedUsage _logger = logging.getLogger(__name__) @@ -116,6 +116,28 @@ def judge(bundle: Bundle): # Do the set-up for the judgement. collector = OutputManager(bundle.out) collector.add(StartJudgement()) + + messages = [] + if DeprecatedUsage.INPUT_FILES in bundle.suite.deprecated: + messages.append( + ExtendedMessage( + f"WARNING: You are using YAML syntax to specify input files with the key 'files'. This usage is deprecated! Try using 'input_files' instead.", + permission=Permission.STAFF, + ) + ) + + + if DeprecatedUsage.OUTPUT_FILES in bundle.suite.deprecated: + messages.append( + ExtendedMessage( + f"WARNING: You are using YAML syntax to specify output files with the key 'file'. This usage is deprecated! Try using 'output_files' instead.", + permission=Permission.STAFF, + ) + ) + + if messages: + collector.add_messages(messages) + max_time = float(bundle.config.time_limit) * 0.9 start = time.perf_counter() diff --git a/tested/testsuite.py b/tested/testsuite.py index 50940d60..d82a43ec 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -783,12 +783,17 @@ class ExecutionMode(StrEnum): PRECOMPILATION = "batch" INDIVIDUAL = "context" +@unique +class DeprecatedUsage(StrEnum): + INPUT_FILES = "input_files" + OUTPUT_FILES = "output_files" @define class Suite(WithFeatures, WithFunctions): """General test suite, which is used to run tests of some code.""" tabs: list[Tab] = field(factory=list) + deprecated: list[DeprecatedUsage] = field(factory=list) namespace: str = "submission" def get_used_features(self) -> FeatureSet: From ca853113aa6d50f80380147900a2819236b84df4 Mon Sep 17 00:00:00 2001 From: breblanc Date: Thu, 17 Apr 2025 21:03:29 +0200 Subject: [PATCH 49/77] Fixing linting and typing --- tested/dsl/translate_parser.py | 65 +++++++++++++++++++++++----------- tested/judge/core.py | 7 ++-- tested/testsuite.py | 2 ++ 3 files changed, 50 insertions(+), 24 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 2c9f6273..50d75312 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -49,6 +49,7 @@ from tested.testsuite import ( Context, CustomCheckOracle, + DeprecatedUsage, EmptyChannel, EvaluationFunction, ExceptionOutputChannel, @@ -71,7 +72,7 @@ TextChannelType, TextData, TextOutputChannel, - ValueOutputChannel, DeprecatedUsage, + ValueOutputChannel, ) from tested.utils import get_args, recursive_dict_merge @@ -279,12 +280,12 @@ def deepen_context( in a new context for the new level. :param new_level: The new object from the DSL to get information from. - :param workdir: The working directory where all files are located. + :param workdir: The working directory where all files are located. :return: A new context. """ if new_level is None: - return self + return self, set() deprecated_usage = set() the_files = self.files @@ -293,9 +294,14 @@ def deepen_context( if "files" in new_level: key = "files" deprecated_usage.add(DeprecatedUsage.INPUT_FILES) - assert isinstance(new_level[key], list) + + files = new_level[key] + assert isinstance(files, list) additional_files = { - _convert_file(f, workdir=workdir, deprecated=len(deprecated_usage) == 0) for f in new_level[key] + _convert_file( + f, workdir=workdir, deprecated_usage=len(deprecated_usage) == 0 + ) + for f in files } the_files = list(set(self.files) | additional_files) @@ -433,22 +439,25 @@ def _convert_value(value: YamlObject) -> Value: return _tested_type_to_value(tested_type) -def _convert_file(link_file: YamlDict, workdir: Path | None, deprecated: bool) -> FileUrl: - path_key = "path" if deprecated else "name" +def _convert_file( + link_file: YamlDict, workdir: Path | None, deprecated_usage: bool +) -> FileUrl: + path_key = "path" if deprecated_usage else "name" + path_str = link_file[path_key] - assert isinstance(link_file[path_key], str) + assert isinstance(path_str, str) assert isinstance(link_file["url"], str) content = "" if "content" in link_file: content = link_file["content"] assert isinstance(content, str) if workdir is not None: - full_path = workdir / link_file[path_key] + full_path = workdir / path_str os.makedirs(os.path.dirname(full_path), exist_ok=True) with open(full_path, "w", encoding="utf-8") as f: f.write(content) - return FileUrl(path=link_file[path_key], url=link_file["url"], content=content) + return FileUrl(path=path_str, url=link_file["url"], content=content) def _convert_evaluation_function(stream: dict) -> EvaluationFunction: @@ -539,12 +548,17 @@ def _convert_text_output_channel( return text_output raise TypeError(f"Unknown text oracle type: {stream['oracle']}") + def _convert_file_output_channel_deprecated( stream: YamlObject, context: DslContext, config_name: str ) -> FileOutputChannel: assert isinstance(stream, dict) - data = OutputFileData(content_type=TextChannelType.TEXT, content=str(stream["content"]), path=str(stream["location"])) + data = OutputFileData( + content_type=TextChannelType.TEXT, + content=str(stream["content"]), + path=str(stream["location"]), + ) if "oracle" not in stream or stream["oracle"] == "builtin": config = context.merge_inheritable_with_specific_config(stream, config_name) @@ -794,13 +808,16 @@ def _convert_testcase( else: the_description = None - return Testcase( - description=the_description, - input=the_input, - output=output, - link_files=context.files, - line_comment=line_comment, - ), deprecated_usage + return ( + Testcase( + description=the_description, + input=the_input, + output=output, + link_files=context.files, + line_comment=line_comment, + ), + deprecated_usage, + ) def _convert_context( @@ -816,7 +833,9 @@ def _convert_context( return Context(testcases=testcases), deprecated_usage -def _convert_tab(tab: YamlDict, context: DslContext, workdir: Path | None) -> tuple[Tab, set[DeprecatedUsage]]: +def _convert_tab( + tab: YamlDict, context: DslContext, workdir: Path | None +) -> tuple[Tab, set[DeprecatedUsage]]: """ Translate a DSL tab to a full test suite tab. @@ -839,7 +858,9 @@ def _convert_tab(tab: YamlDict, context: DslContext, workdir: Path | None) -> tu assert "unit" in tab # We have testcases N.S. / contexts O.S. assert isinstance(tab["cases"], list) - contexts, du = _convert_dsl_list(tab["cases"], context, workdir, _convert_context) + contexts, du = _convert_dsl_list( + tab["cases"], context, workdir, _convert_context + ) deprecated_usage.update(du) elif "testcases" in tab: # We have scripts N.S. / testcases O.S. @@ -869,7 +890,9 @@ def _convert_dsl_list( dsl_list: list, context: DslContext, workdir: Path | None, - converter: Callable[[YamlDict, DslContext, Path | None], tuple[T, set[DeprecatedUsage]]], + converter: Callable[ + [YamlDict, DslContext, Path | None], tuple[T, set[DeprecatedUsage]] + ], ) -> tuple[list[T], set[DeprecatedUsage]]: """ Convert a list of YAML objects into a test suite object. diff --git a/tested/judge/core.py b/tested/judge/core.py index ad1b59f8..2a22eff1 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -10,13 +10,15 @@ CloseContext, CloseJudgement, CloseTab, + ExtendedMessage, Metadata, + Permission, StartContext, StartJudgement, StartTab, Status, StatusMessage, - report_update, ExtendedMessage, Permission, + report_update, ) from tested.features import is_supported from tested.internationalization import get_i18n_string, set_locale @@ -46,7 +48,7 @@ generate_statement, ) from tested.serialisation import Statement -from tested.testsuite import LanguageLiterals, MainInput, TextData, DeprecatedUsage +from tested.testsuite import DeprecatedUsage, LanguageLiterals, MainInput, TextData _logger = logging.getLogger(__name__) @@ -126,7 +128,6 @@ def judge(bundle: Bundle): ) ) - if DeprecatedUsage.OUTPUT_FILES in bundle.suite.deprecated: messages.append( ExtendedMessage( diff --git a/tested/testsuite.py b/tested/testsuite.py index d82a43ec..690cd4b6 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -783,11 +783,13 @@ class ExecutionMode(StrEnum): PRECOMPILATION = "batch" INDIVIDUAL = "context" + @unique class DeprecatedUsage(StrEnum): INPUT_FILES = "input_files" OUTPUT_FILES = "output_files" + @define class Suite(WithFeatures, WithFunctions): """General test suite, which is used to run tests of some code.""" From d664c3c4c304bdaa1cc050c02f30f0ef88ef5a49 Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 18 Apr 2025 11:34:24 +0200 Subject: [PATCH 50/77] Fixed dsl_yaml tests --- tests/test_dsl_yaml.py | 18 +++++++++++------- tests/tested-draft7.json | 3 ++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 14dd0578..7b49aa00 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -86,7 +86,9 @@ def test_parse_one_tab_ctx_with_files(): - tab: "Ctx" testcases: - arguments: [ "--arg", "argument" ] - stdin: !path "input.text" + stdin: + path: "input.text" + url: "media/input.text" stdout: !path "output.text" stderr: content: !path "error.text" @@ -103,12 +105,14 @@ def test_parse_one_tab_ctx_with_files(): assert len(context.testcases) == 1 tc = context.testcases[0] assert tc.is_main_testcase() - assert tc.input.stdin.data == "input.text" + assert tc.input.stdin.path == "input.text" + assert tc.input.stdin.url == "media/input.text" assert tc.input.stdin.type == TextChannelType.FILE + assert tc.input.stdin.data is None assert tc.input.arguments == ["--arg", "argument"] - assert tc.output.stderr.data == "error.text" + assert tc.output.stderr.path == "error.text" assert tc.output.stderr.type == TextChannelType.FILE - assert tc.output.stdout.data == "output.text" + assert tc.output.stdout.path == "output.text" assert tc.output.stdout.type == TextChannelType.FILE assert tc.output.exit_code.value == 1 @@ -1218,9 +1222,9 @@ def test_files_are_propagated(): yaml_str = """ - tab: "Config ctx" input_files: - - name: "test" + - path: "test" url: "test.md" - - name: "two" + - path: "two" url: "two.md" testcases: - arguments: [ '-a', '2.125', '1.212' ] @@ -1228,7 +1232,7 @@ def test_files_are_propagated(): - arguments: [ '-a', '2.125', '1.212' ] stdout: "3.337" input_files: - - name: "test" + - path: "test" url: "twooo.md" """ json_str = translate_to_test_suite(yaml_str) diff --git a/tests/tested-draft7.json b/tests/tested-draft7.json index 1e49ec74..0f16450e 100644 --- a/tests/tested-draft7.json +++ b/tests/tested-draft7.json @@ -28,7 +28,8 @@ "object", "string", "oracle", - "expression" + "expression", + "path" ] }, "stringArray": { From 8a8d81da2d698733a730da6e43281e3cbf626419 Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 18 Apr 2025 11:50:54 +0200 Subject: [PATCH 51/77] Fixed builtin oracle tests --- tests/test_oracles_builtin.py | 151 ++++++++++++---------------------- 1 file changed, 52 insertions(+), 99 deletions(-) diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index 852aa117..875a4400 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -24,7 +24,6 @@ from tested.testsuite import ( ExceptionOutputChannel, ExpectedException, - FileOutputChannel, OutputFileData, Suite, SupportedLanguage, @@ -150,20 +149,16 @@ def test_file_oracle_full_wrong( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - output_data=[ - OutputFileData( - content="expected.txt", - path="expected.txt", - content_type=TextChannelType.FILE, - ) - ] + channel = OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, ) result = evaluate_file(config, channel, "") s.assert_called_once_with(ANY, "expected\nexpected", "actual\nactual") assert result.result.enum == Status.WRONG - assert result.readable_expected == "--- ---\nexpected\nexpected" - assert result.readable_actual == "--- ---\nactual\nactual" + assert result.readable_expected == "expected\nexpected" + assert result.readable_actual == "actual\nactual" def test_file_oracle_full_correct( @@ -178,64 +173,42 @@ def test_file_oracle_full_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - output_data=[ - OutputFileData( - content="expected.txt", - path="expected.txt", - content_type=TextChannelType.FILE, - ) - ] + channel = OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, ) result = evaluate_file(config, channel, "") s.assert_called_once_with(ANY, "expected\nexpected", "expected\nexpected") assert result.result.enum == Status.CORRECT - assert result.readable_expected == "--- ---\nexpected\nexpected" - assert result.readable_actual == "--- ---\nexpected\nexpected" + assert result.readable_expected == "expected\nexpected" + assert result.readable_actual == "expected\nexpected" -def test_file_oracle_full_correct_with_mixed_content( +def test_file_oracle_full_correct_with_text_content( tmp_path: Path, pytestconfig: pytest.Config, mocker: MockerFixture ): config = oracle_config(tmp_path, pytestconfig, {"mode": "full"}) - s = mocker.spy(tested.oracles.text, name="_text_comparison") # type: ignore[reportAttributeAccessIssue] + s = mocker.spy( + tested.oracles.text, name="_text_comparison" # type: ignore[reportAttributeAccessIssue] + ) mock_files = [ mocker.mock_open(read_data=content).return_value - for content in [ - "expected\nexpected", - "expected\nexpected", - "expected\nexpected", - "expected\nexpected", - ] + for content in ["expected\nexpected", "expected\nexpected"] ] mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - output_data=[ - OutputFileData( - content="expected.txt", - path="expected.txt", - content_type=TextChannelType.FILE, - ), - OutputFileData( - content="expected\nexpected", - path="expected.txt", - content_type=TextChannelType.TEXT, - ), - ] + channel = OutputFileData( + content="expected\nexpected", + path="expected.txt", + content_type=TextChannelType.FILE, ) result = evaluate_file(config, channel, "") - s.assert_called_with(ANY, "expected\nexpected", "expected\nexpected") + s.assert_called_once_with(ANY, "expected\nexpected", "expected\nexpected") assert result.result.enum == Status.CORRECT - assert ( - result.readable_expected - == "--- ---\nexpected\nexpected\n--- ---\nexpected\nexpected" - ) - assert ( - result.readable_actual - == "--- ---\nexpected\nexpected\n--- ---\nexpected\nexpected" - ) + assert result.readable_expected == "expected\nexpected" + assert result.readable_actual == "expected\nexpected" def test_file_oracle_line_wrong( @@ -252,22 +225,18 @@ def test_file_oracle_line_wrong( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - output_data=[ - OutputFileData( - content="expected.txt", - path="expected.txt", - content_type=TextChannelType.FILE, - ) - ] + channel = OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "actual") s.assert_any_call(ANY, "expected2", "actual2") assert s.call_count == 2 assert result.result.enum == Status.WRONG - assert result.readable_expected == "--- ---\nexpected\nexpected2" - assert result.readable_actual == "--- ---\nactual\nactual2" + assert result.readable_expected == "expected\nexpected2" + assert result.readable_actual == "actual\nactual2" def test_file_oracle_line_correct( @@ -284,22 +253,18 @@ def test_file_oracle_line_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - output_data=[ - OutputFileData( - content="expected.txt", - path="expected.txt", - content_type=TextChannelType.FILE, - ) - ] + channel = OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "expected") s.assert_any_call(ANY, "expected2", "expected2") assert s.call_count == 2 assert result.result.enum == Status.CORRECT - assert result.readable_expected == "--- ---\nexpected\nexpected2" - assert result.readable_actual == "--- ---\nexpected\nexpected2" + assert result.readable_expected == "expected\nexpected2" + assert result.readable_actual == "expected\nexpected2" def test_file_oracle_strip_lines_correct( @@ -316,22 +281,18 @@ def test_file_oracle_strip_lines_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - output_data=[ - OutputFileData( - content="expected.txt", - path="expected.txt", - content_type=TextChannelType.FILE, - ) - ] + channel = OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "expected") s.assert_any_call(ANY, "expected2", "expected2") assert s.call_count == 2 assert result.result.enum == Status.CORRECT - assert result.readable_expected == "--- ---\nexpected\nexpected2\n" - assert result.readable_actual == "--- ---\nexpected\nexpected2" + assert result.readable_expected == "expected\nexpected2\n" + assert result.readable_actual == "expected\nexpected2" def test_file_oracle_dont_strip_lines_correct( @@ -348,36 +309,28 @@ def test_file_oracle_dont_strip_lines_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - output_data=[ - OutputFileData( - content="expected.txt", - path="expected.txt", - content_type=TextChannelType.FILE, - ) - ] + channel = OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected\n", "expected\n") s.assert_any_call(ANY, "expected2\n", "expected2\n") assert s.call_count == 2 assert result.result.enum == Status.CORRECT - assert result.readable_expected == "--- ---\nexpected\nexpected2\n" - assert result.readable_actual == "--- ---\nexpected\nexpected2\n" + assert result.readable_expected == "expected\nexpected2\n" + assert result.readable_actual == "expected\nexpected2\n" def test_correct_error_actual_not_found(tmp_path: Path, pytestconfig: pytest.Config): config = oracle_config( tmp_path, pytestconfig, {"mode": "line", "stripNewlines": False} ) - channel = FileOutputChannel( - output_data=[ - OutputFileData( - content="Hallo world!", - path="expected.txt", - content_type=TextChannelType.TEXT, - ) - ] + channel = OutputFileData( + content="Hallo world!", + path="expected.txt", + content_type=TextChannelType.TEXT, ) result = evaluate_file(config, channel, "") assert result.result.enum == Status.RUNTIME_ERROR From 3b950e85d80654b0c57ee2de9a047a43c240b18f Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 18 Apr 2025 12:04:30 +0200 Subject: [PATCH 52/77] removed some unused imports --- tested/dsl/translate_parser.py | 1 - tested/judge/core.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index cce6f32a..adb4d6c2 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -14,7 +14,6 @@ from jsonschema.protocols import Validator from jsonschema.validators import extend as extend_validator from jsonschema.validators import validator_for -from typing_extensions import deprecated from tested.datatypes import ( AdvancedNumericTypes, diff --git a/tested/judge/core.py b/tested/judge/core.py index 2a22eff1..b7676f96 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -48,7 +48,7 @@ generate_statement, ) from tested.serialisation import Statement -from tested.testsuite import DeprecatedUsage, LanguageLiterals, MainInput, TextData +from tested.testsuite import DeprecatedUsage, LanguageLiterals _logger = logging.getLogger(__name__) From e637459d36266eb5bea485099932b80e1b7e4deb Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 18 Apr 2025 12:06:16 +0200 Subject: [PATCH 53/77] fixed linting --- tests/test_oracles_builtin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index 875a4400..f25e7cc4 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -190,7 +190,7 @@ def test_file_oracle_full_correct_with_text_content( ): config = oracle_config(tmp_path, pytestconfig, {"mode": "full"}) s = mocker.spy( - tested.oracles.text, name="_text_comparison" # type: ignore[reportAttributeAccessIssue] + tested.oracles.text, name="_text_comparison" # type: ignore[reportAttributeAccessIssue] ) mock_files = [ mocker.mock_open(read_data=content).return_value From 98c339d6551a99704c18bbe92b46796b44cffdee Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 18 Apr 2025 15:44:21 +0200 Subject: [PATCH 54/77] Wrote a few more tests --- tested/dsl/translate_parser.py | 4 +- tests/test_dsl_yaml.py | 80 ++++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index adb4d6c2..3cc522d2 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -461,7 +461,9 @@ def _convert_file( assert isinstance(content, str) if workdir is not None: full_path = workdir / path_str - os.makedirs(os.path.dirname(full_path), exist_ok=True) + dir_name = os.path.dirname(full_path) + if dir_name: + os.makedirs(os.path.dirname(full_path), exist_ok=True) with open(full_path, "w", encoding="utf-8") as f: f.write(content) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 7044e6fa..fbb4708d 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -1,10 +1,12 @@ -# type: ignore[reportAttributeAccessIssue] import json +import os from pathlib import Path import pytest from jsonschema.validators import validator_for +from pytest_mock import MockerFixture +import tested from tested.datatypes import ( AdvancedNumericTypes, AdvancedSequenceTypes, @@ -41,7 +43,7 @@ TextChannelType, TextOutputChannel, ValueOutputChannel, - parse_test_suite, + parse_test_suite, DeprecatedUsage, ) from tested.utils import get_args @@ -574,7 +576,8 @@ def test_tab_config_trickles_down_stderr(): namespace: "solution" testcases: - arguments: [ "--arg", "argument" ] - stdin: "Input string" + stdin: + content: "Input string" stdout: "Output string" stderr: "Error string" exit_code: 1 @@ -785,6 +788,58 @@ def test_value_built_in_checks_implied(): type=BasicStringTypes.TEXT, data="hallo" ) +def test_using_deprecated_file(): + yaml_str = f""" + - tab: 'Test' + contexts: + - testcases: + - statement: 'test()' + file: + content: "Hello world!" + location: "test.txt" + """ + json_str = translate_to_test_suite(yaml_str) + suite = parse_test_suite(json_str) + using_deprecated = suite.deprecated + assert DeprecatedUsage.OUTPUT_FILES in using_deprecated + assert len(suite.tabs) == 1 + tab = suite.tabs[0] + assert len(tab.contexts) == 1 + testcases = tab.contexts[0].testcases + assert len(testcases) == 1 + test = testcases[0] + assert isinstance(test.input, FunctionCall) + assert isinstance(test.output.file, FileOutputChannel) + assert test.output.file.output_data[0].path == "test.txt" + assert test.output.file.output_data[0].content == "Hello world!" + assert test.output.file.output_data[0].content_type == TextChannelType.TEXT + +def test_using_deprecated_files(): + yaml_str = f""" + - tab: 'Test' + contexts: + - testcases: + - expression: 'test("hello.txt")' + return: "Hello world!" + files: + - url: "media/hello.txt" + name: "hello.txt" + """ + json_str = translate_to_test_suite(yaml_str) + suite = parse_test_suite(json_str) + using_deprecated = suite.deprecated + assert DeprecatedUsage.INPUT_FILES in using_deprecated + assert len(suite.tabs) == 1 + tab = suite.tabs[0] + assert len(tab.contexts) == 1 + testcases = tab.contexts[0].testcases + assert len(testcases) == 1 + test = testcases[0] + assert isinstance(test.input, FunctionCall) + assert len(test.link_files) == 1 + assert test.link_files[0].path == "hello.txt" + assert test.link_files[0].url == "media/hello.txt" + def test_output_files_custom_check_correct(): yaml_str = f""" @@ -1246,6 +1301,25 @@ def test_files_are_propagated(): FileUrl(path="two", url="two.md"), } +def test_input_file_created(tmp_path: Path, pytestconfig: pytest.Config): + yaml_str = f""" + - tab: 'Test' + contexts: + - testcases: + - expression: 'test("hello.txt")' + return: "Hello world!" + input_files: + - url: "media/hello.txt" + content: "Hello world!" + path: "hello.txt" + """ + + os.chdir(tmp_path) + translate_to_test_suite(yaml_str) + + with open("hello.txt", "r", encoding="utf-8") as f: + assert f.read() == "Hello world!" + def test_newlines_are_added_to_stdout(): yaml_str = """ From b16f3622953422fdbab2ffd822638ee63361d329 Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 18 Apr 2025 18:40:05 +0200 Subject: [PATCH 55/77] made extra tests for stdin --- tested/testsuite.py | 6 ++-- tests/test_dsl_yaml.py | 9 ++++-- tests/test_functionality.py | 59 ++++++++++++++++++++++++++++++++++++- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/tested/testsuite.py b/tested/testsuite.py index 5fea4e67..7152454e 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -310,11 +310,9 @@ def get_data_as_string(self, resources: Path) -> str: if output_data.content_type == TextChannelType.FILE: file_path = _resolve_path(resources, output_data.content) with open(file_path, "r") as file: - file_content.append(f"--- <{output_data.path}> ---\n{file.read()}") + file_content.append(f"{file.read()}") else: - file_content.append( - f"--- <{output_data.path}> ---\n{output_data.content[i]}" - ) + file_content.append(f"{output_data.content[i]}") return "\n".join(file_content) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index fbb4708d..9816a997 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -1,12 +1,11 @@ +# type: ignore[reportAttributeAccessIssue] import json import os from pathlib import Path import pytest from jsonschema.validators import validator_for -from pytest_mock import MockerFixture -import tested from tested.datatypes import ( AdvancedNumericTypes, AdvancedSequenceTypes, @@ -33,6 +32,7 @@ ) from tested.testsuite import ( CustomCheckOracle, + DeprecatedUsage, FileOutputChannel, FileUrl, GenericTextOracle, @@ -43,7 +43,7 @@ TextChannelType, TextOutputChannel, ValueOutputChannel, - parse_test_suite, DeprecatedUsage, + parse_test_suite, ) from tested.utils import get_args @@ -788,6 +788,7 @@ def test_value_built_in_checks_implied(): type=BasicStringTypes.TEXT, data="hallo" ) + def test_using_deprecated_file(): yaml_str = f""" - tab: 'Test' @@ -814,6 +815,7 @@ def test_using_deprecated_file(): assert test.output.file.output_data[0].content == "Hello world!" assert test.output.file.output_data[0].content_type == TextChannelType.TEXT + def test_using_deprecated_files(): yaml_str = f""" - tab: 'Test' @@ -1301,6 +1303,7 @@ def test_files_are_propagated(): FileUrl(path="two", url="two.md"), } + def test_input_file_created(tmp_path: Path, pytestconfig: pytest.Config): yaml_str = f""" - tab: 'Test' diff --git a/tests/test_functionality.py b/tests/test_functionality.py index da5aeb80..2c7fc06c 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -18,7 +18,15 @@ from tested.judge.execution import ExecutionResult from tested.languages import LANGUAGES, get_language from tested.languages.generation import get_readable_input -from tested.testsuite import Context, MainInput, Suite, Tab, Testcase, TextData +from tested.testsuite import ( + Context, + MainInput, + Suite, + Tab, + Testcase, + TextChannelType, + TextData, +) from tests.language_markers import ( ALL_LANGUAGES, ALL_SPECIFIC_LANGUAGES, @@ -815,3 +823,52 @@ def test_stdin_token_is_unique(tmp_path: Path, pytestconfig: pytest.Config): assert ( actual.description == "$ submission hello << 'STDINN'\nOne line\nSTDIN\nSTDINN" ) + + +def test_stdin_with_path(tmp_path: Path, pytestconfig: pytest.Config): + conf = configuration( + pytestconfig, + "echo-function", + "bash", + tmp_path, + "two.yaml", + "top-level-output", + ) + the_input = Testcase( + input=MainInput( + arguments=["hello"], + stdin=TextData( + data="One line\n", + path="line.txt", + url="media/line.txt", + type=TextChannelType.FILE, + ), + ) + ) + suite = Suite(tabs=[Tab(contexts=[Context(testcases=[the_input])], name="hallo")]) + bundle = create_bundle(conf, sys.stdout, suite) + actual, _ = get_readable_input(bundle, the_input) + + assert ( + actual.description + == '$ submission hello < line.txt' + ) + + +def test_stdin_with_one_line(tmp_path: Path, pytestconfig: pytest.Config): + conf = configuration( + pytestconfig, + "echo-function", + "bash", + tmp_path, + "two.yaml", + "top-level-output", + ) + the_input = Testcase( + input=MainInput(arguments=["hello"], stdin=TextData(data="One line\n")) + ) + suite = Suite(tabs=[Tab(contexts=[Context(testcases=[the_input])], name="hallo")]) + bundle = create_bundle(conf, sys.stdout, suite) + actual, _ = get_readable_input(bundle, the_input) + + assert actual.description == "submission hello <<< One line" From 4a9a96153cb46215f71a5a68275d0e1324015fe1 Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 18 Apr 2025 19:10:10 +0200 Subject: [PATCH 56/77] Fixed test --- tests/test_functionality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_functionality.py b/tests/test_functionality.py index 2c7fc06c..cd488058 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -871,4 +871,4 @@ def test_stdin_with_one_line(tmp_path: Path, pytestconfig: pytest.Config): bundle = create_bundle(conf, sys.stdout, suite) actual, _ = get_readable_input(bundle, the_input) - assert actual.description == "submission hello <<< One line" + assert actual.description == "$ submission hello <<< One line" From 86642ac41745d94fd82667dac38c81d28c916428 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 20 Apr 2025 14:15:27 +0200 Subject: [PATCH 57/77] Added changes for stdout and stderr --- tested/dsl/schema-strict.json | 89 ++++++++++++++++++++++++++++++++++ tested/dsl/schema.json | 89 ++++++++++++++++++++++++++++++++++ tested/dsl/translate_parser.py | 29 ++++++----- tested/languages/generation.py | 30 ++++++++++++ 4 files changed, 222 insertions(+), 15 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 01086366..d0a043ce 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -699,6 +699,7 @@ ] } ], + "additionalProperties" : false, "properties" : { "content": { "$ref" : "#/definitions/textualType", @@ -715,9 +716,43 @@ } } }, + { + "type" : "object", + "description" : "Built-in oracle for text values.", + "required" : [ + "path", + "url" + ], + "additionalProperties" : false, + "properties" : { + "content": { + "$ref" : "#/definitions/textualType", + "description" : "Content or relative path to the file in the `evaluation` folder of an exercise." + }, + "data" : { + "$ref" : "#/definitions/textualType" + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "url": { + "type": "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/textConfigurationOptions" + } + } + }, { "type" : "object", "description" : "Custom oracle for text values.", + "additionalProperties" : false, "oneOf": [ { "required" : [ @@ -768,6 +803,60 @@ } } } + }, + { + "type" : "object", + "description" : "Custom oracle for text values.", + "additionalProperties" : false, + "required" : [ + "oracle", + "file", + "path", + "url" + ], + "properties" : { + "content": { + "$ref" : "#/definitions/textualType" + }, + "data" : { + "$ref" : "#/definitions/textualType" + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "url": { + "type": "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + }, + "oracle" : { + "const" : "custom_check" + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } } ] }, diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 750d9a81..25db2cda 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -709,6 +709,7 @@ ] } ], + "additionalProperties" : false, "properties" : { "content": { "$ref" : "#/definitions/textualType", @@ -725,9 +726,43 @@ } } }, + { + "type" : "object", + "description" : "Built-in oracle for text values.", + "required" : [ + "path", + "url" + ], + "additionalProperties" : false, + "properties" : { + "content": { + "$ref" : "#/definitions/textualType", + "description" : "Content or relative path to the file in the `evaluation` folder of an exercise." + }, + "data" : { + "$ref" : "#/definitions/textualType" + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "url": { + "type": "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/textConfigurationOptions" + } + } + }, { "type" : "object", "description" : "Custom oracle for text values.", + "additionalProperties" : false, "oneOf": [ { "required" : [ @@ -778,6 +813,60 @@ } } } + }, + { + "type" : "object", + "description" : "Custom oracle for text values.", + "additionalProperties" : false, + "required" : [ + "oracle", + "file", + "path", + "url" + ], + "properties" : { + "content": { + "$ref" : "#/definitions/textualType" + }, + "data" : { + "$ref" : "#/definitions/textualType" + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "url": { + "type": "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + }, + "oracle" : { + "const" : "custom_check" + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } } ] }, diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 3cc522d2..be0d6e51 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -516,31 +516,30 @@ def _convert_text_output_channel( ) -> TextOutputChannel: # Get the config applicable to this level. # Either attempt to get it from an object, or using the inherited options as is. + url = "" path = None - + data = None if isinstance(stream, str): config = context.config.get(config_name, dict()) - raw_data = stream - if isinstance(raw_data, PathString): - path = raw_data + data = stream else: assert isinstance(stream, dict) - raw_data = stream.get("content", stream.get("data")) - if not isinstance(raw_data, PathString): - config = context.merge_inheritable_with_specific_config(stream, config_name) - else: - config = context.config.get(config_name, dict()) - path = raw_data - assert raw_data is not None + config = context.merge_inheritable_with_specific_config(stream, config_name) + if "path" in stream: + path = stream["path"] + url = stream["url"] + + if "content" in stream or "data" in stream: + data = stream.get("content", stream.get("data")) + # Normalize the data if necessary. if config.get("normalizeTrailingNewlines", True) and path is None: - data = _ensure_trailing_newline(str(raw_data)) - else: - data = str(raw_data) + assert data is not None + data = _ensure_trailing_newline(str(data)) if path is not None: - text_output = TextOutputChannel(data=None, path=data, type=TextChannelType.FILE) + text_output = TextOutputChannel(data=data, path=path, url=url, type=TextChannelType.FILE) else: text_output = TextOutputChannel(data=data) diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 13948cfa..a1f13361 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -84,6 +84,12 @@ def _get_heredoc_token(stdin: str) -> str: delimiter = delimiter + "N" return delimiter +def append_stdin_stderr(text: str, stdout: str, stderr: str) -> str: + if stdout: + text += f" > {stdout}" + if stderr: + text += f" 2> {stderr}" + return text def get_readable_input( bundle: Bundle, case: Testcase @@ -100,10 +106,14 @@ def get_readable_input( """ stdin = "" + stdout = "" + stderr = "" link_files = case.link_files if case.is_main_testcase(): assert isinstance(case.input, MainInput) stdin_data = case.input.stdin + stdout_data = case.output.stdout + stderr_data = case.output.stderr if isinstance(stdin_data, TextData): if stdin_data.type == "file": stdin = stdin_data.path @@ -111,6 +121,20 @@ def get_readable_input( else: stdin = stdin_data.data + if isinstance(stdout_data, TextData): + if stdout_data.type == "file": + stdout = stdout_data.path + link_files.append(FileUrl(path=stdout, url=stdout_data.url)) + else: + stdout = stdout_data.data + + if isinstance(stderr_data, TextData): + if stderr_data.type == "file": + stderr = stderr_data.path + link_files.append(FileUrl(path=stderr, url=stderr_data.url)) + else: + stderr = stderr_data.data + format_ = "text" # By default, we use text as input. if case.description: if isinstance(case.description, ExtendedMessage): @@ -133,6 +157,7 @@ def get_readable_input( if stdin: if isinstance(stdin_data, TextData) and stdin_data.type == "file": text = f"{args} < {stdin}" + text = append_stdin_stderr(text, stdout, stderr) elif case.input.arguments: assert stdin[-1] == "\n", "stdin must end with a newline" if stdin.count("\n") > 1: @@ -140,11 +165,16 @@ def get_readable_input( text = f"{args} << '{delimiter}'\n{stdin}{delimiter}" else: text = f"{args} <<< {stdin.strip()}" + + text = append_stdin_stderr(text, stdout, stderr) else: assert not case.input.arguments text = stdin else: text = args + text = append_stdin_stderr(text, stdout, stderr) + + elif isinstance(case.input, Statement): format_ = bundle.config.programming_language text = generate_statement(bundle, case.input) From 114881f76c983769ad2b9097757c5b19b3277068 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 20 Apr 2025 15:32:15 +0200 Subject: [PATCH 58/77] Fixed tests and other issues --- tested/dsl/translate_parser.py | 11 ++++++----- tested/languages/generation.py | 8 +++----- tests/test_dsl_yaml.py | 9 +++++++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index be0d6e51..238ee891 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -526,12 +526,11 @@ def _convert_text_output_channel( assert isinstance(stream, dict) config = context.merge_inheritable_with_specific_config(stream, config_name) if "path" in stream: - path = stream["path"] - url = stream["url"] + path = str(stream["path"]) + url = str(stream["url"]) if "content" in stream or "data" in stream: - data = stream.get("content", stream.get("data")) - + data = str(stream.get("content", stream.get("data"))) # Normalize the data if necessary. if config.get("normalizeTrailingNewlines", True) and path is None: @@ -539,7 +538,9 @@ def _convert_text_output_channel( data = _ensure_trailing_newline(str(data)) if path is not None: - text_output = TextOutputChannel(data=data, path=path, url=url, type=TextChannelType.FILE) + text_output = TextOutputChannel( + data=data, path=path, url=url, type=TextChannelType.FILE + ) else: text_output = TextOutputChannel(data=data) diff --git a/tested/languages/generation.py b/tested/languages/generation.py index a1f13361..41593719 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -84,6 +84,7 @@ def _get_heredoc_token(stdin: str) -> str: delimiter = delimiter + "N" return delimiter + def append_stdin_stderr(text: str, stdout: str, stderr: str) -> str: if stdout: text += f" > {stdout}" @@ -91,6 +92,7 @@ def append_stdin_stderr(text: str, stdout: str, stderr: str) -> str: text += f" 2> {stderr}" return text + def get_readable_input( bundle: Bundle, case: Testcase ) -> tuple[ExtendedMessage, set[FileUrl]]: @@ -120,20 +122,17 @@ def get_readable_input( link_files.append(FileUrl(path=stdin, url=stdin_data.url)) else: stdin = stdin_data.data + assert stdin is not None if isinstance(stdout_data, TextData): if stdout_data.type == "file": stdout = stdout_data.path link_files.append(FileUrl(path=stdout, url=stdout_data.url)) - else: - stdout = stdout_data.data if isinstance(stderr_data, TextData): if stderr_data.type == "file": stderr = stderr_data.path link_files.append(FileUrl(path=stderr, url=stderr_data.url)) - else: - stderr = stderr_data.data format_ = "text" # By default, we use text as input. if case.description: @@ -174,7 +173,6 @@ def get_readable_input( text = args text = append_stdin_stderr(text, stdout, stderr) - elif isinstance(case.input, Statement): format_ = bundle.config.programming_language text = generate_statement(bundle, case.input) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 9816a997..6c1b5d01 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -91,9 +91,12 @@ def test_parse_one_tab_ctx_with_files(): stdin: path: "input.text" url: "media/input.text" - stdout: !path "output.text" + stdout: + path: "output.text" + url: "media/output.text" stderr: - content: !path "error.text" + path: "error.text" + url: "media/error.text" exit_code: 1 """ json_str = translate_to_test_suite(yaml_str) @@ -113,8 +116,10 @@ def test_parse_one_tab_ctx_with_files(): assert tc.input.stdin.data is None assert tc.input.arguments == ["--arg", "argument"] assert tc.output.stderr.path == "error.text" + assert tc.output.stderr.url == "media/error.text" assert tc.output.stderr.type == TextChannelType.FILE assert tc.output.stdout.path == "output.text" + assert tc.output.stdout.url == "media/output.text" assert tc.output.stdout.type == TextChannelType.FILE assert tc.output.exit_code.value == 1 From 730d8fd532c1cab8363377053bee5ed8a88f8de2 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 20 Apr 2025 16:31:00 +0200 Subject: [PATCH 59/77] Made a few more tests --- tests/test_functionality.py | 89 ++++++++++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/tests/test_functionality.py b/tests/test_functionality.py index cd488058..413f0df0 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -25,7 +25,7 @@ Tab, Testcase, TextChannelType, - TextData, + TextData, Output, TextOutputChannel, ) from tests.language_markers import ( ALL_LANGUAGES, @@ -854,6 +854,91 @@ def test_stdin_with_path(tmp_path: Path, pytestconfig: pytest.Config): == '$ submission hello < line.txt' ) +def test_stdin_with_stdout_stderr(tmp_path: Path, pytestconfig: pytest.Config): + conf = configuration( + pytestconfig, + "echo-function", + "bash", + tmp_path, + "two.yaml", + "top-level-output", + ) + the_input = Testcase( + input=MainInput( + arguments=["hello"], + stdin=TextData( + data="One line\n", + path="line.txt", + url="media/line.txt", + type=TextChannelType.FILE, + ), + + ), + output=Output( + stdout=TextOutputChannel( + data=None, + path="out.txt", + url="media/out.txt", + type=TextChannelType.FILE, + ), + stderr=TextOutputChannel( + data=None, + path="error.txt", + url="media/error.txt", + type=TextChannelType.FILE, + ) + ) + ) + suite = Suite(tabs=[Tab(contexts=[Context(testcases=[the_input])], name="hallo")]) + bundle = create_bundle(conf, sys.stdout, suite) + actual, _ = get_readable_input(bundle, the_input) + + assert ( + actual.description + == '$ submission hello < line.txt > out.txt 2> error.txt' + ) + +def test_inline_stdin_with_stdout_stderr(tmp_path: Path, pytestconfig: pytest.Config): + conf = configuration( + pytestconfig, + "echo-function", + "bash", + tmp_path, + "two.yaml", + "top-level-output", + ) + the_input = Testcase( + input=MainInput( + stdin=TextData( + data="One line\n", + type=TextChannelType.TEXT, + ), + + ), + output=Output( + stdout=TextOutputChannel( + data=None, + path="out.txt", + url="media/out.txt", + type=TextChannelType.FILE, + ), + stderr=TextOutputChannel( + data=None, + path="error.txt", + url="media/error.txt", + type=TextChannelType.FILE, + ) + ) + ) + suite = Suite(tabs=[Tab(contexts=[Context(testcases=[the_input])], name="hallo")]) + bundle = create_bundle(conf, sys.stdout, suite) + actual, _ = get_readable_input(bundle, the_input) + + assert ( + actual.description + == 'One line\n' + ) + def test_stdin_with_one_line(tmp_path: Path, pytestconfig: pytest.Config): conf = configuration( @@ -872,3 +957,5 @@ def test_stdin_with_one_line(tmp_path: Path, pytestconfig: pytest.Config): actual, _ = get_readable_input(bundle, the_input) assert actual.description == "$ submission hello <<< One line" + + From 03df8c0f338c53a576f9f90103c0732920abaefa Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 20 Apr 2025 17:21:29 +0200 Subject: [PATCH 60/77] Fixed linting --- tests/test_functionality.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/test_functionality.py b/tests/test_functionality.py index 413f0df0..4fb6069e 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -21,11 +21,13 @@ from tested.testsuite import ( Context, MainInput, + Output, Suite, Tab, Testcase, TextChannelType, - TextData, Output, TextOutputChannel, + TextData, + TextOutputChannel, ) from tests.language_markers import ( ALL_LANGUAGES, @@ -854,6 +856,7 @@ def test_stdin_with_path(tmp_path: Path, pytestconfig: pytest.Config): == '$ submission hello < line.txt' ) + def test_stdin_with_stdout_stderr(tmp_path: Path, pytestconfig: pytest.Config): conf = configuration( pytestconfig, @@ -872,7 +875,6 @@ def test_stdin_with_stdout_stderr(tmp_path: Path, pytestconfig: pytest.Config): url="media/line.txt", type=TextChannelType.FILE, ), - ), output=Output( stdout=TextOutputChannel( @@ -886,8 +888,8 @@ def test_stdin_with_stdout_stderr(tmp_path: Path, pytestconfig: pytest.Config): path="error.txt", url="media/error.txt", type=TextChannelType.FILE, - ) - ) + ), + ), ) suite = Suite(tabs=[Tab(contexts=[Context(testcases=[the_input])], name="hallo")]) bundle = create_bundle(conf, sys.stdout, suite) @@ -898,6 +900,7 @@ def test_stdin_with_stdout_stderr(tmp_path: Path, pytestconfig: pytest.Config): == '$ submission hello < line.txt > out.txt 2> error.txt' ) + def test_inline_stdin_with_stdout_stderr(tmp_path: Path, pytestconfig: pytest.Config): conf = configuration( pytestconfig, @@ -913,7 +916,6 @@ def test_inline_stdin_with_stdout_stderr(tmp_path: Path, pytestconfig: pytest.Co data="One line\n", type=TextChannelType.TEXT, ), - ), output=Output( stdout=TextOutputChannel( @@ -927,17 +929,14 @@ def test_inline_stdin_with_stdout_stderr(tmp_path: Path, pytestconfig: pytest.Co path="error.txt", url="media/error.txt", type=TextChannelType.FILE, - ) - ) + ), + ), ) suite = Suite(tabs=[Tab(contexts=[Context(testcases=[the_input])], name="hallo")]) bundle = create_bundle(conf, sys.stdout, suite) actual, _ = get_readable_input(bundle, the_input) - assert ( - actual.description - == 'One line\n' - ) + assert actual.description == "One line\n" def test_stdin_with_one_line(tmp_path: Path, pytestconfig: pytest.Config): @@ -957,5 +956,3 @@ def test_stdin_with_one_line(tmp_path: Path, pytestconfig: pytest.Config): actual, _ = get_readable_input(bundle, the_input) assert actual.description == "$ submission hello <<< One line" - - From 6a0808dc9ad0ff74e3e556d775e95a88dbdd599e Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 25 Apr 2025 19:14:42 +0200 Subject: [PATCH 61/77] slight name change --- tested/dsl/translate_parser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 238ee891..1f696db9 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -307,7 +307,7 @@ def deepen_context( assert isinstance(files, list) additional_files = { _convert_file( - f, workdir=workdir, deprecated_usage=len(deprecated_usage) == 0 + f, workdir=workdir, not_deprecated_usage=len(deprecated_usage) == 0 ) for f in files } @@ -448,9 +448,9 @@ def _convert_value(value: YamlObject) -> Value: def _convert_file( - link_file: YamlDict, workdir: Path | None, deprecated_usage: bool + link_file: YamlDict, workdir: Path | None, not_deprecated_usage: bool ) -> FileUrl: - path_key = "path" if deprecated_usage else "name" + path_key = "path" if not_deprecated_usage else "name" path_str = link_file[path_key] assert isinstance(path_str, str) From 5c9a3893b0bfbeb0606597ebff97641d25649abe Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 25 Apr 2025 21:34:39 +0200 Subject: [PATCH 62/77] updated input_files --- tested/dsl/schema-strict.json | 53 ++++++++++++++++++++++----------- tested/dsl/schema.json | 54 +++++++++++++++++++++++----------- tested/dsl/translate_parser.py | 10 +++++-- tested/judge/core.py | 17 ++++++----- tested/judge/evaluation.py | 12 ++++---- 5 files changed, 97 insertions(+), 49 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index d0a043ce..77189f9e 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -659,25 +659,44 @@ "file" : { "type" : "object", "description" : "A file used in the test suite.", - "required" : [ - "path", - "url" - ], - "properties" : { - "path" : { - "type" : "string", - "description" : "The filename, including the file extension." - }, - "content" : { - "type" : "string", - "description" : "The actual content of the file." + "oneOf" : [ + { + "type" : "object", + "description" : "A file used in the test suite.", + "required" : [ + "path", + "url" + ], + "properties" : { + "path" : { + "type" : "string", + "description" : "The filename, including the file extension." + }, + "url" : { + "type" : "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." + { + "required" : [ + "path", + "content" + ], + "additionalProperties" : false, + "properties" : { + "path" : { + "type" : "string", + "description" : "The filename, including the file extension." + }, + "content" : { + "type" : "string", + "description" : "The actual content of the file." + } + } } - } + ] }, "textOutputChannel" : { "anyOf" : [ diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 25db2cda..1340a43b 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -669,25 +669,45 @@ "file" : { "type" : "object", "description" : "A file used in the test suite.", - "required" : [ - "path", - "url" - ], - "properties" : { - "path" : { - "type" : "string", - "description" : "The filename, including the file extension." - }, - "content" : { - "type" : "string", - "description" : "The actual content of the file." + "oneOf" : [ + { + "type" : "object", + "description" : "A file used in the test suite.", + "required" : [ + "path", + "url" + ], + "additionalProperties" : false, + "properties" : { + "path" : { + "type" : "string", + "description" : "The filename, including the file extension." + }, + "url" : { + "type" : "string", + "format" : "uri", + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." + { + "required" : [ + "path", + "content" + ], + "additionalProperties" : false, + "properties" : { + "path" : { + "type" : "string", + "description" : "The filename, including the file extension." + }, + "content" : { + "type" : "string", + "description" : "The actual content of the file." + } + } } - } + ] }, "textOutputChannel" : { "anyOf" : [ diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 1f696db9..ef5f1843 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -452,10 +452,10 @@ def _convert_file( ) -> FileUrl: path_key = "path" if not_deprecated_usage else "name" path_str = link_file[path_key] - assert isinstance(path_str, str) - assert isinstance(link_file["url"], str) + content = "" + url = "" if "content" in link_file: content = link_file["content"] assert isinstance(content, str) @@ -466,8 +466,12 @@ def _convert_file( os.makedirs(os.path.dirname(full_path), exist_ok=True) with open(full_path, "w", encoding="utf-8") as f: f.write(content) + else: + assert "url" in link_file + assert isinstance(link_file["url"], str) + url = link_file["url"] - return FileUrl(path=path_str, url=link_file["url"], content=content) + return FileUrl(path=path_str, url=url, content=content) def _convert_evaluation_function(stream: dict) -> EvaluationFunction: diff --git a/tested/judge/core.py b/tested/judge/core.py index b7676f96..cbe36f00 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -352,7 +352,7 @@ def _process_results( # Handle the contexts. collector.add(StartContext(description=planned.context.description)) - continue_ = evaluate_context_results( + continue_, seen_files = evaluate_context_results( bundle, context=planned.context, exec_results=context_result, @@ -368,13 +368,16 @@ def _process_results( # since it only differs a bit. meta_statements = [] input_files = [] - for case in planned.context.testcases: - for file in case.link_files: - file_data = {"path": file.path} - if file.url != "": - file_data["url"] = file.url + for file in seen_files: + file_data = {"path": file.path} + if file.url != "": + file_data["url"] = file.url + elif file.content != "": + file_data["content"] = file.content + + input_files.append(file_data) - input_files.append(file_data) + for case in planned.context.testcases: if isinstance(case.input, Statement): stmt = generate_statement(bundle, case.input) meta_statements.append(stmt) diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index fdba3059..2c79d781 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -167,7 +167,7 @@ def evaluate_context_results( compilation_results: CompilationResult, context_dir: Path, collector: OutputManager, -) -> Status | None: +) -> tuple[Status | None, set]: """ Evaluate the results for a single context. @@ -198,7 +198,7 @@ def evaluate_context_results( # Finish the evaluation, since there is nothing we can do. collector.add(CloseTestcase(accepted=False), 0) - return compilation_results.status + return compilation_results.status, set() # There must be execution if compilation succeeded. assert exec_results is not None @@ -256,6 +256,7 @@ def evaluate_context_results( # All files that will be used in this context. all_files = context.get_files() + all_seen = set() # Begin processing the normal testcases. for i, testcase in enumerate(context.testcases): @@ -264,6 +265,7 @@ def evaluate_context_results( readable_input, seen = get_readable_input(bundle, testcase) all_files = all_files - seen t_col = TestcaseCollector(StartTestcase(description=readable_input)) + all_seen.update(seen) # Get the functions output = testcase.output @@ -369,10 +371,10 @@ def evaluate_context_results( collector.add(_link_files_message(all_files)) if exec_results.timeout: - return Status.TIME_LIMIT_EXCEEDED + return Status.TIME_LIMIT_EXCEEDED, all_seen if exec_results.memory: - return Status.MEMORY_LIMIT_EXCEEDED - return None + return Status.MEMORY_LIMIT_EXCEEDED, all_seen + return None, all_seen def _link_files_message(link_files: Collection[FileUrl]) -> AppendMessage: From 858819d233544185a0c481dad822848e187b94c6 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 26 Apr 2025 16:01:51 +0200 Subject: [PATCH 63/77] updated stdin, stdout and stderr --- tested/dsl/schema-strict.json | 33 +++++++++------------------- tested/dsl/schema.json | 31 +++++++-------------------- tested/dsl/translate_parser.py | 19 +++++++++++------ tested/languages/generation.py | 39 ++++++++++++++++++++++++++-------- 4 files changed, 60 insertions(+), 62 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 77189f9e..f7ccdfec 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -342,11 +342,6 @@ "path": { "type": "string", "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." - }, - "url": { - "type": "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." } } }, @@ -481,6 +476,7 @@ { "type": "object", "required": ["content"], + "additionalProperties": false, "properties": { "content": { "type" : [ @@ -495,17 +491,13 @@ "path": { "type": "string", "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." - }, - "url": { - "type": "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." } } }, { "type": "object", "required": ["path", "url"], + "additionalProperties": false, "properties": { "path": { "type": "string", @@ -724,6 +716,10 @@ "$ref" : "#/definitions/textualType", "description" : "Content or relative path to the file in the `evaluation` folder of an exercise." }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, "data" : { "$ref" : "#/definitions/textualType" }, @@ -744,13 +740,6 @@ ], "additionalProperties" : false, "properties" : { - "content": { - "$ref" : "#/definitions/textualType", - "description" : "Content or relative path to the file in the `evaluation` folder of an exercise." - }, - "data" : { - "$ref" : "#/definitions/textualType" - }, "path": { "type": "string", "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." @@ -795,6 +784,10 @@ "data" : { "$ref" : "#/definitions/textualType" }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, "oracle" : { "const" : "custom_check" }, @@ -834,12 +827,6 @@ "url" ], "properties" : { - "content": { - "$ref" : "#/definitions/textualType" - }, - "data" : { - "$ref" : "#/definitions/textualType" - }, "path": { "type": "string", "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 1340a43b..630728d4 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -342,11 +342,6 @@ "path": { "type": "string", "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." - }, - "url": { - "type": "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." } } }, @@ -496,11 +491,6 @@ "path": { "type": "string", "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." - }, - "url": { - "type": "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." } } }, @@ -738,6 +728,10 @@ "data" : { "$ref" : "#/definitions/textualType" }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, "oracle" : { "const" : "builtin" }, @@ -755,13 +749,6 @@ ], "additionalProperties" : false, "properties" : { - "content": { - "$ref" : "#/definitions/textualType", - "description" : "Content or relative path to the file in the `evaluation` folder of an exercise." - }, - "data" : { - "$ref" : "#/definitions/textualType" - }, "path": { "type": "string", "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." @@ -806,6 +793,10 @@ "data" : { "$ref" : "#/definitions/textualType" }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, "oracle" : { "const" : "custom_check" }, @@ -845,12 +836,6 @@ "url" ], "properties" : { - "content": { - "$ref" : "#/definitions/textualType" - }, - "data" : { - "$ref" : "#/definitions/textualType" - }, "path": { "type": "string", "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index ef5f1843..6f8ef413 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -531,9 +531,12 @@ def _convert_text_output_channel( config = context.merge_inheritable_with_specific_config(stream, config_name) if "path" in stream: path = str(stream["path"]) - url = str(stream["url"]) - if "content" in stream or "data" in stream: + if "url" in stream: + assert "path" in stream + url = str(stream["url"]) + else: + assert "content" in stream or "data" in stream data = str(stream.get("content", stream.get("data"))) # Normalize the data if necessary. @@ -743,16 +746,18 @@ def _convert_testcase( data = _ensure_trailing_newline(stdin_data) else: assert isinstance(stdin_data, dict) + if "path" in stdin_data: + path = stdin_data["path"] + assert isinstance(path, str) + if "content" in stdin_data: content = stdin_data["content"] assert isinstance(content, str) data = _ensure_trailing_newline(content) - - if "path" in stdin_data: - assert "url" in stdin_data - path = stdin_data["path"] + else: + assert "url" in stdin_data and "path" in stdin_data url = stdin_data["url"] - assert isinstance(path, str) and isinstance(url, str) + assert isinstance(url, str) if path: stdin = TextData( diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 41593719..fca48f41 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -119,20 +119,42 @@ def get_readable_input( if isinstance(stdin_data, TextData): if stdin_data.type == "file": stdin = stdin_data.path - link_files.append(FileUrl(path=stdin, url=stdin_data.url)) + link_files.append( + FileUrl( + path=stdin, + url=stdin_data.url, + content=stdin_data.data if stdin_data.data is not None else "", + ) + ) else: stdin = stdin_data.data assert stdin is not None if isinstance(stdout_data, TextData): - if stdout_data.type == "file": + if stdout_data.type == "file" and (stdout_data.url or stdout_data.data): stdout = stdout_data.path - link_files.append(FileUrl(path=stdout, url=stdout_data.url)) - - if isinstance(stderr_data, TextData): + link_files.append( + FileUrl( + path=stdout, + url=stdout_data.url, + content=( + stdout_data.data if stdout_data.data is not None else "" + ), + ) + ) + + if isinstance(stderr_data, TextData) and (stderr_data.url or stderr_data.data): if stderr_data.type == "file": stderr = stderr_data.path - link_files.append(FileUrl(path=stderr, url=stderr_data.url)) + link_files.append( + FileUrl( + path=stderr, + url=stderr_data.url, + content=( + stderr_data.data if stderr_data.data is not None else "" + ), + ) + ) format_ = "text" # By default, we use text as input. if case.description: @@ -157,7 +179,7 @@ def get_readable_input( if isinstance(stdin_data, TextData) and stdin_data.type == "file": text = f"{args} < {stdin}" text = append_stdin_stderr(text, stdout, stderr) - elif case.input.arguments: + elif case.input.arguments or stdout or stderr: assert stdin[-1] == "\n", "stdin must end with a newline" if stdin.count("\n") > 1: delimiter = _get_heredoc_token(stdin) @@ -170,8 +192,7 @@ def get_readable_input( assert not case.input.arguments text = stdin else: - text = args - text = append_stdin_stderr(text, stdout, stderr) + text = append_stdin_stderr(args, stdout, stderr) elif isinstance(case.input, Statement): format_ = bundle.config.programming_language From 07c57ca1737fa9d683ebf4afc1c22f8e91d77ee3 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 26 Apr 2025 16:37:05 +0200 Subject: [PATCH 64/77] fixed test --- tests/test_functionality.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_functionality.py b/tests/test_functionality.py index 4fb6069e..68b35ebe 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -936,7 +936,10 @@ def test_inline_stdin_with_stdout_stderr(tmp_path: Path, pytestconfig: pytest.Co bundle = create_bundle(conf, sys.stdout, suite) actual, _ = get_readable_input(bundle, the_input) - assert actual.description == "One line\n" + assert ( + actual.description + == '$ submission <<< One line > out.txt 2> error.txt' + ) def test_stdin_with_one_line(tmp_path: Path, pytestconfig: pytest.Config): From 80ff8477588f00847066b17939acae4920f13970 Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 9 May 2025 11:31:18 +0200 Subject: [PATCH 65/77] input files no longer use url --- tested/dodona.py | 2 +- tested/dsl/schema-strict.json | 48 +++++++++------------------------ tested/dsl/schema.json | 49 +++++++++------------------------- tested/dsl/translate_parser.py | 7 +---- tested/judge/core.py | 6 ++--- tested/languages/generation.py | 3 --- tested/testsuite.py | 1 - 7 files changed, 28 insertions(+), 88 deletions(-) diff --git a/tested/dodona.py b/tested/dodona.py index 15dcbb96..2bcc92ad 100644 --- a/tested/dodona.py +++ b/tested/dodona.py @@ -39,7 +39,7 @@ class Metadata: """Currently only used for the Python tutor""" statements: str | None - input_files: list[dict[str, str]] | None + files: list[dict[str, str]] | None Message = ExtendedMessage | str diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index f7ccdfec..8da5e3ee 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -651,44 +651,20 @@ "file" : { "type" : "object", "description" : "A file used in the test suite.", - "oneOf" : [ - { - "type" : "object", - "description" : "A file used in the test suite.", - "required" : [ - "path", - "url" - ], - "properties" : { - "path" : { - "type" : "string", - "description" : "The filename, including the file extension." - }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." - } - } + "required" : [ + "path" + ], + "additionalProperties" : false, + "properties" : { + "path" : { + "type" : "string", + "description" : "The filename, including the file extension." }, - { - "required" : [ - "path", - "content" - ], - "additionalProperties" : false, - "properties" : { - "path" : { - "type" : "string", - "description" : "The filename, including the file extension." - }, - "content" : { - "type" : "string", - "description" : "The actual content of the file." - } - } + "content" : { + "type" : "string", + "description" : "The actual content of the file." } - ] + } }, "textOutputChannel" : { "anyOf" : [ diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 630728d4..71ed8e63 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -659,45 +659,20 @@ "file" : { "type" : "object", "description" : "A file used in the test suite.", - "oneOf" : [ - { - "type" : "object", - "description" : "A file used in the test suite.", - "required" : [ - "path", - "url" - ], - "additionalProperties" : false, - "properties" : { - "path" : { - "type" : "string", - "description" : "The filename, including the file extension." - }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." - } - } + "required" : [ + "path" + ], + "additionalProperties" : false, + "properties" : { + "path" : { + "type" : "string", + "description" : "The filename, including the file extension." }, - { - "required" : [ - "path", - "content" - ], - "additionalProperties" : false, - "properties" : { - "path" : { - "type" : "string", - "description" : "The filename, including the file extension." - }, - "content" : { - "type" : "string", - "description" : "The actual content of the file." - } - } + "content" : { + "type" : "string", + "description" : "The actual content of the file." } - ] + } }, "textOutputChannel" : { "anyOf" : [ diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 6f8ef413..f9338881 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -455,7 +455,6 @@ def _convert_file( assert isinstance(path_str, str) content = "" - url = "" if "content" in link_file: content = link_file["content"] assert isinstance(content, str) @@ -466,12 +465,8 @@ def _convert_file( os.makedirs(os.path.dirname(full_path), exist_ok=True) with open(full_path, "w", encoding="utf-8") as f: f.write(content) - else: - assert "url" in link_file - assert isinstance(link_file["url"], str) - url = link_file["url"] - return FileUrl(path=path_str, url=url, content=content) + return FileUrl(path=path_str, content=content) def _convert_evaluation_function(stream: dict) -> EvaluationFunction: diff --git a/tested/judge/core.py b/tested/judge/core.py index cbe36f00..27d3cb9e 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -370,9 +370,7 @@ def _process_results( input_files = [] for file in seen_files: file_data = {"path": file.path} - if file.url != "": - file_data["url"] = file.url - elif file.content != "": + if file.content != "": file_data["content"] = file.content input_files.append(file_data) @@ -400,7 +398,7 @@ def _process_results( CloseContext( data=Metadata( statements=meta_statements, - input_files=input_files, + files=input_files, ) ), planned.context_index, diff --git a/tested/languages/generation.py b/tested/languages/generation.py index fca48f41..e963445f 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -122,7 +122,6 @@ def get_readable_input( link_files.append( FileUrl( path=stdin, - url=stdin_data.url, content=stdin_data.data if stdin_data.data is not None else "", ) ) @@ -136,7 +135,6 @@ def get_readable_input( link_files.append( FileUrl( path=stdout, - url=stdout_data.url, content=( stdout_data.data if stdout_data.data is not None else "" ), @@ -149,7 +147,6 @@ def get_readable_input( link_files.append( FileUrl( path=stderr, - url=stderr_data.url, content=( stderr_data.data if stderr_data.data is not None else "" ), diff --git a/tested/testsuite.py b/tested/testsuite.py index 7152454e..142d47e1 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -556,7 +556,6 @@ def get_functions(self) -> Iterable[FunctionCall]: class FileUrl: path: str content: str = "" - url: str = "" @ignore_field(get_converter(), "essential") From fd15d8c152a66e97899dc5c6abcebdeccd1d54e9 Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 9 May 2025 11:35:21 +0200 Subject: [PATCH 66/77] small update for output files --- tested/judge/evaluation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index 2c79d781..eba5c870 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -136,7 +136,7 @@ def _evaluate_channel( expected = evaluation_result.readable_expected expected_channel = channel if isinstance(output_element, OutputFileData): - expected_channel = f"File: {output_element.path}" + expected_channel = f"file: {output_element.path}" out.add(StartTest(expected=expected, channel=expected_channel)) # Report any messages we received. From f44048443d68141b376a2167521588b650d6a54b Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 9 May 2025 12:12:05 +0200 Subject: [PATCH 67/77] removed url from stdin --- tested/dsl/schema-strict.json | 7 +------ tested/dsl/schema.json | 14 ++------------ tested/dsl/translate_parser.py | 9 +-------- tested/languages/generation.py | 4 ++-- tested/testsuite.py | 1 - 5 files changed, 6 insertions(+), 29 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 8da5e3ee..7e6a1209 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -348,16 +348,11 @@ { "type": "object", "additionalProperties" : false, - "required": ["path", "url"], + "required": ["path"], "properties": { "path": { "type": "string", "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." - }, - "url": { - "type": "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." } } } diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 71ed8e63..cd95b9a3 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -348,16 +348,11 @@ { "type": "object", "additionalProperties" : false, - "required": ["path", "url"], + "required": ["path"], "properties": { "path": { "type": "string", "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." - }, - "url": { - "type": "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." } } } @@ -497,16 +492,11 @@ { "type": "object", "additionalProperties" : false, - "required": ["path", "url"], + "required": ["path"], "properties": { "path": { "type": "string", "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." - }, - "url": { - "type": "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." } } } diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index f9338881..30ce2ac6 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -736,7 +736,6 @@ def _convert_testcase( stdin_data = testcase["stdin"] data = None path = "" - url = "" if isinstance(stdin_data, str): data = _ensure_trailing_newline(stdin_data) else: @@ -749,15 +748,9 @@ def _convert_testcase( content = stdin_data["content"] assert isinstance(content, str) data = _ensure_trailing_newline(content) - else: - assert "url" in stdin_data and "path" in stdin_data - url = stdin_data["url"] - assert isinstance(url, str) if path: - stdin = TextData( - data=data, path=path, url=url, type=TextChannelType.FILE - ) + stdin = TextData(data=data, path=path, type=TextChannelType.FILE) else: stdin = TextData(data=data) else: diff --git a/tested/languages/generation.py b/tested/languages/generation.py index e963445f..b1918637 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -130,7 +130,7 @@ def get_readable_input( assert stdin is not None if isinstance(stdout_data, TextData): - if stdout_data.type == "file" and (stdout_data.url or stdout_data.data): + if stdout_data.type == "file" and stdout_data.data: stdout = stdout_data.path link_files.append( FileUrl( @@ -141,7 +141,7 @@ def get_readable_input( ) ) - if isinstance(stderr_data, TextData) and (stderr_data.url or stderr_data.data): + if isinstance(stderr_data, TextData) and stderr_data.data: if stderr_data.type == "file": stderr = stderr_data.path link_files.append( diff --git a/tested/testsuite.py b/tested/testsuite.py index 142d47e1..5d22fb9a 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -249,7 +249,6 @@ class TextData(WithFeatures): data: str | None path: str = "" - url: str = "" type: TextChannelType = TextChannelType.TEXT def get_data_as_string(self, working_directory: Path) -> str: From 55990536431d6405684684352a6718e2b490597a Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 9 May 2025 13:05:07 +0200 Subject: [PATCH 68/77] changed stdout en stderr --- tested/dsl/schema-strict.json | 18 +------ tested/dsl/schema.json | 18 +------ tested/dsl/translate_parser.py | 13 ++--- tested/languages/generation.py | 43 +---------------- tests/test_functionality.py | 86 ---------------------------------- 5 files changed, 10 insertions(+), 168 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 7e6a1209..9d090e91 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -705,21 +705,13 @@ { "type" : "object", "description" : "Built-in oracle for text values.", - "required" : [ - "path", - "url" - ], + "required" : ["path"], "additionalProperties" : false, "properties" : { "path": { "type": "string", "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." }, - "url": { - "type": "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." - }, "oracle" : { "const" : "builtin" }, @@ -794,19 +786,13 @@ "required" : [ "oracle", "file", - "path", - "url" + "path" ], "properties" : { "path": { "type": "string", "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." }, - "url": { - "type": "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." - }, "oracle" : { "const" : "custom_check" }, diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index cd95b9a3..a4ad2826 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -708,21 +708,13 @@ { "type" : "object", "description" : "Built-in oracle for text values.", - "required" : [ - "path", - "url" - ], + "required" : ["path"], "additionalProperties" : false, "properties" : { "path": { "type": "string", "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." }, - "url": { - "type": "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." - }, "oracle" : { "const" : "builtin" }, @@ -797,19 +789,13 @@ "required" : [ "oracle", "file", - "path", - "url" + "path" ], "properties" : { "path": { "type": "string", "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." }, - "url": { - "type": "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." - }, "oracle" : { "const" : "custom_check" }, diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 30ce2ac6..2ff312c7 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -515,7 +515,6 @@ def _convert_text_output_channel( ) -> TextOutputChannel: # Get the config applicable to this level. # Either attempt to get it from an object, or using the inherited options as is. - url = "" path = None data = None if isinstance(stream, str): @@ -527,22 +526,18 @@ def _convert_text_output_channel( if "path" in stream: path = str(stream["path"]) - if "url" in stream: - assert "path" in stream - url = str(stream["url"]) - else: - assert "content" in stream or "data" in stream + if "content" in stream or "data" in stream: data = str(stream.get("content", stream.get("data"))) + assert path or data + # Normalize the data if necessary. if config.get("normalizeTrailingNewlines", True) and path is None: assert data is not None data = _ensure_trailing_newline(str(data)) if path is not None: - text_output = TextOutputChannel( - data=data, path=path, url=url, type=TextChannelType.FILE - ) + text_output = TextOutputChannel(data=data, path=path, type=TextChannelType.FILE) else: text_output = TextOutputChannel(data=data) diff --git a/tested/languages/generation.py b/tested/languages/generation.py index b1918637..6a2b6546 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -85,14 +85,6 @@ def _get_heredoc_token(stdin: str) -> str: return delimiter -def append_stdin_stderr(text: str, stdout: str, stderr: str) -> str: - if stdout: - text += f" > {stdout}" - if stderr: - text += f" 2> {stderr}" - return text - - def get_readable_input( bundle: Bundle, case: Testcase ) -> tuple[ExtendedMessage, set[FileUrl]]: @@ -108,14 +100,10 @@ def get_readable_input( """ stdin = "" - stdout = "" - stderr = "" link_files = case.link_files if case.is_main_testcase(): assert isinstance(case.input, MainInput) stdin_data = case.input.stdin - stdout_data = case.output.stdout - stderr_data = case.output.stderr if isinstance(stdin_data, TextData): if stdin_data.type == "file": stdin = stdin_data.path @@ -129,30 +117,6 @@ def get_readable_input( stdin = stdin_data.data assert stdin is not None - if isinstance(stdout_data, TextData): - if stdout_data.type == "file" and stdout_data.data: - stdout = stdout_data.path - link_files.append( - FileUrl( - path=stdout, - content=( - stdout_data.data if stdout_data.data is not None else "" - ), - ) - ) - - if isinstance(stderr_data, TextData) and stderr_data.data: - if stderr_data.type == "file": - stderr = stderr_data.path - link_files.append( - FileUrl( - path=stderr, - content=( - stderr_data.data if stderr_data.data is not None else "" - ), - ) - ) - format_ = "text" # By default, we use text as input. if case.description: if isinstance(case.description, ExtendedMessage): @@ -175,21 +139,18 @@ def get_readable_input( if stdin: if isinstance(stdin_data, TextData) and stdin_data.type == "file": text = f"{args} < {stdin}" - text = append_stdin_stderr(text, stdout, stderr) - elif case.input.arguments or stdout or stderr: + elif case.input.arguments: assert stdin[-1] == "\n", "stdin must end with a newline" if stdin.count("\n") > 1: delimiter = _get_heredoc_token(stdin) text = f"{args} << '{delimiter}'\n{stdin}{delimiter}" else: text = f"{args} <<< {stdin.strip()}" - - text = append_stdin_stderr(text, stdout, stderr) else: assert not case.input.arguments text = stdin else: - text = append_stdin_stderr(args, stdout, stderr) + text = args elif isinstance(case.input, Statement): format_ = bundle.config.programming_language diff --git a/tests/test_functionality.py b/tests/test_functionality.py index 68b35ebe..12f0f7f6 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -842,7 +842,6 @@ def test_stdin_with_path(tmp_path: Path, pytestconfig: pytest.Config): stdin=TextData( data="One line\n", path="line.txt", - url="media/line.txt", type=TextChannelType.FILE, ), ) @@ -857,91 +856,6 @@ def test_stdin_with_path(tmp_path: Path, pytestconfig: pytest.Config): ) -def test_stdin_with_stdout_stderr(tmp_path: Path, pytestconfig: pytest.Config): - conf = configuration( - pytestconfig, - "echo-function", - "bash", - tmp_path, - "two.yaml", - "top-level-output", - ) - the_input = Testcase( - input=MainInput( - arguments=["hello"], - stdin=TextData( - data="One line\n", - path="line.txt", - url="media/line.txt", - type=TextChannelType.FILE, - ), - ), - output=Output( - stdout=TextOutputChannel( - data=None, - path="out.txt", - url="media/out.txt", - type=TextChannelType.FILE, - ), - stderr=TextOutputChannel( - data=None, - path="error.txt", - url="media/error.txt", - type=TextChannelType.FILE, - ), - ), - ) - suite = Suite(tabs=[Tab(contexts=[Context(testcases=[the_input])], name="hallo")]) - bundle = create_bundle(conf, sys.stdout, suite) - actual, _ = get_readable_input(bundle, the_input) - - assert ( - actual.description - == '$ submission hello < line.txt > out.txt 2> error.txt' - ) - - -def test_inline_stdin_with_stdout_stderr(tmp_path: Path, pytestconfig: pytest.Config): - conf = configuration( - pytestconfig, - "echo-function", - "bash", - tmp_path, - "two.yaml", - "top-level-output", - ) - the_input = Testcase( - input=MainInput( - stdin=TextData( - data="One line\n", - type=TextChannelType.TEXT, - ), - ), - output=Output( - stdout=TextOutputChannel( - data=None, - path="out.txt", - url="media/out.txt", - type=TextChannelType.FILE, - ), - stderr=TextOutputChannel( - data=None, - path="error.txt", - url="media/error.txt", - type=TextChannelType.FILE, - ), - ), - ) - suite = Suite(tabs=[Tab(contexts=[Context(testcases=[the_input])], name="hallo")]) - bundle = create_bundle(conf, sys.stdout, suite) - actual, _ = get_readable_input(bundle, the_input) - - assert ( - actual.description - == '$ submission <<< One line > out.txt 2> error.txt' - ) - - def test_stdin_with_one_line(tmp_path: Path, pytestconfig: pytest.Config): conf = configuration( pytestconfig, From 1cab274c88cc7ba4603370aa0ab65a8c18cc6356 Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 9 May 2025 13:39:52 +0200 Subject: [PATCH 69/77] fixed tests --- tests/test_dsl_yaml.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 6c1b5d01..498a3a92 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -90,13 +90,10 @@ def test_parse_one_tab_ctx_with_files(): - arguments: [ "--arg", "argument" ] stdin: path: "input.text" - url: "media/input.text" stdout: path: "output.text" - url: "media/output.text" stderr: path: "error.text" - url: "media/error.text" exit_code: 1 """ json_str = translate_to_test_suite(yaml_str) @@ -111,15 +108,12 @@ def test_parse_one_tab_ctx_with_files(): tc = context.testcases[0] assert tc.is_main_testcase() assert tc.input.stdin.path == "input.text" - assert tc.input.stdin.url == "media/input.text" assert tc.input.stdin.type == TextChannelType.FILE assert tc.input.stdin.data is None assert tc.input.arguments == ["--arg", "argument"] assert tc.output.stderr.path == "error.text" - assert tc.output.stderr.url == "media/error.text" assert tc.output.stderr.type == TextChannelType.FILE assert tc.output.stdout.path == "output.text" - assert tc.output.stdout.url == "media/output.text" assert tc.output.stdout.type == TextChannelType.FILE assert tc.output.exit_code.value == 1 @@ -845,7 +839,6 @@ def test_using_deprecated_files(): assert isinstance(test.input, FunctionCall) assert len(test.link_files) == 1 assert test.link_files[0].path == "hello.txt" - assert test.link_files[0].url == "media/hello.txt" def test_output_files_custom_check_correct(): @@ -1285,9 +1278,7 @@ def test_files_are_propagated(): - tab: "Config ctx" input_files: - path: "test" - url: "test.md" - path: "two" - url: "two.md" testcases: - arguments: [ '-a', '2.125', '1.212' ] stdout: "3.34" @@ -1295,7 +1286,6 @@ def test_files_are_propagated(): stdout: "3.337" input_files: - path: "test" - url: "twooo.md" """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) @@ -1304,8 +1294,8 @@ def test_files_are_propagated(): testcases0, testcases1 = ctx0.testcases, ctx1.testcases test0, test1 = testcases0[0], testcases1[0] assert set(test0.link_files) == { - FileUrl(path="test", url="test.md"), - FileUrl(path="two", url="two.md"), + FileUrl(path="test"), + FileUrl(path="two"), } @@ -1317,8 +1307,7 @@ def test_input_file_created(tmp_path: Path, pytestconfig: pytest.Config): - expression: 'test("hello.txt")' return: "Hello world!" input_files: - - url: "media/hello.txt" - content: "Hello world!" + - content: "Hello world!" path: "hello.txt" """ From aab198e7ba6b692806a3edcd25bcc5c8960a30eb Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 9 May 2025 15:36:06 +0200 Subject: [PATCH 70/77] Made an extra test --- tests/test_oracles_builtin.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index f25e7cc4..8fc1c418 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -339,6 +339,22 @@ def test_correct_error_actual_not_found(tmp_path: Path, pytestconfig: pytest.Con or result.result.human == "Bestand niet gevonden." ) +def test_correct_error_expected_not_found(tmp_path: Path, pytestconfig: pytest.Config): + config = oracle_config( + tmp_path, pytestconfig, {"mode": "line", "stripNewlines": False} + ) + channel = OutputFileData( + content="actual.txt", + path="expected.txt", + content_type=TextChannelType.FILE, + ) + try: + evaluate_file(config, channel, "") + except ValueError: + print("As expected") + else: + assert False + def test_exception_oracle_only_messages_correct( tmp_path: Path, pytestconfig: pytest.Config From 2b90afd2be32614b4424ab34fac19fdd41fff52c Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 9 May 2025 15:37:51 +0200 Subject: [PATCH 71/77] Fixed linting --- tests/test_oracles_builtin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index 8fc1c418..a3d5acb5 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -339,6 +339,7 @@ def test_correct_error_actual_not_found(tmp_path: Path, pytestconfig: pytest.Con or result.result.human == "Bestand niet gevonden." ) + def test_correct_error_expected_not_found(tmp_path: Path, pytestconfig: pytest.Config): config = oracle_config( tmp_path, pytestconfig, {"mode": "line", "stripNewlines": False} From 2c06c20e43812a63172312c072b01216cf51fe20 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 12 May 2025 13:15:36 +0200 Subject: [PATCH 72/77] changed how deprecated messages are handled --- tested/configs.py | 13 ++- tested/descriptions/renderer.py | 2 +- tested/dodona.py | 2 +- tested/dsl/translate_parser.py | 136 ++++++++++++++++++++------------ tested/judge/core.py | 26 +----- tested/main.py | 7 +- tested/testsuite.py | 7 -- tested/utils.py | 12 ++- tests/test_dsl_yaml.py | 30 ++++--- tests/test_functionality.py | 2 - 10 files changed, 139 insertions(+), 98 deletions(-) diff --git a/tested/configs.py b/tested/configs.py index 0cb0cc03..5e58010f 100644 --- a/tested/configs.py +++ b/tested/configs.py @@ -8,6 +8,7 @@ from attrs import define, evolve, field +from tested.dodona import ExtendedMessage from tested.parsing import fallback_field, get_converter from tested.testsuite import ExecutionMode, Suite, SupportedLanguage from tested.utils import get_identifier, smart_close @@ -129,6 +130,7 @@ class Bundle: language: "Language" global_config: GlobalConfig out: IO + messages: set[ExtendedMessage] = set() @property def config(self) -> DodonaConfig: @@ -207,6 +209,7 @@ def create_bundle( output: IO, suite: Suite, language: str | None = None, + messages: set[ExtendedMessage] | None = None, ) -> Bundle: """ Create a configuration bundle. @@ -216,7 +219,7 @@ def create_bundle( :param suite: The test suite. :param language: Optional programming language. If None, the one from the Dodona configuration will be used. - + :param messages: Messages generated out of the translate parser. :return: The configuration bundle. """ import tested.languages as langs @@ -232,4 +235,10 @@ def create_bundle( suite=suite, ) lang_config = langs.get_language(global_config, language) - return Bundle(language=lang_config, global_config=global_config, out=output) + + if messages is None: + messages = set() + + return Bundle( + language=lang_config, global_config=global_config, out=output, messages=messages + ) diff --git a/tested/descriptions/renderer.py b/tested/descriptions/renderer.py index 5690f12c..9552c682 100644 --- a/tested/descriptions/renderer.py +++ b/tested/descriptions/renderer.py @@ -87,7 +87,7 @@ def _render_dsl_statements(self, element: block.FencedCode) -> str: # Get all actual tests tests = [] - for tab in parsed_dsl.tabs: + for tab in parsed_dsl.data.tabs: for context in tab.contexts: for testcase in context.testcases: tests.append(testcase) diff --git a/tested/dodona.py b/tested/dodona.py index 2bcc92ad..8c27e929 100644 --- a/tested/dodona.py +++ b/tested/dodona.py @@ -27,7 +27,7 @@ class Permission(StrEnum): ZEUS = auto() -@define +@define(frozen=True) class ExtendedMessage: description: str format: str = "text" diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 2ff312c7..0e4b9edd 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -32,7 +32,7 @@ StringTypes, resolve_to_basic, ) -from tested.dodona import ExtendedMessage +from tested.dodona import ExtendedMessage, Permission from tested.dsl.ast_translator import InvalidDslError, extract_comment, parse_string from tested.parsing import get_converter, suite_to_json from tested.serialisation import ( @@ -48,7 +48,6 @@ from tested.testsuite import ( Context, CustomCheckOracle, - DeprecatedUsage, EmptyChannel, EvaluationFunction, ExceptionOutputChannel, @@ -73,7 +72,7 @@ TextOutputChannel, ValueOutputChannel, ) -from tested.utils import get_args, recursive_dict_merge +from tested.utils import DataWithMessage, get_args, recursive_dict_merge YamlDict = dict[str, "YamlObject"] @@ -282,7 +281,7 @@ class DslContext: def deepen_context( self, new_level: YamlDict | None, workdir: Path | None - ) -> tuple["DslContext", set[DeprecatedUsage]]: + ) -> DataWithMessage["DslContext"]: """ Merge certain fields of the new object with the current context, resulting in a new context for the new level. @@ -293,7 +292,7 @@ def deepen_context( :return: A new context. """ if new_level is None: - return self, set() + return DataWithMessage(data=self, messages=set()) deprecated_usage = set() the_files = self.files @@ -301,7 +300,12 @@ def deepen_context( key = "input_files" if "files" in new_level: key = "files" - deprecated_usage.add(DeprecatedUsage.INPUT_FILES) + deprecated_usage.add( + ExtendedMessage( + f"WARNING: You are using YAML syntax to specify input files with the key 'files'. This usage is deprecated! Try using 'input_files' instead.", + permission=Permission.STAFF, + ) + ) files = new_level[key] assert isinstance(files, list) @@ -318,7 +322,10 @@ def deepen_context( assert isinstance(new_level["config"], dict) the_config = recursive_dict_merge(the_config, new_level["config"]) - return evolve(self, files=the_files, config=the_config), deprecated_usage + return DataWithMessage( + data=evolve(self, files=the_files, config=the_config), + messages=deprecated_usage, + ) def merge_inheritable_with_specific_config( self, level: YamlDict, config_name: str @@ -697,8 +704,10 @@ def _validate_testcase_combinations(testcase: YamlDict): def _convert_testcase( testcase: YamlDict, context: DslContext, workdir: Path | None -) -> tuple[Testcase, set[DeprecatedUsage]]: - context, deprecated_usage = context.deepen_context(testcase, workdir) +) -> DataWithMessage[Testcase]: + context_with_data = context.deepen_context(testcase, workdir) + context = context_with_data.data + deprecated_messages = context_with_data.messages # This is backwards compatability to some extend. # TODO: remove this at some point. @@ -764,7 +773,12 @@ def _convert_testcase( output.stdout = _convert_text_output_channel(stdout, context, "stdout") if (file := testcase.get("file")) is not None: output.file = _convert_file_output_channel_deprecated(file, context, "file") - deprecated_usage.add(DeprecatedUsage.OUTPUT_FILES) + deprecated_messages.add( + ExtendedMessage( + "WARNING: You are using YAML syntax to specify output files with the key 'file'. This usage is deprecated! Try using 'output_files' instead.", + permission=Permission.STAFF, + ) + ) if (file := testcase.get("output_files")) is not None: output.file = _convert_file_output_channel(file, context, "output_files") if (stderr := testcase.get("stderr")) is not None: @@ -810,34 +824,39 @@ def _convert_testcase( else: the_description = None - return ( - Testcase( + return DataWithMessage( + data=Testcase( description=the_description, input=the_input, output=output, link_files=context.files, line_comment=line_comment, ), - deprecated_usage, + messages=deprecated_messages, ) def _convert_context( context: YamlDict, dsl_context: DslContext, workdir: Path | None -) -> tuple[Context, set[DeprecatedUsage]]: - dsl_context, deprecated_usage = dsl_context.deepen_context(context, workdir) +) -> DataWithMessage[Context]: + dsl_context_with_messages = dsl_context.deepen_context(context, workdir) + dsl_context = dsl_context_with_messages.data + deprecated_messages = dsl_context_with_messages.messages raw_testcases = context.get("script", context.get("testcases")) assert isinstance(raw_testcases, list) - testcases, du = _convert_dsl_list( + testcases = _convert_dsl_list( raw_testcases, dsl_context, workdir, _convert_testcase ) - deprecated_usage.update(du) - return Context(testcases=testcases), deprecated_usage + deprecated_messages.update(testcases.messages) + return DataWithMessage( + data=Context(testcases=testcases.data), + messages=deprecated_messages, + ) def _convert_tab( tab: YamlDict, context: DslContext, workdir: Path | None -) -> tuple[Tab, set[DeprecatedUsage]]: +) -> DataWithMessage[Tab]: """ Translate a DSL tab to a full test suite tab. @@ -845,44 +864,51 @@ def _convert_tab( :param context: The context with config for the parent level. :return: A full tab. """ - context, deprecated_usage = context.deepen_context(tab, workdir) + context_with_messages = context.deepen_context(tab, workdir) + context = context_with_messages.data + deprecated_messages = context_with_messages.messages name = tab.get("unit", tab.get("tab")) assert isinstance(name, str) # The tab can have testcases or contexts. if "contexts" in tab: assert isinstance(tab["contexts"], list) - contexts, du = _convert_dsl_list( + contexts_with_messages = _convert_dsl_list( tab["contexts"], context, workdir, _convert_context ) - deprecated_usage.update(du) + contexts = contexts_with_messages.data + deprecated_messages.update(contexts_with_messages.messages) elif "cases" in tab: assert "unit" in tab # We have testcases N.S. / contexts O.S. assert isinstance(tab["cases"], list) - contexts, du = _convert_dsl_list( + contexts_with_messages = _convert_dsl_list( tab["cases"], context, workdir, _convert_context ) - deprecated_usage.update(du) + contexts = contexts_with_messages.data + deprecated_messages.update(contexts_with_messages.messages) elif "testcases" in tab: # We have scripts N.S. / testcases O.S. assert "tab" in tab assert isinstance(tab["testcases"], list) - testcases, du = _convert_dsl_list( + testcases = _convert_dsl_list( tab["testcases"], context, workdir, _convert_testcase ) - deprecated_usage.update(du) - contexts = [Context(testcases=[t]) for t in testcases] + deprecated_messages.update(testcases.messages) + contexts = [Context(testcases=[t]) for t in testcases.data] else: assert "scripts" in tab assert isinstance(tab["scripts"], list) - testcases, du = _convert_dsl_list( + testcases = _convert_dsl_list( tab["scripts"], context, workdir, _convert_testcase ) - deprecated_usage.update(du) - contexts = [Context(testcases=[t]) for t in testcases] + deprecated_messages.update(testcases.messages) + contexts = [Context(testcases=[t]) for t in testcases.data] - return Tab(name=name, contexts=contexts), deprecated_usage + return DataWithMessage( + data=Tab(name=name, contexts=contexts), + messages=deprecated_messages, + ) T = TypeVar("T") @@ -892,24 +918,27 @@ def _convert_dsl_list( dsl_list: list, context: DslContext, workdir: Path | None, - converter: Callable[ - [YamlDict, DslContext, Path | None], tuple[T, set[DeprecatedUsage]] - ], -) -> tuple[list[T], set[DeprecatedUsage]]: + converter: Callable[[YamlDict, DslContext, Path | None], DataWithMessage[T]], +) -> DataWithMessage[list[T]]: """ Convert a list of YAML objects into a test suite object. """ objects = [] - deprecated_usage = set() + deprecated_messages = set() for dsl_object in dsl_list: assert isinstance(dsl_object, dict) - ob, du = converter(dsl_object, context, workdir) - deprecated_usage.update(du) - objects.append(ob) - return objects, deprecated_usage + ob = converter(dsl_object, context, workdir) + deprecated_messages.update(ob.messages) + objects.append(ob.data) + return DataWithMessage( + data=objects, + messages=deprecated_messages, + ) -def _convert_dsl(dsl_object: YamlObject, workdir: Path | None) -> Suite: +def _convert_dsl( + dsl_object: YamlObject, workdir: Path | None +) -> DataWithMessage[Suite]: """ Translate a DSL test suite into a full test suite. @@ -920,7 +949,7 @@ def _convert_dsl(dsl_object: YamlObject, workdir: Path | None) -> Suite: :return: A full test suite. """ context = DslContext() - deprecated_usage = set() + deprecated_messages = set() if isinstance(dsl_object, list): namespace = None @@ -928,24 +957,31 @@ def _convert_dsl(dsl_object: YamlObject, workdir: Path | None) -> Suite: else: assert isinstance(dsl_object, dict) namespace = dsl_object.get("namespace") - context, du = context.deepen_context(dsl_object, workdir) - deprecated_usage.update(du) + context_with_messages = context.deepen_context(dsl_object, workdir) + context = context_with_messages.data + deprecated_messages.update(context_with_messages.messages) tab_list = dsl_object.get("units", dsl_object.get("tabs")) assert isinstance(tab_list, list) if (language := dsl_object.get("language", "tested")) != "tested": language = SupportedLanguage(language) context = evolve(context, language=language) - tabs, du = _convert_dsl_list(tab_list, context, workdir, _convert_tab) - deprecated_usage.update(du) + tabs = _convert_dsl_list(tab_list, context, workdir, _convert_tab) + deprecated_messages.update(tabs.messages) if namespace: assert isinstance(namespace, str) - return Suite(tabs=tabs, deprecated=list(deprecated_usage), namespace=namespace) + return DataWithMessage( + data=Suite(tabs=tabs.data, namespace=namespace), + messages=deprecated_messages, + ) else: - return Suite(tabs=tabs, deprecated=list(deprecated_usage)) + return DataWithMessage( + data=Suite(tabs=tabs.data), + messages=deprecated_messages, + ) -def parse_dsl(dsl_string: str, workdir: Path | None = None) -> Suite: +def parse_dsl(dsl_string: str, workdir: Path | None = None) -> DataWithMessage[Suite]: """ Parse a string containing a DSL test suite into our representation, a test suite. @@ -968,4 +1004,4 @@ def translate_to_test_suite(dsl_string: str) -> str: :return: The test suite. """ suite = parse_dsl(dsl_string, Path(".")) - return suite_to_json(suite) + return suite_to_json(suite.data) diff --git a/tested/judge/core.py b/tested/judge/core.py index 27d3cb9e..9e6f1aab 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -10,9 +10,7 @@ CloseContext, CloseJudgement, CloseTab, - ExtendedMessage, Metadata, - Permission, StartContext, StartJudgement, StartTab, @@ -48,7 +46,7 @@ generate_statement, ) from tested.serialisation import Statement -from tested.testsuite import DeprecatedUsage, LanguageLiterals +from tested.testsuite import LanguageLiterals _logger = logging.getLogger(__name__) @@ -118,26 +116,8 @@ def judge(bundle: Bundle): # Do the set-up for the judgement. collector = OutputManager(bundle.out) collector.add(StartJudgement()) - - messages = [] - if DeprecatedUsage.INPUT_FILES in bundle.suite.deprecated: - messages.append( - ExtendedMessage( - f"WARNING: You are using YAML syntax to specify input files with the key 'files'. This usage is deprecated! Try using 'input_files' instead.", - permission=Permission.STAFF, - ) - ) - - if DeprecatedUsage.OUTPUT_FILES in bundle.suite.deprecated: - messages.append( - ExtendedMessage( - f"WARNING: You are using YAML syntax to specify output files with the key 'file'. This usage is deprecated! Try using 'output_files' instead.", - permission=Permission.STAFF, - ) - ) - - if messages: - collector.add_messages(messages) + if bundle.messages: + collector.add_messages(bundle.messages) max_time = float(bundle.config.time_limit) * 0.9 start = time.perf_counter() diff --git a/tested/main.py b/tested/main.py index 2c5313c8..45fc4ef2 100644 --- a/tested/main.py +++ b/tested/main.py @@ -29,11 +29,14 @@ def run(config: DodonaConfig, judge_output: IO): _, ext = os.path.splitext(config.test_suite) is_yaml = ext.lower() in (".yaml", ".yml") + messages = set() if is_yaml: - suite = parse_dsl(textual_suite, config.workdir) + suite_with_message = parse_dsl(textual_suite, config.workdir) + messages = suite_with_message.messages + suite = suite_with_message.data else: suite = parse_test_suite(textual_suite) - pack = create_bundle(config, judge_output, suite) + pack = create_bundle(config, judge_output, suite, messages=messages) from .judge import judge judge(pack) diff --git a/tested/testsuite.py b/tested/testsuite.py index 5d22fb9a..44f9813b 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -781,18 +781,11 @@ class ExecutionMode(StrEnum): INDIVIDUAL = "context" -@unique -class DeprecatedUsage(StrEnum): - INPUT_FILES = "input_files" - OUTPUT_FILES = "output_files" - - @define class Suite(WithFeatures, WithFunctions): """General test suite, which is used to run tests of some code.""" tabs: list[Tab] = field(factory=list) - deprecated: list[DeprecatedUsage] = field(factory=list) namespace: str = "submission" def get_used_features(self) -> FeatureSet: diff --git a/tested/utils.py b/tested/utils.py index e913c8b6..6b5c838e 100644 --- a/tested/utils.py +++ b/tested/utils.py @@ -7,9 +7,13 @@ from collections.abc import Callable, Iterable from itertools import zip_longest from pathlib import Path -from typing import IO, TYPE_CHECKING, Any, TypeGuard, TypeVar +from typing import IO, TYPE_CHECKING, Any, Generic, TypeGuard, TypeVar from typing import get_args as typing_get_args +from attr import define + +from tested.dodona import ExtendedMessage + if TYPE_CHECKING: from tested.serialisation import Assignment @@ -56,6 +60,12 @@ def get_identifier() -> str: T = TypeVar("T") +@define +class DataWithMessage(Generic[T]): + data: T + messages: set[ExtendedMessage] + + def get_args(type_: Any) -> tuple[Any, ...]: """ Get the args of a type or the type itself. diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 498a3a92..27844637 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -20,6 +20,7 @@ SequenceTypes, StringTypes, ) +from tested.dodona import Permission from tested.dsl import parse_dsl, translate_to_test_suite from tested.dsl.translate_parser import load_schema_validator from tested.serialisation import ( @@ -32,7 +33,6 @@ ) from tested.testsuite import ( CustomCheckOracle, - DeprecatedUsage, FileOutputChannel, FileUrl, GenericTextOracle, @@ -798,10 +798,16 @@ def test_using_deprecated_file(): content: "Hello world!" location: "test.txt" """ - json_str = translate_to_test_suite(yaml_str) - suite = parse_test_suite(json_str) - using_deprecated = suite.deprecated - assert DeprecatedUsage.OUTPUT_FILES in using_deprecated + suite_with_data = parse_dsl(yaml_str) + suite = suite_with_data.data + messages = list(suite_with_data.messages) + assert len(messages) == 1 + deprecated_message = messages[0] + assert deprecated_message.permission == Permission.STAFF + assert ( + deprecated_message.description + == "WARNING: You are using YAML syntax to specify output files with the key 'file'. This usage is deprecated! Try using 'output_files' instead." + ) assert len(suite.tabs) == 1 tab = suite.tabs[0] assert len(tab.contexts) == 1 @@ -826,10 +832,16 @@ def test_using_deprecated_files(): - url: "media/hello.txt" name: "hello.txt" """ - json_str = translate_to_test_suite(yaml_str) - suite = parse_test_suite(json_str) - using_deprecated = suite.deprecated - assert DeprecatedUsage.INPUT_FILES in using_deprecated + suite_with_data = parse_dsl(yaml_str) + suite = suite_with_data.data + messages = list(suite_with_data.messages) + assert len(messages) == 1 + deprecated_message = messages[0] + assert deprecated_message.permission == Permission.STAFF + assert ( + deprecated_message.description + == "WARNING: You are using YAML syntax to specify input files with the key 'files'. This usage is deprecated! Try using 'input_files' instead." + ) assert len(suite.tabs) == 1 tab = suite.tabs[0] assert len(tab.contexts) == 1 diff --git a/tests/test_functionality.py b/tests/test_functionality.py index 12f0f7f6..31279ae7 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -21,13 +21,11 @@ from tested.testsuite import ( Context, MainInput, - Output, Suite, Tab, Testcase, TextChannelType, TextData, - TextOutputChannel, ) from tests.language_markers import ( ALL_LANGUAGES, From 04c28ad02f12a87f6f84caaa888c1ffb7722a044 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 12 May 2025 18:50:34 +0200 Subject: [PATCH 73/77] solved some of the suggested issues --- tested/descriptions/renderer.py | 4 +- tested/dodona.py | 3 +- tested/dsl/schema-strict.json | 142 +++++++++++--------------------- tested/dsl/schema.json | 137 +++++++++++------------------- tested/judge/core.py | 14 +++- tested/judge/evaluation.py | 69 ++++++++-------- tested/oracles/text.py | 18 ++-- 7 files changed, 155 insertions(+), 232 deletions(-) diff --git a/tested/descriptions/renderer.py b/tested/descriptions/renderer.py index 9552c682..1c9c8ecd 100644 --- a/tested/descriptions/renderer.py +++ b/tested/descriptions/renderer.py @@ -83,11 +83,11 @@ def _render_dsl_statements(self, element: block.FencedCode) -> str: rendered_dsl = self.render_children(element) # Parse the DSL - parsed_dsl = parse_dsl(rendered_dsl) + parsed_dsl = parse_dsl(rendered_dsl).data # Get all actual tests tests = [] - for tab in parsed_dsl.data.tabs: + for tab in parsed_dsl.tabs: for context in tab.contexts: for testcase in context.testcases: tests.append(testcase) diff --git a/tested/dodona.py b/tested/dodona.py index 8c27e929..3ece258a 100644 --- a/tested/dodona.py +++ b/tested/dodona.py @@ -36,9 +36,10 @@ class ExtendedMessage: @define class Metadata: - """Currently only used for the Python tutor""" + """Currently used for the Python tutor and rendering files in Dodona.""" statements: str | None + stdin: str | None files: list[dict[str, str]] | None diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 9d090e91..c8d66690 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -314,49 +314,7 @@ "$ref" : "#/definitions/message" }, "stdin" : { - "description" : "Stdin for this context", - "oneOf": [ - { - "type" : [ - "string", - "number", - "integer", - "boolean" - ] - }, - { - "type": "object", - "additionalProperties" : false, - "required": ["content"], - "properties": { - "content": { - "type" : [ - "string", - "number", - "integer", - "boolean", - "object" - ], - "description": "The actual content that will be used for stdin." - }, - "path": { - "type": "string", - "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." - } - } - }, - { - "type": "object", - "additionalProperties" : false, - "required": ["path"], - "properties": { - "path": { - "type": "string", - "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." - } - } - } - ] + "$ref" : "#/definitions/stdinData" }, "arguments" : { "type" : "array", @@ -458,54 +416,7 @@ "$ref" : "#/definitions/message" }, "stdin" : { - "description" : "Stdin for this context", - "oneOf": [ - { - "type" : [ - "string", - "number", - "integer", - "boolean" - ] - }, - { - "type": "object", - "required": ["content"], - "additionalProperties": false, - "properties": { - "content": { - "type" : [ - "string", - "number", - "integer", - "boolean", - "object" - ], - "description": "The actual content that will be used for stdin." - }, - "path": { - "type": "string", - "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." - } - } - }, - { - "type": "object", - "required": ["path", "url"], - "additionalProperties": false, - "properties": { - "path": { - "type": "string", - "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." - }, - "url": { - "type": "string", - "format" : "uri", - "description" : "Relative path to the file in the `evaluation` folder of an exercise." - } - } - } - ] + "$ref" : "#/definitions/stdinData" }, "arguments" : { "type" : "array", @@ -591,6 +502,51 @@ } } }, + "stdinData" : { + "description" : "Stdin for this context", + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required": ["content"], + "additionalProperties": false, + "properties": { + "content": { + "type" : [ + "string", + "number", + "integer", + "boolean", + "object" + ], + "description": "The actual content that will be used for stdin." + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + } + } + }, + { + "type": "object", + "required": ["path"], + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + } + } + } + ] + }, "expressionOrStatement" : { "oneOf" : [ { @@ -634,7 +590,7 @@ "properties" : { "name" : { "type" : "string", - "description" : "The filename, including the file extension." + "description" : "The filename, including the file extension. It is also the relative path to a file in the working directory." }, "url" : { "type" : "string", @@ -653,7 +609,7 @@ "properties" : { "path" : { "type" : "string", - "description" : "The filename, including the file extension." + "description" : "The filename, including the file extension. It is also the relative path to a file in the working directory." }, "content" : { "type" : "string", diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index a4ad2826..f9aa0564 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -314,49 +314,7 @@ "$ref" : "#/definitions/message" }, "stdin" : { - "description" : "Stdin for this context", - "oneOf": [ - { - "type" : [ - "string", - "number", - "integer", - "boolean" - ] - }, - { - "type": "object", - "additionalProperties" : false, - "required": ["content"], - "properties": { - "content": { - "type" : [ - "string", - "number", - "integer", - "boolean", - "object" - ], - "description": "The actual content that will be used for stdin." - }, - "path": { - "type": "string", - "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." - } - } - }, - { - "type": "object", - "additionalProperties" : false, - "required": ["path"], - "properties": { - "path": { - "type": "string", - "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." - } - } - } - ] + "$ref" : "#/definitions/stdinData" }, "arguments" : { "type" : "array", @@ -458,49 +416,7 @@ "$ref" : "#/definitions/message" }, "stdin" : { - "description" : "Stdin for this context", - "oneOf": [ - { - "type" : [ - "string", - "number", - "integer", - "boolean" - ] - }, - { - "type": "object", - "additionalProperties" : false, - "required": ["content"], - "properties": { - "content": { - "type" : [ - "string", - "number", - "integer", - "boolean", - "object" - ], - "description": "The actual content that will be used for stdin." - }, - "path": { - "type": "string", - "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." - } - } - }, - { - "type": "object", - "additionalProperties" : false, - "required": ["path"], - "properties": { - "path": { - "type": "string", - "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." - } - } - } - ] + "$ref" : "#/definitions/stdinData" }, "arguments" : { "type" : "array", @@ -594,6 +510,51 @@ } } }, + "stdinData" : { + "description" : "Stdin for this context", + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties" : false, + "required": ["content"], + "properties": { + "content": { + "type" : [ + "string", + "number", + "integer", + "boolean", + "object" + ], + "description": "The actual content that will be used for stdin." + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + } + } + }, + { + "type": "object", + "additionalProperties" : false, + "required": ["path"], + "properties": { + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + } + } + } + ] + }, "expressionOrStatement" : { "oneOf" : [ { @@ -637,7 +598,7 @@ "properties" : { "name" : { "type" : "string", - "description" : "The filename, including the file extension." + "description" : "The filename, including the file extension. It is also the relative path to a file in the working directory." }, "url" : { "type" : "string", @@ -656,7 +617,7 @@ "properties" : { "path" : { "type" : "string", - "description" : "The filename, including the file extension." + "description" : "The filename, including the file extension. It is also the relative path to a file in the working directory." }, "content" : { "type" : "string", diff --git a/tested/judge/core.py b/tested/judge/core.py index 9e6f1aab..da947598 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -46,7 +46,7 @@ generate_statement, ) from tested.serialisation import Statement -from tested.testsuite import LanguageLiterals +from tested.testsuite import LanguageLiterals, MainInput, TextData _logger = logging.getLogger(__name__) @@ -348,6 +348,7 @@ def _process_results( # since it only differs a bit. meta_statements = [] input_files = [] + meta_stdin = None for file in seen_files: file_data = {"path": file.path} if file.content != "": @@ -356,7 +357,13 @@ def _process_results( input_files.append(file_data) for case in planned.context.testcases: - if isinstance(case.input, Statement): + if case.is_main_testcase(): + assert isinstance(case.input, MainInput) + if isinstance(case.input.stdin, TextData): + meta_stdin = case.input.stdin.get_data_as_string( + bundle.config.resources + ) + elif isinstance(case.input, Statement): stmt = generate_statement(bundle, case.input) meta_statements.append(stmt) elif isinstance(case.input, LanguageLiterals): @@ -377,8 +384,7 @@ def _process_results( collector.add( CloseContext( data=Metadata( - statements=meta_statements, - files=input_files, + statements=meta_statements, files=input_files, stdin=meta_stdin ) ), planned.context_index, diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index eba5c870..d7af5fb3 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Literal +from exercise.opgaven.reeks07.zendmasten.preparation.outputprocessor import outputBlok from tested.configs import Bundle from tested.dodona import ( AppendMessage, @@ -110,18 +111,17 @@ def _evaluate_channel( evaluator = get_oracle( bundle, context_directory, output, testcase, unexpected_status=unexpected_status ) - # Run the oracle. + if channel == Channel.FILE and output != "ignored": assert isinstance(output, FileOutputChannel) - new_output = output.output_data + output_channels = output.output_data else: - new_output = [output] + output_channels = [output] - missing = False - if actual is None: - missing = True + missing = actual is None - for output_element in new_output: + for output_element in output_channels: + # Run the oracle. evaluation_result = evaluator(output_element, actual if actual else "") status = evaluation_result.result @@ -129,33 +129,36 @@ def _evaluate_channel( is_correct = status.enum == Status.CORRECT should_report_case = should_show(output, channel, evaluation_result) - if not should_report_case and is_correct: - # We do report that a test is correct, to set the status. - return False + if should_report_case or not is_correct: + expected = evaluation_result.readable_expected + expected_channel = channel + if isinstance(output_element, OutputFileData): + expected_channel = f"file: {output_element.path}" + out.add(StartTest(expected=expected, channel=expected_channel)) - expected = evaluation_result.readable_expected - expected_channel = channel - if isinstance(output_element, OutputFileData): - expected_channel = f"file: {output_element.path}" - out.add(StartTest(expected=expected, channel=expected_channel)) - - # Report any messages we received. - for message in evaluation_result.messages: - out.add(AppendMessage(message=message)) - - if actual is None: - out.add(AppendMessage(message=get_i18n_string("judge.evaluation.missing"))) - elif should_report_case and timeout and not is_correct: - status.human = get_i18n_string("judge.evaluation.time-limit") - status.enum = Status.TIME_LIMIT_EXCEEDED - out.add(AppendMessage(message=status.human)) - elif should_report_case and memory and not is_correct: - status.human = get_i18n_string("judge.evaluation.memory-limit") - status.enum = Status.TIME_LIMIT_EXCEEDED - out.add(AppendMessage(message=status.human)) - - # Close the test. - out.add(CloseTest(generated=evaluation_result.readable_actual, status=status)) + # Report any messages we received. + for message in evaluation_result.messages: + out.add(AppendMessage(message=message)) + + if actual is None: + out.add( + AppendMessage(message=get_i18n_string("judge.evaluation.missing")) + ) + elif should_report_case and timeout and not is_correct: + status.human = get_i18n_string("judge.evaluation.time-limit") + status.enum = Status.TIME_LIMIT_EXCEEDED + out.add(AppendMessage(message=status.human)) + elif should_report_case and memory and not is_correct: + status.human = get_i18n_string("judge.evaluation.memory-limit") + status.enum = Status.TIME_LIMIT_EXCEEDED + out.add(AppendMessage(message=status.human)) + + # Close the test. + out.add( + CloseTest(generated=evaluation_result.readable_actual, status=status) + ) + else: + missing = False return missing diff --git a/tested/oracles/text.py b/tested/oracles/text.py index fe324e41..ecc1564f 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -47,9 +47,7 @@ def _file_defaults(config: OracleConfig) -> dict: return defaults -def _text_comparison( - options: dict[str, Any], expected: str, actual: str -) -> tuple[bool, str]: +def compare_text(options: dict[str, Any], expected: str, actual: str) -> OracleResult: # Temporary variables that may modified by the evaluation options, # Don't modify the actual values, otherwise there maybe confusion with the # solution submitted by the student @@ -68,16 +66,14 @@ def _text_comparison( expected_float = float(expected_eval.strip()) if options["applyRounding"]: numbers = int(options["roundTo"]) + # noinspection PyUnboundLocalVariable actual_float = round(actual_float, numbers) expected_float = round(expected_float, numbers) - return math.isclose(actual_float, expected_float), str(expected_float) - - return actual_eval == expected_eval, expected - - -def compare_text(options: dict[str, Any], expected: str, actual: str) -> OracleResult: - - result, expected = _text_comparison(options, expected, actual) + # noinspection PyUnboundLocalVariable + result = math.isclose(actual_float, expected_float) + expected = str(expected_float) + else: + result = actual_eval == expected_eval return OracleResult( result=StatusMessage(enum=Status.CORRECT if result else Status.WRONG), From 8cdd78b4e3caad0618ff714bb12a3088847c6526 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 12 May 2025 18:55:59 +0200 Subject: [PATCH 74/77] remove weird import added by IDE --- tested/judge/evaluation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index d7af5fb3..0da4b314 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -5,7 +5,6 @@ from pathlib import Path from typing import Literal -from exercise.opgaven.reeks07.zendmasten.preparation.outputprocessor import outputBlok from tested.configs import Bundle from tested.dodona import ( AppendMessage, From ef5324245a55544887a0bd72f2cf97754069d2cb Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 13 May 2025 14:38:03 +0200 Subject: [PATCH 75/77] fixed tests --- tests/test_oracles_builtin.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index a3d5acb5..3898969d 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -141,7 +141,7 @@ def test_file_oracle_full_wrong( tmp_path: Path, pytestconfig: pytest.Config, mocker: MockerFixture ): config = oracle_config(tmp_path, pytestconfig, {"mode": "full"}) - s = mocker.spy(tested.oracles.text, name="_text_comparison") # type: ignore[reportAttributeAccessIssue] + s = mocker.spy(tested.oracles.text, name="compare_text") # type: ignore[reportAttributeAccessIssue] mock_files = [ mocker.mock_open(read_data=content).return_value for content in ["expected\nexpected", "actual\nactual"] @@ -165,7 +165,7 @@ def test_file_oracle_full_correct( tmp_path: Path, pytestconfig: pytest.Config, mocker: MockerFixture ): config = oracle_config(tmp_path, pytestconfig, {"mode": "full"}) - s = mocker.spy(tested.oracles.text, name="_text_comparison") # type: ignore[reportAttributeAccessIssue] + s = mocker.spy(tested.oracles.text, name="compare_text") # type: ignore[reportAttributeAccessIssue] mock_files = [ mocker.mock_open(read_data=content).return_value for content in ["expected\nexpected", "expected\nexpected"] @@ -190,7 +190,7 @@ def test_file_oracle_full_correct_with_text_content( ): config = oracle_config(tmp_path, pytestconfig, {"mode": "full"}) s = mocker.spy( - tested.oracles.text, name="_text_comparison" # type: ignore[reportAttributeAccessIssue] + tested.oracles.text, name="compare_text" # type: ignore[reportAttributeAccessIssue] ) mock_files = [ mocker.mock_open(read_data=content).return_value @@ -217,7 +217,7 @@ def test_file_oracle_line_wrong( config = oracle_config( tmp_path, pytestconfig, {"mode": "line", "stripNewlines": True} ) - s = mocker.spy(tested.oracles.text, name="_text_comparison") # type: ignore[reportAttributeAccessIssue] + s = mocker.spy(tested.oracles.text, name="compare_text") # type: ignore[reportAttributeAccessIssue] mock_files = [ mocker.mock_open(read_data=content).return_value for content in ["expected\nexpected2", "actual\nactual2"] @@ -245,7 +245,7 @@ def test_file_oracle_line_correct( config = oracle_config( tmp_path, pytestconfig, {"mode": "line", "stripNewlines": True} ) - s = mocker.spy(tested.oracles.text, name="_text_comparison") # type: ignore[reportAttributeAccessIssue] + s = mocker.spy(tested.oracles.text, name="compare_text") # type: ignore[reportAttributeAccessIssue] mock_files = [ mocker.mock_open(read_data=content).return_value for content in ["expected\nexpected2", "expected\nexpected2"] @@ -273,7 +273,7 @@ def test_file_oracle_strip_lines_correct( config = oracle_config( tmp_path, pytestconfig, {"mode": "line", "stripNewlines": True} ) - s = mocker.spy(tested.oracles.text, name="_text_comparison") # type: ignore[reportAttributeAccessIssue] + s = mocker.spy(tested.oracles.text, name="compare_text") # type: ignore[reportAttributeAccessIssue] mock_files = [ mocker.mock_open(read_data=content).return_value for content in ["expected\nexpected2\n", "expected\nexpected2"] @@ -301,7 +301,7 @@ def test_file_oracle_dont_strip_lines_correct( config = oracle_config( tmp_path, pytestconfig, {"mode": "line", "stripNewlines": False} ) - s = mocker.spy(tested.oracles.text, name="_text_comparison") # type: ignore[reportAttributeAccessIssue] + s = mocker.spy(tested.oracles.text, name="compare_text") # type: ignore[reportAttributeAccessIssue] mock_files = [ mocker.mock_open(read_data=content).return_value for content in ["expected\nexpected2\n", "expected\nexpected2\n"] From 77d78c7140b7f365dbb8e75e83785f06cf9414e8 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 13 May 2025 15:08:35 +0200 Subject: [PATCH 76/77] did a few simple changes --- tested/judge/core.py | 25 ++++++++++--------------- tested/judge/evaluation.py | 12 +++++------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/tested/judge/core.py b/tested/judge/core.py index da947598..fe04fddd 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -332,7 +332,7 @@ def _process_results( # Handle the contexts. collector.add(StartContext(description=planned.context.description)) - continue_, seen_files = evaluate_context_results( + continue_ = evaluate_context_results( bundle, context=planned.context, exec_results=context_result, @@ -342,27 +342,22 @@ def _process_results( ) if bundle.language.supports_debug_information(): - # TODO: this is currently very Python-specific - # See if we need a callback to the language modules in the future. - # TODO: we could probably re-use the "readable_input" function here, - # since it only differs a bit. meta_statements = [] input_files = [] meta_stdin = None - for file in seen_files: - file_data = {"path": file.path} - if file.content != "": - file_data["content"] = file.content - - input_files.append(file_data) for case in planned.context.testcases: + for file in case.link_files: + file_data = {"path": file.path} + if file.content != "": + file_data["content"] = file.content + + input_files.append(file_data) + if case.is_main_testcase(): assert isinstance(case.input, MainInput) - if isinstance(case.input.stdin, TextData): - meta_stdin = case.input.stdin.get_data_as_string( - bundle.config.resources - ) + if isinstance(case.input.stdin, TextData) and case.input.stdin.data is not None: + meta_stdin = case.input.stdin.data elif isinstance(case.input, Statement): stmt = generate_statement(bundle, case.input) meta_statements.append(stmt) diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index 0da4b314..7e2a596c 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -169,7 +169,7 @@ def evaluate_context_results( compilation_results: CompilationResult, context_dir: Path, collector: OutputManager, -) -> tuple[Status | None, set]: +) -> Status | None: """ Evaluate the results for a single context. @@ -200,7 +200,7 @@ def evaluate_context_results( # Finish the evaluation, since there is nothing we can do. collector.add(CloseTestcase(accepted=False), 0) - return compilation_results.status, set() + return compilation_results.status # There must be execution if compilation succeeded. assert exec_results is not None @@ -258,7 +258,6 @@ def evaluate_context_results( # All files that will be used in this context. all_files = context.get_files() - all_seen = set() # Begin processing the normal testcases. for i, testcase in enumerate(context.testcases): @@ -267,7 +266,6 @@ def evaluate_context_results( readable_input, seen = get_readable_input(bundle, testcase) all_files = all_files - seen t_col = TestcaseCollector(StartTestcase(description=readable_input)) - all_seen.update(seen) # Get the functions output = testcase.output @@ -373,10 +371,10 @@ def evaluate_context_results( collector.add(_link_files_message(all_files)) if exec_results.timeout: - return Status.TIME_LIMIT_EXCEEDED, all_seen + return Status.TIME_LIMIT_EXCEEDED if exec_results.memory: - return Status.MEMORY_LIMIT_EXCEEDED, all_seen - return None, all_seen + return Status.MEMORY_LIMIT_EXCEEDED + return None def _link_files_message(link_files: Collection[FileUrl]) -> AppendMessage: From b9d168be7a6e7847dee3dd388528c959e80da0e7 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 13 May 2025 15:49:35 +0200 Subject: [PATCH 77/77] fixed linting --- tested/judge/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tested/judge/core.py b/tested/judge/core.py index fe04fddd..c12a8d19 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -356,7 +356,10 @@ def _process_results( if case.is_main_testcase(): assert isinstance(case.input, MainInput) - if isinstance(case.input.stdin, TextData) and case.input.stdin.data is not None: + if ( + isinstance(case.input.stdin, TextData) + and case.input.stdin.data is not None + ): meta_stdin = case.input.stdin.data elif isinstance(case.input, Statement): stmt = generate_statement(bundle, case.input)