From 54874a14e385c33c3d747d8965b630dbd37626a8 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 12 Apr 2025 17:08:11 +0200 Subject: [PATCH 01/23] added script to make conversion from multilingual json schema to one that can be used in the IDE a lot easier. --- tested/transform_json.py | 76 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 tested/transform_json.py diff --git a/tested/transform_json.py b/tested/transform_json.py new file mode 100644 index 00000000..0432befa --- /dev/null +++ b/tested/transform_json.py @@ -0,0 +1,76 @@ +import json +import os +import sys +from pathlib import Path +from typing import Any + + +def transform(data: Any) -> Any: + if isinstance(data, list): + return [transformed for ele in data if (transformed := transform(ele)) != {}] + elif isinstance(data, dict): + if "return" in data: + # This is necessary since tags aren't recognized in the Json schema + # and such natural_language maps wil always be seen as yamlValue. + assert isinstance(data["return"], dict) and "oneOf" in data["return"] + data["return"]["anyOf"] = data["return"].pop("oneOf") + + if "yamlValue" in data: + data["yamlValue"] = { + "description": "A value represented as YAML.", + } + + if "required" in data and "properties" in data: + required = data["required"] + properties = data["properties"] + assert isinstance(required, list) + assert isinstance(properties, dict) + if "__tag__" in required and "value" in required: + assert "__tag__" in properties and "value" in properties + value = properties["value"] + tag = properties["__tag__"] + assert isinstance(value, dict) + assert isinstance(tag, dict) and "const" in tag + + if tag["const"] == "!programming_language" or tag["const"] == "!expression": + return {} + + if tag["const"] == "!natural_language": + value["not"] = { + "anyOf": [ + {"required": ["description"]}, + {"required": ["value"]}, + {"required": ["types"]}, + ] + } + data = value + + return {k: transform(v) for k, v in data.items()} + return data + +def transform_json(json_file): + _, ext = os.path.splitext(json_file) + assert ext.lower() == ".json", f"expected a json file, got {ext}." + try: + with open(json_file, "r") as stream: + json_stream = json.load(stream) + except FileNotFoundError as e: + print("The json file was not found.") + raise e + + result = transform(json_stream) + + print(json.dumps(result, indent=2)) + with open("output.json", "w", encoding='utf-8') as f: + json.dump(result, f, indent=2) + #print(result) + + + + + +if __name__ == "__main__": + n = len(sys.argv) + assert n > 1, "Expected path to multilingual json schema." + + transform_json(Path(sys.argv[1])) From 2bb7fd7ae43f125a3bbd532d04440a256a4c1253 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 12 Apr 2025 17:13:19 +0200 Subject: [PATCH 02/23] fixed linting --- tested/transform_json.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tested/transform_json.py b/tested/transform_json.py index 0432befa..acb8dad7 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -32,7 +32,10 @@ def transform(data: Any) -> Any: assert isinstance(value, dict) assert isinstance(tag, dict) and "const" in tag - if tag["const"] == "!programming_language" or tag["const"] == "!expression": + if ( + tag["const"] == "!programming_language" + or tag["const"] == "!expression" + ): return {} if tag["const"] == "!natural_language": @@ -48,6 +51,7 @@ def transform(data: Any) -> Any: return {k: transform(v) for k, v in data.items()} return data + def transform_json(json_file): _, ext = os.path.splitext(json_file) assert ext.lower() == ".json", f"expected a json file, got {ext}." @@ -61,12 +65,9 @@ def transform_json(json_file): result = transform(json_stream) print(json.dumps(result, indent=2)) - with open("output.json", "w", encoding='utf-8') as f: + with open("output.json", "w", encoding="utf-8") as f: json.dump(result, f, indent=2) - #print(result) - - - + # print(result) if __name__ == "__main__": From 525373eb04a483bddf4f58624ee9472ac806d544 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 12 Apr 2025 18:49:46 +0200 Subject: [PATCH 03/23] added some extra tests --- tested/transform_json.py | 10 +-- tests/test_preprocess_dsl.py | 145 +++++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 5 deletions(-) diff --git a/tested/transform_json.py b/tested/transform_json.py index acb8dad7..36d1f330 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -10,8 +10,8 @@ def transform(data: Any) -> Any: return [transformed for ele in data if (transformed := transform(ele)) != {}] elif isinstance(data, dict): if "return" in data: - # This is necessary since tags aren't recognized in the Json schema - # and such natural_language maps wil always be seen as yamlValue. + # This is necessary since tags aren't recognized in the Json schema. + # So a natural_language maps wil always be seen as yamlValue. assert isinstance(data["return"], dict) and "oneOf" in data["return"] data["return"]["anyOf"] = data["return"].pop("oneOf") @@ -64,10 +64,10 @@ def transform_json(json_file): result = transform(json_stream) - print(json.dumps(result, indent=2)) - with open("output.json", "w", encoding="utf-8") as f: + with open( + json_file.parent / "multilingual-schema.json", "w", encoding="utf-8" + ) as f: json.dump(result, f, indent=2) - # print(result) if __name__ == "__main__": diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index c00d5fa6..78a8f2a0 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -18,6 +18,7 @@ translate_yaml, validate_pre_dsl, ) +from tested.transform_json import transform def validate_natural_translate(yaml_str: str, translated_yaml_str: str): @@ -666,3 +667,147 @@ def test_template_syntax_error(): return: 11{%{{works}} """.strip() validate_natural_translate(yaml_str, translated_yml) + + +def test_return_json_schema(): + json_schema = { + "return": { + "description": "Expected return value", + "oneOf": [ + {"$ref": "#/definitions/returnOutputChannel"}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/returnOutputChannel" + }, + }, + }, + }, + ], + } + } + + json_schema_expected = { + "return": { + "description": "Expected return value", + "anyOf": [ + {"$ref": "#/definitions/returnOutputChannel"}, + { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/returnOutputChannel" + }, + "not": { + "anyOf": [ + {"required": ["description"]}, + {"required": ["value"]}, + {"required": ["types"]}, + ] + }, + }, + ], + } + } + + result = transform(json_schema) + + assert result == json_schema_expected + + +def test_yaml_value_json_schema(): + json_schema = { + "yamlValue": { + "description": "A value represented as YAML.", + "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, + }, + } + + json_schema_expected = { + "yamlValue": { + "description": "A value represented as YAML.", + } + } + + result = transform(json_schema) + + assert result == json_schema_expected + + +def test_nat_lang_json_schema(): + json_schema = { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": {"type": "object", "additionalProperties": {"type": "string"}}, + }, + } + + json_schema_expected = { + "type": "object", + "additionalProperties": {"type": "string"}, + "not": { + "anyOf": [ + {"required": ["description"]}, + {"required": ["value"]}, + {"required": ["types"]}, + ] + }, + } + + result = transform(json_schema) + + assert result == json_schema_expected + + +def test_expr_json_schema(): + json_schema = { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!expression", + }, + "value": { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + }, + }, + } + + json_schema_expected = {} + + result = transform(json_schema) + + assert result == json_schema_expected + +def test_list_json_schema(): + json_schema = [{}, { + "type" : "string", + "description" : "A statement of expression in Python-like syntax as YAML string." + }] + + json_schema_expected = [{ + "type" : "string", + "description" : "A statement of expression in Python-like syntax as YAML string." + }] + + result = transform(json_schema) + + assert result == json_schema_expected From f5c1a5cb276d087bf414683f683e1e4389c9559a Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 12 Apr 2025 19:58:27 +0200 Subject: [PATCH 04/23] added more text and documentation --- tested/transform_json.py | 12 ++++- tests/test_preprocess_dsl.py | 85 ++++++++++++++++++++++++++++++++---- 2 files changed, 87 insertions(+), 10 deletions(-) diff --git a/tested/transform_json.py b/tested/transform_json.py index 36d1f330..1cb39b6f 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -32,12 +32,16 @@ def transform(data: Any) -> Any: assert isinstance(value, dict) assert isinstance(tag, dict) and "const" in tag + # For !programming_language the same was already defined without the tag. + # The only usage for the expression tag is also redundant. if ( tag["const"] == "!programming_language" or tag["const"] == "!expression" ): return {} + # This to help the validator to somewhat distinguish between + # a natural_language map and a dictionary. if tag["const"] == "!natural_language": value["not"] = { "anyOf": [ @@ -52,7 +56,13 @@ def transform(data: Any) -> Any: return data -def transform_json(json_file): +def transform_json(json_file: Path): + """ + This function transforms the JSON schema used in the DSL translator into + a new JSON schema that can be used to validate the multilingual YAML in your IDE. + + :param json_file: The path to the JSON file. + """ _, ext = os.path.splitext(json_file) assert ext.lower() == ".json", f"expected a json file, got {ext}." try: diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 78a8f2a0..2b5cfbbf 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -18,7 +18,7 @@ translate_yaml, validate_pre_dsl, ) -from tested.transform_json import transform +from tested.transform_json import transform, transform_json def validate_natural_translate(yaml_str: str, translated_yaml_str: str): @@ -797,17 +797,84 @@ def test_expr_json_schema(): assert result == json_schema_expected + def test_list_json_schema(): - json_schema = [{}, { - "type" : "string", - "description" : "A statement of expression in Python-like syntax as YAML string." - }] + json_schema = [ + {}, + { + "type": "string", + "description": "A statement of expression in Python-like syntax as YAML string.", + }, + ] - json_schema_expected = [{ - "type" : "string", - "description" : "A statement of expression in Python-like syntax as YAML string." - }] + json_schema_expected = [ + { + "type": "string", + "description": "A statement of expression in Python-like syntax as YAML string.", + } + ] result = transform(json_schema) assert result == json_schema_expected + + +def test_transform_executed_correct(mocker: MockerFixture): + s = mocker.spy( + tested.transform_json, name="transform" # type: ignore[reportAttributeAccessIssue] + ) + + mock_files = [ + mocker.mock_open(read_data=content).return_value + for content in [ + """ +{ + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + } + } + ] + } +}""" + ] + ] + mock_files.append(mocker.mock_open(read_data="{}").return_value) + mock_files.append(mocker.mock_open().return_value) + mock_opener = mocker.mock_open() + mock_opener.side_effect = mock_files + mocker.patch("builtins.open", mock_opener) + + transform_json(Path("schema.json")) + + assert s.call_count == 25 + + # Check if the file was opened for writing + mock_opener.assert_any_call(Path("multilingual-schema.json"), "w", encoding="utf-8") From afb53c809937a129b8943d47a760f81104681f29 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 13 Apr 2025 19:22:38 +0200 Subject: [PATCH 05/23] Also wrote a script to generate monolingual JSON schemas --- tested/dsl/schema-strict-nat-translation.json | 21 +-- tested/dsl/schema-strict.json | 5 +- tested/transform_json.py | 132 +++++++++++++++++- tests/test_preprocess_dsl.py | 12 +- 4 files changed, 148 insertions(+), 22 deletions(-) diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json index e598cc2b..adfeaa57 100644 --- a/tested/dsl/schema-strict-nat-translation.json +++ b/tested/dsl/schema-strict-nat-translation.json @@ -250,7 +250,7 @@ ], "description" : "The name of this tab." }, - "translation" : { + "translations" : { "type" : "object", "description": "Define translations in the tab scope." }, @@ -331,12 +331,14 @@ "description" : "Defines if the unit/tab is hidden for the student or not" }, "unit" : { - "anyOf" : [ + "oneOf" : [ { - "type" : "string" + "type" : "string", + "description" : "The name of this tab." }, { "type" : "object", + "description" : "The name of this tab.", "required": [ "__tag__", "value" @@ -394,7 +396,7 @@ ] }, "_contextList" : { - "anyOf" : [ + "oneOf" : [ { "type" : "array", "minItems" : 1, @@ -429,7 +431,7 @@ ] }, "_caseList" : { - "anyOf" : [ + "oneOf" : [ { "type" : "array", "minItems" : 1, @@ -464,7 +466,7 @@ ] }, "_testcaseList" : { - "anyOf" : [ + "oneOf" : [ { "type" : "array", "minItems" : 1, @@ -499,7 +501,7 @@ ] }, "_scriptList" : { - "anyOf" : [ + "oneOf" : [ { "type" : "array", "minItems" : 1, @@ -1250,6 +1252,7 @@ "oneOf" : [ { "type" : "string", + "format" : "tested-dsl-expression", "description" : "A statement of expression in Python-like syntax as YAML string." }, { @@ -1296,6 +1299,7 @@ "oneOf" : [ { "type" : "string", + "format" : "tested-dsl-expression", "description" : "A statement of expression in Python-like syntax as YAML string." }, { @@ -1955,7 +1959,8 @@ "kotlin", "python", "runhaskell", - "csharp" + "csharp", + "cpp" ] }, "message" : { diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 0471d175..0659aa42 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -449,6 +449,10 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "file": { + "description" : "Expected files generated by the submission.", + "$ref" : "#/definitions/fileOutputChannel" + }, "exit_code" : { "type" : "integer", "description" : "Expected exit code for the run" @@ -472,7 +476,6 @@ "type": "programming_language" } ], - "type" : "object", "minProperties" : 1, "propertyNames" : { "$ref" : "#/definitions/programmingLanguage" diff --git a/tested/transform_json.py b/tested/transform_json.py index 1cb39b6f..048295fc 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -1,13 +1,119 @@ import json import os import sys +from enum import Enum from pathlib import Path from typing import Any +class SpecialMap(Enum): + NATURAL_LANGUAGE = "natural_language" + PROGRAMMING_LANGUAGE = "programming_language" + ORACLE = "oracle" + EXPRESSION = "expression" + NONE = "none" -def transform(data: Any) -> Any: +def map_kind(element: dict) -> SpecialMap: + if "required" in element and "properties" in element: + required = element["required"] + properties = element["properties"] + assert isinstance(required, list) + assert isinstance(properties, dict) + if "__tag__" in required and "value" in required: + if "__tag__" in properties and "value" in properties: + tag = properties["__tag__"] + if isinstance(tag, dict) and "const" in tag and isinstance(tag["const"], str): + if tag["const"] == "!natural_language": + return SpecialMap.NATURAL_LANGUAGE + elif tag["const"] == "!programming_language": + return SpecialMap.PROGRAMMING_LANGUAGE + elif tag["const"] == "!oracle": + return SpecialMap.ORACLE + elif tag["const"] == "!expression": + return SpecialMap.EXPRESSION + return SpecialMap.NONE + +def change_prog_lang_type(element: dict, prog_lang: dict) -> dict: + if element == prog_lang: + ele_type = element.pop("type") + element["anyOf"] = [ + {"type": ele_type}, + {"type": "programming_language"}, + ] + return element + +def transform_monolingual(data: Any, strict: bool) -> Any: + if isinstance(data, dict): + if "oneOf" in data: + new_one_of = [] + prog_lang = None + for element in data["oneOf"]: + assert isinstance(element, dict) + # Removes all natural translations + kind = map_kind(element) + if kind == SpecialMap.PROGRAMMING_LANGUAGE: + prog_lang = element["properties"]["value"] + elif kind != SpecialMap.NATURAL_LANGUAGE: + if kind == SpecialMap.EXPRESSION or kind == SpecialMap.ORACLE: + element = element["properties"]["value"] + if strict: + if kind == SpecialMap.EXPRESSION: + element["type"] = "expression" + else: + element["type"] = "oracle" + new_one_of.append(element) + + # A programming_langauge map was found. If not strict, just remove. + # If strict, still provide the type option for the corresponding object. + if prog_lang is not None and strict: + new_one_of = [change_prog_lang_type(ele, prog_lang) for ele in new_one_of] + + if len(new_one_of) <= 1: + data.pop("oneOf") + if len(new_one_of) == 1: + for key, value in new_one_of[0].items(): + data[key] = value + else: + data["oneOf"] = new_one_of + + if "expressionOrStatementWithNatTranslation" in data: + data.pop("expressionOrStatementWithNatTranslation") + + if "translations" in data: + data.pop("translations") + + if "$ref" in data: + if isinstance(data["$ref"], str): + if data["$ref"] == "#/definitions/expressionOrStatementWithNatTranslation": + data["$ref"] = "#/definitions/expressionOrStatement" + + if "yamlValue" in data: + if strict: + data["yamlValue"] = { + "description" : "A value represented as YAML.", + "not" : { + "type" : [ + "oracle", + "expression", + "programming_language" + ] + } + } + else: + data["yamlValue"] = { + "description": "A value represented as YAML.", + } + + + return {k: transform_monolingual(v, strict) for k, v in data.items()} + elif isinstance(data, list): + return [transformed for ele in data if + (transformed := transform_monolingual(ele, strict)) != {}] + return data + + +def transform_IDE(data: Any) -> Any: if isinstance(data, list): - return [transformed for ele in data if (transformed := transform(ele)) != {}] + return [transformed for ele in data if (transformed := transform_IDE(ele)) != {}] elif isinstance(data, dict): if "return" in data: # This is necessary since tags aren't recognized in the Json schema. @@ -52,11 +158,11 @@ def transform(data: Any) -> Any: } data = value - return {k: transform(v) for k, v in data.items()} + return {k: transform_IDE(v) for k, v in data.items()} return data -def transform_json(json_file: Path): +def transform_json(json_file: Path, monolingual: bool, strict: bool): """ This function transforms the JSON schema used in the DSL translator into a new JSON schema that can be used to validate the multilingual YAML in your IDE. @@ -72,10 +178,15 @@ def transform_json(json_file: Path): print("The json file was not found.") raise e - result = transform(json_stream) + if not monolingual: + result = transform_IDE(json_stream) + file_name = "multilingual-schema.json" + else: + result = transform_monolingual(json_stream, strict) + file_name = "multilingual-schema.json" with open( - json_file.parent / "multilingual-schema.json", "w", encoding="utf-8" + json_file.parent / file_name, "w", encoding="utf-8" ) as f: json.dump(result, f, indent=2) @@ -83,5 +194,12 @@ def transform_json(json_file: Path): if __name__ == "__main__": n = len(sys.argv) assert n > 1, "Expected path to multilingual json schema." + monolingual = False + strict = False + if n > 2: + assert sys.argv[2] in ("strict", "not-strict") + strict = sys.argv[2] == "strict" + + monolingual = True - transform_json(Path(sys.argv[1])) + transform_json(Path(sys.argv[1]), monolingual=monolingual, strict=strict) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 2b5cfbbf..035f83c0 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -18,7 +18,7 @@ translate_yaml, validate_pre_dsl, ) -from tested.transform_json import transform, transform_json +from tested.transform_json import transform_IDE, transform_json def validate_natural_translate(yaml_str: str, translated_yaml_str: str): @@ -718,7 +718,7 @@ def test_return_json_schema(): } } - result = transform(json_schema) + result = transform_IDE(json_schema) assert result == json_schema_expected @@ -737,7 +737,7 @@ def test_yaml_value_json_schema(): } } - result = transform(json_schema) + result = transform_IDE(json_schema) assert result == json_schema_expected @@ -768,7 +768,7 @@ def test_nat_lang_json_schema(): }, } - result = transform(json_schema) + result = transform_IDE(json_schema) assert result == json_schema_expected @@ -793,7 +793,7 @@ def test_expr_json_schema(): json_schema_expected = {} - result = transform(json_schema) + result = transform_IDE(json_schema) assert result == json_schema_expected @@ -814,7 +814,7 @@ def test_list_json_schema(): } ] - result = transform(json_schema) + result = transform_IDE(json_schema) assert result == json_schema_expected From 4b00b0f8febfc1178ed0830b13d68a6c78572c1b Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 13 Apr 2025 19:51:33 +0200 Subject: [PATCH 06/23] wrote some more tests --- tested/transform_json.py | 49 ++++++----- tests/test_preprocess_dsl.py | 155 ++++++++++++++++++++++++++++++++++- 2 files changed, 182 insertions(+), 22 deletions(-) diff --git a/tested/transform_json.py b/tested/transform_json.py index 048295fc..30e180a9 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Any + class SpecialMap(Enum): NATURAL_LANGUAGE = "natural_language" PROGRAMMING_LANGUAGE = "programming_language" @@ -12,6 +13,7 @@ class SpecialMap(Enum): EXPRESSION = "expression" NONE = "none" + def map_kind(element: dict) -> SpecialMap: if "required" in element and "properties" in element: required = element["required"] @@ -19,9 +21,13 @@ def map_kind(element: dict) -> SpecialMap: assert isinstance(required, list) assert isinstance(properties, dict) if "__tag__" in required and "value" in required: - if "__tag__" in properties and "value" in properties: + if "__tag__" in properties and "value" in properties: tag = properties["__tag__"] - if isinstance(tag, dict) and "const" in tag and isinstance(tag["const"], str): + if ( + isinstance(tag, dict) + and "const" in tag + and isinstance(tag["const"], str) + ): if tag["const"] == "!natural_language": return SpecialMap.NATURAL_LANGUAGE elif tag["const"] == "!programming_language": @@ -32,6 +38,7 @@ def map_kind(element: dict) -> SpecialMap: return SpecialMap.EXPRESSION return SpecialMap.NONE + def change_prog_lang_type(element: dict, prog_lang: dict) -> dict: if element == prog_lang: ele_type = element.pop("type") @@ -41,6 +48,7 @@ def change_prog_lang_type(element: dict, prog_lang: dict) -> dict: ] return element + def transform_monolingual(data: Any, strict: bool) -> Any: if isinstance(data, dict): if "oneOf" in data: @@ -65,7 +73,9 @@ def transform_monolingual(data: Any, strict: bool) -> Any: # A programming_langauge map was found. If not strict, just remove. # If strict, still provide the type option for the corresponding object. if prog_lang is not None and strict: - new_one_of = [change_prog_lang_type(ele, prog_lang) for ele in new_one_of] + new_one_of = [ + change_prog_lang_type(ele, prog_lang) for ele in new_one_of + ] if len(new_one_of) <= 1: data.pop("oneOf") @@ -83,37 +93,38 @@ def transform_monolingual(data: Any, strict: bool) -> Any: if "$ref" in data: if isinstance(data["$ref"], str): - if data["$ref"] == "#/definitions/expressionOrStatementWithNatTranslation": + if ( + data["$ref"] + == "#/definitions/expressionOrStatementWithNatTranslation" + ): data["$ref"] = "#/definitions/expressionOrStatement" if "yamlValue" in data: if strict: data["yamlValue"] = { - "description" : "A value represented as YAML.", - "not" : { - "type" : [ - "oracle", - "expression", - "programming_language" - ] - } + "description": "A value represented as YAML.", + "not": {"type": ["oracle", "expression", "programming_language"]}, } else: data["yamlValue"] = { "description": "A value represented as YAML.", } - return {k: transform_monolingual(v, strict) for k, v in data.items()} elif isinstance(data, list): - return [transformed for ele in data if - (transformed := transform_monolingual(ele, strict)) != {}] + return [ + transformed + for ele in data + if (transformed := transform_monolingual(ele, strict)) != {} + ] return data def transform_IDE(data: Any) -> Any: if isinstance(data, list): - return [transformed for ele in data if (transformed := transform_IDE(ele)) != {}] + return [ + transformed for ele in data if (transformed := transform_IDE(ele)) != {} + ] elif isinstance(data, dict): if "return" in data: # This is necessary since tags aren't recognized in the Json schema. @@ -162,7 +173,7 @@ def transform_IDE(data: Any) -> Any: return data -def transform_json(json_file: Path, monolingual: bool, strict: bool): +def transform_json(json_file: Path, monolingual: bool, strict: bool): """ This function transforms the JSON schema used in the DSL translator into a new JSON schema that can be used to validate the multilingual YAML in your IDE. @@ -185,9 +196,7 @@ def transform_json(json_file: Path, monolingual: bool, strict: bool): result = transform_monolingual(json_stream, strict) file_name = "multilingual-schema.json" - with open( - json_file.parent / file_name, "w", encoding="utf-8" - ) as f: + with open(json_file.parent / file_name, "w", encoding="utf-8") as f: json.dump(result, f, indent=2) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 035f83c0..ed5cf605 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -18,7 +18,7 @@ translate_yaml, validate_pre_dsl, ) -from tested.transform_json import transform_IDE, transform_json +from tested.transform_json import transform_IDE, transform_json, transform_monolingual def validate_natural_translate(yaml_str: str, translated_yaml_str: str): @@ -872,9 +872,160 @@ def test_transform_executed_correct(mocker: MockerFixture): mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - transform_json(Path("schema.json")) + transform_json(Path("schema.json"), False, False) assert s.call_count == 25 # Check if the file was opened for writing mock_opener.assert_any_call(Path("multilingual-schema.json"), "w", encoding="utf-8") + + +def test_json_rm_nat_lang_json_schema(): + json_schema = { + "oneOf": [ + {"type": "array", "minItems": 1, "items": {"$ref": "#/definitions/tab"}}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/definitions/tab"}, + }, + }, + }, + }, + ] + } + + json_schema_expected = { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/definitions/tab"}, + } + + result = transform_monolingual(json_schema, True) + + assert result == json_schema_expected + + +def test_json_rm_prog_lang_json_schema(): + json_schema = { + "expressionOrStatement": { + "oneOf": [ + { + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string.", + }, + { + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": {"$ref": "#/definitions/programmingLanguage"}, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim.", + }, + }, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!programming_language", + }, + "value": { + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim.", + }, + }, + }, + }, + ] + }, + } + + json_schema_expected = { + "expressionOrStatement": { + "oneOf": [ + { + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string.", + }, + { + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": {"$ref": "#/definitions/programmingLanguage"}, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim.", + }, + "anyOf": [{"type": "object"}, {"type": "programming_language"}], + }, + ] + }, + } + + result = transform_monolingual(json_schema, True) + + assert result == json_schema_expected + + +def test_json_schema_edge_cases(): + json_schema = { + "expressionOrStatementWithNatTranslation": {}, + "translations": {}, + "$ref": "#/definitions/expressionOrStatementWithNatTranslation", + "yamlValue": { + "description": "A value represented as YAML.", + "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, + }, + } + + json_schema_expected = { + "$ref": "#/definitions/expressionOrStatement", + "yamlValue": { + "description": "A value represented as YAML.", + "not": {"type": ["oracle", "expression", "programming_language"]}, + }, + } + + result = transform_monolingual(json_schema, True) + + assert result == json_schema_expected + + json_schema = { + "yamlValue": { + "description": "A value represented as YAML.", + "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, + }, + } + + json_schema_expected = { + "yamlValue": { + "description": "A value represented as YAML.", + } + } + + result = transform_monolingual(json_schema, False) + + assert result == json_schema_expected From 943a0affbfa362913687d1115f77f49e32e3ed9c Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 14 Apr 2025 16:13:31 +0200 Subject: [PATCH 07/23] added extra bit to the schema --- tested/dsl/schema-strict-nat-translation.json | 58 +++++++++++++++++-- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json index adfeaa57..51f0647c 100644 --- a/tested/dsl/schema-strict-nat-translation.json +++ b/tested/dsl/schema-strict-nat-translation.json @@ -1310,8 +1310,33 @@ "$ref" : "#/definitions/programmingLanguage" }, "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "oneOf" : [ + { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + } + } + ] } }, { @@ -1334,8 +1359,33 @@ "$ref" : "#/definitions/programmingLanguage" }, "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "oneOf" : [ + { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + } + } + ] } } } From 431b27df12dd9ad8786e353c8434ec77c45e4f61 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 14 Apr 2025 16:24:53 +0200 Subject: [PATCH 08/23] fixed tests --- tests/test_preprocess_dsl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index ed5cf605..bbedfcfa 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -821,7 +821,7 @@ def test_list_json_schema(): def test_transform_executed_correct(mocker: MockerFixture): s = mocker.spy( - tested.transform_json, name="transform" # type: ignore[reportAttributeAccessIssue] + tested.transform_json, name="transform_IDE" # type: ignore[reportAttributeAccessIssue] ) mock_files = [ From 23b3dbf57e8dd516d09cb18131bb6b56a50a74b4 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 15 Apr 2025 15:46:31 +0200 Subject: [PATCH 09/23] fixed linting --- tested/nat_translation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tested/nat_translation.py b/tested/nat_translation.py index 7b4ebe34..5b656ebe 100644 --- a/tested/nat_translation.py +++ b/tested/nat_translation.py @@ -49,7 +49,6 @@ def custom_representer(dumper, data): if "__tag__" in data: return dumper.represent_with_tag(data["__tag__"], data["value"]) - return dumper.represent_mapping( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data ) From edce398720e25f07b909f8026d0610a1a242d966 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 29 Apr 2025 16:34:43 +0200 Subject: [PATCH 10/23] updated schema-strict.json by generated version --- tested/dsl/schema-strict.json | 1076 ++++++++++++++++----------------- tested/transform_json.py | 30 +- tests/test_preprocess_dsl.py | 12 +- 3 files changed, 555 insertions(+), 563 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 0659aa42..66d8cc0e 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -1,293 +1,297 @@ { - "$id" : "tested:dsl:schema7", - "$schema" : "http://json-schema.org/draft-07/schema#", - "title" : "TESTed-DSL", - "oneOf" : [ + "$id": "tested:dsl:schema7", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TESTed-DSL", + "oneOf": [ { - "$ref" : "#/definitions/_rootObject" + "$ref": "#/definitions/_rootObject" }, { - "$ref" : "#/definitions/_tabList" + "$ref": "#/definitions/_tabList" }, { - "$ref" : "#/definitions/_unitList" + "$ref": "#/definitions/_unitList" } ], - "definitions" : { - "_rootObject" : { - "type" : "object", - "oneOf" : [ + "definitions": { + "_rootObject": { + "type": "object", + "oneOf": [ { - "required" : [ + "required": [ "tabs" ], - "not" : { - "required" : [ + "not": { + "required": [ "units" ] } }, { - "required" : [ + "required": [ "units" ], - "not" : { - "required" : [ + "not": { + "required": [ "tabs" ] } } ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "namespace" : { - "type" : "string", - "description" : "Namespace of the submitted solution, in `snake_case`" + "namespace": { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" }, - "tabs" : { - "$ref" : "#/definitions/_tabList" + "tabs": { + "$ref": "#/definitions/_tabList" }, - "units" : { - "$ref" : "#/definitions/_unitList" + "units": { + "$ref": "#/definitions/_unitList" }, - "language" : { - "description" : "Indicate that all code is in a specific language.", - "oneOf" : [ + "language": { + "description": "Indicate that all code is in a specific language.", + "oneOf": [ { - "$ref" : "#/definitions/programmingLanguage" + "$ref": "#/definitions/programmingLanguage" }, { - "const" : "tested" + "const": "tested" } ] }, - "definitions" : { - "description" : "Define hashes to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define hashes to use elsewhere.", + "type": "object" }, "config": { "$ref": "#/definitions/inheritableConfigObject" } } }, - "_tabList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/tab" + "_tabList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/tab" } }, - "_unitList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/unit" + "_unitList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/unit" } }, - "tab" : { - "type" : "object", - "description" : "A tab in the test suite.", - "required" : [ + "tab": { + "type": "object", + "description": "A tab in the test suite.", + "required": [ "tab" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "tab" : { - "type" : "string", - "description" : "The name of this tab." + "tab": { + "description": "The name of this tab.", + "type": "string" }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { "$ref": "#/definitions/inheritableConfigObject" } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "contexts" ], - "properties" : { - "contexts" : { - "$ref" : "#/definitions/_contextList" + "properties": { + "contexts": { + "$ref": "#/definitions/_contextList" } } }, { - "required" : [ + "required": [ "testcases" ], - "properties" : { - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "properties": { + "testcases": { + "$ref": "#/definitions/_testcaseList" } } } ] }, - "unit" : { - "type" : "object", - "description" : "A unit in the test suite.", - "required" : [ + "unit": { + "type": "object", + "description": "A unit in the test suite.", + "required": [ "unit" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "unit" : { - "type" : "string", - "description" : "The name of this tab." + "unit": { + "description": "The name of this tab.", + "type": "string" }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { "$ref": "#/definitions/inheritableConfigObject" } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "cases" ], - "properties" : { - "cases" : { - "$ref" : "#/definitions/_caseList" + "properties": { + "cases": { + "$ref": "#/definitions/_caseList" } } }, { - "required" : [ + "required": [ "scripts" ], - "properties" : { - "scripts" : { - "$ref" : "#/definitions/_scriptList" + "properties": { + "scripts": { + "$ref": "#/definitions/_scriptList" } } } ] }, - "_contextList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/context" + "_contextList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" } }, - "_caseList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/case" + "_caseList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" } }, - "_testcaseList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/testcase" + "_testcaseList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" } }, - "_scriptList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/script" + "_scriptList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/script" } }, - "context" : { - "type" : "object", - "description" : "A set of testcase in the same context.", - "required" : [ + "context": { + "type": "object", + "description": "A set of testcase in the same context.", + "required": [ "testcases" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "context" : { - "type" : "string", - "description" : "Description of this context." + "context": { + "type": "string", + "description": "Description of this context." }, - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "testcases": { + "$ref": "#/definitions/_testcaseList" } } }, - "case" : { - "type" : "object", - "description" : "A test case.", - "required" : [ + "case": { + "type": "object", + "description": "A test case.", + "required": [ "script" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "context" : { - "type" : "string", - "description" : "Description of this context." + "context": { + "type": "string", + "description": "Description of this context." }, - "script" : { - "$ref" : "#/definitions/_scriptList" + "script": { + "$ref": "#/definitions/_scriptList" } } }, - "testcase" : { - "type" : "object", - "description" : "An individual test for a statement or expression", - "additionalProperties" : false, - "properties" : { - "description" : { - "$ref" : "#/definitions/message" - }, - "stdin" : { - "description" : "Stdin for this context", - "type" : [ + "testcase": { + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/definitions/message" + }, + "stdin": { + "description": "Stdin for this context", + "type": [ "string", "number", "integer", "boolean" ] }, - "arguments" : { - "type" : "array", - "description" : "Array of program call arguments", - "items" : { - "type" : [ + "arguments": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ "string", "number", "integer", @@ -295,95 +299,68 @@ ] } }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "type" : "string", - "description" : "Message of the expected exception." - }, - { - "type" : "object", - "required" : [ - "types" - ], - "properties" : { - "message" : { - "type" : "string", - "description" : "Message of the expected exception." - }, - "types" : { - "minProperties" : 1, - "description" : "Language mapping of expected exception types.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string" - } - } - } - } - ] + "exception": { + "description": "Expected exception message", + "$ref": "#/definitions/exceptionChannel" }, - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "return" : { - "description" : "Expected return value", - "$ref" : "#/definitions/returnOutputChannel" + "return": { + "description": "Expected return value", + "$ref": "#/definitions/returnOutputChannel" }, - "stderr" : { - "description" : "Expected output at stderr", - "$ref" : "#/definitions/textOutputChannel" + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/definitions/textOutputChannel" }, - "stdout" : { - "description" : "Expected output at stdout", - "$ref" : "#/definitions/textOutputChannel" + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/definitions/textOutputChannel" }, "file": { - "description" : "Expected files generated by the submission.", - "$ref" : "#/definitions/fileOutputChannel" + "description": "Expected files generated by the submission.", + "$ref": "#/definitions/fileOutputChannel" }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" } } }, - "script" : { - "type" : "object", - "description" : "An individual test (script) for a statement or expression", - "properties" : { - "description" : { - "$ref" : "#/definitions/message" - }, - "stdin" : { - "description" : "Stdin for this context", - "type" : [ + "script": { + "type": "object", + "description": "An individual test (script) for a statement or expression", + "properties": { + "description": { + "$ref": "#/definitions/message" + }, + "stdin": { + "description": "Stdin for this context", + "type": [ "string", "number", "integer", "boolean" ] }, - "arguments" : { - "type" : "array", - "description" : "Array of program call arguments", - "items" : { - "type" : [ + "arguments": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ "string", "number", "integer", @@ -391,83 +368,64 @@ ] } }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "type" : "string", - "description" : "Message of the expected exception." - }, - { - "type" : "object", - "required" : [ - "types" - ], - "properties" : { - "message" : { - "type" : "string", - "description" : "Message of the expected exception." - }, - "types" : { - "minProperties" : 1, - "description" : "Language mapping of expected exception types.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string" - } - } - } - } - ] + "exception": { + "description": "Expected exception message", + "$ref": "#/definitions/exceptionChannel" }, - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "return" : { - "description" : "Expected return value", - "$ref" : "#/definitions/returnOutputChannel" + "return": { + "description": "Expected return value", + "$ref": "#/definitions/returnOutputChannel" }, - "stderr" : { - "description" : "Expected output at stderr", - "$ref" : "#/definitions/textOutputChannel" + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/definitions/textOutputChannel" }, - "stdout" : { - "description" : "Expected output at stdout", - "$ref" : "#/definitions/textOutputChannel" + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/definitions/textOutputChannel" }, "file": { - "description" : "Expected files generated by the submission.", - "$ref" : "#/definitions/fileOutputChannel" + "description": "Expected files generated by the submission.", + "$ref": "#/definitions/fileOutputChannel" }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" } } }, - "expressionOrStatement" : { - "oneOf" : [ + "expressionOrStatement": { + "oneOf": [ { - "type" : "string", - "format" : "tested-dsl-expression", - "description" : "A statement of expression in Python-like syntax as YAML string." + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." }, { - "description" : "Programming-language-specific statement or expression.", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, "anyOf": [ { "type": "object" @@ -475,108 +433,131 @@ { "type": "programming_language" } - ], - "minProperties" : 1, - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - } + ] } ] }, - "yamlValueOrPythonExpression" : { - "oneOf" : [ + "yamlValueOrPythonExpression": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValue" + "$ref": "#/definitions/yamlValue" }, { - "type" : "expression", - "format" : "tested-dsl-expression", - "description" : "An expression in Python-syntax." + "type": "expression", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax." } ] }, - "file" : { - "type" : "object", - "description" : "A file used in the test suite.", - "required" : [ + "file": { + "type": "object", + "description": "A file used in the test suite.", + "required": [ "name", "url" ], - "properties" : { - "name" : { - "type" : "string", - "description" : "The filename, including the file extension." - }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." + "properties": { + "name": { + "type": "string", + "description": "The filename, including the file extension." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." } } }, - "textOutputChannel" : { - "anyOf" : [ + "exceptionChannel": { + "oneOf": [ + { + "type": "string", + "description": "Message of the expected exception." + }, + { + "type": "object", + "required": [ + "types" + ], + "properties": { + "message": { + "description": "Message of the expected exception.", + "type": "string" + }, + "types": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" + }, + "items": { + "type": "string" + } + } + } + } + ] + }, + "textOutputChannel": { + "anyOf": [ { - "$ref" : "#/definitions/textualType" + "$ref": "#/definitions/textualType" }, { - "type" : "object", - "description" : "Built-in oracle for text values.", - "required" : [ + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ "data" ], - "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "properties": { + "data": { + "$ref": "#/definitions/textualType" }, - "oracle" : { - "const" : "builtin" + "oracle": { + "const": "builtin" }, - "config" : { - "$ref" : "#/definitions/textConfigurationOptions" + "config": { + "$ref": "#/definitions/textConfigurationOptions" } } }, { - "type" : "object", - "description" : "Custom oracle for text values.", - "required" : [ + "type": "object", + "description": "Custom oracle for text values.", + "required": [ "oracle", "file", "data" ], - "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "properties": { + "data": { + "$ref": "#/definitions/textualType" }, - "oracle" : { - "const" : "custom_check" + "oracle": { + "const": "custom_check" }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/definitions/programmingLanguage" } } } @@ -584,202 +565,202 @@ ] }, "fileOutputChannel": { - "anyOf" : [ + "anyOf": [ { - "type" : "object", - "description" : "Built-in oracle for files.", - "required" : [ + "type": "object", + "description": "Built-in oracle for files.", + "required": [ "content", "location" ], - "properties" : { - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "properties": { + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." }, - "oracle" : { - "const" : "builtin" + "oracle": { + "const": "builtin" }, - "config" : { - "$ref" : "#/definitions/fileConfigurationOptions" + "config": { + "$ref": "#/definitions/fileConfigurationOptions" } } }, { - "type" : "object", - "description" : "Custom oracle for file values.", - "required" : [ + "type": "object", + "description": "Custom oracle for file values.", + "required": [ "oracle", "content", "location", "file" ], - "properties" : { - "oracle" : { - "const" : "custom_check" + "properties": { + "oracle": { + "const": "custom_check" }, - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/definitions/programmingLanguage" } } } } ] }, - "returnOutputChannel" : { - "oneOf" : [ + "returnOutputChannel": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/definitions/yamlValueOrPythonExpression" }, { - "type" : "oracle", - "additionalProperties" : false, - "required" : [ + "type": "oracle", + "additionalProperties": false, + "required": [ "value" ], - "properties" : { - "oracle" : { - "const" : "builtin" + "properties": { + "oracle": { + "const": "builtin" }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } } }, { - "type" : "oracle", - "additionalProperties" : false, - "required" : [ + "type": "oracle", + "additionalProperties": false, + "required": [ "value", "oracle", "file" ], - "properties" : { - "oracle" : { - "const" : "custom_check" + "properties": { + "oracle": { + "const": "custom_check" }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/definitions/yamlValueOrPythonExpression" }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/definitions/programmingLanguage" } } } }, { - "type" : "oracle", - "additionalProperties" : false, - "required" : [ + "type": "oracle", + "additionalProperties": false, + "required": [ "oracle", "functions" ], - "properties" : { - "oracle" : { - "const" : "specific_check" + "properties": { + "oracle": { + "const": "specific_check" }, - "functions" : { - "minProperties" : 1, - "description" : "Language mapping of oracle functions.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "functions": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" }, - "items" : { - "type" : "object", - "required" : [ + "items": { + "type": "object", + "required": [ "file" ], - "properties" : { - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" } } } }, - "arguments" : { - "minProperties" : 1, - "description" : "Language mapping of oracle arguments.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "arguments": { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" }, - "items" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." } } }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } } } ] }, - "programmingLanguage" : { - "type" : "string", - "description" : "One of the programming languages supported by TESTed.", - "enum" : [ + "programmingLanguage": { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ "bash", "c", "haskell", @@ -793,96 +774,99 @@ "cpp" ] }, - "message" : { - "oneOf" : [ + "message": { + "oneOf": [ { - "type" : "string", - "description" : "A simple message to display." + "type": "string", + "description": "A simple message to display." }, { - "type" : "object", - "required" : [ + "type": "object", + "required": [ "description" ], - "properties" : { - "description" : { - "type" : "string", - "description" : "The message to display." + "properties": { + "description": { + "description": "The message to display.", + "type": "string" }, - "format" : { - "type" : "string", - "default" : "text", - "description" : "The format of the message, either a programming language, 'text' or 'html'." + "format": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." } } } ] }, - "textConfigurationOptions" : { - "type" : "object", - "description" : "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", - "minProperties" : 1, - "properties" : { - "applyRounding" : { - "description" : "Apply rounding when comparing as float", - "type" : "boolean" - }, - "caseInsensitive" : { - "description" : "Ignore case when comparing strings", - "type" : "boolean" - }, - "ignoreWhitespace" : { - "description" : "Ignore trailing whitespace", - "type" : "boolean" - }, - "normalizeTrailingNewlines" : { - "description" : "Normalize trailing newlines", - "type" : "boolean" - }, - "roundTo" : { - "description" : "The number of decimals to round at, when applying the rounding on floats", - "type" : "integer" - }, - "tryFloatingPoint" : { - "description" : "Try comparing text as floating point numbers", - "type" : "boolean" - }, - "hideExpected" : { - "description" : "Hide the expected value in feedback (default: false), not recommended to use!", - "type" : "boolean" + "textConfigurationOptions": { + "type": "object", + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, + "properties": { + "applyRounding": { + "description": "Apply rounding when comparing as float", + "type": "boolean" + }, + "caseInsensitive": { + "description": "Ignore case when comparing strings", + "type": "boolean" + }, + "ignoreWhitespace": { + "description": "Ignore trailing whitespace", + "type": "boolean" + }, + "normalizeTrailingNewlines": { + "description": "Normalize trailing newlines", + "type": "boolean" + }, + "roundTo": { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + }, + "tryFloatingPoint": { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + }, + "hideExpected": { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" } } }, "fileConfigurationOptions": { - "anyOf" : [ + "anyOf": [ { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/definitions/textConfigurationOptions" }, { - "type" : "object", - "properties" : { + "type": "object", + "properties": { "mode": { - "type" : "string", - "enum" : ["full", "line"], - "default" : "full" + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" } } } ] }, - "textualType" : { - "description" : "Simple textual value, converted to string.", - "type" : [ + "textualType": { + "description": "Simple textual value, converted to string.", + "type": [ "string", "number", "integer", "boolean" ] }, - "yamlValue" : { - "description" : "A value represented as YAML.", - "not" : { - "type" : [ + "yamlValue": { + "description": "A value represented as YAML.", + "not": { + "type": [ "oracle", "expression", "programming_language" @@ -891,17 +875,17 @@ }, "inheritableConfigObject": { "type": "object", - "properties" : { + "properties": { "stdout": { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/definitions/textConfigurationOptions" }, "stderr": { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/definitions/textConfigurationOptions" }, "file": { - "$ref" : "#/definitions/fileConfigurationOptions" + "$ref": "#/definitions/fileConfigurationOptions" } } } } -} +} \ No newline at end of file diff --git a/tested/transform_json.py b/tested/transform_json.py index 30e180a9..0900a39f 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -49,7 +49,7 @@ def change_prog_lang_type(element: dict, prog_lang: dict) -> dict: return element -def transform_monolingual(data: Any, strict: bool) -> Any: +def transform_monolingual(data: Any, strict_schema: bool) -> Any: if isinstance(data, dict): if "oneOf" in data: new_one_of = [] @@ -63,7 +63,7 @@ def transform_monolingual(data: Any, strict: bool) -> Any: elif kind != SpecialMap.NATURAL_LANGUAGE: if kind == SpecialMap.EXPRESSION or kind == SpecialMap.ORACLE: element = element["properties"]["value"] - if strict: + if strict_schema: if kind == SpecialMap.EXPRESSION: element["type"] = "expression" else: @@ -72,7 +72,7 @@ def transform_monolingual(data: Any, strict: bool) -> Any: # A programming_langauge map was found. If not strict, just remove. # If strict, still provide the type option for the corresponding object. - if prog_lang is not None and strict: + if prog_lang is not None and strict_schema: new_one_of = [ change_prog_lang_type(ele, prog_lang) for ele in new_one_of ] @@ -85,6 +85,7 @@ def transform_monolingual(data: Any, strict: bool) -> Any: else: data["oneOf"] = new_one_of + # THe next changes are a few edge cases. if "expressionOrStatementWithNatTranslation" in data: data.pop("expressionOrStatementWithNatTranslation") @@ -100,7 +101,7 @@ def transform_monolingual(data: Any, strict: bool) -> Any: data["$ref"] = "#/definitions/expressionOrStatement" if "yamlValue" in data: - if strict: + if strict_schema: data["yamlValue"] = { "description": "A value represented as YAML.", "not": {"type": ["oracle", "expression", "programming_language"]}, @@ -110,20 +111,20 @@ def transform_monolingual(data: Any, strict: bool) -> Any: "description": "A value represented as YAML.", } - return {k: transform_monolingual(v, strict) for k, v in data.items()} + return {k: transform_monolingual(v, strict_schema) for k, v in data.items()} elif isinstance(data, list): return [ transformed for ele in data - if (transformed := transform_monolingual(ele, strict)) != {} + if (transformed := transform_monolingual(ele, strict_schema)) != {} ] return data -def transform_IDE(data: Any) -> Any: +def transform_ide(data: Any) -> Any: if isinstance(data, list): return [ - transformed for ele in data if (transformed := transform_IDE(ele)) != {} + transformed for ele in data if (transformed := transform_ide(ele)) != {} ] elif isinstance(data, dict): if "return" in data: @@ -137,6 +138,7 @@ def transform_IDE(data: Any) -> Any: "description": "A value represented as YAML.", } + # This is a speciale structure for tags. if "required" in data and "properties" in data: required = data["required"] properties = data["properties"] @@ -169,7 +171,7 @@ def transform_IDE(data: Any) -> Any: } data = value - return {k: transform_IDE(v) for k, v in data.items()} + return {k: transform_ide(v) for k, v in data.items()} return data @@ -179,6 +181,9 @@ def transform_json(json_file: Path, monolingual: bool, strict: bool): a new JSON schema that can be used to validate the multilingual YAML in your IDE. :param json_file: The path to the JSON file. + :param monolingual: indicator if it will generate a monolingual schema. + :param strict: indicator if it will generate a strict schema. This only applies + to monolingual transforms """ _, ext = os.path.splitext(json_file) assert ext.lower() == ".json", f"expected a json file, got {ext}." @@ -190,11 +195,14 @@ def transform_json(json_file: Path, monolingual: bool, strict: bool): raise e if not monolingual: - result = transform_IDE(json_stream) + result = transform_ide(json_stream) file_name = "multilingual-schema.json" else: result = transform_monolingual(json_stream, strict) - file_name = "multilingual-schema.json" + if strict: + file_name = "schema-strict.json" + else: + file_name = "multilingual-schema.json" with open(json_file.parent / file_name, "w", encoding="utf-8") as f: json.dump(result, f, indent=2) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 2a4a45d6..e72cd0e1 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -19,7 +19,7 @@ translate_yaml, validate_pre_dsl, ) -from tested.transform_json import transform_IDE, transform_json, transform_monolingual +from tested.transform_json import transform_ide, transform_json, transform_monolingual def validate_natural_translate(yaml_str: str, translated_yaml_str: str): @@ -692,7 +692,7 @@ def test_return_json_schema(): } } - result = transform_IDE(json_schema) + result = transform_ide(json_schema) assert result == json_schema_expected @@ -711,7 +711,7 @@ def test_yaml_value_json_schema(): } } - result = transform_IDE(json_schema) + result = transform_ide(json_schema) assert result == json_schema_expected @@ -742,7 +742,7 @@ def test_nat_lang_json_schema(): }, } - result = transform_IDE(json_schema) + result = transform_ide(json_schema) assert result == json_schema_expected @@ -767,7 +767,7 @@ def test_expr_json_schema(): json_schema_expected = {} - result = transform_IDE(json_schema) + result = transform_ide(json_schema) assert result == json_schema_expected @@ -788,7 +788,7 @@ def test_list_json_schema(): } ] - result = transform_IDE(json_schema) + result = transform_ide(json_schema) assert result == json_schema_expected From 66acff4c625f30ccf06671732ec06bcd8773e660 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 29 Apr 2025 17:07:49 +0200 Subject: [PATCH 11/23] updated schema.json by generated version --- tested/dsl/schema.json | 1068 +++++++++++++++++----------------- tested/transform_json.py | 6 +- tests/test_preprocess_dsl.py | 2 +- 3 files changed, 532 insertions(+), 544 deletions(-) diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index f8d066bc..6d8943c4 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -1,293 +1,297 @@ { - "$id" : "tested:dsl:schema7", - "$schema" : "http://json-schema.org/draft-07/schema#", - "title" : "TESTed-DSL", - "oneOf" : [ + "$id": "tested:dsl:schema7", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TESTed-DSL", + "oneOf": [ { - "$ref" : "#/definitions/_rootObject" + "$ref": "#/definitions/_rootObject" }, { - "$ref" : "#/definitions/_tabList" + "$ref": "#/definitions/_tabList" }, { - "$ref" : "#/definitions/_unitList" + "$ref": "#/definitions/_unitList" } ], - "definitions" : { - "_rootObject" : { - "type" : "object", - "oneOf" : [ + "definitions": { + "_rootObject": { + "type": "object", + "oneOf": [ { - "required" : [ + "required": [ "tabs" ], - "not" : { - "required" : [ + "not": { + "required": [ "units" ] } }, { - "required" : [ + "required": [ "units" ], - "not" : { - "required" : [ + "not": { + "required": [ "tabs" ] } } ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "namespace" : { - "type" : "string", - "description" : "Namespace of the submitted solution, in `snake_case`" + "namespace": { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" }, - "tabs" : { - "$ref" : "#/definitions/_tabList" + "tabs": { + "$ref": "#/definitions/_tabList" }, - "units" : { - "$ref" : "#/definitions/_unitList" + "units": { + "$ref": "#/definitions/_unitList" }, - "language" : { - "description" : "Indicate that all code is in a specific language.", - "oneOf" : [ + "language": { + "description": "Indicate that all code is in a specific language.", + "oneOf": [ { - "$ref" : "#/definitions/programmingLanguage" + "$ref": "#/definitions/programmingLanguage" }, { - "const" : "tested" + "const": "tested" } ] }, - "definitions" : { - "description" : "Define hashes to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define hashes to use elsewhere.", + "type": "object" }, "config": { "$ref": "#/definitions/inheritableConfigObject" } } }, - "_tabList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/tab" + "_tabList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/tab" } }, - "_unitList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/unit" + "_unitList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/unit" } }, - "tab" : { - "type" : "object", - "description" : "A tab in the test suite.", - "required" : [ + "tab": { + "type": "object", + "description": "A tab in the test suite.", + "required": [ "tab" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "tab" : { - "type" : "string", - "description" : "The name of this tab." + "tab": { + "description": "The name of this tab.", + "type": "string" }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { "$ref": "#/definitions/inheritableConfigObject" } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "contexts" ], - "properties" : { - "contexts" : { - "$ref" : "#/definitions/_contextList" + "properties": { + "contexts": { + "$ref": "#/definitions/_contextList" } } }, { - "required" : [ + "required": [ "testcases" ], - "properties" : { - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "properties": { + "testcases": { + "$ref": "#/definitions/_testcaseList" } } } ] }, - "unit" : { - "type" : "object", - "description" : "A unit in the test suite.", - "required" : [ + "unit": { + "type": "object", + "description": "A unit in the test suite.", + "required": [ "unit" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "unit" : { - "type" : "string", - "description" : "The name of this tab." + "unit": { + "description": "The name of this tab.", + "type": "string" }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { "$ref": "#/definitions/inheritableConfigObject" } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "cases" ], - "properties" : { - "cases" : { - "$ref" : "#/definitions/_caseList" + "properties": { + "cases": { + "$ref": "#/definitions/_caseList" } } }, { - "required" : [ + "required": [ "scripts" ], - "properties" : { - "scripts" : { - "$ref" : "#/definitions/_scriptList" + "properties": { + "scripts": { + "$ref": "#/definitions/_scriptList" } } } ] }, - "_contextList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/context" + "_contextList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" } }, - "_caseList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/case" + "_caseList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" } }, - "_testcaseList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/testcase" + "_testcaseList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" } }, - "_scriptList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/script" + "_scriptList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/script" } }, - "context" : { - "type" : "object", - "description" : "A set of testcase in the same context.", - "required" : [ + "context": { + "type": "object", + "description": "A set of testcase in the same context.", + "required": [ "testcases" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "context" : { - "type" : "string", - "description" : "Description of this context." + "context": { + "type": "string", + "description": "Description of this context." }, - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "testcases": { + "$ref": "#/definitions/_testcaseList" } } }, - "case" : { - "type" : "object", - "description" : "A test case.", - "required" : [ + "case": { + "type": "object", + "description": "A test case.", + "required": [ "script" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "context" : { - "type" : "string", - "description" : "Description of this context." + "context": { + "type": "string", + "description": "Description of this context." }, - "script" : { - "$ref" : "#/definitions/_scriptList" + "script": { + "$ref": "#/definitions/_scriptList" } } }, - "testcase" : { - "type" : "object", - "description" : "An individual test for a statement or expression", - "additionalProperties" : false, - "properties" : { - "description" : { - "$ref" : "#/definitions/message" - }, - "stdin" : { - "description" : "Stdin for this context", - "type" : [ + "testcase": { + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/definitions/message" + }, + "stdin": { + "description": "Stdin for this context", + "type": [ "string", "number", "integer", "boolean" ] }, - "arguments" : { - "type" : "array", - "description" : "Array of program call arguments", - "items" : { - "type" : [ + "arguments": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ "string", "number", "integer", @@ -295,91 +299,68 @@ ] } }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "type" : "string", - "description" : "Message of the expected exception." - }, - { - "type" : "object", - "required" : [ - "types" - ], - "properties" : { - "message" : { - "type" : "string", - "description" : "Message of the expected exception." - }, - "types" : { - "minProperties" : 1, - "description" : "Language mapping of expected exception types.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string" - } - } - } - } - ] + "exception": { + "description": "Expected exception message", + "$ref": "#/definitions/exceptionChannel" }, - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "return" : { - "description" : "Expected return value", - "$ref" : "#/definitions/returnOutputChannel" + "return": { + "description": "Expected return value", + "$ref": "#/definitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/definitions/textOutputChannel" }, - "stderr" : { - "description" : "Expected output at stderr", - "$ref" : "#/definitions/textOutputChannel" + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/definitions/textOutputChannel" }, - "stdout" : { - "description" : "Expected output at stdout", - "$ref" : "#/definitions/textOutputChannel" + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/definitions/fileOutputChannel" }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" } } }, - "script" : { - "type" : "object", - "description" : "An individual test (script) for a statement or expression", - "properties" : { - "description" : { - "$ref" : "#/definitions/message" - }, - "stdin" : { - "description" : "Stdin for this context", - "type" : [ + "script": { + "type": "object", + "description": "An individual test (script) for a statement or expression", + "properties": { + "description": { + "$ref": "#/definitions/message" + }, + "stdin": { + "description": "Stdin for this context", + "type": [ "string", "number", "integer", "boolean" ] }, - "arguments" : { - "type" : "array", - "description" : "Array of program call arguments", - "items" : { - "type" : [ + "arguments": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ "string", "number", "integer", @@ -387,185 +368,189 @@ ] } }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "type" : "string", - "description" : "Message of the expected exception." - }, - { - "type" : "object", - "required" : [ - "types" - ], - "properties" : { - "message" : { - "type" : "string", - "description" : "Message of the expected exception." - }, - "types" : { - "minProperties" : 1, - "description" : "Language mapping of expected exception types.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string" - } - } - } - } - ] + "exception": { + "description": "Expected exception message", + "$ref": "#/definitions/exceptionChannel" }, - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "return" : { - "description" : "Expected return value", - "$ref" : "#/definitions/returnOutputChannel" + "return": { + "description": "Expected return value", + "$ref": "#/definitions/returnOutputChannel" }, - "stderr" : { - "description" : "Expected output at stderr", - "$ref" : "#/definitions/textOutputChannel" + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/definitions/textOutputChannel" }, - "stdout" : { - "description" : "Expected output at stdout", - "$ref" : "#/definitions/textOutputChannel" + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/definitions/textOutputChannel" }, "file": { - "description" : "Expected files generated by the submission.", - "$ref" : "#/definitions/fileOutputChannel" + "description": "Expected files generated by the submission.", + "$ref": "#/definitions/fileOutputChannel" }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" } } }, - "expressionOrStatement" : { - "oneOf" : [ + "expressionOrStatement": { + "oneOf": [ { - "type" : "string", - "format" : "tested-dsl-expression", - "description" : "A statement of expression in Python-like syntax as YAML string." + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." }, { - "description" : "Programming-language-specific statement or expression.", - "type" : "object", - "minProperties" : 1, - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" }, - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." } } ] }, - "yamlValueOrPythonExpression" : { - "oneOf" : [ + "yamlValueOrPythonExpression": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValue" + "$ref": "#/definitions/yamlValue" }, { - "type" : "string", - "format" : "tested-dsl-expression", - "description" : "An expression in Python-syntax." + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax." } ] }, - "file" : { - "type" : "object", - "description" : "A file used in the test suite.", - "required" : [ + "file": { + "type": "object", + "description": "A file used in the test suite.", + "required": [ "name", "url" ], - "properties" : { - "name" : { - "type" : "string", - "description" : "The filename, including the file extension." - }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." + "properties": { + "name": { + "type": "string", + "description": "The filename, including the file extension." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." } } }, - "textOutputChannel" : { - "anyOf" : [ + "exceptionChannel": { + "oneOf": [ + { + "type": "string", + "description": "Message of the expected exception." + }, + { + "type": "object", + "required": [ + "types" + ], + "properties": { + "message": { + "description": "Message of the expected exception.", + "type": "string" + }, + "types": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" + }, + "items": { + "type": "string" + } + } + } + } + ] + }, + "textOutputChannel": { + "anyOf": [ { - "$ref" : "#/definitions/textualType" + "$ref": "#/definitions/textualType" }, { - "type" : "object", - "description" : "Built-in oracle for text values.", - "required" : [ + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ "data" ], - "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "properties": { + "data": { + "$ref": "#/definitions/textualType" }, - "oracle" : { - "const" : "builtin" + "oracle": { + "const": "builtin" }, - "config" : { - "$ref" : "#/definitions/textConfigurationOptions" + "config": { + "$ref": "#/definitions/textConfigurationOptions" } } }, { - "type" : "object", - "description" : "Custom oracle for text values.", - "required" : [ + "type": "object", + "description": "Custom oracle for text values.", + "required": [ "oracle", "file", "data" ], - "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "properties": { + "data": { + "$ref": "#/definitions/textualType" }, - "oracle" : { - "const" : "custom_check" + "oracle": { + "const": "custom_check" }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/definitions/programmingLanguage" } } } @@ -573,202 +558,202 @@ ] }, "fileOutputChannel": { - "anyOf" : [ + "anyOf": [ { - "type" : "object", - "description" : "Built-in oracle for files.", - "required" : [ + "type": "object", + "description": "Built-in oracle for files.", + "required": [ "content", "location" ], - "properties" : { - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "properties": { + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." }, - "oracle" : { - "const" : "builtin" + "oracle": { + "const": "builtin" }, - "config" : { - "$ref" : "#/definitions/fileConfigurationOptions" + "config": { + "$ref": "#/definitions/fileConfigurationOptions" } } }, { - "type" : "object", - "description" : "Custom oracle for file values.", - "required" : [ + "type": "object", + "description": "Custom oracle for file values.", + "required": [ "oracle", "content", "location", "file" ], - "properties" : { - "oracle" : { - "const" : "custom_check" + "properties": { + "oracle": { + "const": "custom_check" }, - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/definitions/programmingLanguage" } } } } ] }, - "returnOutputChannel" : { - "oneOf" : [ + "returnOutputChannel": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/definitions/yamlValueOrPythonExpression" }, { - "type" : "object", - "additionalProperties" : false, - "required" : [ + "type": "object", + "additionalProperties": false, + "required": [ "value" ], - "properties" : { - "oracle" : { - "const" : "builtin" + "properties": { + "oracle": { + "const": "builtin" }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } } }, { - "type" : "object", - "additionalProperties" : false, - "required" : [ + "type": "object", + "additionalProperties": false, + "required": [ "value", "oracle", "file" ], - "properties" : { - "oracle" : { - "const" : "custom_check" + "properties": { + "oracle": { + "const": "custom_check" }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/definitions/yamlValueOrPythonExpression" }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/definitions/programmingLanguage" } } } }, { - "type" : "object", - "additionalProperties" : false, - "required" : [ + "type": "object", + "additionalProperties": false, + "required": [ "oracle", "functions" ], - "properties" : { - "oracle" : { - "const" : "specific_check" + "properties": { + "oracle": { + "const": "specific_check" }, - "functions" : { - "minProperties" : 1, - "description" : "Language mapping of oracle functions.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "functions": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" }, - "items" : { - "type" : "object", - "required" : [ + "items": { + "type": "object", + "required": [ "file" ], - "properties" : { - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" } } } }, - "arguments" : { - "minProperties" : 1, - "description" : "Language mapping of oracle arguments.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "arguments": { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" }, - "items" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." } } }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } } } ] }, - "programmingLanguage" : { - "type" : "string", - "description" : "One of the programming languages supported by TESTed.", - "enum" : [ + "programmingLanguage": { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ "bash", "c", "haskell", @@ -782,108 +767,111 @@ "cpp" ] }, - "message" : { - "oneOf" : [ + "message": { + "oneOf": [ { - "type" : "string", - "description" : "A simple message to display." + "type": "string", + "description": "A simple message to display." }, { - "type" : "object", - "required" : [ + "type": "object", + "required": [ "description" ], - "properties" : { - "description" : { - "type" : "string", - "description" : "The message to display." + "properties": { + "description": { + "description": "The message to display.", + "type": "string" }, - "format" : { - "type" : "string", - "default" : "text", - "description" : "The format of the message, either a programming language, 'text' or 'html'." + "format": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." } } } ] }, - "textConfigurationOptions" : { - "type" : "object", - "description" : "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", - "minProperties" : 1, - "properties" : { - "applyRounding" : { - "description" : "Apply rounding when comparing as float", - "type" : "boolean" - }, - "caseInsensitive" : { - "description" : "Ignore case when comparing strings", - "type" : "boolean" - }, - "ignoreWhitespace" : { - "description" : "Ignore trailing whitespace", - "type" : "boolean" - }, - "normalizeTrailingNewlines" : { - "description" : "Normalize trailing newlines", - "type" : "boolean" - }, - "roundTo" : { - "description" : "The number of decimals to round at, when applying the rounding on floats", - "type" : "integer" - }, - "tryFloatingPoint" : { - "description" : "Try comparing text as floating point numbers", - "type" : "boolean" - }, - "hideExpected" : { - "description" : "Hide the expected value in feedback (default: false), not recommended to use!", - "type" : "boolean" + "textConfigurationOptions": { + "type": "object", + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, + "properties": { + "applyRounding": { + "description": "Apply rounding when comparing as float", + "type": "boolean" + }, + "caseInsensitive": { + "description": "Ignore case when comparing strings", + "type": "boolean" + }, + "ignoreWhitespace": { + "description": "Ignore trailing whitespace", + "type": "boolean" + }, + "normalizeTrailingNewlines": { + "description": "Normalize trailing newlines", + "type": "boolean" + }, + "roundTo": { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + }, + "tryFloatingPoint": { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + }, + "hideExpected": { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" } } }, "fileConfigurationOptions": { - "anyOf" : [ + "anyOf": [ { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/definitions/textConfigurationOptions" }, { - "type" : "object", - "properties" : { + "type": "object", + "properties": { "mode": { - "type" : "string", - "enum" : ["full", "line"], - "default" : "full" + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" } } } ] }, - "textualType" : { - "description" : "Simple textual value, converted to string.", - "type" : [ + "textualType": { + "description": "Simple textual value, converted to string.", + "type": [ "string", "number", "integer", "boolean" ] }, - "yamlValue" : { - "description" : "A value represented as YAML." + "yamlValue": { + "description": "A value represented as YAML." }, "inheritableConfigObject": { "type": "object", - "properties" : { + "properties": { "stdout": { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/definitions/textConfigurationOptions" }, "stderr": { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/definitions/textConfigurationOptions" }, "file": { - "$ref" : "#/definitions/fileConfigurationOptions" + "$ref": "#/definitions/fileConfigurationOptions" } } } } -} +} \ No newline at end of file diff --git a/tested/transform_json.py b/tested/transform_json.py index 0900a39f..c47d9cd9 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -1,12 +1,12 @@ import json import os import sys -from enum import Enum +from enum import StrEnum from pathlib import Path from typing import Any -class SpecialMap(Enum): +class SpecialMap(StrEnum): NATURAL_LANGUAGE = "natural_language" PROGRAMMING_LANGUAGE = "programming_language" ORACLE = "oracle" @@ -202,7 +202,7 @@ def transform_json(json_file: Path, monolingual: bool, strict: bool): if strict: file_name = "schema-strict.json" else: - file_name = "multilingual-schema.json" + file_name = "schema.json" with open(json_file.parent / file_name, "w", encoding="utf-8") as f: json.dump(result, f, indent=2) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index e72cd0e1..136748ca 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -795,7 +795,7 @@ def test_list_json_schema(): def test_transform_executed_correct(mocker: MockerFixture): s = mocker.spy( - tested.transform_json, name="transform_IDE" # type: ignore[reportAttributeAccessIssue] + tested.transform_json, name="transform_ide" # type: ignore[reportAttributeAccessIssue] ) mock_files = [ From 88b0196d45718e639a64a4636214810a0951f0a8 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 29 Apr 2025 17:17:16 +0200 Subject: [PATCH 12/23] small change --- tested/transform_json.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tested/transform_json.py b/tested/transform_json.py index c47d9cd9..ab1e059d 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -137,6 +137,7 @@ def transform_ide(data: Any) -> Any: data["yamlValue"] = { "description": "A value represented as YAML.", } + # This is a speciale structure for tags. if "required" in data and "properties" in data: From 2b65c0e42e9fb361471c58fb60b49d7f1f8b15b4 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 29 Apr 2025 17:20:59 +0200 Subject: [PATCH 13/23] fixed linting --- tested/transform_json.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tested/transform_json.py b/tested/transform_json.py index ab1e059d..c47d9cd9 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -137,7 +137,6 @@ def transform_ide(data: Any) -> Any: data["yamlValue"] = { "description": "A value represented as YAML.", } - # This is a speciale structure for tags. if "required" in data and "properties" in data: From 6149de9178fac39c8757778dfee108fbcf470e0b Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 29 Apr 2025 18:17:04 +0200 Subject: [PATCH 14/23] Wrote a few more tests --- tests/test_preprocess_dsl.py | 110 +++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 136748ca..588ae864 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -964,6 +964,116 @@ def test_json_rm_prog_lang_json_schema(): assert result == json_schema_expected +def test_strict_json_schema_oracle(): + json_schema = { + "oneOf": [ + {"$ref": "#/definitions/yamlValueOrPythonExpression"}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle", + }, + "value": { + "type": "object", + "additionalProperties": False, + "required": ["value"], + "properties": { + "oracle": {"const": "builtin"}, + "value": { + "oneOf": [ + { + "$ref": "#/definitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/yamlValueOrPythonExpression" + }, + }, + }, + }, + ] + }, + }, + }, + }, + }, + ] + } + + json_schema_expected = { + "oneOf": [ + {"$ref": "#/definitions/yamlValueOrPythonExpression"}, + { + "type": "oracle", + "additionalProperties": False, + "required": ["value"], + "properties": { + "oracle": {"const": "builtin"}, + "value": {"$ref": "#/definitions/yamlValueOrPythonExpression"}, + }, + }, + ] + } + + result = transform_monolingual(json_schema, True) + + assert result == json_schema_expected + + +def test_strict_json_schema_expression(): + json_schema = { + "oneOf": [ + {"$ref": "#/definitions/yamlValue"}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!expression", + }, + "value": { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + }, + }, + }, + ] + } + + json_schema_expected = { + "oneOf": [ + {"$ref": "#/definitions/yamlValue"}, + { + "type": "expression", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + }, + ] + } + + result = transform_monolingual(json_schema, True) + print(result) + + assert result == json_schema_expected + + def test_json_schema_edge_cases(): json_schema = { "expressionOrStatementWithNatTranslation": {}, From 1364569eb7a66122cdf0ee392991b84b5ed3fb31 Mon Sep 17 00:00:00 2001 From: Brent Blanckaert <125136419+BrentBlanckaert@users.noreply.github.com> Date: Tue, 29 Apr 2025 18:21:21 +0200 Subject: [PATCH 15/23] Update tested/transform_json.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tested/transform_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tested/transform_json.py b/tested/transform_json.py index c47d9cd9..29945bf2 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -70,7 +70,7 @@ def transform_monolingual(data: Any, strict_schema: bool) -> Any: element["type"] = "oracle" new_one_of.append(element) - # A programming_langauge map was found. If not strict, just remove. + # A programming_language map was found. If not strict, just remove. # If strict, still provide the type option for the corresponding object. if prog_lang is not None and strict_schema: new_one_of = [ From 60c215c2cf1b187d5ccbf7d55f165b4f386df297 Mon Sep 17 00:00:00 2001 From: Brent Blanckaert <125136419+BrentBlanckaert@users.noreply.github.com> Date: Tue, 29 Apr 2025 18:21:57 +0200 Subject: [PATCH 16/23] Update tested/transform_json.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tested/transform_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tested/transform_json.py b/tested/transform_json.py index 29945bf2..6d35c2af 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -85,7 +85,7 @@ def transform_monolingual(data: Any, strict_schema: bool) -> Any: else: data["oneOf"] = new_one_of - # THe next changes are a few edge cases. + # The next changes are a few edge cases. if "expressionOrStatementWithNatTranslation" in data: data.pop("expressionOrStatementWithNatTranslation") From 6f047c66ac23f9371b0b91ca995a8b45706d1a6d Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 29 Apr 2025 19:00:59 +0200 Subject: [PATCH 17/23] added an extra test --- tests/test_preprocess_dsl.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 588ae864..209b66ae 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -1113,3 +1113,13 @@ def test_json_schema_edge_cases(): result = transform_monolingual(json_schema, False) assert result == json_schema_expected + + +def test_exception_when_file_not_found(): + + try: + transform_json(Path("test.json"), False, False) + except FileNotFoundError: + print("As expected") + else: + assert False, "Expected FileNotFoundError error" From 4b301c02af54e9e9a7b318ed5fc3a0d9d6eb0f52 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 12 May 2025 11:54:34 +0200 Subject: [PATCH 18/23] update changes --- tested/dsl/schema-strict.json | 33 ++++++++----- tested/transform_json.py | 88 +++++++++++++++++++++++++++++++---- 2 files changed, 101 insertions(+), 20 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 66d8cc0e..66806c9d 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -417,21 +417,30 @@ "description": "A statement of expression in Python-like syntax as YAML string." }, { - "description": "Programming-language-specific statement or expression.", - "minProperties": 1, - "propertyNames": { - "$ref": "#/definitions/programmingLanguage" - }, - "items": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." - }, "anyOf": [ { - "type": "object" + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } }, { - "type": "programming_language" + "type": "programming_language", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } } ] } @@ -888,4 +897,4 @@ } } } -} \ No newline at end of file +} diff --git a/tested/transform_json.py b/tested/transform_json.py index 6d35c2af..18a471f6 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -175,6 +175,70 @@ def transform_ide(data: Any) -> Any: return data +SPECIAL_CASES = ["return", "stderr", "stdout", "file", "exception", "statement", "expression"] +SPECIAL_TYPES = ["expression", "programming_language", "oracle"] + +def make_tag_structure(data: Any, tag: str ="!natural_language") -> dict: + return { + "oneOf": [ + data, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": tag, + }, + "value": {"type": "object", "additionalProperties": data}, + }, + }, + ] + } + +def make_translations_map() -> dict: + return { + "type" : "object", + "description": "Define translations in the global scope." + } + + +def transform_json_translation(data: Any) -> Any: + if isinstance(data, dict): + if "type" in data and data["type"] != "object": + if data["type"] in SPECIAL_TYPES: + return make_tag_structure(data, f"!{data['type']}") + return make_tag_structure(data) + + for case in SPECIAL_CASES: + if case in data: + data[case] = make_tag_structure(data[case]) + + if "_rootObject" in data: + assert "properties" in data["_rootObject"] + data["_rootObject"]["properties"]["translations"] = make_translations_map() + + if "tab" in data and "properties" in data["tab"]: + data["tab"]["properties"]["translations"] = make_translations_map() + + if "unit" in data and "properties" in data["unit"]: + data["unit"]["properties"]["translations"] = make_translations_map() + + if "context" in data and "properties" in data["context"]: + data["context"]["properties"]["translations"] = make_translations_map() + + if "case" in data and "properties" in data["case"]: + data["case"]["properties"]["translations"] = make_translations_map() + + return {key: transform_json_translation(value) for key, value in data.items()} + + if isinstance(data, list): + return [transform_json_translation(value) for value in data] + return data + + + def transform_json(json_file: Path, monolingual: bool, strict: bool): """ This function transforms the JSON schema used in the DSL translator into @@ -211,12 +275,20 @@ def transform_json(json_file: Path, monolingual: bool, strict: bool): if __name__ == "__main__": n = len(sys.argv) assert n > 1, "Expected path to multilingual json schema." - monolingual = False - strict = False - if n > 2: - assert sys.argv[2] in ("strict", "not-strict") - strict = sys.argv[2] == "strict" - - monolingual = True + jf = Path(sys.argv[1]) + with open(jf, "r") as stream: + json_stream = json.load(stream) + res = transform_json_translation(json_stream) + fn = "multilingual-schema2.json" + with open(jf.parent / fn, "w", encoding="utf-8") as f: + json.dump(res, f, indent=2) - transform_json(Path(sys.argv[1]), monolingual=monolingual, strict=strict) + # monolingual = False + # strict = False + # if n > 2: + # assert sys.argv[2] in ("strict", "not-strict") + # strict = sys.argv[2] == "strict" +# + # monolingual = True +# + # transform_json(Path(sys.argv[1]), monolingual=monolingual, strict=strict) From d061c404a16431daea8f7ddcf9423e74224f4119 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 13 May 2025 14:26:14 +0200 Subject: [PATCH 19/23] commit current changes --- tested/dsl/schema-strict.json | 108 ++++++++++++++-------------- tested/transform_json.py | 130 +++++++++++++++++++++------------- 2 files changed, 135 insertions(+), 103 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 66806c9d..c486cf5e 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -43,7 +43,7 @@ "description": "A list of files used in the test suite.", "type": "array", "items": { - "$ref": "#/definitions/file" + "$ref": "#/subDefinitions/file" } }, "namespace": { @@ -60,7 +60,7 @@ "description": "Indicate that all code is in a specific language.", "oneOf": [ { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" }, { "const": "tested" @@ -72,7 +72,7 @@ "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" } } }, @@ -101,7 +101,7 @@ "description": "A list of files used in the test suite.", "type": "array", "items": { - "$ref": "#/definitions/file" + "$ref": "#/subDefinitions/file" } }, "hidden": { @@ -117,7 +117,7 @@ "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" } }, "oneOf": [ @@ -154,7 +154,7 @@ "description": "A list of files used in the test suite.", "type": "array", "items": { - "$ref": "#/definitions/file" + "$ref": "#/subDefinitions/file" } }, "hidden": { @@ -170,7 +170,7 @@ "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" } }, "oneOf": [ @@ -235,7 +235,7 @@ "description": "A list of files used in the test suite.", "type": "array", "items": { - "$ref": "#/definitions/file" + "$ref": "#/subDefinitions/file" } }, "context": { @@ -258,7 +258,7 @@ "description": "A list of files used in the test suite.", "type": "array", "items": { - "$ref": "#/definitions/file" + "$ref": "#/subDefinitions/file" } }, "context": { @@ -276,7 +276,7 @@ "additionalProperties": false, "properties": { "description": { - "$ref": "#/definitions/message" + "$ref": "#/subDefinitions/message" }, "stdin": { "description": "Stdin for this context", @@ -301,38 +301,38 @@ }, "statement": { "description": "The statement to evaluate.", - "$ref": "#/definitions/expressionOrStatement" + "$ref": "#/subDefinitions/expressionOrStatement" }, "expression": { "description": "The expression to evaluate.", - "$ref": "#/definitions/expressionOrStatement" + "$ref": "#/subDefinitions/expressionOrStatement" }, "exception": { "description": "Expected exception message", - "$ref": "#/definitions/exceptionChannel" + "$ref": "#/subDefinitions/exceptionChannel" }, "files": { "description": "A list of files used in the test suite.", "type": "array", "items": { - "$ref": "#/definitions/file" + "$ref": "#/subDefinitions/file" } }, "return": { "description": "Expected return value", - "$ref": "#/definitions/returnOutputChannel" + "$ref": "#/subDefinitions/returnOutputChannel" }, "stderr": { "description": "Expected output at stderr", - "$ref": "#/definitions/textOutputChannel" + "$ref": "#/subDefinitions/textOutputChannel" }, "stdout": { "description": "Expected output at stdout", - "$ref": "#/definitions/textOutputChannel" + "$ref": "#/subDefinitions/textOutputChannel" }, "file": { "description": "Expected files generated by the submission.", - "$ref": "#/definitions/fileOutputChannel" + "$ref": "#/subDefinitions/fileOutputChannel" }, "exit_code": { "type": "integer", @@ -345,7 +345,7 @@ "description": "An individual test (script) for a statement or expression", "properties": { "description": { - "$ref": "#/definitions/message" + "$ref": "#/subDefinitions/message" }, "stdin": { "description": "Stdin for this context", @@ -370,45 +370,47 @@ }, "statement": { "description": "The statement to evaluate.", - "$ref": "#/definitions/expressionOrStatement" + "$ref": "#/subDefinitions/expressionOrStatement" }, "expression": { "description": "The expression to evaluate.", - "$ref": "#/definitions/expressionOrStatement" + "$ref": "#/subDefinitions/expressionOrStatement" }, "exception": { "description": "Expected exception message", - "$ref": "#/definitions/exceptionChannel" + "$ref": "#/subDefinitions/exceptionChannel" }, "files": { "description": "A list of files used in the test suite.", "type": "array", "items": { - "$ref": "#/definitions/file" + "$ref": "#/subDefinitions/file" } }, "return": { "description": "Expected return value", - "$ref": "#/definitions/returnOutputChannel" + "$ref": "#/subDefinitions/returnOutputChannel" }, "stderr": { "description": "Expected output at stderr", - "$ref": "#/definitions/textOutputChannel" + "$ref": "#/subDefinitions/textOutputChannel" }, "stdout": { "description": "Expected output at stdout", - "$ref": "#/definitions/textOutputChannel" + "$ref": "#/subDefinitions/textOutputChannel" }, "file": { "description": "Expected files generated by the submission.", - "$ref": "#/definitions/fileOutputChannel" + "$ref": "#/subDefinitions/fileOutputChannel" }, "exit_code": { "type": "integer", "description": "Expected exit code for the run" } } - }, + } + }, + "subDefinitions" : { "expressionOrStatement": { "oneOf": [ { @@ -423,7 +425,7 @@ "description": "Programming-language-specific statement or expression.", "minProperties": 1, "propertyNames": { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" }, "items": { "type": "string", @@ -435,7 +437,7 @@ "description": "Programming-language-specific statement or expression.", "minProperties": 1, "propertyNames": { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" }, "items": { "type": "string", @@ -449,7 +451,7 @@ "yamlValueOrPythonExpression": { "oneOf": [ { - "$ref": "#/definitions/yamlValue" + "$ref": "#/subDefinitions/yamlValue" }, { "type": "expression", @@ -498,7 +500,7 @@ "description": "Language mapping of expected exception types.", "type": "object", "propertyNames": { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" }, "items": { "type": "string" @@ -511,7 +513,7 @@ "textOutputChannel": { "anyOf": [ { - "$ref": "#/definitions/textualType" + "$ref": "#/subDefinitions/textualType" }, { "type": "object", @@ -521,13 +523,13 @@ ], "properties": { "data": { - "$ref": "#/definitions/textualType" + "$ref": "#/subDefinitions/textualType" }, "oracle": { "const": "builtin" }, "config": { - "$ref": "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" } } }, @@ -541,7 +543,7 @@ ], "properties": { "data": { - "$ref": "#/definitions/textualType" + "$ref": "#/subDefinitions/textualType" }, "oracle": { "const": "custom_check" @@ -559,14 +561,14 @@ "type": "array", "description": "List of YAML (or tagged expression) values to use as arguments to the function.", "items": { - "$ref": "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } }, "languages": { "type": "array", "description": "Which programming languages are supported by this oracle.", "items": { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" } } } @@ -595,7 +597,7 @@ "const": "builtin" }, "config": { - "$ref": "#/definitions/fileConfigurationOptions" + "$ref": "#/subDefinitions/fileConfigurationOptions" } } }, @@ -633,14 +635,14 @@ "type": "array", "description": "List of YAML (or tagged expression) values to use as arguments to the function.", "items": { - "$ref": "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } }, "languages": { "type": "array", "description": "Which programming languages are supported by this oracle.", "items": { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" } } } @@ -650,7 +652,7 @@ "returnOutputChannel": { "oneOf": [ { - "$ref": "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, { "type": "oracle", @@ -663,7 +665,7 @@ "const": "builtin" }, "value": { - "$ref": "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } }, @@ -680,7 +682,7 @@ "const": "custom_check" }, "value": { - "$ref": "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, "file": { "type": "string", @@ -695,14 +697,14 @@ "description": "List of YAML (or tagged expression) values to use as arguments to the function.", "type": "array", "items": { - "$ref": "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } }, "languages": { "type": "array", "description": "Which programming languages are supported by this oracle.", "items": { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" } } } @@ -723,7 +725,7 @@ "description": "Language mapping of oracle functions.", "type": "object", "propertyNames": { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" }, "items": { "type": "object", @@ -748,7 +750,7 @@ "description": "Language mapping of oracle arguments.", "type": "object", "propertyNames": { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" }, "items": { "type": "array", @@ -760,7 +762,7 @@ } }, "value": { - "$ref": "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } } @@ -846,7 +848,7 @@ "fileConfigurationOptions": { "anyOf": [ { - "$ref": "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" }, { "type": "object", @@ -886,13 +888,13 @@ "type": "object", "properties": { "stdout": { - "$ref": "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" }, "stderr": { - "$ref": "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" }, "file": { - "$ref": "#/definitions/fileConfigurationOptions" + "$ref": "#/subDefinitions/fileConfigurationOptions" } } } diff --git a/tested/transform_json.py b/tested/transform_json.py index 18a471f6..a5ebb0f0 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -175,70 +175,100 @@ def transform_ide(data: Any) -> Any: return data -SPECIAL_CASES = ["return", "stderr", "stdout", "file", "exception", "statement", "expression"] +SPECIAL_CASES = [ + "return", + "stderr", + "stdout", + "file", + "exception", + "statement", + "expression", +] SPECIAL_TYPES = ["expression", "programming_language", "oracle"] -def make_tag_structure(data: Any, tag: str ="!natural_language") -> dict: - return { - "oneOf": [ - data, - { - "type": "object", - "required": ["__tag__", "value"], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": tag, - }, - "value": {"type": "object", "additionalProperties": data}, - }, +def flatten_one_of_stack(data: list) -> list: + new_one_stack = [] + for ele in data: + if isinstance(ele, dict) and len(ele.keys()) == 1 and "oneOf" in ele: + assert isinstance(ele["oneOf"], list) + new_one_stack.extend(ele["oneOf"]) + else: + new_one_stack.append(ele) + return new_one_stack + + +def make_tag_structure(data: Any, data_with_inner_translations: Any = None, tag: str = "!natural_language") -> dict: + base = { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": tag, }, - ] + "value": {"type": "object", "additionalProperties": data}, + }, } -def make_translations_map() -> dict: - return { - "type" : "object", - "description": "Define translations in the global scope." - } + if tag == "!natural_language" and data_with_inner_translations is not None: + return {"oneOf": [data_with_inner_translations, base]} + return base -def transform_json_translation(data: Any) -> Any: - if isinstance(data, dict): - if "type" in data and data["type"] != "object": - if data["type"] in SPECIAL_TYPES: - return make_tag_structure(data, f"!{data['type']}") - return make_tag_structure(data) - - for case in SPECIAL_CASES: - if case in data: - data[case] = make_tag_structure(data[case]) - if "_rootObject" in data: - assert "properties" in data["_rootObject"] - data["_rootObject"]["properties"]["translations"] = make_translations_map() - - if "tab" in data and "properties" in data["tab"]: - data["tab"]["properties"]["translations"] = make_translations_map() +def make_translations_map() -> dict: + return {"type": "object", "description": "Define translations in the global scope."} - if "unit" in data and "properties" in data["unit"]: - data["unit"]["properties"]["translations"] = make_translations_map() - if "context" in data and "properties" in data["context"]: - data["context"]["properties"]["translations"] = make_translations_map() +def transform_json_translation(data: Any, in_sub_def: bool) -> Any: + if isinstance(data, dict): + # Standard creation of the special tag structure for translations. + new_data = { + key: transform_json_translation( + value, in_sub_def or key == "subDefinitions" + ) + for key, value in data.items() + } + if "type" in data and (data["type"] != "object" or in_sub_def): + if data["type"] in SPECIAL_TYPES: + tag = data.pop("type") + new_data.pop("type") + # translations applied to inner part + tag_data_with_inner_translations = make_tag_structure( + new_data, + tag=f"!{tag}", + ) + # translations not applied to inner part + tag_data = make_tag_structure( + data, + tag=f"!{tag}", + ) + return make_tag_structure( + data=tag_data, + data_with_inner_translations=tag_data_with_inner_translations, + ) + + return make_tag_structure(data, new_data) + data = new_data + + # Add the translations maps + targets = ["_rootObject", "tab", "unit", "context", "case"] + for target in targets: + if target in data and "properties" in data[target]: + data[target]["properties"]["translations"] = make_translations_map() - if "case" in data and "properties" in data["case"]: - data["case"]["properties"]["translations"] = make_translations_map() + if "oneOf" in data: + assert isinstance(data["oneOf"], list) + data["oneOf"] = flatten_one_of_stack(data["oneOf"]) - return {key: transform_json_translation(value) for key, value in data.items()} + return data if isinstance(data, list): - return [transform_json_translation(value) for value in data] + return [transform_json_translation(value, in_sub_def) for value in data] return data - def transform_json(json_file: Path, monolingual: bool, strict: bool): """ This function transforms the JSON schema used in the DSL translator into @@ -278,7 +308,7 @@ def transform_json(json_file: Path, monolingual: bool, strict: bool): jf = Path(sys.argv[1]) with open(jf, "r") as stream: json_stream = json.load(stream) - res = transform_json_translation(json_stream) + res = transform_json_translation(json_stream, False) fn = "multilingual-schema2.json" with open(jf.parent / fn, "w", encoding="utf-8") as f: json.dump(res, f, indent=2) @@ -289,6 +319,6 @@ def transform_json(json_file: Path, monolingual: bool, strict: bool): # assert sys.argv[2] in ("strict", "not-strict") # strict = sys.argv[2] == "strict" # - # monolingual = True +# monolingual = True # - # transform_json(Path(sys.argv[1]), monolingual=monolingual, strict=strict) +# transform_json(Path(sys.argv[1]), monolingual=monolingual, strict=strict) From 76d67710ec98312e5d2dbe394c8474e57c3d94d6 Mon Sep 17 00:00:00 2001 From: breblanc Date: Wed, 14 May 2025 19:10:04 +0200 Subject: [PATCH 20/23] Made new scripts starting from schema-strict.json --- tested/dsl/multilingual-schema.json | 3813 +++++++++++++++ tested/dsl/schema-strict-nat-translation.json | 4139 +++++++++++------ tested/dsl/schema.json | 137 +- tested/transform_json.py | 286 +- tests/test_preprocess_dsl.py | 521 +-- 5 files changed, 7066 insertions(+), 1830 deletions(-) create mode 100644 tested/dsl/multilingual-schema.json diff --git a/tested/dsl/multilingual-schema.json b/tested/dsl/multilingual-schema.json new file mode 100644 index 00000000..b32f98d4 --- /dev/null +++ b/tested/dsl/multilingual-schema.json @@ -0,0 +1,3813 @@ +{ + "$id": "tested:dsl:schema7", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TESTed-DSL", + "oneOf": [ + { + "$ref": "#/definitions/_rootObject" + }, + { + "$ref": "#/definitions/_tabList" + }, + { + "$ref": "#/definitions/_unitList" + } + ], + "definitions": { + "_rootObject": { + "type": "object", + "oneOf": [ + { + "required": [ + "tabs" + ], + "not": { + "required": [ + "units" + ] + } + }, + { + "required": [ + "units" + ], + "not": { + "required": [ + "tabs" + ] + } + } + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "namespace": { + "oneOf": [ + { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "tabs": { + "$ref": "#/definitions/_tabList" + }, + "units": { + "$ref": "#/definitions/_unitList" + }, + "language": { + "description": "Indicate that all code is in a specific language.", + "oneOf": [ + { + "$ref": "#/subDefinitions/programmingLanguage" + }, + { + "const": "tested" + } + ] + }, + "definitions": { + "description": "Define hashes to use elsewhere.", + "type": "object" + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." + } + } + }, + "_tabList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/tab" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/tab" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "_unitList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/unit" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/unit" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "tab": { + "type": "object", + "description": "A tab in the test suite.", + "required": [ + "tab" + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" + }, + "tab": { + "oneOf": [ + { + "description": "The name of this tab.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The name of this tab.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." + } + }, + "oneOf": [ + { + "required": [ + "contexts" + ], + "properties": { + "contexts": { + "$ref": "#/definitions/_contextList" + } + } + }, + { + "required": [ + "testcases" + ], + "properties": { + "testcases": { + "$ref": "#/definitions/_testcaseList" + } + } + } + ] + }, + "unit": { + "type": "object", + "description": "A unit in the test suite.", + "required": [ + "unit" + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" + }, + "unit": { + "oneOf": [ + { + "description": "The name of this tab.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The name of this tab.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." + } + }, + "oneOf": [ + { + "required": [ + "cases" + ], + "properties": { + "cases": { + "$ref": "#/definitions/_caseList" + } + } + }, + { + "required": [ + "scripts" + ], + "properties": { + "scripts": { + "$ref": "#/definitions/_scriptList" + } + } + } + ] + }, + "_contextList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "_caseList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "_testcaseList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "_scriptList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/script" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/script" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "context": { + "type": "object", + "description": "A set of testcase in the same context.", + "required": [ + "testcases" + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "context": { + "oneOf": [ + { + "type": "string", + "description": "Description of this context." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Description of this context." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "testcases": { + "$ref": "#/definitions/_testcaseList" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." + } + } + }, + "case": { + "type": "object", + "description": "A test case.", + "required": [ + "script" + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "context": { + "oneOf": [ + { + "type": "string", + "description": "Description of this context." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Description of this context." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "script": { + "$ref": "#/definitions/_scriptList" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." + } + } + }, + "testcase": { + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { + "oneOf": [ + { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties": { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "description": "Array of program call arguments", + "type": "array", + "items": { + "oneOf": [ + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" + }, + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" + } + } + }, + "script": { + "type": "object", + "description": "An individual test (script) for a statement or expression", + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { + "oneOf": [ + { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties": { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "description": "Array of program call arguments", + "type": "array", + "items": { + "oneOf": [ + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" + }, + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" + } + } + } + }, + "subDefinitions": { + "expressionOrStatement": { + "oneOf": [ + { + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + }, + { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + } + ] + }, + "yamlValueOrPythonExpression": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValue" + }, + { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax." + } + ] + }, + "file": { + "oneOf": [ + { + "type": "object", + "description": "A file used in the test suite.", + "required": [ + "name", + "url" + ], + "properties": { + "name": { + "oneOf": [ + { + "type": "string", + "description": "The filename, including the file extension." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The filename, including the file extension." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "url": { + "oneOf": [ + { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "A file used in the test suite.", + "required": [ + "name", + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "The filename, including the file extension." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "exceptionChannel": { + "oneOf": [ + { + "type": "string", + "description": "Message of the expected exception." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Message of the expected exception." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + }, + { + "type": "object", + "required": [ + "types" + ], + "properties": { + "message": { + "oneOf": [ + { + "description": "Message of the expected exception.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "Message of the expected exception.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "types": { + "oneOf": [ + { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "types" + ], + "properties": { + "message": { + "description": "Message of the expected exception.", + "type": "string" + }, + "types": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string" + } + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "textOutputChannel": { + "anyOf": [ + { + "$ref": "#/subDefinitions/textualType" + }, + { + "oneOf": [ + { + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/textConfigurationOptions" + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/textConfigurationOptions" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "oneOf": [ + { + "type": "object", + "description": "Custom oracle for text values.", + "required": [ + "oracle", + "file", + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "custom_check" + }, + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "languages": { + "oneOf": [ + { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Custom oracle for text values.", + "required": [ + "oracle", + "file", + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "custom_check" + }, + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + }, + "fileOutputChannel": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "description": "Built-in oracle for files.", + "required": [ + "content", + "location" + ], + "properties": { + "content": { + "oneOf": [ + { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "location": { + "oneOf": [ + { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Built-in oracle for files.", + "required": [ + "content", + "location" + ], + "properties": { + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "oneOf": [ + { + "type": "object", + "description": "Custom oracle for file values.", + "required": [ + "oracle", + "content", + "location", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "content": { + "oneOf": [ + { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "location": { + "oneOf": [ + { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "languages": { + "oneOf": [ + { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Custom oracle for file values.", + "required": [ + "oracle", + "content", + "location", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + }, + "returnOutputChannel": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + }, + { + "additionalProperties": false, + "required": [ + "value" + ], + "properties": { + "oracle": { + "const": "builtin" + }, + "value": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + "type": "object" + }, + { + "type": "object", + "additionalProperties": { + "additionalProperties": false, + "required": [ + "value" + ], + "properties": { + "oracle": { + "const": "builtin" + }, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "type": "object" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + }, + { + "additionalProperties": false, + "required": [ + "value", + "oracle", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "value": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "languages": { + "oneOf": [ + { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + "type": "object" + }, + { + "type": "object", + "additionalProperties": { + "additionalProperties": false, + "required": [ + "value", + "oracle", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "arguments": { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + }, + "type": "object" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + }, + { + "additionalProperties": false, + "required": [ + "oracle", + "functions" + ], + "properties": { + "oracle": { + "const": "specific_check" + }, + "functions": { + "oneOf": [ + { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "value": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + "type": "object" + }, + { + "type": "object", + "additionalProperties": { + "additionalProperties": false, + "required": [ + "oracle", + "functions" + ], + "properties": { + "oracle": { + "const": "specific_check" + }, + "functions": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + }, + "arguments": { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + }, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "type": "object" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "programmingLanguage": { + "oneOf": [ + { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ + "bash", + "c", + "haskell", + "java", + "javascript", + "typescript", + "kotlin", + "python", + "runhaskell", + "csharp", + "cpp" + ] + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ + "bash", + "c", + "haskell", + "java", + "javascript", + "typescript", + "kotlin", + "python", + "runhaskell", + "csharp", + "cpp" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "message": { + "oneOf": [ + { + "type": "string", + "description": "A simple message to display." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A simple message to display." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + }, + { + "type": "object", + "required": [ + "description" + ], + "properties": { + "description": { + "oneOf": [ + { + "description": "The message to display.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The message to display.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "format": { + "oneOf": [ + { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "description" + ], + "properties": { + "description": { + "description": "The message to display.", + "type": "string" + }, + "format": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "textConfigurationOptions": { + "oneOf": [ + { + "type": "object", + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, + "properties": { + "applyRounding": { + "description": "Apply rounding when comparing as float", + "type": "boolean" + }, + "caseInsensitive": { + "description": "Ignore case when comparing strings", + "type": "boolean" + }, + "ignoreWhitespace": { + "description": "Ignore trailing whitespace", + "type": "boolean" + }, + "normalizeTrailingNewlines": { + "description": "Normalize trailing newlines", + "type": "boolean" + }, + "roundTo": { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + }, + "tryFloatingPoint": { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + }, + "hideExpected": { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, + "properties": { + "applyRounding": { + "description": "Apply rounding when comparing as float", + "type": "boolean" + }, + "caseInsensitive": { + "description": "Ignore case when comparing strings", + "type": "boolean" + }, + "ignoreWhitespace": { + "description": "Ignore trailing whitespace", + "type": "boolean" + }, + "normalizeTrailingNewlines": { + "description": "Normalize trailing newlines", + "type": "boolean" + }, + "roundTo": { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + }, + "tryFloatingPoint": { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + }, + "hideExpected": { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "fileConfigurationOptions": { + "anyOf": [ + { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + { + "oneOf": [ + { + "type": "object", + "properties": { + "mode": { + "oneOf": [ + { + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + }, + "textualType": { + "oneOf": [ + { + "description": "Simple textual value, converted to string.", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties": { + "description": "Simple textual value, converted to string.", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "yamlValue": { + "description": "A value represented as YAML." + }, + "inheritableConfigObject": { + "oneOf": [ + { + "type": "object", + "properties": { + "stdout": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "stderr": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "file": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "stdout": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "stderr": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "file": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } +} \ No newline at end of file diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json index 51f0647c..65083de8 100644 --- a/tested/dsl/schema-strict-nat-translation.json +++ b/tested/dsl/schema-strict-nat-translation.json @@ -1,71 +1,72 @@ { - "$id" : "tested:dsl:schema7", - "$schema" : "http://json-schema.org/draft-07/schema#", - "title" : "TESTed-DSL", - "oneOf" : [ + "$id": "tested:dsl:schema7", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TESTed-DSL", + "oneOf": [ { - "$ref" : "#/definitions/_rootObject" + "$ref": "#/definitions/_rootObject" }, { - "$ref" : "#/definitions/_tabList" + "$ref": "#/definitions/_tabList" }, { - "$ref" : "#/definitions/_unitList" + "$ref": "#/definitions/_unitList" } ], - "definitions" : { - "_rootObject" : { - "type" : "object", - "oneOf" : [ + "definitions": { + "_rootObject": { + "type": "object", + "oneOf": [ { - "required" : [ + "required": [ "tabs" ], - "not" : { - "required" : [ + "not": { + "required": [ "units" ] } }, { - "required" : [ + "required": [ "units" ], - "not" : { - "required" : [ + "not": { + "required": [ "tabs" ] } } ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "oneOf" : [ + "properties": { + "files": { + "oneOf": [ { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } } } @@ -73,68 +74,93 @@ } ] }, - "namespace" : { - "type" : "string", - "description" : "Namespace of the submitted solution, in `snake_case`" + "namespace": { + "oneOf": [ + { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" + } + } + } + } + ] }, - "tabs" : { - "$ref" : "#/definitions/_tabList" + "tabs": { + "$ref": "#/definitions/_tabList" }, - "units" : { - "$ref" : "#/definitions/_unitList" + "units": { + "$ref": "#/definitions/_unitList" }, - "language" : { - "description" : "Indicate that all code is in a specific language.", - "oneOf" : [ + "language": { + "description": "Indicate that all code is in a specific language.", + "oneOf": [ { - "$ref" : "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" }, { - "const" : "tested" + "const": "tested" } ] }, - "translations" : { - "type" : "object", - "description": "Define translations in the global scope." - }, - "definitions" : { - "description" : "Define hashes to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define hashes to use elsewhere.", + "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." } } }, - "_tabList" : { - "oneOf" : [ + "_tabList": { + "oneOf": [ { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/tab" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/tab" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/tab" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/tab" } } } @@ -142,34 +168,34 @@ } ] }, - "_unitList" : { - "oneOf" : [ + "_unitList": { + "oneOf": [ { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/unit" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/unit" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/unit" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/unit" } } } @@ -177,40 +203,41 @@ } ] }, - "tab" : { - "type" : "object", - "description" : "A tab in the test suite.", - "required" : [ + "tab": { + "type": "object", + "description": "A tab in the test suite.", + "required": [ "tab" ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "oneOf" : [ + "properties": { + "files": { + "oneOf": [ { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } } } @@ -218,107 +245,109 @@ } ] }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "tab" : { - "oneOf" : [ + "tab": { + "oneOf": [ { - "type" : "string" + "description": "The name of this tab.", + "type": "string" }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "string" + "description": "The name of this tab.", + "type": "string" } } } } - ], - "description" : "The name of this tab." - }, - "translations" : { - "type" : "object", - "description": "Define translations in the tab scope." + ] }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "contexts" ], - "properties" : { - "contexts" : { - "$ref" : "#/definitions/_contextList" + "properties": { + "contexts": { + "$ref": "#/definitions/_contextList" } } }, { - "required" : [ + "required": [ "testcases" ], - "properties" : { - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "properties": { + "testcases": { + "$ref": "#/definitions/_testcaseList" } } } ] }, - "unit" : { - "type" : "object", - "description" : "A unit in the test suite.", - "required" : [ + "unit": { + "type": "object", + "description": "A unit in the test suite.", + "required": [ "unit" ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "oneOf" : [ + "properties": { + "files": { + "oneOf": [ { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } } } @@ -326,103 +355,102 @@ } ] }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "unit" : { - "oneOf" : [ + "unit": { + "oneOf": [ { - "type" : "string", - "description" : "The name of this tab." + "description": "The name of this tab.", + "type": "string" }, { - "type" : "object", - "description" : "The name of this tab.", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "string" + "description": "The name of this tab.", + "type": "string" } } } } - ], - "description" : "The name of this tab." - }, - "translations" : { - "type" : "object", - "description": "Define translations in the unit scope." + ] }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "cases" ], - "properties" : { - "cases" : { - "$ref" : "#/definitions/_caseList" + "properties": { + "cases": { + "$ref": "#/definitions/_caseList" } } }, { - "required" : [ + "required": [ "scripts" ], - "properties" : { - "scripts" : { - "$ref" : "#/definitions/_scriptList" + "properties": { + "scripts": { + "$ref": "#/definitions/_scriptList" } } } ] }, - "_contextList" : { - "oneOf" : [ + "_contextList": { + "oneOf": [ { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/context" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/context" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" } } } @@ -430,34 +458,34 @@ } ] }, - "_caseList" : { - "oneOf" : [ + "_caseList": { + "oneOf": [ { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/case" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/case" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" } } } @@ -465,34 +493,34 @@ } ] }, - "_testcaseList" : { - "oneOf" : [ + "_testcaseList": { + "oneOf": [ { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/testcase" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/testcase" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" } } } @@ -500,34 +528,34 @@ } ] }, - "_scriptList" : { - "oneOf" : [ + "_scriptList": { + "oneOf": [ { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/script" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/script" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/script" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/script" } } } @@ -535,40 +563,41 @@ } ] }, - "context" : { - "type" : "object", - "description" : "A set of testcase in the same context.", - "required" : [ + "context": { + "type": "object", + "description": "A set of testcase in the same context.", + "required": [ "testcases" ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "oneOf" : [ + "properties": { + "files": { + "oneOf": [ { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } } } @@ -576,53 +605,79 @@ } ] }, - "translations" : { - "type" : "object", - "description": "Define translations in the context scope." + "context": { + "oneOf": [ + { + "type": "string", + "description": "Description of this context." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Description of this context." + } + } + } + } + ] }, - "context" : { - "type" : "string", - "description" : "Description of this context." + "testcases": { + "$ref": "#/definitions/_testcaseList" }, - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "translations": { + "type": "object", + "description": "Define translations in the global scope." } } }, - "case" : { - "type" : "object", - "description" : "A test case.", - "required" : [ + "case": { + "type": "object", + "description": "A test case.", + "required": [ "script" ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "oneOf" : [ + "properties": { + "files": { + "oneOf": [ { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } } } @@ -630,56 +685,57 @@ } ] }, - "translations" : { - "type" : "object", - "description": "Define translations in the case scope." - }, - "context" : { - "type" : "string", - "description" : "Description of this context." - }, - "script" : { - "$ref" : "#/definitions/_scriptList" - } - } - }, - "testcase" : { - "type" : "object", - "description" : "An individual test for a statement or expression", - "additionalProperties" : false, - "properties" : { - "description" : { + "context": { "oneOf": [ { - "$ref" : "#/definitions/message" + "type": "string", + "description": "Description of this context." }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/message" + "type": "string", + "description": "Description of this context." } } } } ] }, - "stdin" : { - "description" : "Stdin for this context", + "script": { + "$ref": "#/definitions/_scriptList" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." + } + } + }, + "testcase": { + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { "oneOf": [ { - "type" : [ + "description": "Stdin for this context", + "type": [ "string", "number", "integer", @@ -687,21 +743,22 @@ ] }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : [ + "description": "Stdin for this context", + "type": [ "string", "number", "integer", @@ -713,37 +770,68 @@ } ] }, - "arguments" : { + "arguments": { "oneOf": [ { - "type" : "array", - "items" : { - "type" : [ - "string", - "number", - "integer", - "boolean" + "description": "Array of program call arguments", + "type": "array", + "items": { + "oneOf": [ + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + } + } + } ] } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "type" : [ + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ "string", "number", "integer", @@ -754,72 +842,48 @@ } } } - ], - "description" : "Array of program call arguments" + ] }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "$ref" : "#/definitions/exceptionChannel" - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/exceptionChannel" - } - } - } - } - ] + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" }, - "files" : { - "description" : "A list of files used in the test suite.", - "oneOf" : [ + "files": { + "oneOf": [ { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } } } @@ -827,1035 +891,1842 @@ } ] }, - "return" : { - "description" : "Expected return value", - "oneOf" : [ + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" + } + } + }, + "script": { + "type": "object", + "description": "An individual test (script) for a statement or expression", + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { + "oneOf": [ { - "$ref" : "#/definitions/returnOutputChannel" + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/returnOutputChannel" + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] } } } } ] - }, - "stderr" : { - "description" : "Expected output at stderr", - "oneOf" : [ - { - "$ref" : "#/definitions/textOutputChannel" - }, + "arguments": { + "oneOf": [ { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/textOutputChannel" + "description": "Array of program call arguments", + "type": "array", + "items": { + "oneOf": [ + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + } + } } - } + ] } - } - ] - }, - "stdout" : { - "description" : "Expected output at stdout", - "oneOf" : [ - { - "$ref" : "#/definitions/textOutputChannel" }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/textOutputChannel" + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } } } } } ] }, - "file": { - "description" : "Expected files generated by the submission.", - "oneOf" : [ + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" + }, + "files": { + "oneOf": [ { - "$ref" : "#/definitions/fileOutputChannel" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/fileOutputChannel" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } } } } } ] }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" } } - }, - "script" : { - "type" : "object", - "description" : "An individual test (script) for a statement or expression", - "properties" : { - "description" : { - "oneOf": [ - { - "$ref" : "#/definitions/message" + } + }, + "subDefinitions": { + "expressionOrStatement": { + "oneOf": [ + { + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/message" - } - } + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." } } - ] + } }, - "stdin" : { - "description" : "Stdin for this context", - "oneOf": [ - { - "type" : [ - "string", - "number", - "integer", - "boolean" - ] - }, + { + "anyOf": [ { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ + "oneOf": [ + { "type": "object", - "additionalProperties": { - "type" : [ - "string", - "number", - "integer", - "boolean" + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + } + } ] } - } - } - } - ] - }, - "arguments" : { - "oneOf": [ - { - "type" : "array", - "items" : { - "type" : [ - "string", - "number", - "integer", - "boolean" - ] - } - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" }, - "value":{ + { "type": "object", - "additionalProperties": { - "type" : "array", - "items" : { - "type" : [ - "string", - "number", - "integer", - "boolean" - ] + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } } } } - } - } - ], - "description" : "Array of program call arguments" - }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" - }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" - }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "$ref" : "#/definitions/exceptionChannel" - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/exceptionChannel" - } - } - } - } - ] - }, - "files" : { - "description" : "A list of files used in the test suite.", - "oneOf" : [ - { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" - } + ] }, { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ + "oneOf": [ + { "type": "object", - "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!programming_language" + }, + "value": { + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + } + } + ] + }, + "type": "object" } } - } - } - } - ] - }, - "return" : { - "description" : "Expected return value", - "oneOf" : [ - { - "$ref" : "#/definitions/returnOutputChannel" - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/returnOutputChannel" - } - } - } - } - ] - - }, - "stderr" : { - "description" : "Expected output at stderr", - "oneOf" : [ - { - "$ref" : "#/definitions/textOutputChannel" - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/textOutputChannel" - } - } - } - } - ] - }, - "stdout" : { - "description" : "Expected output at stdout", - "oneOf" : [ - { - "$ref" : "#/definitions/textOutputChannel" - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/textOutputChannel" - } - } - } - } - ] - }, - "file": { - "description" : "Expected files generated by the submission.", - "oneOf" : [ - { - "$ref" : "#/definitions/fileOutputChannel" - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" }, - "value":{ + { "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/textOutputChannel" + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!programming_language" + }, + "value": { + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + "type": "object" + } + } + } + } } } - } + ] } ] - }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" } - } + ] }, - "expressionOrStatement" : { - "oneOf" : [ + "yamlValueOrPythonExpression": { + "oneOf": [ { - "type" : "string", - "format" : "tested-dsl-expression", - "description" : "A statement of expression in Python-like syntax as YAML string." + "$ref": "#/subDefinitions/yamlValue" }, { "type": "object", - "description" : "Programming-language-specific statement or expression.", - "minProperties" : 1, - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - } - }, - { - "type" : "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!programming_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!expression" }, - "value":{ - "type": "object", - "description" : "Programming-language-specific statement or expression.", - "minProperties" : 1, - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - } + "value": { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax." } } } ] }, - "expressionOrStatementWithNatTranslation" : { - "oneOf" : [ - { - "type" : "string", - "format" : "tested-dsl-expression", - "description" : "A statement of expression in Python-like syntax as YAML string." - }, + "file": { + "oneOf": [ { "type": "object", - "description" : "Programming-language-specific statement or expression.", - "minProperties" : 1, - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "oneOf" : [ - { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "description": "A file used in the test suite.", + "required": [ + "name", + "url" + ], + "properties": { + "name": { + "oneOf": [ + { + "type": "string", + "description": "The filename, including the file extension." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The filename, including the file extension." + } } } } - } - ] - } - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!programming_language" + ] }, - "value":{ - "type": "object", - "description" : "Programming-language-specific statement or expression.", - "minProperties" : 1, - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "oneOf" : [ - { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - } + "url": { + "oneOf": [ + { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." } } } - ] - } + } + ] } } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/expressionOrStatement" + "type": "object", + "description": "A file used in the test suite.", + "required": [ + "name", + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "The filename, including the file extension." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." + } + } } } } } ] }, - "yamlValueOrPythonExpression" : { - "oneOf" : [ + "exceptionChannel": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValue" + "type": "string", + "description": "Message of the expected exception." }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!expression" - }, - "value":{ "type": "string", - "format" : "tested-dsl-expression", - "description" : "An expression in Python-syntax." + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Message of the expected exception." + } } } - } - ] - }, - "file" : { - "type" : "object", - "description" : "A file used in the test suite.", - "required" : [ - "name", - "url" - ], - "properties" : { - "name" : { - "type" : "string", - "description" : "The filename, including the file extension." - }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." - } - } - }, - "exceptionChannel" : { - "oneOf" : [ - { - "type" : "string", - "description" : "Message of the expected exception." }, { - "type" : "object", - "required" : [ + "type": "object", + "required": [ "types" ], - "properties" : { - "message" : { - "oneOf" : [ + "properties": { + "message": { + "oneOf": [ { - "type" : "string" + "description": "Message of the expected exception.", + "type": "string" }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "string" + "description": "Message of the expected exception.", + "type": "string" } } } } - ], - "description" : "Message of the expected exception." + ] }, - "types" : { - "minProperties" : 1, - "description" : "Language mapping of expected exception types.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string" + "types": { + "oneOf": [ + { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string" + } + } + } + } + } + ] + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "types" + ], + "properties": { + "message": { + "description": "Message of the expected exception.", + "type": "string" + }, + "types": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string" + } + } + } } } } } ] }, - "textOutputChannel" : { - "anyOf" : [ + "textOutputChannel": { + "anyOf": [ { - "$ref" : "#/definitions/textualType" + "$ref": "#/subDefinitions/textualType" }, { - "type" : "object", - "description" : "Built-in oracle for text values.", - "required" : [ - "data" - ], - "properties" : { - "data" : { - "oneOf" : [ - { - "$ref" : "#/definitions/textualType" + "oneOf": [ + { + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/textConfigurationOptions" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/textConfigurationOptions" + } + } + } + } + } + } + ] + }, + { + "oneOf": [ + { + "type": "object", + "description": "Custom oracle for text values.", + "required": [ + "oracle", + "file", + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "custom_check" + }, + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + } + } + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + } + ] + }, + "languages": { + "oneOf": [ + { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + } + } + } + ] + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Custom oracle for text values.", + "required": [ + "oracle", + "file", + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "custom_check" + }, + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + } + } + } + } + } + ] + } + ] + }, + "fileOutputChannel": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "description": "Built-in oracle for files.", + "required": [ + "content", + "location" + ], + "properties": { + "content": { + "oneOf": [ + { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + } + } + } + } + ] + }, + "location": { + "oneOf": [ + { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + } + } + } + } + ] + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Built-in oracle for files.", + "required": [ + "content", + "location" + ], + "properties": { + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } + } + } + } + } + ] + }, + { + "oneOf": [ + { + "type": "object", + "description": "Custom oracle for file values.", + "required": [ + "oracle", + "content", + "location", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "content": { + "oneOf": [ + { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + } + } + } + } + ] + }, + "location": { + "oneOf": [ + { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + } + } + } + } + ] + }, + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + } + } + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + } + ] + }, + "languages": { + "oneOf": [ + { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } }, - "value":{ + { "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/textualType" + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + } } } - } + ] } - ] - }, - "oracle" : { - "const" : "builtin" + } }, - "config" : { - "$ref" : "#/definitions/textConfigurationOptions" - } - } - }, - { - "type" : "object", - "description" : "Custom oracle for text values.", - "required" : [ - "oracle", - "file", - "data" - ], - "properties" : { - "data" : { - "oneOf" : [ - { - "$ref" : "#/definitions/textualType" + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/textualType" + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Custom oracle for file values.", + "required": [ + "oracle", + "content", + "location", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } } } } } - ] - }, - "oracle" : { - "const" : "custom_check" - }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." - }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" - }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" - } - }, - "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" } } - } + ] } ] }, - "fileOutputChannel": { - "anyOf" : [ + "returnOutputChannel": { + "oneOf": [ { - "type" : "object", - "description" : "Built-in oracle for files.", - "required" : [ - "content", - "location" - ], - "properties" : { - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." - }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." - }, - "oracle" : { - "const" : "builtin" - }, - "config" : { - "$ref" : "#/definitions/fileConfigurationOptions" - } - } + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, { - "type" : "object", - "description" : "Custom oracle for file values.", - "required" : [ - "oracle", - "content", - "location", - "file" + "type": "object", + "required": [ + "__tag__", + "value" ], - "properties" : { - "oracle" : { - "const" : "custom_check" - }, - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." - }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." - }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." - }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" - }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" - } + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } } - } - ] - }, - "returnOutputChannel" : { - "oneOf" : [ - { - "$ref" : "#/definitions/yamlValueOrPythonExpression" }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!oracle" + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" }, - "value":{ - "type": "object", - "additionalProperties" : false, - "required" : [ + "value": { + "additionalProperties": false, + "required": [ "value" ], - "properties" : { - "oracle" : { - "const" : "builtin" + "properties": { + "oracle": { + "const": "builtin" }, - "value" : { - "oneOf" : [ + "value": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } } } ] } - } + }, + "type": "object" } } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!oracle" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", - "additionalProperties" : false, - "required" : [ + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" + }, + "value": { + "additionalProperties": false, + "required": [ + "value" + ], + "properties": { + "oracle": { + "const": "builtin" + }, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "type": "object" + } + } + } + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" + }, + "value": { + "additionalProperties": false, + "required": [ "value", "oracle", "file" ], - "properties" : { - "oracle" : { - "const" : "custom_check" + "properties": { + "oracle": { + "const": "custom_check" }, - "value" : { - "oneOf" : [ + "value": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } } } ] }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + } + } + } + } + ] }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + ] }, - "arguments" : { - "oneOf" : [ + "arguments": { + "oneOf": [ { - "type" : "array", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + ] } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } } } } - ], - "description" : "List of YAML (or tagged expression) values to use as arguments to the function." + ] }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "oneOf": [ + { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + } + } + } + ] + } + }, + "type": "object" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" + }, + "value": { + "additionalProperties": false, + "required": [ + "value", + "oracle", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "arguments": { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + }, + "type": "object" } } } @@ -1863,98 +2734,286 @@ } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!oracle" + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" }, - "value":{ - "type": "object", - "additionalProperties" : false, - "required" : [ + "value": { + "additionalProperties": false, + "required": [ "oracle", "functions" ], - "properties" : { - "oracle" : { - "const" : "specific_check" + "properties": { + "oracle": { + "const": "specific_check" }, - "functions" : { - "minProperties" : 1, - "description" : "Language mapping of oracle functions.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "object", - "required" : [ - "file" - ], - "properties" : { - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "functions": { + "oneOf": [ + { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "items": { + "oneOf": [ + { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + } + } + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + ] + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + } } } - } + ] }, - "arguments" : { - "oneOf" : [ + "arguments": { + "oneOf": [ { - "minProperties" : 1, - "description" : "Language mapping of oracle arguments.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" }, - "items" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - } + "items": { + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + } + } + } + ] } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "minProperties" : 1, - "description" : "Language mapping of oracle arguments.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" }, - "items" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." } } } @@ -1963,188 +3022,586 @@ } ] }, - "value" : { - "oneOf" : [ + "value": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } } } ] } + }, + "type": "object" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" + }, + "value": { + "additionalProperties": false, + "required": [ + "oracle", + "functions" + ], + "properties": { + "oracle": { + "const": "specific_check" + }, + "functions": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + }, + "arguments": { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + }, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "type": "object" + } + } } } } } ] }, - "programmingLanguage" : { - "type" : "string", - "description" : "One of the programming languages supported by TESTed.", - "enum" : [ - "bash", - "c", - "haskell", - "java", - "javascript", - "typescript", - "kotlin", - "python", - "runhaskell", - "csharp", - "cpp" + "programmingLanguage": { + "oneOf": [ + { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ + "bash", + "c", + "haskell", + "java", + "javascript", + "typescript", + "kotlin", + "python", + "runhaskell", + "csharp", + "cpp" + ] + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ + "bash", + "c", + "haskell", + "java", + "javascript", + "typescript", + "kotlin", + "python", + "runhaskell", + "csharp", + "cpp" + ] + } + } + } + } ] }, - "message" : { - "oneOf" : [ + "message": { + "oneOf": [ + { + "type": "string", + "description": "A simple message to display." + }, { - "type" : "string", - "description" : "A simple message to display." + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A simple message to display." + } + } + } }, { - "type" : "object", - "required" : [ + "type": "object", + "required": [ "description" ], - "properties" : { - "description" : { + "properties": { + "description": { "oneOf": [ { - "type" : "object", + "description": "The message to display.", + "type": "string" + }, + { + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "string" + "description": "The message to display.", + "type": "string" } } } + } + ] + }, + "format": { + "oneOf": [ + { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." }, { - "type" : "string" + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + } + } + } } - ], - "description" : "The message to display." + ] + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "format" : { - "type" : "string", - "default" : "text", - "description" : "The format of the message, either a programming language, 'text' or 'html'." + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "description" + ], + "properties": { + "description": { + "description": "The message to display.", + "type": "string" + }, + "format": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + } + } + } } } } ] }, - "textConfigurationOptions" : { - "type" : "object", - "description" : "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", - "minProperties" : 1, - "properties" : { - "applyRounding" : { - "description" : "Apply rounding when comparing as float", - "type" : "boolean" - }, - "caseInsensitive" : { - "description" : "Ignore case when comparing strings", - "type" : "boolean" - }, - "ignoreWhitespace" : { - "description" : "Ignore trailing whitespace", - "type" : "boolean" - }, - "normalizeTrailingNewlines" : { - "description" : "Normalize trailing newlines", - "type" : "boolean" - }, - "roundTo" : { - "description" : "The number of decimals to round at, when applying the rounding on floats", - "type" : "integer" - }, - "tryFloatingPoint" : { - "description" : "Try comparing text as floating point numbers", - "type" : "boolean" - }, - "hideExpected" : { - "description" : "Hide the expected value in feedback (default: false), not recommended to use!", - "type" : "boolean" + "textConfigurationOptions": { + "oneOf": [ + { + "type": "object", + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, + "properties": { + "applyRounding": { + "description": "Apply rounding when comparing as float", + "type": "boolean" + }, + "caseInsensitive": { + "description": "Ignore case when comparing strings", + "type": "boolean" + }, + "ignoreWhitespace": { + "description": "Ignore trailing whitespace", + "type": "boolean" + }, + "normalizeTrailingNewlines": { + "description": "Normalize trailing newlines", + "type": "boolean" + }, + "roundTo": { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + }, + "tryFloatingPoint": { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + }, + "hideExpected": { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, + "properties": { + "applyRounding": { + "description": "Apply rounding when comparing as float", + "type": "boolean" + }, + "caseInsensitive": { + "description": "Ignore case when comparing strings", + "type": "boolean" + }, + "ignoreWhitespace": { + "description": "Ignore trailing whitespace", + "type": "boolean" + }, + "normalizeTrailingNewlines": { + "description": "Normalize trailing newlines", + "type": "boolean" + }, + "roundTo": { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + }, + "tryFloatingPoint": { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + }, + "hideExpected": { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" + } + } + } + } + } } - } + ] }, "fileConfigurationOptions": { - "anyOf" : [ + "anyOf": [ { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" }, { - "type" : "object", - "properties" : { - "mode": { - "type" : "string", - "enum" : ["full", "line"], - "default" : "full" + "oneOf": [ + { + "type": "object", + "properties": { + "mode": { + "oneOf": [ + { + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" + } + } + } + } + ] + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" + } + } + } + } + } } - } + ] } ] }, - "textualType" : { - "description" : "Simple textual value, converted to string.", - "type" : [ - "string", - "number", - "integer", - "boolean" + "textualType": { + "oneOf": [ + { + "description": "Simple textual value, converted to string.", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "Simple textual value, converted to string.", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + } + } + } ] }, - "yamlValue" : { - "description" : "A value represented as YAML.", - "not" : { + "yamlValue": { + "description": "A value represented as YAML.", + "not": { "properties": { - "__tag__": { "type": "string" } + "__tag__": { + "type": "string" + } }, "type": "object" } }, "inheritableConfigObject": { - "type": "object", - "properties" : { - "stdout": { - "$ref" : "#/definitions/textConfigurationOptions" - }, - "stderr": { - "$ref" : "#/definitions/textConfigurationOptions" + "oneOf": [ + { + "type": "object", + "properties": { + "stdout": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "stderr": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "file": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } }, - "file": { - "$ref" : "#/definitions/fileConfigurationOptions" + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "stdout": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "stderr": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "file": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } + } + } + } } - } + ] } } -} +} \ No newline at end of file diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 6d8943c4..23125eb3 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -43,7 +43,7 @@ "description": "A list of files used in the test suite.", "type": "array", "items": { - "$ref": "#/definitions/file" + "$ref": "#/subDefinitions/file" } }, "namespace": { @@ -60,7 +60,7 @@ "description": "Indicate that all code is in a specific language.", "oneOf": [ { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" }, { "const": "tested" @@ -72,7 +72,7 @@ "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" } } }, @@ -101,7 +101,7 @@ "description": "A list of files used in the test suite.", "type": "array", "items": { - "$ref": "#/definitions/file" + "$ref": "#/subDefinitions/file" } }, "hidden": { @@ -117,7 +117,7 @@ "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" } }, "oneOf": [ @@ -154,7 +154,7 @@ "description": "A list of files used in the test suite.", "type": "array", "items": { - "$ref": "#/definitions/file" + "$ref": "#/subDefinitions/file" } }, "hidden": { @@ -170,7 +170,7 @@ "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" } }, "oneOf": [ @@ -235,7 +235,7 @@ "description": "A list of files used in the test suite.", "type": "array", "items": { - "$ref": "#/definitions/file" + "$ref": "#/subDefinitions/file" } }, "context": { @@ -258,7 +258,7 @@ "description": "A list of files used in the test suite.", "type": "array", "items": { - "$ref": "#/definitions/file" + "$ref": "#/subDefinitions/file" } }, "context": { @@ -276,7 +276,7 @@ "additionalProperties": false, "properties": { "description": { - "$ref": "#/definitions/message" + "$ref": "#/subDefinitions/message" }, "stdin": { "description": "Stdin for this context", @@ -301,38 +301,38 @@ }, "statement": { "description": "The statement to evaluate.", - "$ref": "#/definitions/expressionOrStatement" + "$ref": "#/subDefinitions/expressionOrStatement" }, "expression": { "description": "The expression to evaluate.", - "$ref": "#/definitions/expressionOrStatement" + "$ref": "#/subDefinitions/expressionOrStatement" }, "exception": { "description": "Expected exception message", - "$ref": "#/definitions/exceptionChannel" + "$ref": "#/subDefinitions/exceptionChannel" }, "files": { "description": "A list of files used in the test suite.", "type": "array", "items": { - "$ref": "#/definitions/file" + "$ref": "#/subDefinitions/file" } }, "return": { "description": "Expected return value", - "$ref": "#/definitions/returnOutputChannel" + "$ref": "#/subDefinitions/returnOutputChannel" }, "stderr": { "description": "Expected output at stderr", - "$ref": "#/definitions/textOutputChannel" + "$ref": "#/subDefinitions/textOutputChannel" }, "stdout": { "description": "Expected output at stdout", - "$ref": "#/definitions/textOutputChannel" + "$ref": "#/subDefinitions/textOutputChannel" }, "file": { "description": "Expected files generated by the submission.", - "$ref": "#/definitions/fileOutputChannel" + "$ref": "#/subDefinitions/fileOutputChannel" }, "exit_code": { "type": "integer", @@ -345,7 +345,7 @@ "description": "An individual test (script) for a statement or expression", "properties": { "description": { - "$ref": "#/definitions/message" + "$ref": "#/subDefinitions/message" }, "stdin": { "description": "Stdin for this context", @@ -370,45 +370,47 @@ }, "statement": { "description": "The statement to evaluate.", - "$ref": "#/definitions/expressionOrStatement" + "$ref": "#/subDefinitions/expressionOrStatement" }, "expression": { "description": "The expression to evaluate.", - "$ref": "#/definitions/expressionOrStatement" + "$ref": "#/subDefinitions/expressionOrStatement" }, "exception": { "description": "Expected exception message", - "$ref": "#/definitions/exceptionChannel" + "$ref": "#/subDefinitions/exceptionChannel" }, "files": { "description": "A list of files used in the test suite.", "type": "array", "items": { - "$ref": "#/definitions/file" + "$ref": "#/subDefinitions/file" } }, "return": { "description": "Expected return value", - "$ref": "#/definitions/returnOutputChannel" + "$ref": "#/subDefinitions/returnOutputChannel" }, "stderr": { "description": "Expected output at stderr", - "$ref": "#/definitions/textOutputChannel" + "$ref": "#/subDefinitions/textOutputChannel" }, "stdout": { "description": "Expected output at stdout", - "$ref": "#/definitions/textOutputChannel" + "$ref": "#/subDefinitions/textOutputChannel" }, "file": { "description": "Expected files generated by the submission.", - "$ref": "#/definitions/fileOutputChannel" + "$ref": "#/subDefinitions/fileOutputChannel" }, "exit_code": { "type": "integer", "description": "Expected exit code for the run" } } - }, + } + }, + "subDefinitions": { "expressionOrStatement": { "oneOf": [ { @@ -417,23 +419,27 @@ "description": "A statement of expression in Python-like syntax as YAML string." }, { - "type": "object", - "description": "Programming-language-specific statement or expression.", - "minProperties": 1, - "propertyNames": { - "$ref": "#/definitions/programmingLanguage" - }, - "items": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." - } + "anyOf": [ + { + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + ] } ] }, "yamlValueOrPythonExpression": { "oneOf": [ { - "$ref": "#/definitions/yamlValue" + "$ref": "#/subDefinitions/yamlValue" }, { "type": "string", @@ -482,7 +488,7 @@ "description": "Language mapping of expected exception types.", "type": "object", "propertyNames": { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" }, "items": { "type": "string" @@ -495,7 +501,7 @@ "textOutputChannel": { "anyOf": [ { - "$ref": "#/definitions/textualType" + "$ref": "#/subDefinitions/textualType" }, { "type": "object", @@ -505,13 +511,13 @@ ], "properties": { "data": { - "$ref": "#/definitions/textualType" + "$ref": "#/subDefinitions/textualType" }, "oracle": { "const": "builtin" }, "config": { - "$ref": "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" } } }, @@ -525,7 +531,7 @@ ], "properties": { "data": { - "$ref": "#/definitions/textualType" + "$ref": "#/subDefinitions/textualType" }, "oracle": { "const": "custom_check" @@ -543,14 +549,14 @@ "type": "array", "description": "List of YAML (or tagged expression) values to use as arguments to the function.", "items": { - "$ref": "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } }, "languages": { "type": "array", "description": "Which programming languages are supported by this oracle.", "items": { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" } } } @@ -579,7 +585,7 @@ "const": "builtin" }, "config": { - "$ref": "#/definitions/fileConfigurationOptions" + "$ref": "#/subDefinitions/fileConfigurationOptions" } } }, @@ -617,14 +623,14 @@ "type": "array", "description": "List of YAML (or tagged expression) values to use as arguments to the function.", "items": { - "$ref": "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } }, "languages": { "type": "array", "description": "Which programming languages are supported by this oracle.", "items": { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" } } } @@ -634,7 +640,7 @@ "returnOutputChannel": { "oneOf": [ { - "$ref": "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, { "type": "object", @@ -647,7 +653,7 @@ "const": "builtin" }, "value": { - "$ref": "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } }, @@ -664,7 +670,7 @@ "const": "custom_check" }, "value": { - "$ref": "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, "file": { "type": "string", @@ -679,14 +685,14 @@ "description": "List of YAML (or tagged expression) values to use as arguments to the function.", "type": "array", "items": { - "$ref": "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } }, "languages": { "type": "array", "description": "Which programming languages are supported by this oracle.", "items": { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" } } } @@ -707,7 +713,7 @@ "description": "Language mapping of oracle functions.", "type": "object", "propertyNames": { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" }, "items": { "type": "object", @@ -732,7 +738,7 @@ "description": "Language mapping of oracle arguments.", "type": "object", "propertyNames": { - "$ref": "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" }, "items": { "type": "array", @@ -744,7 +750,7 @@ } }, "value": { - "$ref": "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } } @@ -830,7 +836,7 @@ "fileConfigurationOptions": { "anyOf": [ { - "$ref": "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" }, { "type": "object", @@ -857,19 +863,26 @@ ] }, "yamlValue": { - "description": "A value represented as YAML." + "description": "A value represented as YAML.", + "not": { + "type": [ + "oracle", + "expression", + "programming_language" + ] + } }, "inheritableConfigObject": { "type": "object", "properties": { "stdout": { - "$ref": "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" }, "stderr": { - "$ref": "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" }, "file": { - "$ref": "#/definitions/fileConfigurationOptions" + "$ref": "#/subDefinitions/fileConfigurationOptions" } } } diff --git a/tested/transform_json.py b/tested/transform_json.py index a5ebb0f0..ecffea0f 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -1,123 +1,29 @@ import json import os import sys -from enum import StrEnum from pathlib import Path from typing import Any +SPECIAL_TYPES = ["expression", "programming_language", "oracle"] -class SpecialMap(StrEnum): - NATURAL_LANGUAGE = "natural_language" - PROGRAMMING_LANGUAGE = "programming_language" - ORACLE = "oracle" - EXPRESSION = "expression" - NONE = "none" - - -def map_kind(element: dict) -> SpecialMap: - if "required" in element and "properties" in element: - required = element["required"] - properties = element["properties"] - assert isinstance(required, list) - assert isinstance(properties, dict) - if "__tag__" in required and "value" in required: - if "__tag__" in properties and "value" in properties: - tag = properties["__tag__"] - if ( - isinstance(tag, dict) - and "const" in tag - and isinstance(tag["const"], str) - ): - if tag["const"] == "!natural_language": - return SpecialMap.NATURAL_LANGUAGE - elif tag["const"] == "!programming_language": - return SpecialMap.PROGRAMMING_LANGUAGE - elif tag["const"] == "!oracle": - return SpecialMap.ORACLE - elif tag["const"] == "!expression": - return SpecialMap.EXPRESSION - return SpecialMap.NONE - - -def change_prog_lang_type(element: dict, prog_lang: dict) -> dict: - if element == prog_lang: - ele_type = element.pop("type") - element["anyOf"] = [ - {"type": ele_type}, - {"type": "programming_language"}, - ] - return element - - -def transform_monolingual(data: Any, strict_schema: bool) -> Any: - if isinstance(data, dict): - if "oneOf" in data: - new_one_of = [] - prog_lang = None - for element in data["oneOf"]: - assert isinstance(element, dict) - # Removes all natural translations - kind = map_kind(element) - if kind == SpecialMap.PROGRAMMING_LANGUAGE: - prog_lang = element["properties"]["value"] - elif kind != SpecialMap.NATURAL_LANGUAGE: - if kind == SpecialMap.EXPRESSION or kind == SpecialMap.ORACLE: - element = element["properties"]["value"] - if strict_schema: - if kind == SpecialMap.EXPRESSION: - element["type"] = "expression" - else: - element["type"] = "oracle" - new_one_of.append(element) - - # A programming_language map was found. If not strict, just remove. - # If strict, still provide the type option for the corresponding object. - if prog_lang is not None and strict_schema: - new_one_of = [ - change_prog_lang_type(ele, prog_lang) for ele in new_one_of - ] - - if len(new_one_of) <= 1: - data.pop("oneOf") - if len(new_one_of) == 1: - for key, value in new_one_of[0].items(): - data[key] = value - else: - data["oneOf"] = new_one_of - - # The next changes are a few edge cases. - if "expressionOrStatementWithNatTranslation" in data: - data.pop("expressionOrStatementWithNatTranslation") - - if "translations" in data: - data.pop("translations") - - if "$ref" in data: - if isinstance(data["$ref"], str): - if ( - data["$ref"] - == "#/definitions/expressionOrStatementWithNatTranslation" - ): - data["$ref"] = "#/definitions/expressionOrStatement" - - if "yamlValue" in data: - if strict_schema: - data["yamlValue"] = { - "description": "A value represented as YAML.", - "not": {"type": ["oracle", "expression", "programming_language"]}, - } - else: - data["yamlValue"] = { - "description": "A value represented as YAML.", - } - return {k: transform_monolingual(v, strict_schema) for k, v in data.items()} - elif isinstance(data, list): +def transform_non_strict(data: Any) -> Any: + if isinstance(data, list): return [ transformed for ele in data - if (transformed := transform_monolingual(ele, strict_schema)) != {} + if (transformed := transform_non_strict(ele)) != {} ] + elif isinstance(data, dict): + if "type" in data and data["type"] in SPECIAL_TYPES: + if data["type"] == "expression": + data["type"] = "string" + elif data["type"] == "oracle": + data["type"] = "object" + else: + return {} + + return {k: transform_non_strict(v) for k, v in data.items()} return data @@ -127,12 +33,6 @@ def transform_ide(data: Any) -> Any: transformed for ele in data if (transformed := transform_ide(ele)) != {} ] elif isinstance(data, dict): - if "return" in data: - # This is necessary since tags aren't recognized in the Json schema. - # So a natural_language maps wil always be seen as yamlValue. - assert isinstance(data["return"], dict) and "oneOf" in data["return"] - data["return"]["anyOf"] = data["return"].pop("oneOf") - if "yamlValue" in data: data["yamlValue"] = { "description": "A value represented as YAML.", @@ -153,14 +53,18 @@ def transform_ide(data: Any) -> Any: # For !programming_language the same was already defined without the tag. # The only usage for the expression tag is also redundant. + if tag["const"] == "!programming_language": + return {} + + value = {k: transform_ide(v) for k, v in value.items()} if ( - tag["const"] == "!programming_language" - or tag["const"] == "!expression" + "additionalProperties" in value + and value["additionalProperties"] == {} ): return {} # This to help the validator to somewhat distinguish between - # a natural_language map and a dictionary. + # a natural_language map and another YAML-map. if tag["const"] == "!natural_language": value["not"] = { "anyOf": [ @@ -169,22 +73,18 @@ def transform_ide(data: Any) -> Any: {"required": ["types"]}, ] } - data = value - return {k: transform_ide(v) for k, v in data.items()} - return data + if "oneOf" in value and value["oneOf"] == []: + return {} + return value + + data = {k: transform_ide(v) for k, v in data.items()} + if "oneOf" in data and data["oneOf"] == []: + return {} + return data + return data -SPECIAL_CASES = [ - "return", - "stderr", - "stdout", - "file", - "exception", - "statement", - "expression", -] -SPECIAL_TYPES = ["expression", "programming_language", "oracle"] def flatten_one_of_stack(data: list) -> list: new_one_stack = [] @@ -197,7 +97,10 @@ def flatten_one_of_stack(data: list) -> list: return new_one_stack -def make_tag_structure(data: Any, data_with_inner_translations: Any = None, tag: str = "!natural_language") -> dict: +def make_tag_structure( + data: Any, data_with_inner_translations: Any = None, tag: str = "!natural_language" +) -> dict: + base = { "type": "object", "required": ["__tag__", "value"], @@ -207,12 +110,22 @@ def make_tag_structure(data: Any, data_with_inner_translations: Any = None, tag: "description": "The tag used in the yaml", "const": tag, }, - "value": {"type": "object", "additionalProperties": data}, }, } - if tag == "!natural_language" and data_with_inner_translations is not None: - return {"oneOf": [data_with_inner_translations, base]} + if tag in ["!oracle", "!programming_language", "!expression"]: + base["properties"]["value"] = data + base["properties"]["value"]["type"] = "object" + if tag == "!expression": + base["properties"]["value"]["type"] = "string" + + elif tag == "!natural_language": + base["properties"]["value"] = { + "type": "object", + "additionalProperties": data, + } + if data_with_inner_translations is not None: + return {"oneOf": [data_with_inner_translations, base]} return base @@ -221,16 +134,20 @@ def make_translations_map() -> dict: return {"type": "object", "description": "Define translations in the global scope."} -def transform_json_translation(data: Any, in_sub_def: bool) -> Any: +def transform_json_preprocessor(data: Any, in_sub_def: bool) -> Any: if isinstance(data, dict): # Standard creation of the special tag structure for translations. new_data = { - key: transform_json_translation( + key: transform_json_preprocessor( value, in_sub_def or key == "subDefinitions" ) for key, value in data.items() } - if "type" in data and (data["type"] != "object" or in_sub_def): + if ( + "type" in data + and (data["type"] != "object" or in_sub_def) + and data["type"] not in ["boolean", "integer"] + ): if data["type"] in SPECIAL_TYPES: tag = data.pop("type") new_data.pop("type") @@ -258,26 +175,64 @@ def transform_json_translation(data: Any, in_sub_def: bool) -> Any: if target in data and "properties" in data[target]: data[target]["properties"]["translations"] = make_translations_map() + # Flatten the oneOf structures if "oneOf" in data: assert isinstance(data["oneOf"], list) data["oneOf"] = flatten_one_of_stack(data["oneOf"]) + # Edge cases for return + if "yamlValueOrPythonExpression" in data: + data["yamlValueOrPythonExpression"] = { + "oneOf": [ + {"$ref": "#/subDefinitions/yamlValue"}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!expression", + }, + "value": { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + }, + }, + }, + ] + } + + if "yamlValue" in data: + data["yamlValue"] = { + "description": "A value represented as YAML.", + "not": { + "properties": {"__tag__": {"type": "string"}}, + "type": "object", + }, + } + + if ( + "$ref" in data + and data["$ref"] == "#/subDefinitions/yamlValueOrPythonExpression" + ): + data = make_tag_structure(data, data) return data if isinstance(data, list): - return [transform_json_translation(value, in_sub_def) for value in data] + return [transform_json_preprocessor(value, in_sub_def) for value in data] return data -def transform_json(json_file: Path, monolingual: bool, strict: bool): +def transform_json(json_file: Path, multilingual: bool, ide: bool): """ This function transforms the JSON schema used in the DSL translator into a new JSON schema that can be used to validate the multilingual YAML in your IDE. :param json_file: The path to the JSON file. - :param monolingual: indicator if it will generate a monolingual schema. - :param strict: indicator if it will generate a strict schema. This only applies - to monolingual transforms + :param multilingual: indicator if it will generate a multilingual schema. + :param ide: indicates if it will generate a JSON-schema for the IDE of the user. """ _, ext = os.path.splitext(json_file) assert ext.lower() == ".json", f"expected a json file, got {ext}." @@ -288,15 +243,16 @@ def transform_json(json_file: Path, monolingual: bool, strict: bool): print("The json file was not found.") raise e - if not monolingual: - result = transform_ide(json_stream) - file_name = "multilingual-schema.json" + if not multilingual: + result = transform_non_strict(json_stream) + file_name = "schema.json" else: - result = transform_monolingual(json_stream, strict) - if strict: - file_name = "schema-strict.json" + result = transform_json_preprocessor(json_stream, False) + if ide: + result = transform_ide(result) + file_name = "multilingual-schema.json" else: - file_name = "schema.json" + file_name = "schema-strict-nat-translation.json" with open(json_file.parent / file_name, "w", encoding="utf-8") as f: json.dump(result, f, indent=2) @@ -304,21 +260,17 @@ def transform_json(json_file: Path, monolingual: bool, strict: bool): if __name__ == "__main__": n = len(sys.argv) - assert n > 1, "Expected path to multilingual json schema." - jf = Path(sys.argv[1]) - with open(jf, "r") as stream: - json_stream = json.load(stream) - res = transform_json_translation(json_stream, False) - fn = "multilingual-schema2.json" - with open(jf.parent / fn, "w", encoding="utf-8") as f: - json.dump(res, f, indent=2) - - # monolingual = False - # strict = False - # if n > 2: - # assert sys.argv[2] in ("strict", "not-strict") - # strict = sys.argv[2] == "strict" -# -# monolingual = True -# -# transform_json(Path(sys.argv[1]), monolingual=monolingual, strict=strict) + assert n >= 1 + + multilingual_param = False + ide_param = False + if n >= 2: + assert sys.argv[1] in ("validation", "ide") + multilingual_param = True + ide_param = sys.argv[1] == "ide" + + transform_json( + Path("./tested/dsl/schema-strict.json"), + multilingual=multilingual_param, + ide=ide_param, + ) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 209b66ae..b111b632 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -19,12 +19,13 @@ translate_yaml, validate_pre_dsl, ) -from tested.transform_json import transform_ide, transform_json, transform_monolingual +from tested.transform_json import transform_ide, transform_json def validate_natural_translate(yaml_str: str, translated_yaml_str: str): enviroment = create_enviroment() yaml_object = parse_yaml(yaml_str) + validate_pre_dsl(yaml_object) translated_dsl = translate_yaml(yaml_object, {}, "en", enviroment) translated_yaml = convert_to_yaml(translated_dsl) assert translated_yaml.strip() == translated_yaml_str @@ -854,265 +855,265 @@ def test_transform_executed_correct(mocker: MockerFixture): mock_opener.assert_any_call(Path("multilingual-schema.json"), "w", encoding="utf-8") -def test_json_rm_nat_lang_json_schema(): - json_schema = { - "oneOf": [ - {"type": "array", "minItems": 1, "items": {"$ref": "#/definitions/tab"}}, - { - "type": "object", - "required": ["__tag__", "value"], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language", - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "array", - "minItems": 1, - "items": {"$ref": "#/definitions/tab"}, - }, - }, - }, - }, - ] - } - - json_schema_expected = { - "type": "array", - "minItems": 1, - "items": {"$ref": "#/definitions/tab"}, - } - - result = transform_monolingual(json_schema, True) - - assert result == json_schema_expected - - -def test_json_rm_prog_lang_json_schema(): - json_schema = { - "expressionOrStatement": { - "oneOf": [ - { - "type": "string", - "format": "tested-dsl-expression", - "description": "A statement of expression in Python-like syntax as YAML string.", - }, - { - "type": "object", - "description": "Programming-language-specific statement or expression.", - "minProperties": 1, - "propertyNames": {"$ref": "#/definitions/programmingLanguage"}, - "items": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim.", - }, - }, - { - "type": "object", - "required": ["__tag__", "value"], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!programming_language", - }, - "value": { - "type": "object", - "description": "Programming-language-specific statement or expression.", - "minProperties": 1, - "propertyNames": { - "$ref": "#/definitions/programmingLanguage" - }, - "items": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim.", - }, - }, - }, - }, - ] - }, - } - - json_schema_expected = { - "expressionOrStatement": { - "oneOf": [ - { - "type": "string", - "format": "tested-dsl-expression", - "description": "A statement of expression in Python-like syntax as YAML string.", - }, - { - "description": "Programming-language-specific statement or expression.", - "minProperties": 1, - "propertyNames": {"$ref": "#/definitions/programmingLanguage"}, - "items": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim.", - }, - "anyOf": [{"type": "object"}, {"type": "programming_language"}], - }, - ] - }, - } - - result = transform_monolingual(json_schema, True) - - assert result == json_schema_expected - - -def test_strict_json_schema_oracle(): - json_schema = { - "oneOf": [ - {"$ref": "#/definitions/yamlValueOrPythonExpression"}, - { - "type": "object", - "required": ["__tag__", "value"], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!oracle", - }, - "value": { - "type": "object", - "additionalProperties": False, - "required": ["value"], - "properties": { - "oracle": {"const": "builtin"}, - "value": { - "oneOf": [ - { - "$ref": "#/definitions/yamlValueOrPythonExpression" - }, - { - "type": "object", - "required": ["__tag__", "value"], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language", - }, - "value": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/yamlValueOrPythonExpression" - }, - }, - }, - }, - ] - }, - }, - }, - }, - }, - ] - } - - json_schema_expected = { - "oneOf": [ - {"$ref": "#/definitions/yamlValueOrPythonExpression"}, - { - "type": "oracle", - "additionalProperties": False, - "required": ["value"], - "properties": { - "oracle": {"const": "builtin"}, - "value": {"$ref": "#/definitions/yamlValueOrPythonExpression"}, - }, - }, - ] - } - - result = transform_monolingual(json_schema, True) - - assert result == json_schema_expected - - -def test_strict_json_schema_expression(): - json_schema = { - "oneOf": [ - {"$ref": "#/definitions/yamlValue"}, - { - "type": "object", - "required": ["__tag__", "value"], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!expression", - }, - "value": { - "type": "string", - "format": "tested-dsl-expression", - "description": "An expression in Python-syntax.", - }, - }, - }, - ] - } - - json_schema_expected = { - "oneOf": [ - {"$ref": "#/definitions/yamlValue"}, - { - "type": "expression", - "format": "tested-dsl-expression", - "description": "An expression in Python-syntax.", - }, - ] - } - - result = transform_monolingual(json_schema, True) - print(result) - - assert result == json_schema_expected - - -def test_json_schema_edge_cases(): - json_schema = { - "expressionOrStatementWithNatTranslation": {}, - "translations": {}, - "$ref": "#/definitions/expressionOrStatementWithNatTranslation", - "yamlValue": { - "description": "A value represented as YAML.", - "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, - }, - } - - json_schema_expected = { - "$ref": "#/definitions/expressionOrStatement", - "yamlValue": { - "description": "A value represented as YAML.", - "not": {"type": ["oracle", "expression", "programming_language"]}, - }, - } - - result = transform_monolingual(json_schema, True) - - assert result == json_schema_expected - - json_schema = { - "yamlValue": { - "description": "A value represented as YAML.", - "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, - }, - } - - json_schema_expected = { - "yamlValue": { - "description": "A value represented as YAML.", - } - } - - result = transform_monolingual(json_schema, False) - - assert result == json_schema_expected +# def test_json_rm_nat_lang_json_schema(): +# json_schema = { +# "oneOf": [ +# {"type": "array", "minItems": 1, "items": {"$ref": "#/definitions/tab"}}, +# { +# "type": "object", +# "required": ["__tag__", "value"], +# "properties": { +# "__tag__": { +# "type": "string", +# "description": "The tag used in the yaml", +# "const": "!natural_language", +# }, +# "value": { +# "type": "object", +# "additionalProperties": { +# "type": "array", +# "minItems": 1, +# "items": {"$ref": "#/definitions/tab"}, +# }, +# }, +# }, +# }, +# ] +# } +# +# json_schema_expected = { +# "type": "array", +# "minItems": 1, +# "items": {"$ref": "#/definitions/tab"}, +# } +# +# result = transform_monolingual(json_schema, True) +# +# assert result == json_schema_expected +# +# +# def test_json_rm_prog_lang_json_schema(): +# json_schema = { +# "expressionOrStatement": { +# "oneOf": [ +# { +# "type": "string", +# "format": "tested-dsl-expression", +# "description": "A statement of expression in Python-like syntax as YAML string.", +# }, +# { +# "type": "object", +# "description": "Programming-language-specific statement or expression.", +# "minProperties": 1, +# "propertyNames": {"$ref": "#/definitions/programmingLanguage"}, +# "items": { +# "type": "string", +# "description": "A language-specific literal, which will be used verbatim.", +# }, +# }, +# { +# "type": "object", +# "required": ["__tag__", "value"], +# "properties": { +# "__tag__": { +# "type": "string", +# "description": "The tag used in the yaml", +# "const": "!programming_language", +# }, +# "value": { +# "type": "object", +# "description": "Programming-language-specific statement or expression.", +# "minProperties": 1, +# "propertyNames": { +# "$ref": "#/definitions/programmingLanguage" +# }, +# "items": { +# "type": "string", +# "description": "A language-specific literal, which will be used verbatim.", +# }, +# }, +# }, +# }, +# ] +# }, +# } +# +# json_schema_expected = { +# "expressionOrStatement": { +# "oneOf": [ +# { +# "type": "string", +# "format": "tested-dsl-expression", +# "description": "A statement of expression in Python-like syntax as YAML string.", +# }, +# { +# "description": "Programming-language-specific statement or expression.", +# "minProperties": 1, +# "propertyNames": {"$ref": "#/definitions/programmingLanguage"}, +# "items": { +# "type": "string", +# "description": "A language-specific literal, which will be used verbatim.", +# }, +# "anyOf": [{"type": "object"}, {"type": "programming_language"}], +# }, +# ] +# }, +# } +# +# result = transform_monolingual(json_schema, True) +# +# assert result == json_schema_expected +# +# +# def test_strict_json_schema_oracle(): +# json_schema = { +# "oneOf": [ +# {"$ref": "#/definitions/yamlValueOrPythonExpression"}, +# { +# "type": "object", +# "required": ["__tag__", "value"], +# "properties": { +# "__tag__": { +# "type": "string", +# "description": "The tag used in the yaml", +# "const": "!oracle", +# }, +# "value": { +# "type": "object", +# "additionalProperties": False, +# "required": ["value"], +# "properties": { +# "oracle": {"const": "builtin"}, +# "value": { +# "oneOf": [ +# { +# "$ref": "#/definitions/yamlValueOrPythonExpression" +# }, +# { +# "type": "object", +# "required": ["__tag__", "value"], +# "properties": { +# "__tag__": { +# "type": "string", +# "description": "The tag used in the yaml", +# "const": "!natural_language", +# }, +# "value": { +# "type": "object", +# "additionalProperties": { +# "$ref": "#/definitions/yamlValueOrPythonExpression" +# }, +# }, +# }, +# }, +# ] +# }, +# }, +# }, +# }, +# }, +# ] +# } +# +# json_schema_expected = { +# "oneOf": [ +# {"$ref": "#/definitions/yamlValueOrPythonExpression"}, +# { +# "type": "oracle", +# "additionalProperties": False, +# "required": ["value"], +# "properties": { +# "oracle": {"const": "builtin"}, +# "value": {"$ref": "#/definitions/yamlValueOrPythonExpression"}, +# }, +# }, +# ] +# } +# +# result = transform_monolingual(json_schema, True) +# +# assert result == json_schema_expected +# +# +# def test_strict_json_schema_expression(): +# json_schema = { +# "oneOf": [ +# {"$ref": "#/definitions/yamlValue"}, +# { +# "type": "object", +# "required": ["__tag__", "value"], +# "properties": { +# "__tag__": { +# "type": "string", +# "description": "The tag used in the yaml", +# "const": "!expression", +# }, +# "value": { +# "type": "string", +# "format": "tested-dsl-expression", +# "description": "An expression in Python-syntax.", +# }, +# }, +# }, +# ] +# } +# +# json_schema_expected = { +# "oneOf": [ +# {"$ref": "#/definitions/yamlValue"}, +# { +# "type": "expression", +# "format": "tested-dsl-expression", +# "description": "An expression in Python-syntax.", +# }, +# ] +# } +# +# result = transform_monolingual(json_schema, True) +# print(result) +# +# assert result == json_schema_expected +# +# +# def test_json_schema_edge_cases(): +# json_schema = { +# "expressionOrStatementWithNatTranslation": {}, +# "translations": {}, +# "$ref": "#/definitions/expressionOrStatementWithNatTranslation", +# "yamlValue": { +# "description": "A value represented as YAML.", +# "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, +# }, +# } +# +# json_schema_expected = { +# "$ref": "#/definitions/expressionOrStatement", +# "yamlValue": { +# "description": "A value represented as YAML.", +# "not": {"type": ["oracle", "expression", "programming_language"]}, +# }, +# } +# +# result = transform_monolingual(json_schema, True) +# +# assert result == json_schema_expected +# +# json_schema = { +# "yamlValue": { +# "description": "A value represented as YAML.", +# "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, +# }, +# } +# +# json_schema_expected = { +# "yamlValue": { +# "description": "A value represented as YAML.", +# } +# } +# +# result = transform_monolingual(json_schema, False) +# +# assert result == json_schema_expected def test_exception_when_file_not_found(): From e3fd250bce70741b76d96422544a6395f06940d1 Mon Sep 17 00:00:00 2001 From: breblanc Date: Wed, 14 May 2025 20:22:06 +0200 Subject: [PATCH 21/23] made some tests --- tests/test_preprocess_dsl.py | 483 ----------------------------------- 1 file changed, 483 deletions(-) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index b111b632..2357ca2a 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -19,7 +19,6 @@ translate_yaml, validate_pre_dsl, ) -from tested.transform_json import transform_ide, transform_json def validate_natural_translate(yaml_str: str, translated_yaml_str: str): @@ -642,485 +641,3 @@ def test_template_syntax_error(): return: 11{%{{works}} """.strip() validate_natural_translate(yaml_str, translated_yml) - - -def test_return_json_schema(): - json_schema = { - "return": { - "description": "Expected return value", - "oneOf": [ - {"$ref": "#/definitions/returnOutputChannel"}, - { - "type": "object", - "required": ["__tag__", "value"], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language", - }, - "value": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/returnOutputChannel" - }, - }, - }, - }, - ], - } - } - - json_schema_expected = { - "return": { - "description": "Expected return value", - "anyOf": [ - {"$ref": "#/definitions/returnOutputChannel"}, - { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/returnOutputChannel" - }, - "not": { - "anyOf": [ - {"required": ["description"]}, - {"required": ["value"]}, - {"required": ["types"]}, - ] - }, - }, - ], - } - } - - result = transform_ide(json_schema) - - assert result == json_schema_expected - - -def test_yaml_value_json_schema(): - json_schema = { - "yamlValue": { - "description": "A value represented as YAML.", - "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, - }, - } - - json_schema_expected = { - "yamlValue": { - "description": "A value represented as YAML.", - } - } - - result = transform_ide(json_schema) - - assert result == json_schema_expected - - -def test_nat_lang_json_schema(): - json_schema = { - "type": "object", - "required": ["__tag__", "value"], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language", - }, - "value": {"type": "object", "additionalProperties": {"type": "string"}}, - }, - } - - json_schema_expected = { - "type": "object", - "additionalProperties": {"type": "string"}, - "not": { - "anyOf": [ - {"required": ["description"]}, - {"required": ["value"]}, - {"required": ["types"]}, - ] - }, - } - - result = transform_ide(json_schema) - - assert result == json_schema_expected - - -def test_expr_json_schema(): - json_schema = { - "type": "object", - "required": ["__tag__", "value"], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!expression", - }, - "value": { - "type": "string", - "format": "tested-dsl-expression", - "description": "An expression in Python-syntax.", - }, - }, - } - - json_schema_expected = {} - - result = transform_ide(json_schema) - - assert result == json_schema_expected - - -def test_list_json_schema(): - json_schema = [ - {}, - { - "type": "string", - "description": "A statement of expression in Python-like syntax as YAML string.", - }, - ] - - json_schema_expected = [ - { - "type": "string", - "description": "A statement of expression in Python-like syntax as YAML string.", - } - ] - - result = transform_ide(json_schema) - - assert result == json_schema_expected - - -def test_transform_executed_correct(mocker: MockerFixture): - s = mocker.spy( - tested.transform_json, name="transform_ide" # type: ignore[reportAttributeAccessIssue] - ) - - mock_files = [ - mocker.mock_open(read_data=content).return_value - for content in [ - """ -{ - "files" : { - "description" : "A list of files used in the test suite.", - "oneOf" : [ - { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" - } - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" - } - } - } - } - } - ] - } -}""" - ] - ] - mock_files.append(mocker.mock_open(read_data="{}").return_value) - mock_files.append(mocker.mock_open().return_value) - mock_opener = mocker.mock_open() - mock_opener.side_effect = mock_files - mocker.patch("builtins.open", mock_opener) - - transform_json(Path("schema.json"), False, False) - - assert s.call_count == 25 - - # Check if the file was opened for writing - mock_opener.assert_any_call(Path("multilingual-schema.json"), "w", encoding="utf-8") - - -# def test_json_rm_nat_lang_json_schema(): -# json_schema = { -# "oneOf": [ -# {"type": "array", "minItems": 1, "items": {"$ref": "#/definitions/tab"}}, -# { -# "type": "object", -# "required": ["__tag__", "value"], -# "properties": { -# "__tag__": { -# "type": "string", -# "description": "The tag used in the yaml", -# "const": "!natural_language", -# }, -# "value": { -# "type": "object", -# "additionalProperties": { -# "type": "array", -# "minItems": 1, -# "items": {"$ref": "#/definitions/tab"}, -# }, -# }, -# }, -# }, -# ] -# } -# -# json_schema_expected = { -# "type": "array", -# "minItems": 1, -# "items": {"$ref": "#/definitions/tab"}, -# } -# -# result = transform_monolingual(json_schema, True) -# -# assert result == json_schema_expected -# -# -# def test_json_rm_prog_lang_json_schema(): -# json_schema = { -# "expressionOrStatement": { -# "oneOf": [ -# { -# "type": "string", -# "format": "tested-dsl-expression", -# "description": "A statement of expression in Python-like syntax as YAML string.", -# }, -# { -# "type": "object", -# "description": "Programming-language-specific statement or expression.", -# "minProperties": 1, -# "propertyNames": {"$ref": "#/definitions/programmingLanguage"}, -# "items": { -# "type": "string", -# "description": "A language-specific literal, which will be used verbatim.", -# }, -# }, -# { -# "type": "object", -# "required": ["__tag__", "value"], -# "properties": { -# "__tag__": { -# "type": "string", -# "description": "The tag used in the yaml", -# "const": "!programming_language", -# }, -# "value": { -# "type": "object", -# "description": "Programming-language-specific statement or expression.", -# "minProperties": 1, -# "propertyNames": { -# "$ref": "#/definitions/programmingLanguage" -# }, -# "items": { -# "type": "string", -# "description": "A language-specific literal, which will be used verbatim.", -# }, -# }, -# }, -# }, -# ] -# }, -# } -# -# json_schema_expected = { -# "expressionOrStatement": { -# "oneOf": [ -# { -# "type": "string", -# "format": "tested-dsl-expression", -# "description": "A statement of expression in Python-like syntax as YAML string.", -# }, -# { -# "description": "Programming-language-specific statement or expression.", -# "minProperties": 1, -# "propertyNames": {"$ref": "#/definitions/programmingLanguage"}, -# "items": { -# "type": "string", -# "description": "A language-specific literal, which will be used verbatim.", -# }, -# "anyOf": [{"type": "object"}, {"type": "programming_language"}], -# }, -# ] -# }, -# } -# -# result = transform_monolingual(json_schema, True) -# -# assert result == json_schema_expected -# -# -# def test_strict_json_schema_oracle(): -# json_schema = { -# "oneOf": [ -# {"$ref": "#/definitions/yamlValueOrPythonExpression"}, -# { -# "type": "object", -# "required": ["__tag__", "value"], -# "properties": { -# "__tag__": { -# "type": "string", -# "description": "The tag used in the yaml", -# "const": "!oracle", -# }, -# "value": { -# "type": "object", -# "additionalProperties": False, -# "required": ["value"], -# "properties": { -# "oracle": {"const": "builtin"}, -# "value": { -# "oneOf": [ -# { -# "$ref": "#/definitions/yamlValueOrPythonExpression" -# }, -# { -# "type": "object", -# "required": ["__tag__", "value"], -# "properties": { -# "__tag__": { -# "type": "string", -# "description": "The tag used in the yaml", -# "const": "!natural_language", -# }, -# "value": { -# "type": "object", -# "additionalProperties": { -# "$ref": "#/definitions/yamlValueOrPythonExpression" -# }, -# }, -# }, -# }, -# ] -# }, -# }, -# }, -# }, -# }, -# ] -# } -# -# json_schema_expected = { -# "oneOf": [ -# {"$ref": "#/definitions/yamlValueOrPythonExpression"}, -# { -# "type": "oracle", -# "additionalProperties": False, -# "required": ["value"], -# "properties": { -# "oracle": {"const": "builtin"}, -# "value": {"$ref": "#/definitions/yamlValueOrPythonExpression"}, -# }, -# }, -# ] -# } -# -# result = transform_monolingual(json_schema, True) -# -# assert result == json_schema_expected -# -# -# def test_strict_json_schema_expression(): -# json_schema = { -# "oneOf": [ -# {"$ref": "#/definitions/yamlValue"}, -# { -# "type": "object", -# "required": ["__tag__", "value"], -# "properties": { -# "__tag__": { -# "type": "string", -# "description": "The tag used in the yaml", -# "const": "!expression", -# }, -# "value": { -# "type": "string", -# "format": "tested-dsl-expression", -# "description": "An expression in Python-syntax.", -# }, -# }, -# }, -# ] -# } -# -# json_schema_expected = { -# "oneOf": [ -# {"$ref": "#/definitions/yamlValue"}, -# { -# "type": "expression", -# "format": "tested-dsl-expression", -# "description": "An expression in Python-syntax.", -# }, -# ] -# } -# -# result = transform_monolingual(json_schema, True) -# print(result) -# -# assert result == json_schema_expected -# -# -# def test_json_schema_edge_cases(): -# json_schema = { -# "expressionOrStatementWithNatTranslation": {}, -# "translations": {}, -# "$ref": "#/definitions/expressionOrStatementWithNatTranslation", -# "yamlValue": { -# "description": "A value represented as YAML.", -# "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, -# }, -# } -# -# json_schema_expected = { -# "$ref": "#/definitions/expressionOrStatement", -# "yamlValue": { -# "description": "A value represented as YAML.", -# "not": {"type": ["oracle", "expression", "programming_language"]}, -# }, -# } -# -# result = transform_monolingual(json_schema, True) -# -# assert result == json_schema_expected -# -# json_schema = { -# "yamlValue": { -# "description": "A value represented as YAML.", -# "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, -# }, -# } -# -# json_schema_expected = { -# "yamlValue": { -# "description": "A value represented as YAML.", -# } -# } -# -# result = transform_monolingual(json_schema, False) -# -# assert result == json_schema_expected - - -def test_exception_when_file_not_found(): - - try: - transform_json(Path("test.json"), False, False) - except FileNotFoundError: - print("As expected") - else: - assert False, "Expected FileNotFoundError error" From 23b34e7ec980a4df499a76dc9a6769ed4dc6b597 Mon Sep 17 00:00:00 2001 From: breblanc Date: Wed, 14 May 2025 20:57:16 +0200 Subject: [PATCH 22/23] small change for test --- tested/dsl/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 23125eb3..fd11feeb 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -887,4 +887,4 @@ } } } -} \ No newline at end of file +} From 59b022e840c8befcb83f71a97d438420f93e311c Mon Sep 17 00:00:00 2001 From: breblanc Date: Wed, 14 May 2025 20:58:10 +0200 Subject: [PATCH 23/23] add actual tests --- tests/test_json_schema_transformer.py | 544 ++++++++++++++++++++++++++ 1 file changed, 544 insertions(+) create mode 100644 tests/test_json_schema_transformer.py diff --git a/tests/test_json_schema_transformer.py b/tests/test_json_schema_transformer.py new file mode 100644 index 00000000..4e98740f --- /dev/null +++ b/tests/test_json_schema_transformer.py @@ -0,0 +1,544 @@ +from pathlib import Path + +from pytest_mock import MockerFixture + +import tested +from tested.transform_json import ( + transform_ide, + transform_json, + transform_json_preprocessor, +) + + +def test_return_json_schema(): + json_schema = { + "return": { + "description": "Expected return value", + "oneOf": [ + {"$ref": "#/definitions/returnOutputChannel"}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/returnOutputChannel" + }, + }, + }, + }, + ], + } + } + + json_schema_expected = { + "return": { + "description": "Expected return value", + "oneOf": [ + {"$ref": "#/definitions/returnOutputChannel"}, + { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/returnOutputChannel" + }, + "not": { + "anyOf": [ + {"required": ["description"]}, + {"required": ["value"]}, + {"required": ["types"]}, + ] + }, + }, + ], + } + } + + result = transform_ide(json_schema) + + assert result == json_schema_expected + + +def test_yaml_value_json_schema(): + json_schema = { + "yamlValue": { + "description": "A value represented as YAML.", + "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, + }, + } + + json_schema_expected = { + "yamlValue": { + "description": "A value represented as YAML.", + } + } + + result = transform_ide(json_schema) + + assert result == json_schema_expected + + +def test_nat_lang_json_schema(): + json_schema = { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": {"type": "object", "additionalProperties": {"type": "string"}}, + }, + } + + json_schema_expected = { + "type": "object", + "additionalProperties": {"type": "string"}, + "not": { + "anyOf": [ + {"required": ["description"]}, + {"required": ["value"]}, + {"required": ["types"]}, + ] + }, + } + + result = transform_ide(json_schema) + + assert result == json_schema_expected + + +def test_expr_json_schema(): + json_schema = { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!expression", + }, + "value": { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + }, + }, + } + + json_schema_expected = { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + } + + result = transform_ide(json_schema) + + assert result == json_schema_expected + + +def test_list_json_schema(): + json_schema = [ + {}, + { + "type": "string", + "description": "A statement of expression in Python-like syntax as YAML string.", + }, + ] + + json_schema_expected = [ + { + "type": "string", + "description": "A statement of expression in Python-like syntax as YAML string.", + } + ] + + result = transform_ide(json_schema) + + assert result == json_schema_expected + + +def test_transform_executed_correct(mocker: MockerFixture): + s = mocker.spy( + tested.transform_json, name="transform_json_preprocessor" # type: ignore[reportAttributeAccessIssue] + ) + + mock_files = [ + mocker.mock_open(read_data=content).return_value + for content in [ + """ +{ + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + } + } + } + } + ] + } +}""" + ] + ] + mock_files.append(mocker.mock_open(read_data="{}").return_value) + mock_files.append(mocker.mock_open().return_value) + mock_opener = mocker.mock_open() + mock_opener.side_effect = mock_files + mocker.patch("builtins.open", mock_opener) + + transform_json(Path("schema.json"), True, False) + + assert s.call_count == 25 + + # Check if the file was opened for writing + mock_opener.assert_any_call( + Path("schema-strict-nat-translation.json"), "w", encoding="utf-8" + ) + + +def test_nat_lang_json_schema_structure(): + json_schema = { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/definitions/tab"}, + } + + json_schema_expected = { + "oneOf": [ + {"type": "array", "minItems": 1, "items": {"$ref": "#/definitions/tab"}}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/definitions/tab"}, + }, + }, + }, + }, + ] + } + + result = transform_json_preprocessor(json_schema, False) + + assert result == json_schema_expected + + +def test_prog_lang_json_schema_structure(): + json_schema = { + "expressionOrStatement": { + "type": "programming_language", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": {"$ref": "#/subDefinitions/programmingLanguage"}, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim.", + }, + }, + } + + json_schema_expected = { + "expressionOrStatement": { + "oneOf": [ + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!programming_language", + }, + "value": { + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "string", + "description": "A language-specific literal, which will be used verbatim.", + }, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim.", + }, + }, + }, + }, + ] + }, + "type": "object", + }, + }, + }, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!programming_language", + }, + "value": { + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim.", + }, + "type": "object", + }, + }, + }, + }, + }, + }, + ] + } + } + + result = transform_json_preprocessor(json_schema, True) + + assert result == json_schema_expected + + +def test_json_schema_oracle(): + json_schema = { + "type": "oracle", + "additionalProperties": False, + "required": ["value"], + "properties": { + "oracle": {"const": "builtin"}, + "value": {"$ref": "#/subDefinitions/yamlValueOrPythonExpression"}, + }, + } + + json_schema_expected = { + "oneOf": [ + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle", + }, + "value": { + "additionalProperties": False, + "required": ["value"], + "properties": { + "oracle": {"const": "builtin"}, + "value": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + }, + }, + }, + ] + }, + }, + "type": "object", + }, + }, + }, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle", + }, + "value": { + "additionalProperties": False, + "required": ["value"], + "properties": { + "oracle": {"const": "builtin"}, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + }, + "type": "object", + }, + }, + }, + }, + }, + }, + ] + } + + result = transform_json_preprocessor(json_schema, True) + + assert result == json_schema_expected + + +def test_json_schema_expression(): + json_schema = { + "yamlValueOrPythonExpression": { + "oneOf": [ + {"$ref": "#/definitions/yamlValue"}, + { + "type": "expression", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + }, + ] + } + } + + json_schema_expected = { + "yamlValueOrPythonExpression": { + "oneOf": [ + {"$ref": "#/subDefinitions/yamlValue"}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!expression", + }, + "value": { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + }, + }, + }, + ] + } + } + + result = transform_json_preprocessor(json_schema, True) + assert result == json_schema_expected + + +def test_json_schema_yaml_value(): + json_schema = { + "yamlValue": { + "description": "A value represented as YAML.", + } + } + + json_schema_expected = { + "yamlValue": { + "description": "A value represented as YAML.", + "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, + }, + } + + result = transform_json_preprocessor(json_schema, False) + + assert result == json_schema_expected + + +def test_exception_when_file_not_found(): + + try: + transform_json(Path("test.json"), False, False) + except FileNotFoundError: + print("As expected") + else: + assert False, "Expected FileNotFoundError error"