diff --git a/tested/dsl/multilingual-schema.json b/tested/dsl/multilingual-schema.json new file mode 100644 index 00000000..b32f98d4 --- /dev/null +++ b/tested/dsl/multilingual-schema.json @@ -0,0 +1,3813 @@ +{ + "$id": "tested:dsl:schema7", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TESTed-DSL", + "oneOf": [ + { + "$ref": "#/definitions/_rootObject" + }, + { + "$ref": "#/definitions/_tabList" + }, + { + "$ref": "#/definitions/_unitList" + } + ], + "definitions": { + "_rootObject": { + "type": "object", + "oneOf": [ + { + "required": [ + "tabs" + ], + "not": { + "required": [ + "units" + ] + } + }, + { + "required": [ + "units" + ], + "not": { + "required": [ + "tabs" + ] + } + } + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "namespace": { + "oneOf": [ + { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "tabs": { + "$ref": "#/definitions/_tabList" + }, + "units": { + "$ref": "#/definitions/_unitList" + }, + "language": { + "description": "Indicate that all code is in a specific language.", + "oneOf": [ + { + "$ref": "#/subDefinitions/programmingLanguage" + }, + { + "const": "tested" + } + ] + }, + "definitions": { + "description": "Define hashes to use elsewhere.", + "type": "object" + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." + } + } + }, + "_tabList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/tab" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/tab" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "_unitList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/unit" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/unit" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "tab": { + "type": "object", + "description": "A tab in the test suite.", + "required": [ + "tab" + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" + }, + "tab": { + "oneOf": [ + { + "description": "The name of this tab.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The name of this tab.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." + } + }, + "oneOf": [ + { + "required": [ + "contexts" + ], + "properties": { + "contexts": { + "$ref": "#/definitions/_contextList" + } + } + }, + { + "required": [ + "testcases" + ], + "properties": { + "testcases": { + "$ref": "#/definitions/_testcaseList" + } + } + } + ] + }, + "unit": { + "type": "object", + "description": "A unit in the test suite.", + "required": [ + "unit" + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" + }, + "unit": { + "oneOf": [ + { + "description": "The name of this tab.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The name of this tab.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." + } + }, + "oneOf": [ + { + "required": [ + "cases" + ], + "properties": { + "cases": { + "$ref": "#/definitions/_caseList" + } + } + }, + { + "required": [ + "scripts" + ], + "properties": { + "scripts": { + "$ref": "#/definitions/_scriptList" + } + } + } + ] + }, + "_contextList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "_caseList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "_testcaseList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "_scriptList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/script" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/script" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "context": { + "type": "object", + "description": "A set of testcase in the same context.", + "required": [ + "testcases" + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "context": { + "oneOf": [ + { + "type": "string", + "description": "Description of this context." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Description of this context." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "testcases": { + "$ref": "#/definitions/_testcaseList" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." + } + } + }, + "case": { + "type": "object", + "description": "A test case.", + "required": [ + "script" + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "context": { + "oneOf": [ + { + "type": "string", + "description": "Description of this context." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Description of this context." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "script": { + "$ref": "#/definitions/_scriptList" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." + } + } + }, + "testcase": { + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { + "oneOf": [ + { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties": { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "description": "Array of program call arguments", + "type": "array", + "items": { + "oneOf": [ + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" + }, + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" + } + } + }, + "script": { + "type": "object", + "description": "An individual test (script) for a statement or expression", + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { + "oneOf": [ + { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties": { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "description": "Array of program call arguments", + "type": "array", + "items": { + "oneOf": [ + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" + }, + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" + } + } + } + }, + "subDefinitions": { + "expressionOrStatement": { + "oneOf": [ + { + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + }, + { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + } + ] + }, + "yamlValueOrPythonExpression": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValue" + }, + { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax." + } + ] + }, + "file": { + "oneOf": [ + { + "type": "object", + "description": "A file used in the test suite.", + "required": [ + "name", + "url" + ], + "properties": { + "name": { + "oneOf": [ + { + "type": "string", + "description": "The filename, including the file extension." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The filename, including the file extension." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "url": { + "oneOf": [ + { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "A file used in the test suite.", + "required": [ + "name", + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "The filename, including the file extension." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "exceptionChannel": { + "oneOf": [ + { + "type": "string", + "description": "Message of the expected exception." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Message of the expected exception." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + }, + { + "type": "object", + "required": [ + "types" + ], + "properties": { + "message": { + "oneOf": [ + { + "description": "Message of the expected exception.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "Message of the expected exception.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "types": { + "oneOf": [ + { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "types" + ], + "properties": { + "message": { + "description": "Message of the expected exception.", + "type": "string" + }, + "types": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string" + } + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "textOutputChannel": { + "anyOf": [ + { + "$ref": "#/subDefinitions/textualType" + }, + { + "oneOf": [ + { + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/textConfigurationOptions" + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/textConfigurationOptions" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "oneOf": [ + { + "type": "object", + "description": "Custom oracle for text values.", + "required": [ + "oracle", + "file", + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "custom_check" + }, + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "languages": { + "oneOf": [ + { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Custom oracle for text values.", + "required": [ + "oracle", + "file", + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "custom_check" + }, + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + }, + "fileOutputChannel": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "description": "Built-in oracle for files.", + "required": [ + "content", + "location" + ], + "properties": { + "content": { + "oneOf": [ + { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "location": { + "oneOf": [ + { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Built-in oracle for files.", + "required": [ + "content", + "location" + ], + "properties": { + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "oneOf": [ + { + "type": "object", + "description": "Custom oracle for file values.", + "required": [ + "oracle", + "content", + "location", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "content": { + "oneOf": [ + { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "location": { + "oneOf": [ + { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "languages": { + "oneOf": [ + { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Custom oracle for file values.", + "required": [ + "oracle", + "content", + "location", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + }, + "returnOutputChannel": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + }, + { + "additionalProperties": false, + "required": [ + "value" + ], + "properties": { + "oracle": { + "const": "builtin" + }, + "value": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + "type": "object" + }, + { + "type": "object", + "additionalProperties": { + "additionalProperties": false, + "required": [ + "value" + ], + "properties": { + "oracle": { + "const": "builtin" + }, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "type": "object" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + }, + { + "additionalProperties": false, + "required": [ + "value", + "oracle", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "value": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "languages": { + "oneOf": [ + { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + "type": "object" + }, + { + "type": "object", + "additionalProperties": { + "additionalProperties": false, + "required": [ + "value", + "oracle", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "arguments": { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + }, + "type": "object" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + }, + { + "additionalProperties": false, + "required": [ + "oracle", + "functions" + ], + "properties": { + "oracle": { + "const": "specific_check" + }, + "functions": { + "oneOf": [ + { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "value": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + "type": "object" + }, + { + "type": "object", + "additionalProperties": { + "additionalProperties": false, + "required": [ + "oracle", + "functions" + ], + "properties": { + "oracle": { + "const": "specific_check" + }, + "functions": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + }, + "arguments": { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + }, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "type": "object" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "programmingLanguage": { + "oneOf": [ + { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ + "bash", + "c", + "haskell", + "java", + "javascript", + "typescript", + "kotlin", + "python", + "runhaskell", + "csharp", + "cpp" + ] + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ + "bash", + "c", + "haskell", + "java", + "javascript", + "typescript", + "kotlin", + "python", + "runhaskell", + "csharp", + "cpp" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "message": { + "oneOf": [ + { + "type": "string", + "description": "A simple message to display." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A simple message to display." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + }, + { + "type": "object", + "required": [ + "description" + ], + "properties": { + "description": { + "oneOf": [ + { + "description": "The message to display.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The message to display.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "format": { + "oneOf": [ + { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "description" + ], + "properties": { + "description": { + "description": "The message to display.", + "type": "string" + }, + "format": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "textConfigurationOptions": { + "oneOf": [ + { + "type": "object", + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, + "properties": { + "applyRounding": { + "description": "Apply rounding when comparing as float", + "type": "boolean" + }, + "caseInsensitive": { + "description": "Ignore case when comparing strings", + "type": "boolean" + }, + "ignoreWhitespace": { + "description": "Ignore trailing whitespace", + "type": "boolean" + }, + "normalizeTrailingNewlines": { + "description": "Normalize trailing newlines", + "type": "boolean" + }, + "roundTo": { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + }, + "tryFloatingPoint": { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + }, + "hideExpected": { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, + "properties": { + "applyRounding": { + "description": "Apply rounding when comparing as float", + "type": "boolean" + }, + "caseInsensitive": { + "description": "Ignore case when comparing strings", + "type": "boolean" + }, + "ignoreWhitespace": { + "description": "Ignore trailing whitespace", + "type": "boolean" + }, + "normalizeTrailingNewlines": { + "description": "Normalize trailing newlines", + "type": "boolean" + }, + "roundTo": { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + }, + "tryFloatingPoint": { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + }, + "hideExpected": { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "fileConfigurationOptions": { + "anyOf": [ + { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + { + "oneOf": [ + { + "type": "object", + "properties": { + "mode": { + "oneOf": [ + { + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + }, + "textualType": { + "oneOf": [ + { + "description": "Simple textual value, converted to string.", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties": { + "description": "Simple textual value, converted to string.", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "yamlValue": { + "description": "A value represented as YAML." + }, + "inheritableConfigObject": { + "oneOf": [ + { + "type": "object", + "properties": { + "stdout": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "stderr": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "file": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "stdout": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "stderr": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "file": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } +} \ No newline at end of file diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json index 51f0647c..65083de8 100644 --- a/tested/dsl/schema-strict-nat-translation.json +++ b/tested/dsl/schema-strict-nat-translation.json @@ -1,71 +1,72 @@ { - "$id" : "tested:dsl:schema7", - "$schema" : "http://json-schema.org/draft-07/schema#", - "title" : "TESTed-DSL", - "oneOf" : [ + "$id": "tested:dsl:schema7", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TESTed-DSL", + "oneOf": [ { - "$ref" : "#/definitions/_rootObject" + "$ref": "#/definitions/_rootObject" }, { - "$ref" : "#/definitions/_tabList" + "$ref": "#/definitions/_tabList" }, { - "$ref" : "#/definitions/_unitList" + "$ref": "#/definitions/_unitList" } ], - "definitions" : { - "_rootObject" : { - "type" : "object", - "oneOf" : [ + "definitions": { + "_rootObject": { + "type": "object", + "oneOf": [ { - "required" : [ + "required": [ "tabs" ], - "not" : { - "required" : [ + "not": { + "required": [ "units" ] } }, { - "required" : [ + "required": [ "units" ], - "not" : { - "required" : [ + "not": { + "required": [ "tabs" ] } } ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "oneOf" : [ + "properties": { + "files": { + "oneOf": [ { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } } } @@ -73,68 +74,93 @@ } ] }, - "namespace" : { - "type" : "string", - "description" : "Namespace of the submitted solution, in `snake_case`" + "namespace": { + "oneOf": [ + { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" + } + } + } + } + ] }, - "tabs" : { - "$ref" : "#/definitions/_tabList" + "tabs": { + "$ref": "#/definitions/_tabList" }, - "units" : { - "$ref" : "#/definitions/_unitList" + "units": { + "$ref": "#/definitions/_unitList" }, - "language" : { - "description" : "Indicate that all code is in a specific language.", - "oneOf" : [ + "language": { + "description": "Indicate that all code is in a specific language.", + "oneOf": [ { - "$ref" : "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" }, { - "const" : "tested" + "const": "tested" } ] }, - "translations" : { - "type" : "object", - "description": "Define translations in the global scope." - }, - "definitions" : { - "description" : "Define hashes to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define hashes to use elsewhere.", + "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." } } }, - "_tabList" : { - "oneOf" : [ + "_tabList": { + "oneOf": [ { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/tab" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/tab" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/tab" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/tab" } } } @@ -142,34 +168,34 @@ } ] }, - "_unitList" : { - "oneOf" : [ + "_unitList": { + "oneOf": [ { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/unit" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/unit" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/unit" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/unit" } } } @@ -177,40 +203,41 @@ } ] }, - "tab" : { - "type" : "object", - "description" : "A tab in the test suite.", - "required" : [ + "tab": { + "type": "object", + "description": "A tab in the test suite.", + "required": [ "tab" ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "oneOf" : [ + "properties": { + "files": { + "oneOf": [ { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } } } @@ -218,107 +245,109 @@ } ] }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "tab" : { - "oneOf" : [ + "tab": { + "oneOf": [ { - "type" : "string" + "description": "The name of this tab.", + "type": "string" }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "string" + "description": "The name of this tab.", + "type": "string" } } } } - ], - "description" : "The name of this tab." - }, - "translations" : { - "type" : "object", - "description": "Define translations in the tab scope." + ] }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "contexts" ], - "properties" : { - "contexts" : { - "$ref" : "#/definitions/_contextList" + "properties": { + "contexts": { + "$ref": "#/definitions/_contextList" } } }, { - "required" : [ + "required": [ "testcases" ], - "properties" : { - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "properties": { + "testcases": { + "$ref": "#/definitions/_testcaseList" } } } ] }, - "unit" : { - "type" : "object", - "description" : "A unit in the test suite.", - "required" : [ + "unit": { + "type": "object", + "description": "A unit in the test suite.", + "required": [ "unit" ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "oneOf" : [ + "properties": { + "files": { + "oneOf": [ { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } } } @@ -326,103 +355,102 @@ } ] }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "unit" : { - "oneOf" : [ + "unit": { + "oneOf": [ { - "type" : "string", - "description" : "The name of this tab." + "description": "The name of this tab.", + "type": "string" }, { - "type" : "object", - "description" : "The name of this tab.", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "string" + "description": "The name of this tab.", + "type": "string" } } } } - ], - "description" : "The name of this tab." - }, - "translations" : { - "type" : "object", - "description": "Define translations in the unit scope." + ] }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "cases" ], - "properties" : { - "cases" : { - "$ref" : "#/definitions/_caseList" + "properties": { + "cases": { + "$ref": "#/definitions/_caseList" } } }, { - "required" : [ + "required": [ "scripts" ], - "properties" : { - "scripts" : { - "$ref" : "#/definitions/_scriptList" + "properties": { + "scripts": { + "$ref": "#/definitions/_scriptList" } } } ] }, - "_contextList" : { - "oneOf" : [ + "_contextList": { + "oneOf": [ { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/context" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/context" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" } } } @@ -430,34 +458,34 @@ } ] }, - "_caseList" : { - "oneOf" : [ + "_caseList": { + "oneOf": [ { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/case" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/case" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" } } } @@ -465,34 +493,34 @@ } ] }, - "_testcaseList" : { - "oneOf" : [ + "_testcaseList": { + "oneOf": [ { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/testcase" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/testcase" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" } } } @@ -500,34 +528,34 @@ } ] }, - "_scriptList" : { - "oneOf" : [ + "_scriptList": { + "oneOf": [ { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/script" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/script" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/script" + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/script" } } } @@ -535,40 +563,41 @@ } ] }, - "context" : { - "type" : "object", - "description" : "A set of testcase in the same context.", - "required" : [ + "context": { + "type": "object", + "description": "A set of testcase in the same context.", + "required": [ "testcases" ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "oneOf" : [ + "properties": { + "files": { + "oneOf": [ { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } } } @@ -576,53 +605,79 @@ } ] }, - "translations" : { - "type" : "object", - "description": "Define translations in the context scope." + "context": { + "oneOf": [ + { + "type": "string", + "description": "Description of this context." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Description of this context." + } + } + } + } + ] }, - "context" : { - "type" : "string", - "description" : "Description of this context." + "testcases": { + "$ref": "#/definitions/_testcaseList" }, - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "translations": { + "type": "object", + "description": "Define translations in the global scope." } } }, - "case" : { - "type" : "object", - "description" : "A test case.", - "required" : [ + "case": { + "type": "object", + "description": "A test case.", + "required": [ "script" ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "oneOf" : [ + "properties": { + "files": { + "oneOf": [ { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } } } @@ -630,56 +685,57 @@ } ] }, - "translations" : { - "type" : "object", - "description": "Define translations in the case scope." - }, - "context" : { - "type" : "string", - "description" : "Description of this context." - }, - "script" : { - "$ref" : "#/definitions/_scriptList" - } - } - }, - "testcase" : { - "type" : "object", - "description" : "An individual test for a statement or expression", - "additionalProperties" : false, - "properties" : { - "description" : { + "context": { "oneOf": [ { - "$ref" : "#/definitions/message" + "type": "string", + "description": "Description of this context." }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/message" + "type": "string", + "description": "Description of this context." } } } } ] }, - "stdin" : { - "description" : "Stdin for this context", + "script": { + "$ref": "#/definitions/_scriptList" + }, + "translations": { + "type": "object", + "description": "Define translations in the global scope." + } + } + }, + "testcase": { + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { "oneOf": [ { - "type" : [ + "description": "Stdin for this context", + "type": [ "string", "number", "integer", @@ -687,21 +743,22 @@ ] }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : [ + "description": "Stdin for this context", + "type": [ "string", "number", "integer", @@ -713,37 +770,68 @@ } ] }, - "arguments" : { + "arguments": { "oneOf": [ { - "type" : "array", - "items" : { - "type" : [ - "string", - "number", - "integer", - "boolean" + "description": "Array of program call arguments", + "type": "array", + "items": { + "oneOf": [ + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + } + } + } ] } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "type" : [ + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ "string", "number", "integer", @@ -754,72 +842,48 @@ } } } - ], - "description" : "Array of program call arguments" + ] }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "$ref" : "#/definitions/exceptionChannel" - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/exceptionChannel" - } - } - } - } - ] + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" }, - "files" : { - "description" : "A list of files used in the test suite.", - "oneOf" : [ + "files": { + "oneOf": [ { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } } } @@ -827,1035 +891,1842 @@ } ] }, - "return" : { - "description" : "Expected return value", - "oneOf" : [ + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" + } + } + }, + "script": { + "type": "object", + "description": "An individual test (script) for a statement or expression", + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { + "oneOf": [ { - "$ref" : "#/definitions/returnOutputChannel" + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/returnOutputChannel" + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] } } } } ] - }, - "stderr" : { - "description" : "Expected output at stderr", - "oneOf" : [ - { - "$ref" : "#/definitions/textOutputChannel" - }, + "arguments": { + "oneOf": [ { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/textOutputChannel" + "description": "Array of program call arguments", + "type": "array", + "items": { + "oneOf": [ + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + } + } } - } + ] } - } - ] - }, - "stdout" : { - "description" : "Expected output at stdout", - "oneOf" : [ - { - "$ref" : "#/definitions/textOutputChannel" }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/textOutputChannel" + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } } } } } ] }, - "file": { - "description" : "Expected files generated by the submission.", - "oneOf" : [ + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" + }, + "files": { + "oneOf": [ { - "$ref" : "#/definitions/fileOutputChannel" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/fileOutputChannel" + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } } } } } ] }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" } } - }, - "script" : { - "type" : "object", - "description" : "An individual test (script) for a statement or expression", - "properties" : { - "description" : { - "oneOf": [ - { - "$ref" : "#/definitions/message" + } + }, + "subDefinitions": { + "expressionOrStatement": { + "oneOf": [ + { + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/message" - } - } + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." } } - ] + } }, - "stdin" : { - "description" : "Stdin for this context", - "oneOf": [ - { - "type" : [ - "string", - "number", - "integer", - "boolean" - ] - }, + { + "anyOf": [ { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ + "oneOf": [ + { "type": "object", - "additionalProperties": { - "type" : [ - "string", - "number", - "integer", - "boolean" + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + } + } ] } - } - } - } - ] - }, - "arguments" : { - "oneOf": [ - { - "type" : "array", - "items" : { - "type" : [ - "string", - "number", - "integer", - "boolean" - ] - } - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" }, - "value":{ + { "type": "object", - "additionalProperties": { - "type" : "array", - "items" : { - "type" : [ - "string", - "number", - "integer", - "boolean" - ] + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } } } } - } - } - ], - "description" : "Array of program call arguments" - }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" - }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" - }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "$ref" : "#/definitions/exceptionChannel" - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/exceptionChannel" - } - } - } - } - ] - }, - "files" : { - "description" : "A list of files used in the test suite.", - "oneOf" : [ - { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" - } + ] }, { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ + "oneOf": [ + { "type": "object", - "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!programming_language" + }, + "value": { + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + } + } + ] + }, + "type": "object" } } - } - } - } - ] - }, - "return" : { - "description" : "Expected return value", - "oneOf" : [ - { - "$ref" : "#/definitions/returnOutputChannel" - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/returnOutputChannel" - } - } - } - } - ] - - }, - "stderr" : { - "description" : "Expected output at stderr", - "oneOf" : [ - { - "$ref" : "#/definitions/textOutputChannel" - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/textOutputChannel" - } - } - } - } - ] - }, - "stdout" : { - "description" : "Expected output at stdout", - "oneOf" : [ - { - "$ref" : "#/definitions/textOutputChannel" - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/textOutputChannel" - } - } - } - } - ] - }, - "file": { - "description" : "Expected files generated by the submission.", - "oneOf" : [ - { - "$ref" : "#/definitions/fileOutputChannel" - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" }, - "value":{ + { "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/textOutputChannel" + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!programming_language" + }, + "value": { + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + "type": "object" + } + } + } + } } } - } + ] } ] - }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" } - } + ] }, - "expressionOrStatement" : { - "oneOf" : [ + "yamlValueOrPythonExpression": { + "oneOf": [ { - "type" : "string", - "format" : "tested-dsl-expression", - "description" : "A statement of expression in Python-like syntax as YAML string." + "$ref": "#/subDefinitions/yamlValue" }, { "type": "object", - "description" : "Programming-language-specific statement or expression.", - "minProperties" : 1, - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - } - }, - { - "type" : "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!programming_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!expression" }, - "value":{ - "type": "object", - "description" : "Programming-language-specific statement or expression.", - "minProperties" : 1, - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - } + "value": { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax." } } } ] }, - "expressionOrStatementWithNatTranslation" : { - "oneOf" : [ - { - "type" : "string", - "format" : "tested-dsl-expression", - "description" : "A statement of expression in Python-like syntax as YAML string." - }, + "file": { + "oneOf": [ { "type": "object", - "description" : "Programming-language-specific statement or expression.", - "minProperties" : 1, - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "oneOf" : [ - { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "description": "A file used in the test suite.", + "required": [ + "name", + "url" + ], + "properties": { + "name": { + "oneOf": [ + { + "type": "string", + "description": "The filename, including the file extension." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The filename, including the file extension." + } } } } - } - ] - } - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!programming_language" + ] }, - "value":{ - "type": "object", - "description" : "Programming-language-specific statement or expression.", - "minProperties" : 1, - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "oneOf" : [ - { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - } + "url": { + "oneOf": [ + { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." } } } - ] - } + } + ] } } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/expressionOrStatement" + "type": "object", + "description": "A file used in the test suite.", + "required": [ + "name", + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "The filename, including the file extension." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." + } + } } } } } ] }, - "yamlValueOrPythonExpression" : { - "oneOf" : [ + "exceptionChannel": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValue" + "type": "string", + "description": "Message of the expected exception." }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!expression" - }, - "value":{ "type": "string", - "format" : "tested-dsl-expression", - "description" : "An expression in Python-syntax." + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Message of the expected exception." + } } } - } - ] - }, - "file" : { - "type" : "object", - "description" : "A file used in the test suite.", - "required" : [ - "name", - "url" - ], - "properties" : { - "name" : { - "type" : "string", - "description" : "The filename, including the file extension." - }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." - } - } - }, - "exceptionChannel" : { - "oneOf" : [ - { - "type" : "string", - "description" : "Message of the expected exception." }, { - "type" : "object", - "required" : [ + "type": "object", + "required": [ "types" ], - "properties" : { - "message" : { - "oneOf" : [ + "properties": { + "message": { + "oneOf": [ { - "type" : "string" + "description": "Message of the expected exception.", + "type": "string" }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "string" + "description": "Message of the expected exception.", + "type": "string" } } } } - ], - "description" : "Message of the expected exception." + ] }, - "types" : { - "minProperties" : 1, - "description" : "Language mapping of expected exception types.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string" + "types": { + "oneOf": [ + { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string" + } + } + } + } + } + ] + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "types" + ], + "properties": { + "message": { + "description": "Message of the expected exception.", + "type": "string" + }, + "types": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string" + } + } + } } } } } ] }, - "textOutputChannel" : { - "anyOf" : [ + "textOutputChannel": { + "anyOf": [ { - "$ref" : "#/definitions/textualType" + "$ref": "#/subDefinitions/textualType" }, { - "type" : "object", - "description" : "Built-in oracle for text values.", - "required" : [ - "data" - ], - "properties" : { - "data" : { - "oneOf" : [ - { - "$ref" : "#/definitions/textualType" + "oneOf": [ + { + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/textConfigurationOptions" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/textConfigurationOptions" + } + } + } + } + } + } + ] + }, + { + "oneOf": [ + { + "type": "object", + "description": "Custom oracle for text values.", + "required": [ + "oracle", + "file", + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "custom_check" + }, + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + } + } + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + } + ] + }, + "languages": { + "oneOf": [ + { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + } + } + } + ] + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Custom oracle for text values.", + "required": [ + "oracle", + "file", + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "custom_check" + }, + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + } + } + } + } + } + ] + } + ] + }, + "fileOutputChannel": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "description": "Built-in oracle for files.", + "required": [ + "content", + "location" + ], + "properties": { + "content": { + "oneOf": [ + { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + } + } + } + } + ] + }, + "location": { + "oneOf": [ + { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + } + } + } + } + ] + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Built-in oracle for files.", + "required": [ + "content", + "location" + ], + "properties": { + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } + } + } + } + } + ] + }, + { + "oneOf": [ + { + "type": "object", + "description": "Custom oracle for file values.", + "required": [ + "oracle", + "content", + "location", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "content": { + "oneOf": [ + { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + } + } + } + } + ] + }, + "location": { + "oneOf": [ + { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + } + } + } + } + ] + }, + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + } + } + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + ] + }, + "arguments": { + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + } + ] + }, + "languages": { + "oneOf": [ + { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } }, - "value":{ + { "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/textualType" + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + } } } - } + ] } - ] - }, - "oracle" : { - "const" : "builtin" + } }, - "config" : { - "$ref" : "#/definitions/textConfigurationOptions" - } - } - }, - { - "type" : "object", - "description" : "Custom oracle for text values.", - "required" : [ - "oracle", - "file", - "data" - ], - "properties" : { - "data" : { - "oneOf" : [ - { - "$ref" : "#/definitions/textualType" + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - { - "type" : "object", - "required": [ - "__tag__", - "value" - ], - "properties" : { - "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" - }, - "value":{ - "type": "object", - "additionalProperties": { - "$ref" : "#/definitions/textualType" + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Custom oracle for file values.", + "required": [ + "oracle", + "content", + "location", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } } } } } - ] - }, - "oracle" : { - "const" : "custom_check" - }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." - }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" - }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" - } - }, - "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" } } - } + ] } ] }, - "fileOutputChannel": { - "anyOf" : [ + "returnOutputChannel": { + "oneOf": [ { - "type" : "object", - "description" : "Built-in oracle for files.", - "required" : [ - "content", - "location" - ], - "properties" : { - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." - }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." - }, - "oracle" : { - "const" : "builtin" - }, - "config" : { - "$ref" : "#/definitions/fileConfigurationOptions" - } - } + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, { - "type" : "object", - "description" : "Custom oracle for file values.", - "required" : [ - "oracle", - "content", - "location", - "file" + "type": "object", + "required": [ + "__tag__", + "value" ], - "properties" : { - "oracle" : { - "const" : "custom_check" - }, - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." - }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." - }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." - }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" - }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" - } + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } } - } - ] - }, - "returnOutputChannel" : { - "oneOf" : [ - { - "$ref" : "#/definitions/yamlValueOrPythonExpression" }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!oracle" + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" }, - "value":{ - "type": "object", - "additionalProperties" : false, - "required" : [ + "value": { + "additionalProperties": false, + "required": [ "value" ], - "properties" : { - "oracle" : { - "const" : "builtin" + "properties": { + "oracle": { + "const": "builtin" }, - "value" : { - "oneOf" : [ + "value": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } } } ] } - } + }, + "type": "object" } } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!oracle" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", - "additionalProperties" : false, - "required" : [ + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" + }, + "value": { + "additionalProperties": false, + "required": [ + "value" + ], + "properties": { + "oracle": { + "const": "builtin" + }, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "type": "object" + } + } + } + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" + }, + "value": { + "additionalProperties": false, + "required": [ "value", "oracle", "file" ], - "properties" : { - "oracle" : { - "const" : "custom_check" + "properties": { + "oracle": { + "const": "custom_check" }, - "value" : { - "oneOf" : [ + "value": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } } } ] }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + } + } + } + } + ] }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + ] }, - "arguments" : { - "oneOf" : [ + "arguments": { + "oneOf": [ { - "type" : "array", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + ] } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "array", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } } } } - ], - "description" : "List of YAML (or tagged expression) values to use as arguments to the function." + ] }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "oneOf": [ + { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + } + } + } + ] + } + }, + "type": "object" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" + }, + "value": { + "additionalProperties": false, + "required": [ + "value", + "oracle", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "arguments": { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + }, + "type": "object" } } } @@ -1863,98 +2734,286 @@ } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!oracle" + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" }, - "value":{ - "type": "object", - "additionalProperties" : false, - "required" : [ + "value": { + "additionalProperties": false, + "required": [ "oracle", "functions" ], - "properties" : { - "oracle" : { - "const" : "specific_check" + "properties": { + "oracle": { + "const": "specific_check" }, - "functions" : { - "minProperties" : 1, - "description" : "Language mapping of oracle functions.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "object", - "required" : [ - "file" - ], - "properties" : { - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "functions": { + "oneOf": [ + { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "items": { + "oneOf": [ + { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + } + } + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + ] + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + } } } - } + ] }, - "arguments" : { - "oneOf" : [ + "arguments": { + "oneOf": [ { - "minProperties" : 1, - "description" : "Language mapping of oracle arguments.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" }, - "items" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - } + "items": { + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + } + } + } + ] } }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "minProperties" : 1, - "description" : "Language mapping of oracle arguments.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" }, - "items" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." } } } @@ -1963,188 +3022,586 @@ } ] }, - "value" : { - "oneOf" : [ + "value": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, { - "type" : "object", + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } } } ] } + }, + "type": "object" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" + }, + "value": { + "additionalProperties": false, + "required": [ + "oracle", + "functions" + ], + "properties": { + "oracle": { + "const": "specific_check" + }, + "functions": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + }, + "arguments": { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + }, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "type": "object" + } + } } } } } ] }, - "programmingLanguage" : { - "type" : "string", - "description" : "One of the programming languages supported by TESTed.", - "enum" : [ - "bash", - "c", - "haskell", - "java", - "javascript", - "typescript", - "kotlin", - "python", - "runhaskell", - "csharp", - "cpp" + "programmingLanguage": { + "oneOf": [ + { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ + "bash", + "c", + "haskell", + "java", + "javascript", + "typescript", + "kotlin", + "python", + "runhaskell", + "csharp", + "cpp" + ] + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ + "bash", + "c", + "haskell", + "java", + "javascript", + "typescript", + "kotlin", + "python", + "runhaskell", + "csharp", + "cpp" + ] + } + } + } + } ] }, - "message" : { - "oneOf" : [ + "message": { + "oneOf": [ + { + "type": "string", + "description": "A simple message to display." + }, { - "type" : "string", - "description" : "A simple message to display." + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A simple message to display." + } + } + } }, { - "type" : "object", - "required" : [ + "type": "object", + "required": [ "description" ], - "properties" : { - "description" : { + "properties": { + "description": { "oneOf": [ { - "type" : "object", + "description": "The message to display.", + "type": "string" + }, + { + "type": "object", "required": [ "__tag__", "value" ], - "properties" : { + "properties": { "__tag__": { - "type" : "string", - "description" : "The tag used in the yaml", - "const": "!natural_language" + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "value":{ + "value": { "type": "object", "additionalProperties": { - "type" : "string" + "description": "The message to display.", + "type": "string" } } } + } + ] + }, + "format": { + "oneOf": [ + { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." }, { - "type" : "string" + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + } + } + } } - ], - "description" : "The message to display." + ] + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "format" : { - "type" : "string", - "default" : "text", - "description" : "The format of the message, either a programming language, 'text' or 'html'." + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "description" + ], + "properties": { + "description": { + "description": "The message to display.", + "type": "string" + }, + "format": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + } + } + } } } } ] }, - "textConfigurationOptions" : { - "type" : "object", - "description" : "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", - "minProperties" : 1, - "properties" : { - "applyRounding" : { - "description" : "Apply rounding when comparing as float", - "type" : "boolean" - }, - "caseInsensitive" : { - "description" : "Ignore case when comparing strings", - "type" : "boolean" - }, - "ignoreWhitespace" : { - "description" : "Ignore trailing whitespace", - "type" : "boolean" - }, - "normalizeTrailingNewlines" : { - "description" : "Normalize trailing newlines", - "type" : "boolean" - }, - "roundTo" : { - "description" : "The number of decimals to round at, when applying the rounding on floats", - "type" : "integer" - }, - "tryFloatingPoint" : { - "description" : "Try comparing text as floating point numbers", - "type" : "boolean" - }, - "hideExpected" : { - "description" : "Hide the expected value in feedback (default: false), not recommended to use!", - "type" : "boolean" + "textConfigurationOptions": { + "oneOf": [ + { + "type": "object", + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, + "properties": { + "applyRounding": { + "description": "Apply rounding when comparing as float", + "type": "boolean" + }, + "caseInsensitive": { + "description": "Ignore case when comparing strings", + "type": "boolean" + }, + "ignoreWhitespace": { + "description": "Ignore trailing whitespace", + "type": "boolean" + }, + "normalizeTrailingNewlines": { + "description": "Normalize trailing newlines", + "type": "boolean" + }, + "roundTo": { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + }, + "tryFloatingPoint": { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + }, + "hideExpected": { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, + "properties": { + "applyRounding": { + "description": "Apply rounding when comparing as float", + "type": "boolean" + }, + "caseInsensitive": { + "description": "Ignore case when comparing strings", + "type": "boolean" + }, + "ignoreWhitespace": { + "description": "Ignore trailing whitespace", + "type": "boolean" + }, + "normalizeTrailingNewlines": { + "description": "Normalize trailing newlines", + "type": "boolean" + }, + "roundTo": { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + }, + "tryFloatingPoint": { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + }, + "hideExpected": { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" + } + } + } + } + } } - } + ] }, "fileConfigurationOptions": { - "anyOf" : [ + "anyOf": [ { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" }, { - "type" : "object", - "properties" : { - "mode": { - "type" : "string", - "enum" : ["full", "line"], - "default" : "full" + "oneOf": [ + { + "type": "object", + "properties": { + "mode": { + "oneOf": [ + { + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" + } + } + } + } + ] + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" + } + } + } + } + } } - } + ] } ] }, - "textualType" : { - "description" : "Simple textual value, converted to string.", - "type" : [ - "string", - "number", - "integer", - "boolean" + "textualType": { + "oneOf": [ + { + "description": "Simple textual value, converted to string.", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "Simple textual value, converted to string.", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + } + } + } ] }, - "yamlValue" : { - "description" : "A value represented as YAML.", - "not" : { + "yamlValue": { + "description": "A value represented as YAML.", + "not": { "properties": { - "__tag__": { "type": "string" } + "__tag__": { + "type": "string" + } }, "type": "object" } }, "inheritableConfigObject": { - "type": "object", - "properties" : { - "stdout": { - "$ref" : "#/definitions/textConfigurationOptions" - }, - "stderr": { - "$ref" : "#/definitions/textConfigurationOptions" + "oneOf": [ + { + "type": "object", + "properties": { + "stdout": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "stderr": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "file": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } }, - "file": { - "$ref" : "#/definitions/fileConfigurationOptions" + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "stdout": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "stderr": { + "$ref": "#/subDefinitions/textConfigurationOptions" + }, + "file": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } + } + } + } } - } + ] } } -} +} \ No newline at end of file diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 0471d175..c486cf5e 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -1,293 +1,297 @@ { - "$id" : "tested:dsl:schema7", - "$schema" : "http://json-schema.org/draft-07/schema#", - "title" : "TESTed-DSL", - "oneOf" : [ + "$id": "tested:dsl:schema7", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TESTed-DSL", + "oneOf": [ { - "$ref" : "#/definitions/_rootObject" + "$ref": "#/definitions/_rootObject" }, { - "$ref" : "#/definitions/_tabList" + "$ref": "#/definitions/_tabList" }, { - "$ref" : "#/definitions/_unitList" + "$ref": "#/definitions/_unitList" } ], - "definitions" : { - "_rootObject" : { - "type" : "object", - "oneOf" : [ + "definitions": { + "_rootObject": { + "type": "object", + "oneOf": [ { - "required" : [ + "required": [ "tabs" ], - "not" : { - "required" : [ + "not": { + "required": [ "units" ] } }, { - "required" : [ + "required": [ "units" ], - "not" : { - "required" : [ + "not": { + "required": [ "tabs" ] } } ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, - "namespace" : { - "type" : "string", - "description" : "Namespace of the submitted solution, in `snake_case`" + "namespace": { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" }, - "tabs" : { - "$ref" : "#/definitions/_tabList" + "tabs": { + "$ref": "#/definitions/_tabList" }, - "units" : { - "$ref" : "#/definitions/_unitList" + "units": { + "$ref": "#/definitions/_unitList" }, - "language" : { - "description" : "Indicate that all code is in a specific language.", - "oneOf" : [ + "language": { + "description": "Indicate that all code is in a specific language.", + "oneOf": [ { - "$ref" : "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" }, { - "const" : "tested" + "const": "tested" } ] }, - "definitions" : { - "description" : "Define hashes to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define hashes to use elsewhere.", + "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" } } }, - "_tabList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/tab" + "_tabList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/tab" } }, - "_unitList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/unit" + "_unitList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/unit" } }, - "tab" : { - "type" : "object", - "description" : "A tab in the test suite.", - "required" : [ + "tab": { + "type": "object", + "description": "A tab in the test suite.", + "required": [ "tab" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "tab" : { - "type" : "string", - "description" : "The name of this tab." + "tab": { + "description": "The name of this tab.", + "type": "string" }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "contexts" ], - "properties" : { - "contexts" : { - "$ref" : "#/definitions/_contextList" + "properties": { + "contexts": { + "$ref": "#/definitions/_contextList" } } }, { - "required" : [ + "required": [ "testcases" ], - "properties" : { - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "properties": { + "testcases": { + "$ref": "#/definitions/_testcaseList" } } } ] }, - "unit" : { - "type" : "object", - "description" : "A unit in the test suite.", - "required" : [ + "unit": { + "type": "object", + "description": "A unit in the test suite.", + "required": [ "unit" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "unit" : { - "type" : "string", - "description" : "The name of this tab." + "unit": { + "description": "The name of this tab.", + "type": "string" }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "cases" ], - "properties" : { - "cases" : { - "$ref" : "#/definitions/_caseList" + "properties": { + "cases": { + "$ref": "#/definitions/_caseList" } } }, { - "required" : [ + "required": [ "scripts" ], - "properties" : { - "scripts" : { - "$ref" : "#/definitions/_scriptList" + "properties": { + "scripts": { + "$ref": "#/definitions/_scriptList" } } } ] }, - "_contextList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/context" + "_contextList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" } }, - "_caseList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/case" + "_caseList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" } }, - "_testcaseList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/testcase" + "_testcaseList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" } }, - "_scriptList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/script" + "_scriptList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/script" } }, - "context" : { - "type" : "object", - "description" : "A set of testcase in the same context.", - "required" : [ + "context": { + "type": "object", + "description": "A set of testcase in the same context.", + "required": [ "testcases" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, - "context" : { - "type" : "string", - "description" : "Description of this context." + "context": { + "type": "string", + "description": "Description of this context." }, - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "testcases": { + "$ref": "#/definitions/_testcaseList" } } }, - "case" : { - "type" : "object", - "description" : "A test case.", - "required" : [ + "case": { + "type": "object", + "description": "A test case.", + "required": [ "script" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, - "context" : { - "type" : "string", - "description" : "Description of this context." + "context": { + "type": "string", + "description": "Description of this context." }, - "script" : { - "$ref" : "#/definitions/_scriptList" + "script": { + "$ref": "#/definitions/_scriptList" } } }, - "testcase" : { - "type" : "object", - "description" : "An individual test for a statement or expression", - "additionalProperties" : false, - "properties" : { - "description" : { - "$ref" : "#/definitions/message" - }, - "stdin" : { - "description" : "Stdin for this context", - "type" : [ + "testcase": { + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { + "description": "Stdin for this context", + "type": [ "string", "number", "integer", "boolean" ] }, - "arguments" : { - "type" : "array", - "description" : "Array of program call arguments", - "items" : { - "type" : [ + "arguments": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ "string", "number", "integer", @@ -295,95 +299,68 @@ ] } }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "type" : "string", - "description" : "Message of the expected exception." - }, - { - "type" : "object", - "required" : [ - "types" - ], - "properties" : { - "message" : { - "type" : "string", - "description" : "Message of the expected exception." - }, - "types" : { - "minProperties" : 1, - "description" : "Language mapping of expected exception types.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string" - } - } - } - } - ] + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" }, - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, - "return" : { - "description" : "Expected return value", - "$ref" : "#/definitions/returnOutputChannel" + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" }, - "stderr" : { - "description" : "Expected output at stderr", - "$ref" : "#/definitions/textOutputChannel" + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" }, - "stdout" : { - "description" : "Expected output at stdout", - "$ref" : "#/definitions/textOutputChannel" + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" }, "file": { - "description" : "Expected files generated by the submission.", - "$ref" : "#/definitions/fileOutputChannel" + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" } } }, - "script" : { - "type" : "object", - "description" : "An individual test (script) for a statement or expression", - "properties" : { - "description" : { - "$ref" : "#/definitions/message" - }, - "stdin" : { - "description" : "Stdin for this context", - "type" : [ + "script": { + "type": "object", + "description": "An individual test (script) for a statement or expression", + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { + "description": "Stdin for this context", + "type": [ "string", "number", "integer", "boolean" ] }, - "arguments" : { - "type" : "array", - "description" : "Array of program call arguments", - "items" : { - "type" : [ + "arguments": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ "string", "number", "integer", @@ -391,189 +368,207 @@ ] } }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "type" : "string", - "description" : "Message of the expected exception." - }, - { - "type" : "object", - "required" : [ - "types" - ], - "properties" : { - "message" : { - "type" : "string", - "description" : "Message of the expected exception." - }, - "types" : { - "minProperties" : 1, - "description" : "Language mapping of expected exception types.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string" - } - } - } - } - ] + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" }, - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, - "return" : { - "description" : "Expected return value", - "$ref" : "#/definitions/returnOutputChannel" + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" }, - "stderr" : { - "description" : "Expected output at stderr", - "$ref" : "#/definitions/textOutputChannel" + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" }, - "stdout" : { - "description" : "Expected output at stdout", - "$ref" : "#/definitions/textOutputChannel" + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" } } - }, - "expressionOrStatement" : { - "oneOf" : [ + } + }, + "subDefinitions" : { + "expressionOrStatement": { + "oneOf": [ { - "type" : "string", - "format" : "tested-dsl-expression", - "description" : "A statement of expression in Python-like syntax as YAML string." + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." }, { - "description" : "Programming-language-specific statement or expression.", "anyOf": [ { - "type": "object" + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } }, { - "type": "programming_language" + "type": "programming_language", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } } - ], - "type" : "object", - "minProperties" : 1, - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - } + ] } ] }, - "yamlValueOrPythonExpression" : { - "oneOf" : [ + "yamlValueOrPythonExpression": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValue" + "$ref": "#/subDefinitions/yamlValue" }, { - "type" : "expression", - "format" : "tested-dsl-expression", - "description" : "An expression in Python-syntax." + "type": "expression", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax." } ] }, - "file" : { - "type" : "object", - "description" : "A file used in the test suite.", - "required" : [ + "file": { + "type": "object", + "description": "A file used in the test suite.", + "required": [ "name", "url" ], - "properties" : { - "name" : { - "type" : "string", - "description" : "The filename, including the file extension." - }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." + "properties": { + "name": { + "type": "string", + "description": "The filename, including the file extension." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." } } }, - "textOutputChannel" : { - "anyOf" : [ + "exceptionChannel": { + "oneOf": [ { - "$ref" : "#/definitions/textualType" + "type": "string", + "description": "Message of the expected exception." }, { - "type" : "object", - "description" : "Built-in oracle for text values.", - "required" : [ + "type": "object", + "required": [ + "types" + ], + "properties": { + "message": { + "description": "Message of the expected exception.", + "type": "string" + }, + "types": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string" + } + } + } + } + ] + }, + "textOutputChannel": { + "anyOf": [ + { + "$ref": "#/subDefinitions/textualType" + }, + { + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ "data" ], - "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" }, - "oracle" : { - "const" : "builtin" + "oracle": { + "const": "builtin" }, - "config" : { - "$ref" : "#/definitions/textConfigurationOptions" + "config": { + "$ref": "#/subDefinitions/textConfigurationOptions" } } }, { - "type" : "object", - "description" : "Custom oracle for text values.", - "required" : [ + "type": "object", + "description": "Custom oracle for text values.", + "required": [ "oracle", "file", "data" ], - "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" }, - "oracle" : { - "const" : "custom_check" + "oracle": { + "const": "custom_check" }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" } } } @@ -581,202 +576,202 @@ ] }, "fileOutputChannel": { - "anyOf" : [ + "anyOf": [ { - "type" : "object", - "description" : "Built-in oracle for files.", - "required" : [ + "type": "object", + "description": "Built-in oracle for files.", + "required": [ "content", "location" ], - "properties" : { - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "properties": { + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." }, - "oracle" : { - "const" : "builtin" + "oracle": { + "const": "builtin" }, - "config" : { - "$ref" : "#/definitions/fileConfigurationOptions" + "config": { + "$ref": "#/subDefinitions/fileConfigurationOptions" } } }, { - "type" : "object", - "description" : "Custom oracle for file values.", - "required" : [ + "type": "object", + "description": "Custom oracle for file values.", + "required": [ "oracle", "content", "location", "file" ], - "properties" : { - "oracle" : { - "const" : "custom_check" + "properties": { + "oracle": { + "const": "custom_check" }, - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" } } } } ] }, - "returnOutputChannel" : { - "oneOf" : [ + "returnOutputChannel": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, { - "type" : "oracle", - "additionalProperties" : false, - "required" : [ + "type": "oracle", + "additionalProperties": false, + "required": [ "value" ], - "properties" : { - "oracle" : { - "const" : "builtin" + "properties": { + "oracle": { + "const": "builtin" }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } }, { - "type" : "oracle", - "additionalProperties" : false, - "required" : [ + "type": "oracle", + "additionalProperties": false, + "required": [ "value", "oracle", "file" ], - "properties" : { - "oracle" : { - "const" : "custom_check" + "properties": { + "oracle": { + "const": "custom_check" }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" } } } }, { - "type" : "oracle", - "additionalProperties" : false, - "required" : [ + "type": "oracle", + "additionalProperties": false, + "required": [ "oracle", "functions" ], - "properties" : { - "oracle" : { - "const" : "specific_check" + "properties": { + "oracle": { + "const": "specific_check" }, - "functions" : { - "minProperties" : 1, - "description" : "Language mapping of oracle functions.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "functions": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" }, - "items" : { - "type" : "object", - "required" : [ + "items": { + "type": "object", + "required": [ "file" ], - "properties" : { - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" } } } }, - "arguments" : { - "minProperties" : 1, - "description" : "Language mapping of oracle arguments.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "arguments": { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" }, - "items" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." } } }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } } ] }, - "programmingLanguage" : { - "type" : "string", - "description" : "One of the programming languages supported by TESTed.", - "enum" : [ + "programmingLanguage": { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ "bash", "c", "haskell", @@ -790,96 +785,99 @@ "cpp" ] }, - "message" : { - "oneOf" : [ + "message": { + "oneOf": [ { - "type" : "string", - "description" : "A simple message to display." + "type": "string", + "description": "A simple message to display." }, { - "type" : "object", - "required" : [ + "type": "object", + "required": [ "description" ], - "properties" : { - "description" : { - "type" : "string", - "description" : "The message to display." + "properties": { + "description": { + "description": "The message to display.", + "type": "string" }, - "format" : { - "type" : "string", - "default" : "text", - "description" : "The format of the message, either a programming language, 'text' or 'html'." + "format": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." } } } ] }, - "textConfigurationOptions" : { - "type" : "object", - "description" : "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", - "minProperties" : 1, - "properties" : { - "applyRounding" : { - "description" : "Apply rounding when comparing as float", - "type" : "boolean" - }, - "caseInsensitive" : { - "description" : "Ignore case when comparing strings", - "type" : "boolean" - }, - "ignoreWhitespace" : { - "description" : "Ignore trailing whitespace", - "type" : "boolean" - }, - "normalizeTrailingNewlines" : { - "description" : "Normalize trailing newlines", - "type" : "boolean" - }, - "roundTo" : { - "description" : "The number of decimals to round at, when applying the rounding on floats", - "type" : "integer" - }, - "tryFloatingPoint" : { - "description" : "Try comparing text as floating point numbers", - "type" : "boolean" - }, - "hideExpected" : { - "description" : "Hide the expected value in feedback (default: false), not recommended to use!", - "type" : "boolean" + "textConfigurationOptions": { + "type": "object", + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, + "properties": { + "applyRounding": { + "description": "Apply rounding when comparing as float", + "type": "boolean" + }, + "caseInsensitive": { + "description": "Ignore case when comparing strings", + "type": "boolean" + }, + "ignoreWhitespace": { + "description": "Ignore trailing whitespace", + "type": "boolean" + }, + "normalizeTrailingNewlines": { + "description": "Normalize trailing newlines", + "type": "boolean" + }, + "roundTo": { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + }, + "tryFloatingPoint": { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + }, + "hideExpected": { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" } } }, "fileConfigurationOptions": { - "anyOf" : [ + "anyOf": [ { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" }, { - "type" : "object", - "properties" : { + "type": "object", + "properties": { "mode": { - "type" : "string", - "enum" : ["full", "line"], - "default" : "full" + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" } } } ] }, - "textualType" : { - "description" : "Simple textual value, converted to string.", - "type" : [ + "textualType": { + "description": "Simple textual value, converted to string.", + "type": [ "string", "number", "integer", "boolean" ] }, - "yamlValue" : { - "description" : "A value represented as YAML.", - "not" : { - "type" : [ + "yamlValue": { + "description": "A value represented as YAML.", + "not": { + "type": [ "oracle", "expression", "programming_language" @@ -888,15 +886,15 @@ }, "inheritableConfigObject": { "type": "object", - "properties" : { + "properties": { "stdout": { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" }, "stderr": { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" }, "file": { - "$ref" : "#/definitions/fileConfigurationOptions" + "$ref": "#/subDefinitions/fileConfigurationOptions" } } } diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index f8d066bc..fd11feeb 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -1,293 +1,297 @@ { - "$id" : "tested:dsl:schema7", - "$schema" : "http://json-schema.org/draft-07/schema#", - "title" : "TESTed-DSL", - "oneOf" : [ + "$id": "tested:dsl:schema7", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TESTed-DSL", + "oneOf": [ { - "$ref" : "#/definitions/_rootObject" + "$ref": "#/definitions/_rootObject" }, { - "$ref" : "#/definitions/_tabList" + "$ref": "#/definitions/_tabList" }, { - "$ref" : "#/definitions/_unitList" + "$ref": "#/definitions/_unitList" } ], - "definitions" : { - "_rootObject" : { - "type" : "object", - "oneOf" : [ + "definitions": { + "_rootObject": { + "type": "object", + "oneOf": [ { - "required" : [ + "required": [ "tabs" ], - "not" : { - "required" : [ + "not": { + "required": [ "units" ] } }, { - "required" : [ + "required": [ "units" ], - "not" : { - "required" : [ + "not": { + "required": [ "tabs" ] } } ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, - "namespace" : { - "type" : "string", - "description" : "Namespace of the submitted solution, in `snake_case`" + "namespace": { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" }, - "tabs" : { - "$ref" : "#/definitions/_tabList" + "tabs": { + "$ref": "#/definitions/_tabList" }, - "units" : { - "$ref" : "#/definitions/_unitList" + "units": { + "$ref": "#/definitions/_unitList" }, - "language" : { - "description" : "Indicate that all code is in a specific language.", - "oneOf" : [ + "language": { + "description": "Indicate that all code is in a specific language.", + "oneOf": [ { - "$ref" : "#/definitions/programmingLanguage" + "$ref": "#/subDefinitions/programmingLanguage" }, { - "const" : "tested" + "const": "tested" } ] }, - "definitions" : { - "description" : "Define hashes to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define hashes to use elsewhere.", + "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" } } }, - "_tabList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/tab" + "_tabList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/tab" } }, - "_unitList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/unit" + "_unitList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/unit" } }, - "tab" : { - "type" : "object", - "description" : "A tab in the test suite.", - "required" : [ + "tab": { + "type": "object", + "description": "A tab in the test suite.", + "required": [ "tab" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "tab" : { - "type" : "string", - "description" : "The name of this tab." + "tab": { + "description": "The name of this tab.", + "type": "string" }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "contexts" ], - "properties" : { - "contexts" : { - "$ref" : "#/definitions/_contextList" + "properties": { + "contexts": { + "$ref": "#/definitions/_contextList" } } }, { - "required" : [ + "required": [ "testcases" ], - "properties" : { - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "properties": { + "testcases": { + "$ref": "#/definitions/_testcaseList" } } } ] }, - "unit" : { - "type" : "object", - "description" : "A unit in the test suite.", - "required" : [ + "unit": { + "type": "object", + "description": "A unit in the test suite.", + "required": [ "unit" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "unit" : { - "type" : "string", - "description" : "The name of this tab." + "unit": { + "description": "The name of this tab.", + "type": "string" }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { - "$ref": "#/definitions/inheritableConfigObject" + "$ref": "#/subDefinitions/inheritableConfigObject" } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "cases" ], - "properties" : { - "cases" : { - "$ref" : "#/definitions/_caseList" + "properties": { + "cases": { + "$ref": "#/definitions/_caseList" } } }, { - "required" : [ + "required": [ "scripts" ], - "properties" : { - "scripts" : { - "$ref" : "#/definitions/_scriptList" + "properties": { + "scripts": { + "$ref": "#/definitions/_scriptList" } } } ] }, - "_contextList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/context" + "_contextList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" } }, - "_caseList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/case" + "_caseList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" } }, - "_testcaseList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/testcase" + "_testcaseList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" } }, - "_scriptList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/script" + "_scriptList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/script" } }, - "context" : { - "type" : "object", - "description" : "A set of testcase in the same context.", - "required" : [ + "context": { + "type": "object", + "description": "A set of testcase in the same context.", + "required": [ "testcases" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, - "context" : { - "type" : "string", - "description" : "Description of this context." + "context": { + "type": "string", + "description": "Description of this context." }, - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "testcases": { + "$ref": "#/definitions/_testcaseList" } } }, - "case" : { - "type" : "object", - "description" : "A test case.", - "required" : [ + "case": { + "type": "object", + "description": "A test case.", + "required": [ "script" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, - "context" : { - "type" : "string", - "description" : "Description of this context." + "context": { + "type": "string", + "description": "Description of this context." }, - "script" : { - "$ref" : "#/definitions/_scriptList" + "script": { + "$ref": "#/definitions/_scriptList" } } }, - "testcase" : { - "type" : "object", - "description" : "An individual test for a statement or expression", - "additionalProperties" : false, - "properties" : { - "description" : { - "$ref" : "#/definitions/message" - }, - "stdin" : { - "description" : "Stdin for this context", - "type" : [ + "testcase": { + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { + "description": "Stdin for this context", + "type": [ "string", "number", "integer", "boolean" ] }, - "arguments" : { - "type" : "array", - "description" : "Array of program call arguments", - "items" : { - "type" : [ + "arguments": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ "string", "number", "integer", @@ -295,91 +299,68 @@ ] } }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "type" : "string", - "description" : "Message of the expected exception." - }, - { - "type" : "object", - "required" : [ - "types" - ], - "properties" : { - "message" : { - "type" : "string", - "description" : "Message of the expected exception." - }, - "types" : { - "minProperties" : 1, - "description" : "Language mapping of expected exception types.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string" - } - } - } - } - ] + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" }, - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, - "return" : { - "description" : "Expected return value", - "$ref" : "#/definitions/returnOutputChannel" + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" }, - "stderr" : { - "description" : "Expected output at stderr", - "$ref" : "#/definitions/textOutputChannel" + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" }, - "stdout" : { - "description" : "Expected output at stdout", - "$ref" : "#/definitions/textOutputChannel" + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" } } }, - "script" : { - "type" : "object", - "description" : "An individual test (script) for a statement or expression", - "properties" : { - "description" : { - "$ref" : "#/definitions/message" - }, - "stdin" : { - "description" : "Stdin for this context", - "type" : [ + "script": { + "type": "object", + "description": "An individual test (script) for a statement or expression", + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { + "description": "Stdin for this context", + "type": [ "string", "number", "integer", "boolean" ] }, - "arguments" : { - "type" : "array", - "description" : "Array of program call arguments", - "items" : { - "type" : [ + "arguments": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ "string", "number", "integer", @@ -387,185 +368,195 @@ ] } }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "type" : "string", - "description" : "Message of the expected exception." - }, - { - "type" : "object", - "required" : [ - "types" - ], - "properties" : { - "message" : { - "type" : "string", - "description" : "Message of the expected exception." - }, - "types" : { - "minProperties" : 1, - "description" : "Language mapping of expected exception types.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string" - } - } - } - } - ] + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" }, - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, - "return" : { - "description" : "Expected return value", - "$ref" : "#/definitions/returnOutputChannel" + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" }, - "stderr" : { - "description" : "Expected output at stderr", - "$ref" : "#/definitions/textOutputChannel" + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" }, - "stdout" : { - "description" : "Expected output at stdout", - "$ref" : "#/definitions/textOutputChannel" + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" }, "file": { - "description" : "Expected files generated by the submission.", - "$ref" : "#/definitions/fileOutputChannel" + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" } } - }, - "expressionOrStatement" : { - "oneOf" : [ + } + }, + "subDefinitions": { + "expressionOrStatement": { + "oneOf": [ { - "type" : "string", - "format" : "tested-dsl-expression", - "description" : "A statement of expression in Python-like syntax as YAML string." + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." }, { - "description" : "Programming-language-specific statement or expression.", - "type" : "object", - "minProperties" : 1, - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - } + "anyOf": [ + { + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + ] } ] }, - "yamlValueOrPythonExpression" : { - "oneOf" : [ + "yamlValueOrPythonExpression": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValue" + "$ref": "#/subDefinitions/yamlValue" }, { - "type" : "string", - "format" : "tested-dsl-expression", - "description" : "An expression in Python-syntax." + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax." } ] }, - "file" : { - "type" : "object", - "description" : "A file used in the test suite.", - "required" : [ + "file": { + "type": "object", + "description": "A file used in the test suite.", + "required": [ "name", "url" ], - "properties" : { - "name" : { - "type" : "string", - "description" : "The filename, including the file extension." - }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." + "properties": { + "name": { + "type": "string", + "description": "The filename, including the file extension." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." } } }, - "textOutputChannel" : { - "anyOf" : [ + "exceptionChannel": { + "oneOf": [ + { + "type": "string", + "description": "Message of the expected exception." + }, + { + "type": "object", + "required": [ + "types" + ], + "properties": { + "message": { + "description": "Message of the expected exception.", + "type": "string" + }, + "types": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string" + } + } + } + } + ] + }, + "textOutputChannel": { + "anyOf": [ { - "$ref" : "#/definitions/textualType" + "$ref": "#/subDefinitions/textualType" }, { - "type" : "object", - "description" : "Built-in oracle for text values.", - "required" : [ + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ "data" ], - "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" }, - "oracle" : { - "const" : "builtin" + "oracle": { + "const": "builtin" }, - "config" : { - "$ref" : "#/definitions/textConfigurationOptions" + "config": { + "$ref": "#/subDefinitions/textConfigurationOptions" } } }, { - "type" : "object", - "description" : "Custom oracle for text values.", - "required" : [ + "type": "object", + "description": "Custom oracle for text values.", + "required": [ "oracle", "file", "data" ], - "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" }, - "oracle" : { - "const" : "custom_check" + "oracle": { + "const": "custom_check" }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" } } } @@ -573,202 +564,202 @@ ] }, "fileOutputChannel": { - "anyOf" : [ + "anyOf": [ { - "type" : "object", - "description" : "Built-in oracle for files.", - "required" : [ + "type": "object", + "description": "Built-in oracle for files.", + "required": [ "content", "location" ], - "properties" : { - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "properties": { + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." }, - "oracle" : { - "const" : "builtin" + "oracle": { + "const": "builtin" }, - "config" : { - "$ref" : "#/definitions/fileConfigurationOptions" + "config": { + "$ref": "#/subDefinitions/fileConfigurationOptions" } } }, { - "type" : "object", - "description" : "Custom oracle for file values.", - "required" : [ + "type": "object", + "description": "Custom oracle for file values.", + "required": [ "oracle", "content", "location", "file" ], - "properties" : { - "oracle" : { - "const" : "custom_check" + "properties": { + "oracle": { + "const": "custom_check" }, - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" } } } } ] }, - "returnOutputChannel" : { - "oneOf" : [ + "returnOutputChannel": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, { - "type" : "object", - "additionalProperties" : false, - "required" : [ + "type": "object", + "additionalProperties": false, + "required": [ "value" ], - "properties" : { - "oracle" : { - "const" : "builtin" + "properties": { + "oracle": { + "const": "builtin" }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } }, { - "type" : "object", - "additionalProperties" : false, - "required" : [ + "type": "object", + "additionalProperties": false, + "required": [ "value", "oracle", "file" ], - "properties" : { - "oracle" : { - "const" : "custom_check" + "properties": { + "oracle": { + "const": "custom_check" }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" } } } }, { - "type" : "object", - "additionalProperties" : false, - "required" : [ + "type": "object", + "additionalProperties": false, + "required": [ "oracle", "functions" ], - "properties" : { - "oracle" : { - "const" : "specific_check" + "properties": { + "oracle": { + "const": "specific_check" }, - "functions" : { - "minProperties" : 1, - "description" : "Language mapping of oracle functions.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "functions": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" }, - "items" : { - "type" : "object", - "required" : [ + "items": { + "type": "object", + "required": [ "file" ], - "properties" : { - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" } } } }, - "arguments" : { - "minProperties" : 1, - "description" : "Language mapping of oracle arguments.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "arguments": { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" }, - "items" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." } } }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } } ] }, - "programmingLanguage" : { - "type" : "string", - "description" : "One of the programming languages supported by TESTed.", - "enum" : [ + "programmingLanguage": { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ "bash", "c", "haskell", @@ -782,106 +773,116 @@ "cpp" ] }, - "message" : { - "oneOf" : [ + "message": { + "oneOf": [ { - "type" : "string", - "description" : "A simple message to display." + "type": "string", + "description": "A simple message to display." }, { - "type" : "object", - "required" : [ + "type": "object", + "required": [ "description" ], - "properties" : { - "description" : { - "type" : "string", - "description" : "The message to display." + "properties": { + "description": { + "description": "The message to display.", + "type": "string" }, - "format" : { - "type" : "string", - "default" : "text", - "description" : "The format of the message, either a programming language, 'text' or 'html'." + "format": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." } } } ] }, - "textConfigurationOptions" : { - "type" : "object", - "description" : "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", - "minProperties" : 1, - "properties" : { - "applyRounding" : { - "description" : "Apply rounding when comparing as float", - "type" : "boolean" - }, - "caseInsensitive" : { - "description" : "Ignore case when comparing strings", - "type" : "boolean" - }, - "ignoreWhitespace" : { - "description" : "Ignore trailing whitespace", - "type" : "boolean" - }, - "normalizeTrailingNewlines" : { - "description" : "Normalize trailing newlines", - "type" : "boolean" - }, - "roundTo" : { - "description" : "The number of decimals to round at, when applying the rounding on floats", - "type" : "integer" - }, - "tryFloatingPoint" : { - "description" : "Try comparing text as floating point numbers", - "type" : "boolean" - }, - "hideExpected" : { - "description" : "Hide the expected value in feedback (default: false), not recommended to use!", - "type" : "boolean" + "textConfigurationOptions": { + "type": "object", + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, + "properties": { + "applyRounding": { + "description": "Apply rounding when comparing as float", + "type": "boolean" + }, + "caseInsensitive": { + "description": "Ignore case when comparing strings", + "type": "boolean" + }, + "ignoreWhitespace": { + "description": "Ignore trailing whitespace", + "type": "boolean" + }, + "normalizeTrailingNewlines": { + "description": "Normalize trailing newlines", + "type": "boolean" + }, + "roundTo": { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + }, + "tryFloatingPoint": { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + }, + "hideExpected": { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" } } }, "fileConfigurationOptions": { - "anyOf" : [ + "anyOf": [ { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" }, { - "type" : "object", - "properties" : { + "type": "object", + "properties": { "mode": { - "type" : "string", - "enum" : ["full", "line"], - "default" : "full" + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" } } } ] }, - "textualType" : { - "description" : "Simple textual value, converted to string.", - "type" : [ + "textualType": { + "description": "Simple textual value, converted to string.", + "type": [ "string", "number", "integer", "boolean" ] }, - "yamlValue" : { - "description" : "A value represented as YAML." + "yamlValue": { + "description": "A value represented as YAML.", + "not": { + "type": [ + "oracle", + "expression", + "programming_language" + ] + } }, "inheritableConfigObject": { "type": "object", - "properties" : { + "properties": { "stdout": { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" }, "stderr": { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/subDefinitions/textConfigurationOptions" }, "file": { - "$ref" : "#/definitions/fileConfigurationOptions" + "$ref": "#/subDefinitions/fileConfigurationOptions" } } } diff --git a/tested/transform_json.py b/tested/transform_json.py new file mode 100644 index 00000000..ecffea0f --- /dev/null +++ b/tested/transform_json.py @@ -0,0 +1,276 @@ +import json +import os +import sys +from pathlib import Path +from typing import Any + +SPECIAL_TYPES = ["expression", "programming_language", "oracle"] + + +def transform_non_strict(data: Any) -> Any: + if isinstance(data, list): + return [ + transformed + for ele in data + if (transformed := transform_non_strict(ele)) != {} + ] + elif isinstance(data, dict): + if "type" in data and data["type"] in SPECIAL_TYPES: + if data["type"] == "expression": + data["type"] = "string" + elif data["type"] == "oracle": + data["type"] = "object" + else: + return {} + + return {k: transform_non_strict(v) for k, v in data.items()} + return data + + +def transform_ide(data: Any) -> Any: + if isinstance(data, list): + return [ + transformed for ele in data if (transformed := transform_ide(ele)) != {} + ] + elif isinstance(data, dict): + if "yamlValue" in data: + data["yamlValue"] = { + "description": "A value represented as YAML.", + } + + # This is a speciale structure for tags. + if "required" in data and "properties" in data: + required = data["required"] + properties = data["properties"] + assert isinstance(required, list) + assert isinstance(properties, dict) + if "__tag__" in required and "value" in required: + assert "__tag__" in properties and "value" in properties + value = properties["value"] + tag = properties["__tag__"] + assert isinstance(value, dict) + assert isinstance(tag, dict) and "const" in tag + + # For !programming_language the same was already defined without the tag. + # The only usage for the expression tag is also redundant. + if tag["const"] == "!programming_language": + return {} + + value = {k: transform_ide(v) for k, v in value.items()} + if ( + "additionalProperties" in value + and value["additionalProperties"] == {} + ): + return {} + + # This to help the validator to somewhat distinguish between + # a natural_language map and another YAML-map. + if tag["const"] == "!natural_language": + value["not"] = { + "anyOf": [ + {"required": ["description"]}, + {"required": ["value"]}, + {"required": ["types"]}, + ] + } + + if "oneOf" in value and value["oneOf"] == []: + return {} + + return value + + data = {k: transform_ide(v) for k, v in data.items()} + if "oneOf" in data and data["oneOf"] == []: + return {} + return data + return data + + +def flatten_one_of_stack(data: list) -> list: + new_one_stack = [] + for ele in data: + if isinstance(ele, dict) and len(ele.keys()) == 1 and "oneOf" in ele: + assert isinstance(ele["oneOf"], list) + new_one_stack.extend(ele["oneOf"]) + else: + new_one_stack.append(ele) + return new_one_stack + + +def make_tag_structure( + data: Any, data_with_inner_translations: Any = None, tag: str = "!natural_language" +) -> dict: + + base = { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": tag, + }, + }, + } + + if tag in ["!oracle", "!programming_language", "!expression"]: + base["properties"]["value"] = data + base["properties"]["value"]["type"] = "object" + if tag == "!expression": + base["properties"]["value"]["type"] = "string" + + elif tag == "!natural_language": + base["properties"]["value"] = { + "type": "object", + "additionalProperties": data, + } + if data_with_inner_translations is not None: + return {"oneOf": [data_with_inner_translations, base]} + + return base + + +def make_translations_map() -> dict: + return {"type": "object", "description": "Define translations in the global scope."} + + +def transform_json_preprocessor(data: Any, in_sub_def: bool) -> Any: + if isinstance(data, dict): + # Standard creation of the special tag structure for translations. + new_data = { + key: transform_json_preprocessor( + value, in_sub_def or key == "subDefinitions" + ) + for key, value in data.items() + } + if ( + "type" in data + and (data["type"] != "object" or in_sub_def) + and data["type"] not in ["boolean", "integer"] + ): + if data["type"] in SPECIAL_TYPES: + tag = data.pop("type") + new_data.pop("type") + # translations applied to inner part + tag_data_with_inner_translations = make_tag_structure( + new_data, + tag=f"!{tag}", + ) + # translations not applied to inner part + tag_data = make_tag_structure( + data, + tag=f"!{tag}", + ) + return make_tag_structure( + data=tag_data, + data_with_inner_translations=tag_data_with_inner_translations, + ) + + return make_tag_structure(data, new_data) + data = new_data + + # Add the translations maps + targets = ["_rootObject", "tab", "unit", "context", "case"] + for target in targets: + if target in data and "properties" in data[target]: + data[target]["properties"]["translations"] = make_translations_map() + + # Flatten the oneOf structures + if "oneOf" in data: + assert isinstance(data["oneOf"], list) + data["oneOf"] = flatten_one_of_stack(data["oneOf"]) + + # Edge cases for return + if "yamlValueOrPythonExpression" in data: + data["yamlValueOrPythonExpression"] = { + "oneOf": [ + {"$ref": "#/subDefinitions/yamlValue"}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!expression", + }, + "value": { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + }, + }, + }, + ] + } + + if "yamlValue" in data: + data["yamlValue"] = { + "description": "A value represented as YAML.", + "not": { + "properties": {"__tag__": {"type": "string"}}, + "type": "object", + }, + } + + if ( + "$ref" in data + and data["$ref"] == "#/subDefinitions/yamlValueOrPythonExpression" + ): + data = make_tag_structure(data, data) + return data + + if isinstance(data, list): + return [transform_json_preprocessor(value, in_sub_def) for value in data] + return data + + +def transform_json(json_file: Path, multilingual: bool, ide: bool): + """ + This function transforms the JSON schema used in the DSL translator into + a new JSON schema that can be used to validate the multilingual YAML in your IDE. + + :param json_file: The path to the JSON file. + :param multilingual: indicator if it will generate a multilingual schema. + :param ide: indicates if it will generate a JSON-schema for the IDE of the user. + """ + _, ext = os.path.splitext(json_file) + assert ext.lower() == ".json", f"expected a json file, got {ext}." + try: + with open(json_file, "r") as stream: + json_stream = json.load(stream) + except FileNotFoundError as e: + print("The json file was not found.") + raise e + + if not multilingual: + result = transform_non_strict(json_stream) + file_name = "schema.json" + else: + result = transform_json_preprocessor(json_stream, False) + if ide: + result = transform_ide(result) + file_name = "multilingual-schema.json" + else: + file_name = "schema-strict-nat-translation.json" + + with open(json_file.parent / file_name, "w", encoding="utf-8") as f: + json.dump(result, f, indent=2) + + +if __name__ == "__main__": + n = len(sys.argv) + assert n >= 1 + + multilingual_param = False + ide_param = False + if n >= 2: + assert sys.argv[1] in ("validation", "ide") + multilingual_param = True + ide_param = sys.argv[1] == "ide" + + transform_json( + Path("./tested/dsl/schema-strict.json"), + multilingual=multilingual_param, + ide=ide_param, + ) diff --git a/tests/test_json_schema_transformer.py b/tests/test_json_schema_transformer.py new file mode 100644 index 00000000..4e98740f --- /dev/null +++ b/tests/test_json_schema_transformer.py @@ -0,0 +1,544 @@ +from pathlib import Path + +from pytest_mock import MockerFixture + +import tested +from tested.transform_json import ( + transform_ide, + transform_json, + transform_json_preprocessor, +) + + +def test_return_json_schema(): + json_schema = { + "return": { + "description": "Expected return value", + "oneOf": [ + {"$ref": "#/definitions/returnOutputChannel"}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/returnOutputChannel" + }, + }, + }, + }, + ], + } + } + + json_schema_expected = { + "return": { + "description": "Expected return value", + "oneOf": [ + {"$ref": "#/definitions/returnOutputChannel"}, + { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/returnOutputChannel" + }, + "not": { + "anyOf": [ + {"required": ["description"]}, + {"required": ["value"]}, + {"required": ["types"]}, + ] + }, + }, + ], + } + } + + result = transform_ide(json_schema) + + assert result == json_schema_expected + + +def test_yaml_value_json_schema(): + json_schema = { + "yamlValue": { + "description": "A value represented as YAML.", + "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, + }, + } + + json_schema_expected = { + "yamlValue": { + "description": "A value represented as YAML.", + } + } + + result = transform_ide(json_schema) + + assert result == json_schema_expected + + +def test_nat_lang_json_schema(): + json_schema = { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": {"type": "object", "additionalProperties": {"type": "string"}}, + }, + } + + json_schema_expected = { + "type": "object", + "additionalProperties": {"type": "string"}, + "not": { + "anyOf": [ + {"required": ["description"]}, + {"required": ["value"]}, + {"required": ["types"]}, + ] + }, + } + + result = transform_ide(json_schema) + + assert result == json_schema_expected + + +def test_expr_json_schema(): + json_schema = { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!expression", + }, + "value": { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + }, + }, + } + + json_schema_expected = { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + } + + result = transform_ide(json_schema) + + assert result == json_schema_expected + + +def test_list_json_schema(): + json_schema = [ + {}, + { + "type": "string", + "description": "A statement of expression in Python-like syntax as YAML string.", + }, + ] + + json_schema_expected = [ + { + "type": "string", + "description": "A statement of expression in Python-like syntax as YAML string.", + } + ] + + result = transform_ide(json_schema) + + assert result == json_schema_expected + + +def test_transform_executed_correct(mocker: MockerFixture): + s = mocker.spy( + tested.transform_json, name="transform_json_preprocessor" # type: ignore[reportAttributeAccessIssue] + ) + + mock_files = [ + mocker.mock_open(read_data=content).return_value + for content in [ + """ +{ + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + } + } + } + } + ] + } +}""" + ] + ] + mock_files.append(mocker.mock_open(read_data="{}").return_value) + mock_files.append(mocker.mock_open().return_value) + mock_opener = mocker.mock_open() + mock_opener.side_effect = mock_files + mocker.patch("builtins.open", mock_opener) + + transform_json(Path("schema.json"), True, False) + + assert s.call_count == 25 + + # Check if the file was opened for writing + mock_opener.assert_any_call( + Path("schema-strict-nat-translation.json"), "w", encoding="utf-8" + ) + + +def test_nat_lang_json_schema_structure(): + json_schema = { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/definitions/tab"}, + } + + json_schema_expected = { + "oneOf": [ + {"type": "array", "minItems": 1, "items": {"$ref": "#/definitions/tab"}}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/definitions/tab"}, + }, + }, + }, + }, + ] + } + + result = transform_json_preprocessor(json_schema, False) + + assert result == json_schema_expected + + +def test_prog_lang_json_schema_structure(): + json_schema = { + "expressionOrStatement": { + "type": "programming_language", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": {"$ref": "#/subDefinitions/programmingLanguage"}, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim.", + }, + }, + } + + json_schema_expected = { + "expressionOrStatement": { + "oneOf": [ + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!programming_language", + }, + "value": { + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "string", + "description": "A language-specific literal, which will be used verbatim.", + }, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim.", + }, + }, + }, + }, + ] + }, + "type": "object", + }, + }, + }, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!programming_language", + }, + "value": { + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim.", + }, + "type": "object", + }, + }, + }, + }, + }, + }, + ] + } + } + + result = transform_json_preprocessor(json_schema, True) + + assert result == json_schema_expected + + +def test_json_schema_oracle(): + json_schema = { + "type": "oracle", + "additionalProperties": False, + "required": ["value"], + "properties": { + "oracle": {"const": "builtin"}, + "value": {"$ref": "#/subDefinitions/yamlValueOrPythonExpression"}, + }, + } + + json_schema_expected = { + "oneOf": [ + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle", + }, + "value": { + "additionalProperties": False, + "required": ["value"], + "properties": { + "oracle": {"const": "builtin"}, + "value": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + }, + }, + }, + ] + }, + }, + "type": "object", + }, + }, + }, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle", + }, + "value": { + "additionalProperties": False, + "required": ["value"], + "properties": { + "oracle": {"const": "builtin"}, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + }, + "type": "object", + }, + }, + }, + }, + }, + }, + ] + } + + result = transform_json_preprocessor(json_schema, True) + + assert result == json_schema_expected + + +def test_json_schema_expression(): + json_schema = { + "yamlValueOrPythonExpression": { + "oneOf": [ + {"$ref": "#/definitions/yamlValue"}, + { + "type": "expression", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + }, + ] + } + } + + json_schema_expected = { + "yamlValueOrPythonExpression": { + "oneOf": [ + {"$ref": "#/subDefinitions/yamlValue"}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!expression", + }, + "value": { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + }, + }, + }, + ] + } + } + + result = transform_json_preprocessor(json_schema, True) + assert result == json_schema_expected + + +def test_json_schema_yaml_value(): + json_schema = { + "yamlValue": { + "description": "A value represented as YAML.", + } + } + + json_schema_expected = { + "yamlValue": { + "description": "A value represented as YAML.", + "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, + }, + } + + result = transform_json_preprocessor(json_schema, False) + + assert result == json_schema_expected + + +def test_exception_when_file_not_found(): + + try: + transform_json(Path("test.json"), False, False) + except FileNotFoundError: + print("As expected") + else: + assert False, "Expected FileNotFoundError error" diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 6a2ed722..2357ca2a 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -24,6 +24,7 @@ def validate_natural_translate(yaml_str: str, translated_yaml_str: str): enviroment = create_enviroment() yaml_object = parse_yaml(yaml_str) + validate_pre_dsl(yaml_object) translated_dsl = translate_yaml(yaml_object, {}, "en", enviroment) translated_yaml = convert_to_yaml(translated_dsl) assert translated_yaml.strip() == translated_yaml_str