Issue #17 introduce initial JSON Schema for benchmark scenario

Author: soxofaan

qa/tools/apex_algorithm_qa_tools/scenarios.py

@@ -6,6 +6,7 @@
 import re
 from typing import List
 
+import jsonschema
 import requests
 from apex_algorithm_qa_tools.common import get_project_root
 
@@ -15,6 +16,11 @@
 # TODO #15 Flatten apex_algorithm_qa_tools to a single module and push as much functionality to https://github.com/ESA-APEx/esa-apex-toolbox-python
 
 
+def _get_benchmark_scenario_schema() -> dict:
+    with open(get_project_root() / "schema/benchmark_scenario.json") as f:
+        return json.load(f)
+
+
 @dataclasses.dataclass(kw_only=True)
 class BenchmarkScenario:
     # TODO #14 need for differentiation between different types of scenarios?
@@ -25,6 +31,9 @@ class BenchmarkScenario:
 
     @classmethod
     def from_dict(cls, data: dict) -> BenchmarkScenario:
+        jsonschema.validate(instance=data, schema=_get_benchmark_scenario_schema())
+        # TODO: also include the `lint_benchmark_scenario` stuff here (maybe with option to toggle deep URL inspection)?
+
         # TODO #14 standardization of these types? What about other types and how to support them?
         assert data["type"] == "openeo"
         return cls(
@@ -55,9 +64,9 @@ def lint_benchmark_scenario(scenario: BenchmarkScenario):
     """
     # TODO #17 use JSON Schema based validation instead of ad-hoc checks?
     # TODO integrate this as a pre-commit hook
-    # TODO raise descriptive exceptions instead of asserts?
+    # TODO #17 raise descriptive exceptions instead of asserts?
     assert re.match(r"^[a-zA-Z0-9_-]+$", scenario.id)
-    # TODO: proper allow-list of backends?
+    # TODO proper allow-list of backends or leave this freeform?
     assert scenario.backend in ["openeofed.dataspace.copernicus.eu"]
     # TODO: more advanced process graph validation?
     assert isinstance(scenario.process_graph, dict)

qa/tools/pyproject.toml

@@ -19,4 +19,5 @@ requires-python = ">=3.10"
 dynamic = ["version"]
 dependencies = [
     "requests>=2.32.0",
+    "jsonschema>=4.0.0",
 ]

qa/unittests/tests/test_scenarios.py

@@ -1,3 +1,4 @@
+import jsonschema
 import pytest
 from apex_algorithm_qa_tools.scenarios import (
     BenchmarkScenario,
@@ -24,3 +25,22 @@ def test_get_benchmark_scenarios():
 )
 def test_lint_scenario(scenario: BenchmarkScenario):
     lint_benchmark_scenario(scenario)
+
+
+class TestBenchmarkScenario:
+    def test_validation_minimal(self):
+        bs = BenchmarkScenario.from_dict(
+            {
+                "id": "foo",
+                "type": "openeo",
+                "backend": "openeo.test",
+                "process_graph": {},
+            }
+        )
+        assert bs.id == "foo"
+        assert bs.backend == "openeo.test"
+        assert bs.process_graph == {}
+
+    def test_validation_missing_essentials(self):
+        with pytest.raises(jsonschema.ValidationError):
+            BenchmarkScenario.from_dict({})

schema/benchmark_scenario.json

@@ -0,0 +1,36 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Benchmark Scenario",
+  "type": "object",
+  "properties": {
+    "id": {
+      "type": "string",
+      "description": "The unique identifier for the benchmark scenario."
+    },
+    "type": {
+      "type": "string",
+      "description": "The type of the benchmark scenario.",
+      "enum": [
+        "openeo"
+      ]
+    },
+    "description": {
+      "type": "string",
+      "description": "A description of the benchmark scenario."
+    },
+    "backend": {
+      "type": "string",
+      "description": "The openEO backend URL to connect to."
+    },
+    "process_graph": {
+      "type": "object",
+      "description": "The openEO process graph to execute."
+    }
+  },
+  "required": [
+    "id",
+    "type",
+    "backend",
+    "process_graph"
+  ]
+}