Tests for the test suite

.editorconfig

@@ -7,3 +7,11 @@ indent_style = spaces
 indent_size = 4
 insert_final_newline = true
 trim_trailing_whitespace = true
+
+[*.json5]
+charset = utf-8
+end_of_line = crlf
+indent_style = spaces
+indent_size = 2
+insert_final_newline = true
+trim_trailing_whitespace = true

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Implementation guide for implementing OGC API - Processes in openEO
+- Unit Tests (see folder `tests`, moved specification tests and CI tools to `dev`)
 - `export_collection`
 - `export_workspace`
 - `run_ogcapi`

absolute.json

@@ -28,12 +28,6 @@
         }
     },
     "examples": [
-        {
-            "arguments": {
-                "x": 0
-            },
-            "returns": 0
-        },
         {
             "arguments": {
                 "x": 3.5
@@ -45,12 +39,6 @@
                 "x": -0.4
             },
             "returns": 0.4
-        },
-        {
-            "arguments": {
-                "x": -3.5
-            },
-            "returns": 3.5
         }
     ],
     "links": [

apply_kernel.json

@@ -1,7 +1,7 @@
 {
     "id": "apply_kernel",
     "summary": "Apply a spatial convolution with a kernel",
-    "description": "Applies a 2D convolution (i.e. a focal operation with a weighted kernel) on the horizontal spatial dimensions (axes `x` and `y`) of a raster data cube.\n\nEach value in the kernel is multiplied with the corresponding pixel value and all products are summed up afterwards. The sum is then multiplied with the factor.\n\nThe process can't handle non-numerical or infinite numerical values in the data cube. Boolean values are converted to integers (`false` = 0, `true` = 1), but all other non-numerical or infinite values are replaced with zeroes by default (see parameter `replace_invalid`).\n\nFor cases requiring more generic focal operations or non-numerical values, see ``apply_neighborhood()``.",
+    "description": "Applies a 2D convolution (i.e. a focal operation with a weighted kernel) on the horizontal spatial dimensions (axes `x` and `y`) of a raster data cube.\n\nEach value in the kernel is multiplied with the corresponding pixel value and all products are summed up afterwards. The sum is then multiplied with the factor.\n\nThe process can't handle non-numerical or infinite numerical values in the data cube. Boolean values are converted to integers (`false` = 0, `true` = 1), but all other non-numerical, NaN, no-data, or infinite values are replaced with zeroes by default (see parameter `replace_invalid`).\n\nFor cases requiring more generic focal operations or non-numerical values, see ``apply_neighborhood()``.",
     "categories": [
         "cubes",
         "math > image filter"
@@ -70,7 +70,7 @@
         },
         {
             "name": "replace_invalid",
-            "description": "This parameter specifies the value to replace non-numerical or infinite numerical values with. By default, those values are replaced with zeroes.",
+            "description": "This parameter specifies the value to replace non-numerical, NaN, no-data, or infinite numerical values with. By default, those values are replaced with zeroes.",
             "schema": {
                 "type": "number"
             },

dev/check-tests.js

@@ -0,0 +1,68 @@
+// Ensure that each test is valid
+
+const fs = require('fs');
+const path = require('path');
+const JSON5 = require('json5');
+const ajv = require('ajv');
+
+const testsDir = path.join(__dirname, '../tests');
+
+const tests = fs.readdirSync(testsDir).filter(file => file.endsWith('.json5'));
+
+const schemaPath = path.join(testsDir, 'schema', 'schema.json');
+const schemaContent = fs.readFileSync(schemaPath, 'utf8');
+const schema = JSON.parse(schemaContent);
+const validate = new ajv().compile(schema);
+
+const results = {};
+for (const testFile of tests) {
+    const testPath = path.join(testsDir, testFile);
+
+    let testData;
+    // Ensure we can load the test file as JSON5
+    try {
+        testData = JSON5.parse(fs.readFileSync(testPath, 'utf8'));
+    } catch (error) {
+        results[testFile] = `Invalid JSON5: ${error.message}`;
+        continue;
+    }
+
+    // Ensure the file is valid against the schema
+    if (!validate(testData)) {
+        results[testFile] = `Schema validation failed: ${validate.errors.map(err => err.message).join(', ')}`;
+        continue;
+    }
+
+    // Make sure the id is the same as filename without extension
+    const expectedId = path.basename(testFile, '.json5');
+    if (testData.id !== expectedId) {
+        results[testFile] = `ID mismatch: expected ${expectedId}, got ${testData.id}`;
+        continue;
+    }
+
+    // Check if experimental is set to the same value as in the process itself
+    let processFile = path.join(__dirname, '../', expectedId + '.json');
+    if (!fs.existsSync(processFile)) {
+        processFile = path.join(__dirname, '../proposals/', expectedId + '.json');
+    }
+    if (fs.existsSync(processFile)) {
+        const processData = JSON.parse(fs.readFileSync(processFile, 'utf8'));
+        const expected = processData.experimental || false;
+        const actual = testData.experimental || false;
+        if (expected !== actual) {
+            results[testFile] = `Experimental flag mismatch: expected ${expected}, got ${actual}`;
+            continue;
+        }
+    }
+}
+
+if (Object.keys(results).length > 0) {
+    console.error('The following test files have issues:');
+    for (const [file, error] of Object.entries(results)) {
+        console.error(`- ${file}: ${error}`);
+    }
+    process.exit(1);
+}
+else {
+    console.log('All test files are valid and match the expected schema.');
+}

dev/has-tests.js

@@ -0,0 +1,44 @@
+// Ensure that each process has a corresponding file in the tests directory.
+// It can be empty, but it must exist to ensure people made that decision consciously.
+
+const fs = require('fs');
+const path = require('path');
+
+const processesDir = path.join(__dirname, '../');
+const proposalsDir = path.join(__dirname, '../proposals');
+const testsDir = path.join(__dirname, '../tests');
+
+const processes = [
+    ...fs.readdirSync(processesDir),
+    ...fs.readdirSync(proposalsDir),
+]
+    .filter(file => file.endsWith('.json'))
+    .map(file => path.basename(file, '.json'));;
+const tests = fs.readdirSync(testsDir)
+    .filter(file => file.endsWith('.json5'))
+    .map(file => path.basename(file, '.json5'));
+
+// Check which tests are missing for the processes
+const missingTests = processes.filter(process => !tests.includes(process));
+
+if (missingTests.length > 0) {
+    console.error('The following processes are missing tests:');
+    missingTests.forEach(process => console.error(`- ${process}`));
+}
+
+// Check whether there are tests for non-existing processes
+const extraTests = tests.filter(test => !processes.includes(test));
+if (extraTests.length > 0) {
+    console.error('\nThe following tests exist without a corresponding process:');
+    extraTests.forEach(test => console.error(`- ${test}`));
+}
+
+// todo: add check that json5 files are valid
+
+if (missingTests.length === 0 && extraTests.length === 0) {
+    console.log('All processes have corresponding tests and vice versa.');
+    process.exit(0);
+}
+else {
+    process.exit(1);
+}

dev/package.json

@@ -23,8 +23,15 @@
     "http-server": "^14.1.1"
   },
   "scripts": {
-    "test": "openeo-processes-lint testConfig.json",
+    "check-tests": "node check-tests.js",
+    "has-tests": "node has-tests.js",
+    "lint": "openeo-processes-lint testConfig.json",
+    "test": "npm run has-tests && npm run check-tests && npm run lint",
     "generate": "concat-json-files \"../{*,proposals/*}.json\" -t \"processes.json\"",
     "start": "npm run generate && http-server -p 9876 -o docs.html -c-1"
+  },
+  "dependencies": {
+    "ajv": "^8.17.1",
+    "json5": "^2.2.3"
   }
 }

tests/README.md

@@ -0,0 +1,119 @@
+# Tests
+
+This folder contains test cases for the openEO processes.
+
+## Assumptions
+
+The test cases assume a couple of things as they are an abstraction and not bound to specific implementations:
+- The JSON Schema type `number` explicitly includes the values `+Infinity`, `-Infinity` and `NaN`.
+- The input and output values for no-data values are `null` by default unless otherwise specified by a runner.
+- Input that is not valid according to the schemas, will be rejected upfront and will not be checked on. For example, the absolute process only tests against the data types `number` and `null`. There are no tests for a boolean or string input.
+- Numerical data types such as uint8 don't matter, i.e. tests don't check for overflows etc. This suite can't provide such tests as the underlying data type is not known.
+- If not otherwise specified for numbers, a precision of 10 decimals is checked so return values should have at least 11 decimals.
+
+## Test Files
+
+To allow for more data types (e.g. infinity and nan for numbers), all the files are encoded in **JSON5** instead of JSON.
+
+The test files have the following schema: [schema/schema.json](./schema/schema.json)
+
+### No-data values
+
+No-data values have a special encoding in tests (see below).
+The encoding is replaced with `null` unless otherwise specified by the runners.
+
+```json5
+{
+  "type": "nodata"
+}
+```
+
+### Datetimes
+
+Datetimes as strings have a varying precision, especially regarding the milliseconds.
+Also, sometimes timezones are differently handled.
+
+Datetimes in return values should be encoded as follows so that the results can be compared better:
+
+```json5
+{
+  "type": "datetime",
+  "value": "2020-01-01T00:00:00Z"
+}
+```
+
+### External references
+
+Arguments and return values can point to external files, e.g.
+
+```json5
+{
+  "$ref": "https://host.example/datacube.json"
+}
+```
+
+The test suite can currently only load JSON and JSON5 files.
+
+### Labeled arrays
+
+Labeled arrays can't be represented in JSON5 and will be provided as an object instead.
+
+```json5
+{
+  "type": "labeled-array",
+  "data": [
+    {
+      "key": "B01",
+      "value": 1.23
+    },
+    {
+      "key": "B02",
+      "value": 0.98
+    }
+    // ...
+  ]
+}
+```
+
+### Datacubes
+
+Datacubes can't be represented in JSON5 and will be provided as an object instead.
+Vector datacubes are currently not supported.
+
+```json5
+{
+  "type": "datacube",
+  "data": [
+    // multi-dimensional array
+    // can be set to `null` if the data values are irrelevant for the test.
+  ],
+  "nodata": [
+    NaN
+  ],
+  "order": ["bands", "t", "y", "x"],
+  "dimensions": {
+    // similar to the STAC datacube extension
+    // properties: type, axis (if type = spatial), values, and reference_system (optional)
+    "bands": {
+      "type": "bands",
+      "values": ["blue","green","red","nir"]
+    },
+    "t": {
+      "type": "temporal",
+      "values": ["2020-06-01T00:00:00Z","2020-06-03T00:00:00Z","2020-06-06T00:00:00Z"]
+    },
+    "y": {
+      "type": "spatial",
+      "axis": "y",
+      "values": [5757495.0,5757485.0,5757475.0,5757465.0],
+      "reference_system": "EPSG:25832"
+    },
+    "x": {
+      "type": "spatial",
+      "axis": "x",
+      "values": [404835.0,404845.0,404855.0,404865.0,404875.0],
+      "reference_system": "EPSG:25832"
+    }
+  }
+}
+```

tests/absolute.json5

@@ -0,0 +1,60 @@
+{
+  "id": "absolute",
+  "level": "L1",
+  "tests": [
+    {
+      "arguments": {
+        "x": 0
+      },
+      "returns": 0
+    },
+    {
+      "arguments": {
+        "x": 1
+      },
+      "returns": 1
+    },
+    {
+      "arguments": {
+        "x": -1
+      },
+      "returns": 1
+    },
+    {
+      "arguments": {
+        "x": 2.5
+      },
+      "returns": 2.5
+    },
+    {
+      "arguments": {
+        "x": -2.5
+      },
+      "returns": 2.5
+    },
+    {
+      "arguments": {
+        "x": NaN
+      },
+      "returns": NaN
+    },
+    {
+      "arguments": {
+        "x": Infinity
+      },
+      "returns": Infinity
+    },
+    {
+      "arguments": {
+        "x": -Infinity
+      },
+      "returns": Infinity
+    },
+    {
+      "arguments": {
+        "x": {"type": "nodata"}
+      },
+      "returns": {"type": "nodata"}
+    }
+  ]
+}