Merge pull request #19 from ESA-APEx/apex5-html-report

add basic HTML+JSON reporting

Author: soxofaan

.github/workflows/benchmarks.yaml

@@ -21,7 +21,17 @@ jobs:
       - name: Test with pytest
         run: |
           cd qa/benchmarks
-          pytest --random-subset=1
+          mkdir report
+          pytest \
+            --random-subset=1 \
+            --html report/report.html --self-contained-html \
+            --track-metrics-report=report/metrics.json
         env:
           OPENEO_AUTH_METHOD: client_credentials
           OPENEO_AUTH_CLIENT_CREDENTIALS_CDSEFED: ${{ secrets.OPENEO_AUTH_CLIENT_CREDENTIALS_CDSEFED }}
+      - name: upload report
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: report
+          path: qa/benchmarks/report/

.gitignore

@@ -166,3 +166,7 @@ poetry.toml
 
 # LSP config files
 pyrightconfig.json
+
+
+tmp*
+temp*

qa/benchmarks/.gitignore

@@ -0,0 +1 @@
+report/

qa/benchmarks/requirements.txt

@@ -7,3 +7,4 @@ requests>=2.32.0
 xarray>=2024.6.0
 netCDF4>=1.7.1
 rioxarray>=0.15.7
+pytest-html>=4.1.1

qa/benchmarks/tests/conftest.py

@@ -10,6 +10,10 @@
 # TODO: how to make sure the logging/printing from this plugin is actually visible by default?
 _log = logging.getLogger(__name__)
 
+pytest_plugins = [
+    "apex_algorithm_qa_tools.pytest_track_metrics",
+]
+
 
 def pytest_addoption(parser):
     parser.addoption(

qa/tools/apex_algorithm_qa_tools/pytest_track_metrics.py

@@ -0,0 +1,140 @@
+"""
+Pytest plugin to track test/benchmark metrics and report them with a JSON file.
+
+
+Usage:
+
+-   Enable the plugin in `conftest.py`:
+
+    ```python
+    pytest_plugins = [
+        "apex_algorithm_qa_tools.pytest_track_metrics",
+    ]
+    ```
+
+-   Use the `track_metric` fixture to record metrics during tests:
+
+    ```python
+    def test_dummy(track_metric):
+        x = 3
+        track_metric("x squared", x*x)
+    ...
+
+-   Run the tests with `--track-metrics-report=path/to/metrics.json`
+    to store metrics in a JSON file
+"""
+
+import json
+import warnings
+from pathlib import Path
+from typing import Any, Callable, List, Tuple, Union
+
+import pytest
+
+_TRACK_METRICS_PATH = "track_metrics_path"
+_TRACK_METRICS_NAME = "track_metrics"
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--track-metrics-report",
+        metavar="PATH",
+        action="store",
+        dest=_TRACK_METRICS_PATH,
+        default=None,
+        help="Path to JSON file to store test/benchmark metrics.",
+    )
+
+
+def pytest_configure(config):
+    track_metrics_path = config.getoption(_TRACK_METRICS_PATH)
+    if (
+        track_metrics_path
+        # Don't register on xdist worker nodes
+        and not hasattr(config, "workerinput")
+    ):
+        config.pluginmanager.register(
+            TrackMetricsReporter(path=track_metrics_path),
+            name=_TRACK_METRICS_NAME,
+        )
+
+
+def pytest_unconfigure(config):
+    if config.pluginmanager.hasplugin(_TRACK_METRICS_NAME):
+        config.pluginmanager.unregister(name=_TRACK_METRICS_NAME)
+
+
+class TrackMetricsReporter:
+    def __init__(
+        self, path: Union[str, Path], user_properties_key: str = "track_metrics"
+    ):
+        self.path = Path(path)
+        self.metrics: List[dict] = []
+        self.user_properties_key = user_properties_key
+
+    def pytest_runtest_logreport(self, report: pytest.TestReport):
+        if report.when == "call":
+            self.metrics.append(
+                {
+                    "nodeid": report.nodeid,
+                    "report": {
+                        "outcome": report.outcome,
+                        "duration": report.duration,
+                        "start": report.start,
+                        "stop": report.stop,
+                    },
+                    "metrics": self.get_metrics(report.user_properties),
+                }
+            )
+
+    def pytest_sessionfinish(self, session):
+        with self.path.open("w", encoding="utf8") as f:
+            json.dump(self.metrics, f, indent=2)
+
+    def pytest_terminal_summary(self, terminalreporter):
+        terminalreporter.write_sep("-", f"Generated track_metrics report: {self.path}")
+
+    def get_metrics(
+        self, user_properties: List[Tuple[str, Any]]
+    ) -> List[Tuple[str, Any]]:
+        """
+        Extract existing test metrics items from user properties
+        or create new one.
+        """
+        for name, value in user_properties:
+            if name == self.user_properties_key:
+                return value
+        # Not found: create it
+        metrics = []
+        user_properties.append((self.user_properties_key, metrics))
+        return metrics
+
+
+@pytest.fixture
+def track_metric(
+    pytestconfig: pytest.Config, request: pytest.FixtureRequest
+) -> Callable[[str, Any], None]:
+    """
+    Fixture to record a metric during tests/benchmarks,
+    which will be stored in the pytest node's "user_properties".
+
+    Returns a callable that expects a metric name and value
+    """
+
+    reporter: Union[TrackMetricsReporter, None] = pytestconfig.pluginmanager.get_plugin(
+        _TRACK_METRICS_NAME
+    )
+
+    if reporter:
+
+        def append(name: str, value: Any):
+            reporter.get_metrics(request.node.user_properties).append((name, value))
+    else:
+        warnings.warn(
+            "The `track_metric` fixture is requested, but no output file is defined (e.g. with `--metrics-tracker-report=path/to/metrics.json`."
+        )
+
+        def append(name: str, value: Any):
+            pass
+
+    return append

qa/unittests/tests/conftest.py

@@ -1,3 +1,7 @@
 import pytest
 
+pytest_plugins = [
+    "pytester",
+]
+
 pytest.register_assert_rewrite("apex_algorithm_qa_tools.scenarios")

qa/unittests/tests/test_pytest_track_metrics.py

@@ -0,0 +1,62 @@
+import json
+import re
+import time
+
+import pytest
+
+
+def test_track_metric_basics(pytester: pytest.Pytester, tmp_path):
+    pytester.makeconftest(
+        """
+        pytest_plugins = [
+            "apex_algorithm_qa_tools.pytest_track_metrics",
+        ]
+        """
+    )
+    pytester.makepyfile(
+        test_addition="""
+            import pytest
+
+            @pytest.mark.parametrize("x", [5, 6])
+            def test_3plus(track_metric, x):
+                track_metric("x squared", x * x)
+                assert 3 + x == 8
+        """
+    )
+
+    metrics_path = tmp_path / "metrics.json"
+    result = pytester.runpytest(f"--track-metrics-report={metrics_path}")
+    result.assert_outcomes(passed=1, failed=1)
+
+    assert metrics_path.exists()
+    result.stdout.re_match_lines([f".*Generated.*{re.escape(str(metrics_path))}.*"])
+
+    with metrics_path.open("r", encoding="utf8") as f:
+        metrics = json.load(f)
+    approx_now = pytest.approx(time.time(), abs=1)
+    assert metrics == [
+        {
+            "nodeid": "test_addition.py::test_3plus[5]",
+            "report": {
+                "outcome": "passed",
+                "duration": pytest.approx(0, abs=1),
+                "start": approx_now,
+                "stop": approx_now,
+            },
+            "metrics": [
+                ["x squared", 25],
+            ],
+        },
+        {
+            "nodeid": "test_addition.py::test_3plus[6]",
+            "report": {
+                "outcome": "failed",
+                "duration": pytest.approx(0, abs=1),
+                "start": approx_now,
+                "stop": approx_now,
+            },
+            "metrics": [
+                ["x squared", 36],
+            ],
+        },
+    ]