update pygeoapi process, add raise_for_status (#133)

tomkralidis · web-flow · commit 58475ac25660 · 2025-06-10T07:50:39.000-04:00
* allow for inline or remote link for pygeoapi process

* add raise_for_status()

* fix ref

* improve data policy failure text
diff --git a/README.md b/README.md
@@ -84,7 +84,8 @@ pywcmp kpi validate --kpi title /path/to/file.json -v INFO
 ...     data = json.load(fh)
 >>> # test ETS
 >>> ts = WMOCoreMetadataProfileTestSuite2(datal)
->>> ts.run_tests()  # raises ValueError error stack on exception
+>>> ts.run_tests()
+>>> ts.raise_for_status()  # raises pywcmp.errors.TestSuiteError on exception with list of errors captured in .errors property
 >>> # test a URL
 >>> from urllib2 import urlopen
 >>> from StringIO import StringIO
diff --git a/pywcmp/pygeoapi_plugin.py b/pywcmp/pygeoapi_plugin.py
@@ -66,7 +66,7 @@
 
 from pywcmp.wcmp2.ets import WMOCoreMetadataProfileTestSuite2
 from pywcmp.wcmp2.kpi import WMOCoreMetadataProfileKeyPerformanceIndicators
-from pywcmp.util import THISDIR
+from pywcmp.util import THISDIR, urlopen_
 
 LOGGER = logging.getLogger(__name__)
 
@@ -76,6 +76,9 @@
 with (THISDIR / 'resources' / 'kpi-report.json').open() as fh:
     KPI_REPORT_SCHEMA = json.load(fh)
 
+with (THISDIR / 'resources' / 'example-ca-eccc-msc.nwp-gdps.json').open() as fh:  # noqa
+    EXAMPLE_WCMP2 = json.load(fh)
+
 
 PROCESS_WCMP2_ETS = {
     'version': '0.1.0',
@@ -97,9 +100,9 @@
     'inputs': {
         'record': {
             'title': 'WCMP2 record',
-            'description': 'WCMP2 record',
+            'description': 'WCMP2 record (can be inline or remote link)',
             'schema': {
-                'type': 'string'
+                'type': ['object', 'string']
             },
             'minOccurs': 1,
             'maxOccurs': 1,
@@ -131,9 +134,7 @@
     },
     'example': {
         'inputs': {
-            'record': {
-                '$ref': 'https://raw.githubusercontent.com/World-Meteorological-Organization/pywcmp/master/tests/data/wcmp2-passing.json'  # noqa
-            },
+            'record': EXAMPLE_WCMP2,
             'fail_on_schema_validation': True
         }
     }
@@ -160,9 +161,9 @@
     'inputs': {
         'record': {
             'title': 'WCMP2 record',
-            'description': 'WCMP2 record',
+            'description': 'WCMP2 record (can be inline or remote link)',
             'schema': {
-                'type': 'string'
+                'type': ['object', 'string']
             },
             'minOccurs': 1,
             'maxOccurs': 1,
@@ -182,9 +183,7 @@
     },
     'example': {
         'inputs': {
-            'record': {
-                '$ref': 'https://raw.githubusercontent.com/World-Meteorological-Organization/pywcmp/master/tests/data/wcmp2-passing.json'  # noqa
-            }
+            'record': EXAMPLE_WCMP2
         }
     }
 }
@@ -216,6 +215,12 @@ def execute(self, data, outputs=None):
             LOGGER.error(msg)
             raise ProcessorExecuteError(msg)
 
+        if isinstance(record, str) and record.startswith('http'):
+            LOGGER.debug('Record is a link')
+            record = json.loads(urlopen_(record).read())
+        else:
+            LOGGER.debug('Record is inline')
+
         LOGGER.debug('Running ETS against record')
         response = WMOCoreMetadataProfileTestSuite2(record).run_tests(
             fail_on_schema_validation=fail_on_schema_validation)
@@ -244,13 +249,20 @@ def execute(self, data, outputs=None):
 
         response = None
         mimetype = 'application/json'
+
         record = data.get('record')
 
         if record is None:
             msg = 'Missing record'
             LOGGER.error(msg)
             raise ProcessorExecuteError(msg)
 
+        if isinstance(record, str) and record.startswith('http'):
+            LOGGER.debug('Record is a link')
+            record = json.loads(urlopen_(record).read())
+        else:
+            LOGGER.debug('Record is inline')
+
         LOGGER.debug('Running KPIs against record')
         kpis = WMOCoreMetadataProfileKeyPerformanceIndicators(record)
         response = kpis.evaluate()
diff --git a/pywcmp/resources/example-ca-eccc-msc.nwp-gdps.json b/pywcmp/resources/example-ca-eccc-msc.nwp-gdps.json
@@ -0,0 +1,159 @@
+{
+    "id": "urn:wmo:md:ca-eccc-msc:nwp.msc_nwp_gdps",
+    "conformsTo": [
+        "http://wis.wmo.int/spec/wcmp/2/conf/core"
+    ],
+    "type": "Feature",
+    "geometry": {
+        "type": "Polygon",
+        "coordinates": [
+            [
+                [
+                    -180,
+                    -90
+                ],
+                [
+                    -180,
+                    90
+                ],
+                [
+                    180,
+                    90
+                ],
+                [
+                    180,
+                    -90
+                ],
+                [
+                    -180,
+                    -90
+                ]
+            ]
+        ]
+    },
+    "time": {
+        "interval": [
+            "1963-10-01",
+            ".."
+        ]
+    },
+    "properties": {
+        "title": "Global Deterministic Prediction System",
+        "description": "The Global Deterministic Prediction System (GDPS) carries out physics calculations to arrive at deterministic predictions of atmospheric elements from the current day out to 10 days into the future. Atmospheric elements include temperature, precipitation, cloud cover, wind speed and direction, humidity and others. This product contains raw numerical results of these calculations. Geographical coverage is global. Data is available at horizontal resolution of about 15 km up to 33 vertical levels. Predictions are performed twice a day.",
+        "themes": [
+            {
+                "concepts": [
+                    {
+                        "id": "Prediction"
+                    },
+                    {
+                        "id": "Global"
+                    },
+                    {
+                        "id": "Deterministic"
+                    },
+                    {
+                        "id": "Weather forecasts"
+                    },
+                    {
+                        "id": "Air temperature"
+                    },
+                    {
+                        "id": "Meteorological data"
+                    }
+                ],
+                "scheme": "https://canada.multites.net/cst"
+            },
+            {
+                "concepts": [
+                    {
+                        "id": "weather",
+                        "title": "Weather",
+                        "url": "https://codes.wmo.int/wis/topic-hierarchy/earth-system-discipline/weather"
+                    }
+                ],
+                "scheme": "https://codes.wmo.int/wis/topic-hierarchy/earth-system-discipline"
+            }
+        ],
+        "contacts": [
+            {
+                "name": "National Inquiry Response Team",
+                "organization": "Government of Canada; Environment and Climate Change Canada; Meteorological Service of Canada",
+                "phones": [
+                    {
+                        "value": "+18199972800"
+                    }
+                ],
+                "emails": [
+                    {
+                        "value": "enviroinfo@ec.gc.ca"
+                    }
+                ],
+                "addresses": [
+                    {
+                        "deliveryPoint": [
+                            "77 Westmorland Street, suite 260"
+                        ],
+                        "city": "Fredericton",
+                        "administrativeArea": "NB",
+                        "postalCode": "E3B 6Z4",
+                        "country": "Canada"
+                    }
+                ],
+                "contactInstructions": "email",
+                "links": [
+                    {
+                        "rel": "canonical",
+                        "type": "text/html",
+                        "href": "https://www.canada.ca/en/environment-climate-change.html"
+                    }
+                ],
+                "roles": [
+                    "host",
+                    "producer"
+                ]
+            }
+        ],
+        "wmo:dataPolicy": "core",
+        "language": {
+            "code": "en"
+        },
+        "type": "dataset",
+        "created": "2018-01-01T11:11:23Z",
+        "updated": "2022-06-17T08:22:24Z"
+    },
+    "links": [
+        {
+            "rel": "data",
+            "href": "https://dd.weather.gc.ca/model_gem_global",
+            "type": "text/html",
+            "title": "MSC Datamart",
+            "distribution": {
+                "availableFormats": [
+                    {
+                        "name": "GRIB2"
+                    }
+                ]
+            }
+        },
+        {
+            "rel": "license",
+            "href": "https://open.canada.ca/en/open-government-licence-canada",
+            "type": "text/html",
+            "title": "Open Government Licence - Canada"
+        },
+        {
+            "rel": "service",
+            "href": "https://geo.weather.gc.ca/geomet?lang=en&service=WMS&request=GetCapabilities&layers=GDPS.ETA_TT",
+            "type": "application/xml",
+            "title": "Air temperature [degrees]"
+        },
+        {
+            "rel": "items",
+            "channel": "origin/a/wis2/ca-eccc-msc/data/core/weather/prediction/forecast/medium-range/deterministic/global",
+            "href": "mqtts://everyone:everyone@globalbroker.meteo.fr:8883",
+            "type": "application/geo+json",
+            "title": "Data notifications"
+        }
+    ]
+}
diff --git a/pywcmp/wcmp2/ets.py b/pywcmp/wcmp2/ets.py
@@ -36,9 +36,11 @@
 from jsonschema.validators import Draft202012Validator
 from shapely.geometry import shape
 
+from pywis_topics.topics import TopicHierarchy
+
 import pywcmp
+from pywcmp.errors import TestSuiteError
 from pywcmp.bundle import WCMP2_FILES
-from pywis_topics.topics import TopicHierarchy
 from pywcmp.util import (get_current_datetime_rfc3339, get_userdir,
                          is_valid_created_datetime)
 
@@ -71,6 +73,7 @@ def __init__(self, data: dict):
 
         self.test_id = None
         self.record = data
+        self.errors = []
 
         self.th = TopicHierarchy(tables=get_userdir())
 
@@ -79,6 +82,7 @@ def run_tests(self, fail_on_schema_validation=False):
 
         results = []
         tests = []
+
         ets_report = {
             'id': str(uuid.uuid4()),
             'report_type': 'ets',
@@ -103,7 +107,10 @@ def run_tests(self, fail_on_schema_validation=False):
                 raise ValueError(msg)
 
         for t in tests:
-            results.append(getattr(self, t)())
+            result = getattr(self, t)()
+            results.append(result)
+            if result['code'] == 'FAILED':
+                self.errors.append(result)
 
         for code in ['PASSED', 'FAILED', 'SKIPPED']:
             r = len([t for t in results if t['code'] == code])
@@ -115,6 +122,16 @@ def run_tests(self, fail_on_schema_validation=False):
 
         return ets_report
 
+    def raise_for_status(self):
+        """
+        Raise error if one or more failures were found during validation.
+
+        :returns: `pywcmp.errors.TestSuiteError` or `None`
+        """
+
+        if len(self.errors) > 0:
+            raise TestSuiteError('Invalid WCMP2 record', self.errors)
+
     def test_requirement_validation(self):
         """
         Validate that a WCMP record is valid to the authoritative WCMP schema.
@@ -450,7 +467,7 @@ def test_requirement_data_policy(self):
                                     if link['rel'] == 'license']
                 if not conditions_links:
                     status['code'] = 'FAILED'
-                    status['message'] = 'missing recommended conditions'
+                    status['message'] = 'recommended data requires a link with rel=license'  # noqa
                     return status
 
         return status
diff --git a/tests/run_tests.py b/tests/run_tests.py
@@ -30,6 +30,7 @@
 import os
 import unittest
 
+from pywcmp.errors import TestSuiteError
 from pywcmp.ets import WMOCoreMetadataProfileTestSuite2
 from pywcmp.wcmp2.kpi import (
     calculate_grade, WMOCoreMetadataProfileKeyPerformanceIndicators)
@@ -120,6 +121,23 @@ def test_fail(self):
             with self.assertRaises(ValueError):
                 ts.run_tests(fail_on_schema_validation=True)
 
+    def test_raise_for_status(self):
+        """Simple test for raise_for_status"""
+
+        with open(get_test_file_path('data/wcmp2-passing.json')) as fh:
+            ts = WMOCoreMetadataProfileTestSuite2(json.load(fh))
+            _ = ts.run_tests(fail_on_schema_validation=True)
+
+            assert ts.raise_for_status() is None
+
+        with open(get_test_file_path('data/wcmp2-failing-invalid-time-resolution.json')) as fh:  # noqa
+            record = json.load(fh)
+            ts = WMOCoreMetadataProfileTestSuite2(record)
+            _ = ts.run_tests(fail_on_schema_validation=True)
+
+            with self.assertRaises(TestSuiteError):
+                ts.raise_for_status()
+
     def test_fail_invalid_time_resolution(self):
         """Simple test for a failing record with an invalid time resolution"""
 
