feat: Add more structure to ocdsmerge notes #468

Author: jpmckinney

process/processors/compiler.py

@@ -1,14 +1,16 @@
 import functools
 import logging
+import warnings
+from collections import Counter
 
-import ocdsmerge_rs.exceptions
 from django.conf import settings
 from ocdsextensionregistry import ProfileBuilder
 from ocdsextensionregistry.exceptions import ExtensionWarning
 from ocdsmerge_rs import Merger
+from ocdsmerge_rs.exceptions import DuplicateIdValueWarning, MergeError, MergeWarning
 
 from process.models import CollectionFile, CollectionNote, CompiledRelease, Data
-from process.util import create_note, create_warnings_note, get_or_create
+from process.util import create_note, get_or_create
 
 logger = logging.getLogger(__name__)
 
@@ -32,15 +34,37 @@ def save_compiled_release(merged, collection, ocid):
 
 
 def compile_releases_by_ocdskit(collection, ocid, releases, extensions):
-    with create_warnings_note(collection, ExtensionWarning):
+    with warnings.catch_warnings(record=True, action="always", category=ExtensionWarning) as wlist:
         merger = _get_merger(frozenset(extensions))
 
+    for w in filter_warnings(wlist, ExtensionWarning):
+        create_note(collection, CollectionNote.Level.WARNING, str(w.message), data={"type": w.category.__name__})
+
     try:
-        with create_warnings_note(collection, ocdsmerge_rs.exceptions.MergeWarning):
+        with warnings.catch_warnings(record=True, action="always", category=MergeWarning) as wlist:
             return merger.create_compiled_release(releases)
-    except ocdsmerge_rs.exceptions.MergeError as e:
+
+        notes = []
+        paths = Counter()
+        for w in filter_warnings(wlist, DuplicateIdValueWarning):  # DuplicateIdValueWarning is the only MergeWarning
+            notes.append(str(w.message))
+            paths[w.message.path] += 1
+
+        if notes:
+            create_note(
+                collection,
+                CollectionNote.Level.WARNING,
+                notes,
+                data={"type": "DuplicateIdValueWarning", "paths": dict(paths)},
+            )
+    except MergeError as e:
         logger.exception("OCID %s can't be compiled, skipping", ocid)
-        create_note(collection, CollectionNote.Level.ERROR, f"OCID {ocid} can't be compiled", data=str(e))
+        create_note(
+            collection,
+            CollectionNote.Level.ERROR,
+            f"OCID {ocid} can't be compiled",
+            data={"type": type(e).__name__, "message": str(e), **vars(e)},
+        )
 
 
 @functools.lru_cache
@@ -51,3 +75,12 @@ def _get_merger(extensions):
     builder = ProfileBuilder(tag, extensions, standard_base_url=url)
     patched_schema = builder.patched_release_schema()
     return Merger(rules=Merger.get_rules(Merger.dereference(patched_schema)))
+
+
+def filter_warnings(wlist, category):
+    """Yield warnings that match category, re-emitting non-matching warnings."""
+    for w in wlist:
+        if issubclass(w.category, category):
+            yield w
+        else:
+            warnings.warn_explicit(w.message, w.category, w.filename, w.lineno, source=w.source)

process/util.py

@@ -2,7 +2,6 @@
 import io
 import logging
 import os
-import warnings
 from contextlib import contextmanager
 from textwrap import fill
 
@@ -151,22 +150,6 @@ def delete_step(name, finish=None, finish_args=(), exception=None, **kwargs):
         finish(*finish_args, exception=exception)
 
 
-@contextmanager
-def create_warnings_note(collection, category):
-    with warnings.catch_warnings(record=True, action="always", category=category) as wlist:
-        yield
-
-    note = []
-    for w in wlist:
-        if issubclass(w.category, category):
-            note.append(str(w.message))
-        else:
-            warnings.warn_explicit(w.message, w.category, w.filename, w.lineno, source=w.source)
-
-    if note:
-        create_note(collection, CollectionNote.Level.WARNING, note)
-
-
 @contextmanager
 def create_logger_note(collection, name):
     stream = io.StringIO()

requirements.txt

@@ -92,7 +92,7 @@ ocdskit==1.4.0
     # via -r requirements.in
 ocdsmerge==0.8.0
     # via ocdskit
-ocdsmerge-rs==0.1.1
+ocdsmerge-rs==0.1.2
     # via -r requirements.in
 odfpy==1.4.1
     # via flattentool

requirements_dev.txt

@@ -134,7 +134,7 @@ ocdsmerge==0.8.0
     # via
     #   -r requirements.txt
     #   ocdskit
-ocdsmerge-rs==0.1.1
+ocdsmerge-rs==0.1.2
     # via -r requirements.txt
 odfpy==1.4.1
     # via