Small workaround to prevent breaking jsons when anonymizing text

Author: rogeriochaves

evaluators/presidio/langevals_presidio/pii_detection.py

@@ -109,6 +109,15 @@ def preload(cls):
 
     def evaluate(self, entry: PresidioPIIDetectionEntry) -> SingleEvaluationResult:
         content = "\n\n".join([entry.input or "", entry.output or ""]).strip()
+        try:
+            json.loads(content)
+            content = (
+                content.replace("\\n", "\n").replace("\\t", "\t").replace("\\r", "\r")
+            )
+            is_valid_json = True
+        except:
+            is_valid_json = False
+
         if not content:
             return EvaluationResultSkipped(details="Input and output are both empty")
 
@@ -137,7 +146,14 @@ def evaluate(self, entry: PresidioPIIDetectionEntry) -> SingleEvaluationResult:
         anonymized_text = anonymizer.anonymize(
             text=content,
             analyzer_results=results,  # type: ignore
-        )
+        ).text
+
+        if is_valid_json:
+            anonymized_text = (
+                anonymized_text.replace("\n", "\\n")
+                .replace("\t", "\\t")
+                .replace("\r", "\\r")
+            )
 
         serialized_results = [result.to_dict() for result in results]
 
@@ -149,6 +165,6 @@ def evaluate(self, entry: PresidioPIIDetectionEntry) -> SingleEvaluationResult:
             ),
             raw_response={
                 "results": serialized_results,
-                "anonymized": anonymized_text.text,
+                "anonymized": anonymized_text,
             },
         )

evaluators/presidio/tests/test_pii_detection.py

@@ -12,25 +12,29 @@
 
 def test_pii_detection():
     entry = PresidioPIIDetectionEntry(input="hey there, my email is foo@bar.com")
-    evaluator = PresidioPIIDetectionEvaluator(
-        settings=PresidioPIIDetectionSettings()
-    )
+    evaluator = PresidioPIIDetectionEvaluator(settings=PresidioPIIDetectionSettings())
     result = evaluator.evaluate(entry)
 
     assert result.status == "processed"
-    assert result.score == 2
+    assert result.score == 1
     assert result.passed is False
-    assert (
-        result.details
-        == "PII detected: EMAIL_ADDRESS (likelihood: 1.0), URL (likelihood: 0.5)"
-    )
+    assert result.details == "PII detected: EMAIL_ADDRESS (likelihood: 1.0)"
 
 
 def test_pii_detection_long_context():
     entry = PresidioPIIDetectionEntry(input="lorem ipsum dolor " * 100000)
-    evaluator = PresidioPIIDetectionEvaluator(
-        settings=PresidioPIIDetectionSettings()
-    )
+    evaluator = PresidioPIIDetectionEvaluator(settings=PresidioPIIDetectionSettings())
 
     with pytest.raises(Exception):
         evaluator.evaluate(entry)
+
+
+def test_keep_jsons_valid():
+    entry = PresidioPIIDetectionEntry(input='{"foo": "bar\\nfoo@bar.com"}')
+    evaluator = PresidioPIIDetectionEvaluator(settings=PresidioPIIDetectionSettings())
+    result = evaluator.evaluate(entry)
+
+    assert result.status == "processed"
+    assert result.passed is False
+    assert result.details == "PII detected: EMAIL_ADDRESS (likelihood: 1.0)"
+    assert result.raw_response["anonymized"] == '{"foo": "bar\\n<EMAIL_ADDRESS>"}'  # type: ignore