Update moderation for input and output separately

Author: richhuth

evaluators/openai/langevals_openai/moderation.py

@@ -68,63 +68,70 @@ class OpenAIModerationEvaluator(
     is_guardrail = True
 
     def evaluate_batch(
-        self, data: list[OpenAIModerationEntry], index = 0
+        self, data: list[OpenAIModerationEntry], index=0
     ) -> BatchEvaluationResult:
         client = OpenAI(api_key=self.get_env("OPENAI_API_KEY"))
 
         results: list[SingleEvaluationResult] = []
 
-        contents = [
-            "\n\n".join([entry.input or "", entry.output or ""]).strip()[0:10_000]
-            for entry in data
-        ]
-        response = client.moderations.create(input=contents)
-        for index, moderation_result in tqdm(enumerate(response.results), position=index):
-            if not contents[index]:
+        contents_input = [entry.input or "" for entry in data]
+        contents_output = [entry.output or "" for entry in data]
+        response_input = client.moderations.create(input=contents_input)
+        response_output = client.moderations.create(input=contents_output)
+
+        for i, (input_result, output_result) in enumerate(
+            zip(response_input.results, response_output.results)
+        ):
+
+            if not contents_input[i] and not contents_output[i]:
                 results.append(
                     EvaluationResultSkipped(details="Input and output are both empty")
                 )
-                continue
-
-            detected_categories = dict(
-                [
-                    item
-                    for item in moderation_result.categories.model_dump().items()
-                    if self.settings.categories.model_dump().get(item[0], False)
-                ]
-            )
-            category_scores = dict(
-                [
-                    item
-                    for item in moderation_result.category_scores.model_dump().items()
-                    if detected_categories.get(item[0], False)
-                ]
-            )
-            highest_categories = sorted(
-                category_scores.items(),
-                key=lambda x: x[1],
-                reverse=True,
+            continue
+
+        # Combine results by taking the maximum scores for each category
+        combined_category_scores = {
+            category: max(
+                input_result.category_scores.model_dump().get(category, 0),
+                output_result.category_scores.model_dump().get(category, 0),
             )
-            score = max(category_scores.values()) if len(category_scores) > 0 else 0
-
-            passed = not any(detected_categories.values())
-
-            details = (
-                (
-                    "Detected "
-                    + ", ".join(
-                        [
-                            f"{category} ({score * 100:.2f}% confidence)"
-                            for category, score in highest_categories
-                        ]
-                    )
+            for category in self.settings.categories.model_dump().keys()
+        }
+
+        detected_categories = {
+            category: score > 0.5  # You may want to adjust this threshold
+            for category, score in combined_category_scores.items()
+            if self.settings.categories.model_dump().get(category, False)
+        }
+
+        highest_categories = sorted(
+            combined_category_scores.items(),
+            key=lambda x: x[1],
+            reverse=True,
+        )
+        score = (
+            max(combined_category_scores.values()) if combined_category_scores else 0
+        )
+
+        passed = not any(detected_categories.values())
+
+        details = (
+            (
+                "Detected "
+                + ", ".join(
+                    [
+                        f"{category} ({score * 100:.2f}% confidence)"
+                        for category, score in highest_categories
+                        if detected_categories.get(category, False)
+                    ]
                 )
-                if not passed
-                else None
             )
+            if not passed
+            else None
+        )
 
-            results.append(
-                OpenAIModerationResult(score=score, passed=passed, details=details)
-            )
+        results.append(
+            OpenAIModerationResult(score=score, passed=passed, details=details)
+        )
 
         return results