fix: return score 1 or 0 for evaluators that were only returning passed true/false

Author: rogeriochaves

evaluators/langevals/langevals_langevals/exact_match.py

@@ -24,7 +24,6 @@ class ExactMatchSettings(EvaluatorSettings):
     )
 
 
-
 class ExactMatchResult(EvaluationResult):
     passed: Optional[bool] = Field(
         default=True,
@@ -58,13 +57,19 @@ def evaluate(self, entry: ExactMatchEntry) -> SingleEvaluationResult:
             expected_output_text = expected_output_text.strip()
 
         if self.settings.remove_punctuation:
-            output_text = ''.join(char for char in output_text if char.isalnum() or char.isspace())
-            expected_output_text = ''.join(char for char in expected_output_text if char.isalnum() or char.isspace())
+            output_text = "".join(
+                char for char in output_text if char.isalnum() or char.isspace()
+            )
+            expected_output_text = "".join(
+                char
+                for char in expected_output_text
+                if char.isalnum() or char.isspace()
+            )
 
         if not self.settings.case_sensitive:
             output_text = output_text.lower()
             expected_output_text = expected_output_text.lower()
 
         passed = output_text == expected_output_text
 
-        return ExactMatchResult(passed=passed)
+        return ExactMatchResult(score=1 if passed else 0, passed=passed)

evaluators/langevals/langevals_langevals/valid_format.py

@@ -71,10 +71,14 @@ def evaluate(self, entry: ValidFormatEntry) -> SingleEvaluationResult:
                         )
                     except ValidationError as e:
                         return ValidFormatResult(
-                            passed=False, details=f"JSON Schema validation failed: {e}"
+                            score=0,
+                            passed=False,
+                            details=f"JSON Schema validation failed: {e}",
                         )
             except json.JSONDecodeError as e:
-                return ValidFormatResult(passed=False, details=f"Invalid JSON: {e}")
+                return ValidFormatResult(
+                    score=0, passed=False, details=f"Invalid JSON: {e}"
+                )
         elif self.settings.format == "markdown":
             try:
                 html_result = markdown.markdown(entry.output)
@@ -93,12 +97,16 @@ def evaluate(self, entry: ValidFormatEntry) -> SingleEvaluationResult:
                         details="No markdown elements found. Text should contain markdown formatting like headers (#), bold (**), lists, etc.",
                     )
             except Exception as e:
-                return ValidFormatResult(passed=False, details=f"Invalid Markdown: {e}")
+                return ValidFormatResult(
+                    score=0, passed=False, details=f"Invalid Markdown: {e}"
+                )
         elif self.settings.format == "python":
             try:
                 ast.parse(entry.output)
             except Exception as e:
-                return ValidFormatResult(passed=False, details=f"Invalid Python: {e}")
+                return ValidFormatResult(
+                    score=0, passed=False, details=f"Invalid Python: {e}"
+                )
         elif self.settings.format == "sql":
             try:
                 try:
@@ -110,6 +118,8 @@ def evaluate(self, entry: ValidFormatEntry) -> SingleEvaluationResult:
                 except Exception:
                     sqlglot.parse(entry.output)
             except Exception as e:
-                return ValidFormatResult(passed=False, details=f"Invalid SQL: {e}")
+                return ValidFormatResult(
+                    score=0, passed=False, details=f"Invalid SQL: {e}"
+                )
 
-        return ValidFormatResult(passed=True)
+        return ValidFormatResult(score=1, passed=True)