Change input/output to output/expected_output on exact_match evaluator

Author: rogeriochaves

evaluators/langevals/langevals_langevals/exact_match.py

@@ -28,21 +28,20 @@ class ExactMatchSettings(EvaluatorSettings):
 class ExactMatchResult(EvaluationResult):
     passed: Optional[bool] = Field(
         default=True,
-        description="True if the output matched the input exactly, False otherwise",
+        description="True if the output matched the expected_output exactly, False otherwise",
     )
 
 
 class ExactMatchEntry(EvaluatorEntry):
-    input: Optional[str] = None
     output: Optional[str] = None
+    expected_output: Optional[str] = None
 
 
 class ExactMatchEvaluator(
     BaseEvaluator[ExactMatchEntry, ExactMatchSettings, ExactMatchResult]
 ):
     """
-    A simple evaluator that checks if the output matches the input exactly, with some 
-    extra bells and whistles to help with whitespace related shenanigans.
+    A simple evaluator that checks if the output matches the expected_output exactly.
     """
 
     name = "Exact Match Evaluator"
@@ -51,21 +50,21 @@ class ExactMatchEvaluator(
     is_guardrail = False
 
     def evaluate(self, entry: ExactMatchEntry) -> SingleEvaluationResult:
-        input_text = entry.input or ""
         output_text = entry.output or ""
+        expected_output_text = entry.expected_output or ""
 
         if self.settings.trim_whitespace:
-            input_text = input_text.strip()
             output_text = output_text.strip()
+            expected_output_text = expected_output_text.strip()
 
         if self.settings.remove_punctuation:
-            input_text = ''.join(char for char in input_text if char.isalnum() or char.isspace())
             output_text = ''.join(char for char in output_text if char.isalnum() or char.isspace())
+            expected_output_text = ''.join(char for char in expected_output_text if char.isalnum() or char.isspace())
 
         if not self.settings.case_sensitive:
-            input_text = input_text.lower()
             output_text = output_text.lower()
+            expected_output_text = expected_output_text.lower()
 
-        passed = input_text == output_text
+        passed = output_text == expected_output_text
 
         return ExactMatchResult(passed=passed)

evaluators/langevals/tests/test_exact_match.py

@@ -7,8 +7,8 @@
 
 def test_langeval_exact_match_evaluator():
     entry = ExactMatchEntry(
-        input="What is the capital of France?",
         output="What is the capital of France?",
+        expected_output="What is the capital of France?",
     )
     settings = ExactMatchSettings()
 
@@ -20,8 +20,8 @@ def test_langeval_exact_match_evaluator():
 
 def test_langeval_exact_match_evaluator_defaults():
     entry = ExactMatchEntry(
-        input="What is the capital of France?",
-        output="What is the capital of the Netherlands?",
+        output="What is the capital of France?",
+        expected_output="What is the capital of the Netherlands?",
     )
     settings = ExactMatchSettings()
 
@@ -33,8 +33,8 @@ def test_langeval_exact_match_evaluator_defaults():
 
 def test_langeval_exact_match_case_sensitive_true():
     entry = ExactMatchEntry(
-        input="Hello World",
-        output="hello world",
+        output="Hello World",
+        expected_output="hello world",
     )
     settings = ExactMatchSettings(case_sensitive=True)
 
@@ -46,8 +46,8 @@ def test_langeval_exact_match_case_sensitive_true():
 
 def test_langeval_exact_match_case_sensitive_false():
     entry = ExactMatchEntry(
-        input="Hello World",
-        output="hello world",
+        output="Hello World",
+        expected_output="hello world",
     )
     settings = ExactMatchSettings(case_sensitive=False)
 
@@ -59,8 +59,8 @@ def test_langeval_exact_match_case_sensitive_false():
 
 def test_langeval_exact_match_trim_whitespace_true():
     entry = ExactMatchEntry(
-        input="  Hello World  ",
-        output="Hello World",
+        output="  Hello World  ",
+        expected_output="Hello World",
     )
     settings = ExactMatchSettings(trim_whitespace=True)
 
@@ -72,8 +72,8 @@ def test_langeval_exact_match_trim_whitespace_true():
 
 def test_langeval_exact_match_trim_whitespace_false():
     entry = ExactMatchEntry(
-        input="  Hello World  ",
-        output="Hello World",
+        output="  Hello World  ",
+        expected_output="Hello World",
     )
     settings = ExactMatchSettings(trim_whitespace=False)
 
@@ -85,8 +85,8 @@ def test_langeval_exact_match_trim_whitespace_false():
 
 def test_langeval_exact_match_remove_punctuation_true():
     entry = ExactMatchEntry(
-        input="Hello, World!",
-        output="Hello World",
+        output="Hello, World!",
+        expected_output="Hello World",
     )
     settings = ExactMatchSettings(remove_punctuation=True)
 
@@ -98,8 +98,8 @@ def test_langeval_exact_match_remove_punctuation_true():
 
 def test_langeval_exact_match_remove_punctuation_false():
     entry = ExactMatchEntry(
-        input="Hello, World!",
-        output="Hello World",
+        output="Hello, World!",
+        expected_output="Hello World",
     )
     settings = ExactMatchSettings(remove_punctuation=False)
 
@@ -111,8 +111,8 @@ def test_langeval_exact_match_remove_punctuation_false():
 
 def test_langeval_exact_match_combined_settings():
     entry = ExactMatchEntry(
-        input="  Hello, World!  ",
-        output="hello world",
+        output="  Hello, World!  ",
+        expected_output="hello world",
     )
     settings = ExactMatchSettings(
         case_sensitive=False,