added exact match evaulator

Author: 0xdeafcafe

evaluators/langevals/langevals_langevals/exact_match.py

@@ -0,0 +1,50 @@
+import ast
+import json
+from typing import Literal, Optional, Dict, Any
+from langevals_core.base_evaluator import (
+    BaseEvaluator,
+    EvaluationResultSkipped,
+    EvaluatorEntry,
+    EvaluationResult,
+    EvaluatorSettings,
+    SingleEvaluationResult,
+    EvaluationResultError,
+)
+import markdown
+from pydantic import Field
+import sqlglot
+
+
+class ExactMatchSettings(EvaluatorSettings):
+    pass
+
+
+class ExactMatchResult(EvaluationResult):
+    passed: Optional[bool] = Field(
+        default=True,
+        description="True if the output matched the input exactly, False otherwise",
+    )
+
+
+class ExactMatchEntry(EvaluatorEntry):
+    input: Optional[str] = None
+    output: Optional[str] = None
+
+
+class ExactMatchEvaluator(
+    BaseEvaluator[ExactMatchEntry, ExactMatchSettings, ExactMatchResult]
+):
+    """
+    A simple evaluator that checks if the output matches the input exactly.
+    """
+
+    name = "Exact Match Evaluator"
+    category = "quality"
+    default_settings = ExactMatchSettings()
+    is_guardrail = False
+
+    def evaluate(self, entry: ExactMatchEntry) -> SingleEvaluationResult:
+        if entry.input == entry.output:
+            return ExactMatchResult(passed=True)
+
+        return ExactMatchResult(passed=False)

evaluators/langevals/tests/test_exact_match.py

@@ -0,0 +1,35 @@
+import dotenv
+
+dotenv.load_dotenv()
+
+from langevals_langevals.exact_match import (
+    ExactMatchEvaluator,
+    ExactMatchEntry,
+    ExactMatchSettings,
+)
+
+
+def test_langeval_exact_match_evaluator_exact():
+    entry = ExactMatchEntry(
+        input="What is the capital of France?",
+        output="What is the capital of France?",
+    )
+    settings = ExactMatchSettings()
+
+    evaluator = ExactMatchEvaluator(settings=settings)
+    result = evaluator.evaluate(entry)
+
+    assert result == True
+
+
+def test_langeval_exact_match_evaluator_different():
+    entry = ExactMatchEntry(
+        input="What is the capital of France?",
+        output="The capital of France is London.",
+    )
+    settings = ExactMatchSettings()
+
+    evaluator = ExactMatchEvaluator(settings=settings)
+    result = evaluator.evaluate(entry)
+
+    assert result == False