Fix ragas relevancy description for better explainability

Author: rogeriochaves

evaluators/ragas/langevals_ragas/lib/common.py

@@ -50,13 +50,17 @@ class _GenericEvaluatorEntry(EvaluatorEntry):
     expected_output: Optional[str]
 
 
-def prepare_llm(evaluator: BaseEvaluator, settings: RagasSettings = RagasSettings()):
+def prepare_llm(
+    evaluator: BaseEvaluator,
+    settings: RagasSettings = RagasSettings(),
+    temperature: float = 0,
+):
     os.environ["AZURE_API_VERSION"] = "2023-07-01-preview"
     if evaluator.env:
         for key, env in evaluator.env.items():
             os.environ[key] = env
 
-    gpt = model_to_langchain(settings.model)
+    gpt = model_to_langchain(settings.model, temperature=temperature)
     llm = LangchainLLMWrapper(langchain_llm=gpt)
 
     if hasattr(settings, "embeddings_model"):

evaluators/ragas/langevals_ragas/lib/model_to_langchain.py

@@ -12,9 +12,15 @@
 
 class LitellmCompletion:
     exception: Optional[Exception] = None
+    temperature: float = 0
+
+    def __init__(self, temperature: float = 0):
+        self.temperature = temperature
 
     def create(self, *args, **kwargs):
         try:
+            if self.temperature:
+                kwargs["temperature"] = self.temperature
             return litellm.completion(*args, **kwargs)
         except Exception as e:
             self.exception = e
@@ -28,16 +34,17 @@ async def create(self, *args, **kwargs):
 
 def model_to_langchain(
     model: str,
+    temperature: float = 0,
 ) -> BaseChatModel:
     if model.startswith("claude-"):
         model = model.replace("claude-", "anthropic/claude-")
 
     return ChatOpenAI(
         model=model,
         api_key="dummy",  # type: ignore
-        client=LitellmCompletion(),
-        async_client=AsyncLitellmCompletion(),
-        temperature=0,
+        temperature=temperature or 0,
+        client=LitellmCompletion(temperature=temperature),
+        async_client=AsyncLitellmCompletion(temperature=temperature),
     )
 
 

evaluators/ragas/langevals_ragas/response_relevancy.py

@@ -1,3 +1,4 @@
+from typing import Sequence
 from langevals_core.base_evaluator import (
     BaseEvaluator,
     EvaluationResult,
@@ -16,6 +17,7 @@
 )
 from pydantic import Field
 from ragas.metrics import ResponseRelevancy
+from ragas.metrics._answer_relevance import ResponseRelevanceOutput
 
 
 class RagasResponseRelevancyEntry(EvaluatorEntry):
@@ -56,7 +58,7 @@ class RagasResponseRelevancyEvaluator(
     is_guardrail = False
 
     def evaluate(self, entry: RagasResponseRelevancyEntry) -> SingleEvaluationResult:
-        llm, embeddings = prepare_llm(self, self.settings)
+        llm, embeddings = prepare_llm(self, self.settings, temperature=0.7)
 
         skip = check_max_tokens(
             input=entry.input,
@@ -69,17 +71,23 @@ def evaluate(self, entry: RagasResponseRelevancyEntry) -> SingleEvaluationResult
         scorer = ResponseRelevancy(llm=llm, embeddings=embeddings)
 
         _original_calculate_similarity = scorer.calculate_similarity
+        _original_calculate_score = scorer._calculate_score
 
-        breakdown = {"similarity": 0, "generated_questions": []}
+        breakdown = {"similarity": 0, "answers": []}
 
         def calculate_similarity(question: str, generated_questions):
             nonlocal breakdown
-            breakdown["generated_questions"] += generated_questions
             similarity = _original_calculate_similarity(question, generated_questions)
             breakdown["similarity"] += similarity
             return similarity
 
+        def _calculate_score(answers: Sequence[ResponseRelevanceOutput], row: dict):
+            nonlocal breakdown
+            breakdown["answers"] += answers
+            return _original_calculate_score(answers, row)
+
         scorer.calculate_similarity = calculate_similarity
+        scorer._calculate_score = _calculate_score
 
         with capture_cost(llm) as cost:
             score = scorer.single_turn_score(
@@ -89,15 +97,19 @@ def calculate_similarity(question: str, generated_questions):
                 )
             )
 
-        generated_questions = "\n- ".join(breakdown["generated_questions"])
+        generated_questions = "\n".join(
+            [f"- {answer.question}" for answer in breakdown["answers"]]
+        )
 
-        if len(breakdown["generated_questions"]) == 0:
+        if len([answer for answer in breakdown["answers"] if answer.question]) == 0:
             return EvaluationResultSkipped(
                 details="No questions could be generated from output.",
             )
 
+        any_noncommittal = any([answer.noncommittal for answer in breakdown["answers"]])
+
         return RagasResult(
             score=score,
             cost=cost,
-            details=f"Questions generated from output:\n{generated_questions}\nSimilarity to original question: {breakdown['similarity']}",
+            details=f"Questions generated from output:\n\n{generated_questions}\n\nSimilarity to original question: {breakdown['similarity']}\nEvasive answer: {any_noncommittal}",
         )