Centralized the choice of the model in LLMEvaluatorSettings

Author: Eugenumber1

evaluators/haystack/langevals_haystack/faithfulness.py

@@ -12,6 +12,7 @@
     EvaluatorSettings,
     SingleEvaluationResult,
     Money,
+    LLMEvaluatorSettings
 )
 from pydantic import BaseModel, Field
 from haystack.components.evaluators import FaithfulnessEvaluator
@@ -28,7 +29,7 @@ class HaystackFaithfulnessEntry(EvaluatorEntry):
     contexts: list[str]
 
 
-class HaystackFaithfulnessSettings(EvaluatorSettings):
+class HaystackFaithfulnessSettings(LLMEvaluatorSettings):
     model: str = Field(
         default="azure/gpt-35-turbo-1106",
         description="The model to use for evaluation.",

evaluators/huggingface/langevals_huggingface/llama_guard.py

@@ -6,6 +6,7 @@
     EvaluatorEntry,
     EvaluationResult,
     EvaluatorSettings,
+    LLMEvaluatorSettings,
     SingleEvaluationResult,
     Money,
 )

evaluators/langevals/langevals_langevals/basic.py

@@ -5,6 +5,7 @@
     EvaluatorEntry,
     EvaluationResult,
     EvaluatorSettings,
+    LLMEvaluatorSettings,
     SingleEvaluationResult,
 )
 from pydantic import BaseModel, Field
@@ -26,7 +27,7 @@ class CustomBasicRule(BaseModel):
     value: str
 
 
-class CustomBasicSettings(EvaluatorSettings):
+class CustomBasicSettings(LLMEvaluatorSettings):
     rules: list[CustomBasicRule] = Field(default=[
         CustomBasicRule(field="output", rule="not_contains", value="artificial intelligence"),
     ], description="List of rules to check, the message must pass all of them")

evaluators/langevals/langevals_langevals/competitor_llm.py

@@ -15,6 +15,7 @@
     EvaluatorEntry,
     EvaluationResult,
     EvaluatorSettings,
+    LLMEvaluatorSettings,
     SingleEvaluationResult,
     EvaluationResultSkipped,
     Money,
@@ -26,7 +27,7 @@ class CompetitorLLMEntry(EvaluatorEntry):
     input: Optional[str] = None
 
 
-class CompetitorLLMSettings(EvaluatorSettings):
+class CompetitorLLMSettings(LLMEvaluatorSettings):
     name: str = Field(default="LangWatch", description="The name of your company")
     description: str = Field(
         default="We are providing an LLM observability and evaluation platform",

evaluators/langevals/langevals_langevals/competitor_llm_function_call.py

@@ -12,7 +12,7 @@
     BaseEvaluator,
     EvaluatorEntry,
     EvaluationResult,
-    EvaluatorSettings,
+    LLMEvaluatorSettings,
     SingleEvaluationResult,
     EvaluationResultSkipped,
     Money,
@@ -24,7 +24,7 @@ class CompetitorLLMFunctionCallEntry(EvaluatorEntry):
     input: Optional[str] = None
 
 
-class CompetitorLLMFunctionCallSettings(EvaluatorSettings):
+class CompetitorLLMFunctionCallSettings(LLMEvaluatorSettings):
     name: str = Field(default="LangWatch", description="The name of your company")
     description: str = Field(
         default="We are providing an LLM observability and evaluation platform",
@@ -34,6 +34,7 @@ class CompetitorLLMFunctionCallSettings(EvaluatorSettings):
         default=["OpenAI", "Google", "Microsoft"],
         description="The competitors that must not be mentioned.",
     )
+    
 
 
 class CompetitorLLMFunctionCallResult(EvaluationResult):

evaluators/langevals/langevals_langevals/llm_boolean.py

@@ -6,13 +6,16 @@
     EvaluatorEntry,
     EvaluationResult,
     EvaluatorSettings,
+    LLMEvaluatorSettings,
     SingleEvaluationResult,
     EvaluationResultSkipped,
     Money,
 )
 from pydantic import BaseModel, Field
 import litellm
-from litellm import ModelResponse, Choices, Message, completion_cost
+from litellm import Choices, Message
+from litellm.files.main import ModelResponse
+from litellm.cost_calculator import completion_cost
 
 
 class CustomLLMBooleanEntry(EvaluatorEntry):
@@ -21,7 +24,7 @@ class CustomLLMBooleanEntry(EvaluatorEntry):
     contexts: Optional[list[str]] = None
 
 
-class CustomLLMBooleanSettings(EvaluatorSettings):
+class CustomLLMBooleanSettings(LLMEvaluatorSettings):
     prompt: str = Field(
         default="You are an LLM evaluator. We need the guarantee that the output answers what is being asked on the input, please evaluate as False if it doesn't",
         description="The system prompt to use for the LLM to run the evaluation",

evaluators/langevals/langevals_langevals/llm_score.py

@@ -5,14 +5,17 @@
     BaseEvaluator,
     EvaluatorEntry,
     EvaluationResult,
+    LLMEvaluatorSettings,
     SingleEvaluationResult,
     EvaluationResultSkipped,
     Money,
     EvaluatorSettings
 )
 from pydantic import Field
 import litellm
-from litellm import ModelResponse, Choices, Message, completion_cost
+from litellm import Choices, Message
+from litellm.files.main import ModelResponse
+from litellm.cost_calculator import completion_cost
 
 
 
@@ -22,7 +25,7 @@ class CustomLLMScoreEntry(EvaluatorEntry):
     contexts: Optional[list[str]] = None
 
 
-class CustomLLMScoreSettings(EvaluatorSettings):
+class CustomLLMScoreSettings(LLMEvaluatorSettings):
     prompt: str = Field(
         default="You are an LLM evaluator. Please score from 0.0 to 1.0 how likely the user is to be satisfied with this answer, from 0.0 being not satisfied at all to 1.0 being completely satisfied",
         description="The system prompt to use for the LLM to run the evaluation",

evaluators/langevals/langevals_langevals/off_topic.py

@@ -1,7 +1,8 @@
 import litellm
-from litellm import get_max_tokens, completion_cost
-from litellm import ModelResponse, Choices, Message
-from litellm.utils import trim_messages
+from litellm import Choices, Message
+from litellm.files.main import ModelResponse
+from litellm.cost_calculator import completion_cost
+from litellm.utils import trim_messages, get_max_tokens
 
 from pydantic import BaseModel, Field
 from typing import Optional, List, Literal, cast
@@ -12,6 +13,7 @@
     BaseEvaluator,
     EvaluatorEntry,
     EvaluationResult,
+    LLMEvaluatorSettings,
     SingleEvaluationResult,
     EvaluationResultSkipped,
     Money,
@@ -28,7 +30,7 @@ class AllowedTopic(BaseModel):
     description: str
 
 
-class OffTopicSettings(EvaluatorSettings):
+class OffTopicSettings(LLMEvaluatorSettings):
     allowed_topics: List[AllowedTopic] = Field(
         default=[
             AllowedTopic(topic="simple_chat", description="Smalltalk with the user"),

evaluators/langevals/langevals_langevals/query_resolution.py

@@ -14,6 +14,7 @@
     EvaluatorEntry,
     EvaluationResult,
     EvaluatorSettings,
+    LLMEvaluatorSettings,
     SingleEvaluationResult,
     EvaluationResultSkipped,
     Money,
@@ -29,7 +30,7 @@ class QueryResolutionEntry(EvaluatorEntry):
     conversation: List[QueryResolutionConversationEntry]
 
 
-class QueryResolutionSettings(EvaluatorSettings):
+class QueryResolutionSettings(LLMEvaluatorSettings):
     pass # maybe specify after how many turns we should run this evaluator?
 
 

evaluators/langevals/langevals_langevals/similarity.py

@@ -4,6 +4,7 @@
     EvaluatorEntry,
     EvaluationResult,
     EvaluatorSettings,
+    LLMEvaluatorSettings,
     SingleEvaluationResult,
     EvaluationResultSkipped,
 )