Remove the temporary fix for gpt-4o, which was actually just messing up the costs calculation

Author: rogeriochaves

evaluators/langevals/langevals_langevals/competitor_llm.py

@@ -153,9 +153,6 @@ def evaluate(self, entry: CompetitorLLMEntry) -> SingleEvaluationResult:
         passed = not arguments["competitor_mentioned"] if "competitor_mentioned" in arguments else True
         confidence = arguments["confidence"] if "confidence" in arguments else 1
         reasoning = arguments["reasoning"] if "reasoning" in arguments else "No reasoning."
-        # Temporary fix for gpt-4o
-        if "gpt-4o" in (response.model or ""):
-            response.model = "openai/gpt-4-turbo"
         cost = completion_cost(completion_response=response, prompt=prompt)
         details = None
         if not passed:

evaluators/langevals/langevals_langevals/competitor_llm_function_call.py

@@ -34,7 +34,7 @@ class CompetitorLLMFunctionCallSettings(LLMEvaluatorSettings):
         default=["OpenAI", "Google", "Microsoft"],
         description="The competitors that must not be mentioned.",
     )
-    
+
 
 
 class CompetitorLLMFunctionCallResult(EvaluationResult):
@@ -77,10 +77,10 @@ def evaluate(self, entry: CompetitorLLMFunctionCallEntry) -> SingleEvaluationRes
         for competitor in self.settings.competitors:
             competitors += competitor + "\n"
         litellm_model = model if vendor == "openai" and model != "gpt-4o" else f"{vendor}/{model}"
-        prompt = f"""Remember that you are an advanced competitor detection system, developed by {your_company_description}. 
-                    Your task is to identify mentions of competitors in any given message. 
+        prompt = f"""Remember that you are an advanced competitor detection system, developed by {your_company_description}.
+                    Your task is to identify mentions of competitors in any given message.
                     The competitors specialize in the same field as your company and are listed below:
-                    
+
                     Competitors:
                     {competitors}
 
@@ -158,9 +158,6 @@ def evaluate(self, entry: CompetitorLLMFunctionCallEntry) -> SingleEvaluationRes
         confidence = arguments["confidence"] if "confidence" in arguments else 1
         reasoning = arguments["reasoning"] if "reasoning" in arguments else "No reasoning."
         print(reasoning)
-        # Temporary fix for gpt-4o
-        if "gpt-4o" in (response.model or ""):
-            response.model = "openai/gpt-4-turbo"
         cost = completion_cost(completion_response=response, prompt=prompt)
         details = None
         if not passed:

evaluators/langevals/langevals_langevals/llm_boolean.py

@@ -130,9 +130,6 @@ def evaluate(self, entry: CustomLLMBooleanEntry) -> SingleEvaluationResult:
         arguments = json.loads(
             cast(Message, choice.message).tool_calls[0].function.arguments
         )
-        # Temporary fix for gpt-4o
-        if "gpt-4o" in (response.model or ""):
-            response.model = "openai/gpt-4-turbo"
         cost = completion_cost(completion_response=response)
 
         return CustomLLMBooleanResult(

evaluators/langevals/langevals_langevals/llm_score.py

@@ -128,9 +128,6 @@ def evaluate(self, entry: CustomLLMScoreEntry) -> SingleEvaluationResult:
         arguments = json.loads(
             cast(Message, choice.message).tool_calls[0].function.arguments
         )
-        # Temporary fix for gpt-4o
-        if "gpt-4o" in (response.model or ""):
-            response.model = "openai/gpt-4-turbo"
         cost = completion_cost(completion_response=response)
 
         return CustomLLMScoreResult(

evaluators/langevals/tests/test_llm_boolean.py

@@ -16,7 +16,7 @@ def test_custom_llm_boolean_evaluator():
         contexts=["London is the capital of France."],
     )
     settings = CustomLLMBooleanSettings(
-        model="openai/gpt-3.5-turbo-0125",
+        model="openai/gpt-4o-mini",
         prompt="You are an LLM evaluator. We need the guarantee that the output is using the provided context and not it's own brain, please evaluate as False if is not.",
     )
 
@@ -37,7 +37,7 @@ def test_custom_llm_boolean_evaluator_skips_if_context_is_too_large():
         contexts=["London is the capital of France."] * 300,
     )
     settings = CustomLLMBooleanSettings(
-        model="openai/gpt-3.5-turbo-0125",
+        model="openai/gpt-4o-mini",
         prompt="You are an LLM evaluator. We need the guarantee that the output is using the provided context and not it's own brain, please evaluate as False if is not.",
         max_tokens=2048,
     )