Use litellm and litellm env variables primarily for all models including langchain adapted ones, to standardize with other langwatch services and allow for custom hosted models

Author: rogeriochaves

evaluators/aws/poetry.lock

@@ -16,7 +16,10 @@ def set_evaluator_model_and_capture_cost(heystack_evaluator: LLMEvaluator, model
     def capture_completions_cost(self, **kwargs):
         kwargs["model"] = model
         response = litellm.completion(**kwargs)
-        cost.amount += litellm.completion_cost(response)
+        amount = litellm.completion_cost(response)
+        if amount is not None:
+            cost.amount += litellm.completion_cost(response)
+
         response = ChatCompletion(**response.model_dump())  # type: ignore
         return response
 

evaluators/azure/poetry.lock

@@ -157,7 +157,6 @@ def evaluate(self, entry: CompetitorLLMFunctionCallEntry) -> SingleEvaluationRes
         passed = not arguments["competitor_mentioned"] if "competitor_mentioned" in arguments else True
         confidence = arguments["confidence"] if "confidence" in arguments else 1
         reasoning = arguments["reasoning"] if "reasoning" in arguments else "No reasoning."
-        print(reasoning)
         cost = completion_cost(completion_response=response, prompt=prompt)
         details = None
         if not passed: