Allow empty input/output on the conversations

Author: rogeriochaves

evaluators/langevals/langevals_langevals/query_resolution.py

@@ -12,6 +12,7 @@
 from langevals_core.base_evaluator import (
     BaseEvaluator,
     EvaluatorEntry,
+    ConversationEntry,
     EvaluationResult,
     EvaluatorSettings,
     LLMEvaluatorSettings,
@@ -21,13 +22,8 @@
 )
 
 
-class QueryResolutionConversationEntry(EvaluatorEntry):
-    input: str
-    output: str
-
-
 class QueryResolutionEntry(EvaluatorEntry):
-    conversation: List[QueryResolutionConversationEntry]
+    conversation: List[ConversationEntry]
 
 
 class QueryResolutionSettings(LLMEvaluatorSettings):

evaluators/langevals/tests/test_query_resolution.py

@@ -2,8 +2,9 @@
 
 dotenv.load_dotenv()
 
+from langevals_core.base_evaluator import ConversationEntry
+
 from langevals_langevals.query_resolution import (
-    QueryResolutionConversationEntry,
     QueryResolutionEntry,
     QueryResolutionSettings,
     QueryResolutionResult,
@@ -12,14 +13,12 @@
 
 
 def test_query_resolution_conversation_evaluator_pass_for_simple_greetings():
-    response1 = QueryResolutionConversationEntry(
+    response1 = ConversationEntry(
         input="Hey, how are you?",
         output="Hello, I am an assistant and I don't have feelings",
     )
     conversation = QueryResolutionEntry(conversation=[response1])
-    settings = QueryResolutionSettings(
-        model="openai/gpt-4o-mini", max_tokens=10000
-    )
+    settings = QueryResolutionSettings(model="openai/gpt-4o-mini", max_tokens=10000)
     evaluator = QueryResolutionEvaluator(settings=settings)
     result = evaluator.evaluate(conversation)
 
@@ -30,18 +29,16 @@ def test_query_resolution_conversation_evaluator_pass_for_simple_greetings():
 
 
 def test_query_resolution_conversation_evaluator_pass():
-    response1 = QueryResolutionConversationEntry(
+    response1 = ConversationEntry(
         input="Hey, how are you?",
         output="Hello, I am an assistant and I don't have feelings",
     )
-    response2 = QueryResolutionConversationEntry(
+    response2 = ConversationEntry(
         input="Okay, is there a president in the Netherlands? Also, tell me what is the system of government in the Netherlands?",
         output="There is no president in the Netherlands. The system of government is constitutional monarchy.",
     )
     conversation = QueryResolutionEntry(conversation=[response1, response2])
-    settings = QueryResolutionSettings(
-        model="openai/gpt-4o-mini", max_tokens=10000
-    )
+    settings = QueryResolutionSettings(model="openai/gpt-4o-mini", max_tokens=10000)
     evaluator = QueryResolutionEvaluator(settings=settings)
     result = evaluator.evaluate(conversation)
 
@@ -52,18 +49,16 @@ def test_query_resolution_conversation_evaluator_pass():
 
 
 def test_query_resolution_conversation_evaluator_fail():
-    response1 = QueryResolutionConversationEntry(
+    response1 = ConversationEntry(
         input="Hey, how are you?",
         output="Hello, I am an assistant and I don't have feelings",
     )
-    response2 = QueryResolutionConversationEntry(
+    response2 = ConversationEntry(
         input="Okay, is there a president in the Netherlands? Also, what equals 2 + 2? How many paws does a standard dog have?",
         output="There is no president in the Netherlands.",
     )
     conversation = QueryResolutionEntry(conversation=[response1, response2])
-    settings = QueryResolutionSettings(
-        model="openai/gpt-4o-mini", max_tokens=10000
-    )
+    settings = QueryResolutionSettings(model="openai/gpt-4o-mini", max_tokens=10000)
     evaluator = QueryResolutionEvaluator(settings=settings)
     result = evaluator.evaluate(conversation)
 
@@ -74,14 +69,12 @@ def test_query_resolution_conversation_evaluator_fail():
 
 
 def test_query_resolution_conversation_evaluator_fails_with_i_dont_know():
-    response1 = QueryResolutionConversationEntry(
+    response1 = ConversationEntry(
         input="What time is it?",
         output="Sorry, I don't have any information about the current time",
     )
     conversation = QueryResolutionEntry(conversation=[response1])
-    settings = QueryResolutionSettings(
-        model="openai/gpt-4o-mini", max_tokens=10000
-    )
+    settings = QueryResolutionSettings(model="openai/gpt-4o-mini", max_tokens=10000)
     evaluator = QueryResolutionEvaluator(settings=settings)
     result = evaluator.evaluate(conversation)
 
@@ -92,12 +85,10 @@ def test_query_resolution_conversation_evaluator_fails_with_i_dont_know():
 
 
 def test_product_sentiment_polarity_evaluator_skipped_for_non_product_related_outputs():
-    response1 = QueryResolutionConversationEntry(input="", output="")
-    response2 = QueryResolutionConversationEntry(input="", output="")
+    response1 = ConversationEntry(input="", output="")
+    response2 = ConversationEntry(input="", output="")
     conversation = QueryResolutionEntry(conversation=[response1, response2])
-    settings = QueryResolutionSettings(
-        model="openai/gpt-4o-mini", max_tokens=10000
-    )
+    settings = QueryResolutionSettings(model="openai/gpt-4o-mini", max_tokens=10000)
     evaluator = QueryResolutionEvaluator(settings=settings)
     result = evaluator.evaluate(conversation)
 

langevals_core/langevals_core/base_evaluator.py

@@ -28,11 +28,14 @@
     "quality", "rag", "safety", "policy", "other", "custom", "similarity"
 ]
 
+
 class EvaluatorSettings(BaseModel):
-     pass
+    pass
+
 
 TSettings = TypeVar("TSettings", bound=EvaluatorSettings)
 
+
 class LLMEvaluatorSettings(EvaluatorSettings):
     model: Literal[
         "openai/gpt-3.5-turbo",
@@ -60,6 +63,12 @@ class LLMEvaluatorSettings(EvaluatorSettings):
         description="Max tokens allowed for evaluation",
     )
 
+
+class ConversationEntry(BaseModel):
+    input: str = Field(default="")
+    output: str = Field(default="")
+
+
 class EvaluatorEntry(BaseModel):
     """
     Entry datapoint for an evaluator, it should contain all the necessary information for the evaluator to run.
@@ -78,7 +87,7 @@ def __init_subclass__(cls, **kwargs):
         super().__init_subclass__(**kwargs)  # Always call super()!
 
         required_fields_types = {
-            "conversation": [EvaluatorEntry, Optional[EvaluatorEntry]],
+            "conversation": [ConversationEntry, Optional[ConversationEntry]],
             "input": [str, Optional[str]],
             "output": [str, Optional[str]],
             "contexts": [