Revert "Revert "Centralized the model option for evaluators by creating EvaluatorSettings in base_evaluator.py, added a couple of default=None fixes to avoid type check complaints""

This reverts commit 993963b.

Author: Eugenumber1

evaluators/aws/langevals_aws/comprehend_pii_detection.py

@@ -4,6 +4,7 @@
 from langevals_core.base_evaluator import (
     BaseEvaluator,
     EvaluatorEntry,
+    EvaluatorSettings,
     SingleEvaluationResult,
     EvaluationResult,
     EvaluationResultSkipped,
@@ -56,7 +57,7 @@ class AWSComprehendEntityTypes(BaseModel):
     IN_VOTER_NUMBER: bool = True
 
 
-class AWSComprehendPIIDetectionSettings(BaseModel):
+class AWSComprehendPIIDetectionSettings(EvaluatorSettings):
     entity_types: AWSComprehendEntityTypes = Field(
         default=AWSComprehendEntityTypes(),
         description="The types of PII to check for in the input.",
@@ -148,7 +149,7 @@ class AWSPIIEntityResults(TypedDict):
 class AWSComprehendPIIDetectionResult(EvaluationResult):
     score: float = Field(description="Amount of PII detected, 0 means no PII detected")
     passed: Optional[bool] = Field(
-        description="If true then no PII was detected, if false then at least one PII was detected"
+        description="If true then no PII was detected, if false then at least one PII was detected", default=None
     )
     raw_response: AWSPIIEntityResults
 

evaluators/aws/poetry.lock

@@ -6,6 +6,7 @@
 from langevals_core.base_evaluator import (
     BaseEvaluator,
     EvaluationResult,
+    EvaluatorSettings,
     SingleEvaluationResult,
     EvaluatorEntry,
     EvaluationResultSkipped,
@@ -25,7 +26,7 @@ class AzureContentSafetyCategories(BaseModel):
     Violence: bool = True
 
 
-class AzureContentSafetySettings(BaseModel):
+class AzureContentSafetySettings(EvaluatorSettings):
     severity_threshold: Literal[1, 2, 3, 4, 5, 6, 7] = Field(
         default=1,
         description="The minimum severity level to consider content as unsafe, from 1 to 7.",

evaluators/azure/langevals_azure/content_safety.py

@@ -3,18 +3,19 @@
 from langevals_core.base_evaluator import (
     BaseEvaluator,
     EvaluationResult,
+    EvaluatorSettings,
     SingleEvaluationResult,
     EvaluatorEntry,
     EvaluationResultSkipped,
 )
-from pydantic import BaseModel, Field
+from pydantic import Field
 
 
 class AzureJailbreakEntry(EvaluatorEntry):
     input: str
 
 
-class AzureJailbreakSettings(BaseModel):
+class AzureJailbreakSettings(EvaluatorSettings):
     pass
 
 

evaluators/azure/langevals_azure/jailbreak.py

@@ -3,11 +3,12 @@
 from langevals_core.base_evaluator import (
     BaseEvaluator,
     EvaluationResult,
+    EvaluatorSettings,
     SingleEvaluationResult,
     EvaluatorEntry,
     EvaluationResultSkipped,
 )
-from pydantic import BaseModel, Field
+from pydantic import Field
 import math
 
 
@@ -16,7 +17,7 @@ class AzurePromptShieldEntry(EvaluatorEntry):
     contexts: Optional[List[str]] = None
 
 
-class AzurePromptShieldSettings(BaseModel):
+class AzurePromptShieldSettings(EvaluatorSettings):
     pass
 
 

evaluators/azure/langevals_azure/prompt_injection.py

@@ -2,9 +2,10 @@
     BaseEvaluator,
     EvaluatorEntry,
     EvaluationResult,
+    EvaluatorSettings,
     SingleEvaluationResult,
 )
-from pydantic import BaseModel, Field
+from pydantic import Field
 
 
 # Type definition of what keys are necessary for each entry to have for the evaluator to process it, in this example
@@ -14,7 +15,7 @@ class ExampleWordCountEntry(EvaluatorEntry):
 
 
 # Generic settings for the evaluator, in this example we don't need any settings, but any fields can be added here
-class ExampleWordCountSettings(BaseModel):
+class ExampleWordCountSettings(EvaluatorSettings):
     pass
 
 

evaluators/azure/poetry.lock

@@ -3,6 +3,7 @@
 from langevals_core.base_evaluator import (
     BaseEvaluator,
     EvaluatorEntry,
+    EvaluatorSettings,
     SingleEvaluationResult,
     EvaluationResult,
     EvaluationResultSkipped,
@@ -29,7 +30,7 @@ class GoogleCloudDLPInfoTypes(BaseModel):
     medical_record_number: bool = True
 
 
-class GoogleCloudDLPPIIDetectionSettings(BaseModel):
+class GoogleCloudDLPPIIDetectionSettings(EvaluatorSettings):
     info_types: GoogleCloudDLPInfoTypes = Field(
         default=GoogleCloudDLPInfoTypes(),
         description="The types of PII to check for in the input.",
@@ -45,7 +46,7 @@ class GoogleCloudDLPPIIDetectionSettings(BaseModel):
 class GoogleCloudDLPPIIDetectionResult(EvaluationResult):
     score: float = Field(description="Amount of PII detected, 0 means no PII detected")
     passed: Optional[bool] = Field(
-        description="If true then no PII was detected, if false then at least one PII was detected"
+        description="If true then no PII was detected, if false then at least one PII was detected", default=None
     )
     raw_response: dict[str, Any]