Update generated ts

Author: rogeriochaves

evaluators/langevals/langevals_langevals/valid_format.py

@@ -8,6 +8,7 @@
     EvaluationResult,
     EvaluatorSettings,
     SingleEvaluationResult,
+    EvaluationResultError,
 )
 import markdown
 from pydantic import Field
@@ -17,7 +18,7 @@
 
 class ValidFormatSettings(EvaluatorSettings):
     format: Literal["json", "markdown", "python", "sql"] = "json"
-    json_schema: Optional[Dict[str, Any]] = Field(
+    json_schema: Optional[str] = Field(
         default=None,
         description="JSON schema to validate against when format is 'json'",
     )
@@ -56,7 +57,18 @@ def evaluate(self, entry: ValidFormatEntry) -> SingleEvaluationResult:
                 parsed_json = json.loads(entry.output)
                 if self.settings.json_schema:
                     try:
-                        validate(instance=parsed_json, schema=self.settings.json_schema)
+                        json_schema = json.loads(self.settings.json_schema)
+                    except Exception as e:
+                        return EvaluationResultError(
+                            details=f"Invalid JSON Schema: {e}",
+                            error_type="json_schema_error",
+                            traceback=[],
+                        )
+                    try:
+                        validate(
+                            instance=parsed_json,
+                            schema=json_schema,
+                        )
                     except ValidationError as e:
                         return ValidFormatResult(
                             passed=False, details=f"JSON Schema validation failed: {e}"

evaluators/langevals/tests/test_valid_format.py

@@ -1,3 +1,4 @@
+import json
 from langevals_langevals.valid_format import (
     ValidFormatEvaluator,
     ValidFormatEntry,
@@ -118,7 +119,7 @@ def test_valid_format_evaluator_json_schema():
         output='{"name": "John Doe", "age": 30, "email": "john@example.com"}'
     )
     evaluator = ValidFormatEvaluator(
-        settings=ValidFormatSettings(format="json", json_schema=schema)
+        settings=ValidFormatSettings(format="json", json_schema=json.dumps(schema))
     )
     result = evaluator.evaluate(entry)