Add format validator

Author: rogeriochaves

evaluators/langevals/langevals_langevals/valid_format.py

@@ -0,0 +1,96 @@
+import ast
+import json
+from typing import Literal, Optional, Dict, Any
+from langevals_core.base_evaluator import (
+    BaseEvaluator,
+    EvaluationResultSkipped,
+    EvaluatorEntry,
+    EvaluationResult,
+    EvaluatorSettings,
+    SingleEvaluationResult,
+)
+import markdown
+from pydantic import Field
+from jsonschema import validate, ValidationError
+import sqlglot
+
+
+class ValidFormatSettings(EvaluatorSettings):
+    format: Literal["json", "markdown", "python", "sql"] = "json"
+    json_schema: Optional[Dict[str, Any]] = Field(
+        default=None,
+        description="JSON schema to validate against when format is 'json'",
+    )
+
+
+class ValidFormatResult(EvaluationResult):
+    passed: Optional[bool] = Field(
+        default=True,
+        description="True if the output is formatted correctly, False otherwise",
+    )
+
+
+class ValidFormatEntry(EvaluatorEntry):
+    output: Optional[str] = None
+
+
+class ValidFormatEvaluator(
+    BaseEvaluator[ValidFormatEntry, ValidFormatSettings, ValidFormatResult]
+):
+    """
+    Allows you to check if the output is a valid json, markdown, python, sql, etc.
+    For JSON, can optionally validate against a provided schema.
+    """
+
+    name = "Valid Format Evaluator"
+    category = "quality"
+    default_settings = ValidFormatSettings()
+    is_guardrail = True
+
+    def evaluate(self, entry: ValidFormatEntry) -> SingleEvaluationResult:
+        if not entry.output:
+            return EvaluationResultSkipped(details="Output is empty")
+
+        if self.settings.format == "json":
+            try:
+                parsed_json = json.loads(entry.output)
+                if self.settings.json_schema:
+                    try:
+                        validate(instance=parsed_json, schema=self.settings.json_schema)
+                    except ValidationError as e:
+                        return ValidFormatResult(
+                            passed=False, details=f"JSON Schema validation failed: {e}"
+                        )
+            except json.JSONDecodeError as e:
+                return ValidFormatResult(passed=False, details=f"Invalid JSON: {e}")
+        elif self.settings.format == "markdown":
+            try:
+                html_result = markdown.markdown(entry.output)
+                # Check if the HTML output is different from plain text
+                # If they're the same, no markdown elements were processed
+                plain_text = entry.output.replace("\n", "").strip()
+                html_without_p = (
+                    html_result.replace("<p>", "")
+                    .replace("</p>", "")
+                    .replace("\n", "")
+                    .strip()
+                )
+                if plain_text == html_without_p:
+                    return ValidFormatResult(
+                        passed=False,
+                        details="No markdown elements found. Text should contain markdown formatting like headers (#), bold (**), lists, etc.",
+                    )
+            except Exception as e:
+                return ValidFormatResult(passed=False, details=f"Invalid Markdown: {e}")
+        elif self.settings.format == "python":
+            try:
+                ast.parse(entry.output)
+            except Exception as e:
+                return ValidFormatResult(passed=False, details=f"Invalid Python: {e}")
+        elif self.settings.format == "sql":
+            try:
+                sqlglot.parse(entry.output)
+            except Exception as e:
+                return ValidFormatResult(passed=False, details=f"Invalid SQL: {e}")
+
+        return ValidFormatResult(passed=True)

evaluators/langevals/poetry.lock

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "langevals-langevals"
 version = "0.1.7"
-description = "LangEvals home-made evaluators."
+description = "LangEvals core evaluators."
 authors = ["Rogerio Chaves <rogerio@langwatch.ai>", "Yevhenii Budnyk <y.budnyk789@gmail.com>"]
 license = "MIT"
 
@@ -12,6 +12,9 @@ langevals-core = { path = "../../langevals_core", develop = true }
 openai = ">=1.27.0"
 numpy = "^1.26.4"
 dspy-ai = "^2.4.9"
+markdown = "^3.7"
+jsonschema = "^4.23.0"
+sqlglot = {extras = ["rs"], version = "^25.32.0"}
 
 [tool.poetry.group.test.dependencies]
 pytest = "^7.4.2"