.

Author: rguan72

backend/onyx/chat/answer_scratchpad.py

@@ -76,7 +76,9 @@ def web_search(query: str, outer_ctx: Dict[str, Any]) -> Dict[str, Any]:
 register_tool(
     ToolSpec(
         name="web_search",
-        description="Search the web for information.",
+        description="""
+        Search the web for information. This tool provides urls and short snippets,
+        but does not fetch the full content of the urls.""",
         parameters={
             "type": "object",
             "properties": {"query": {"type": "string"}},
@@ -110,7 +112,7 @@ def web_fetch(urls: List[str], outer_ctx: Dict[str, Any]) -> Dict[str, Any]:
 register_tool(
     ToolSpec(
         name="web_fetch",
-        description="Fetch the contents of a list of URLs.",
+        description="Fetch the fullcontents of a list of URLs.",
         parameters={
             "type": "object",
             "properties": {"urls": {"type": "array", "items": {"type": "string"}}},
@@ -134,24 +136,45 @@ def reasoning(outer_ctx: Dict[str, Any]) -> Dict[str, Any]:
     revised_messages = [
         {"role": "system", "content": PRIVATE_SCRATCHPAD_SYS},
     ] + messages[1:]
-    results = litellm.completion(
-        model=llm.config.model_name,
+    results = llm_completion(
+        model_name=llm.config.model_name,
         temperature=llm.config.temperature,
         messages=revised_messages,
+        tools=[],
+        stream=False,
     )
     return {"results": results["choices"][0]["message"]["content"]}
 
 
 register_tool(
     ToolSpec(
         name="reasoning",
-        description="Reason about the message history and the goal.",
+        description="""
+        Use this tool for reasoning. Powerful for complex questions and
+        tasks, or questions that require multiple steps to answer.""",
         parameters={"type": "object", "properties": {}, "required": []},
         func=reasoning,
     )
 )
 
 
+@traced(name="llm_completion", type="llm")
+def llm_completion(
+    model_name: str,
+    temperature: float,
+    messages: List[Dict[str, Any]],
+    tools: List[Dict[str, Any]],
+    stream: bool = False,
+) -> Dict[str, Any]:
+    return litellm.completion(
+        model=model_name,
+        temperature=temperature,
+        messages=messages,
+        tools=tools,
+        stream=stream,
+    )
+
+
 def tool_specs_for_openai() -> List[Dict[str, Any]]:
     return [
         {
@@ -197,12 +220,12 @@ def stream_chat_sync(
             yield {"type": "delta", "text": "\n[Timed out while composing reply]"}
             break
         # Start a streaming completion (sync iterator of deltas)
-        stream_iter = litellm.completion(
-            model=llm.config.model_name,
+        stream_iter = llm_completion(
+            model_name=llm.config.model_name,
             temperature=llm.config.temperature,
             messages=messages,
             tools=tools_decl,
-            stream=True,  # iterator of chunks
+            stream=True,
         )
 
         # Accumulate assistant text & tool call chunks

backend/onyx/evals/one_off/create_braintrust_dataset.py

@@ -109,8 +109,7 @@ def parse_csv_file(csv_path: str) -> List[Dict[str, Any]]:
                     records.extend(
                         [
                             {
-                                "question": question
-                                + ". All info is contained in the quesiton. DO NOT ask any clarifying questions.",
+                                "question": question,
                                 "research_type": "DEEP",
                                 "categories": categories,
                                 "expected_depth": expected_depth,

backend/onyx/evals/providers/braintrust.py

@@ -1,6 +1,6 @@
 from collections.abc import Callable
 
-from autoevals import Factuality
+from autoevals.llm import LLMClassifier
 from braintrust import Eval
 from braintrust import EvalCase
 from braintrust import init_dataset
@@ -12,6 +12,33 @@
 from onyx.evals.models import EvalProvider
 
 
+quality_classifier = LLMClassifier(
+    name="quality",
+    prompt_template="""
+    You are a customer doing a trial of the product Onyx. Onyx provides a UI for users to chat with an LLM
+     and search for information, similar to ChatGPT. You think ChatGPT's answer quality is great, and
+     you want to rate Onyx's response relativeto ChatGPT's response.\n
+    [Question]: {{input}}\n
+    [ChatGPT Answer]: {{expected}}\n
+    [Onyx Answer]: {{output}}\n
+
+    Please rate the quality of the Onyx answer relative to the ChatGPT answer on a scale of A to E:
+    A: The Onyx answer is great and is as good or better than the ChatGPT answer.
+    B: The Onyx answer is good and and comparable to the ChatGPT answer.
+    C: The Onyx answer is fair.
+    D: The Onyx answer is poor and is worse than the ChatGPT answer.
+    E: The Onyx answer is terrible and is much worse than the ChatGPT answer.
+    """,
+    choice_scores={
+        "A": 1,
+        "B": 0.75,
+        "C": 0.5,
+        "D": 0.25,
+        "E": 0,
+    },
+)
+
+
 class BraintrustEvalProvider(EvalProvider):
     def eval(
         self,
@@ -34,7 +61,7 @@ def eval(
                 name=BRAINTRUST_PROJECT,
                 data=eval_data,
                 task=task,
-                scores=[Factuality()],
+                scores=[quality_classifier],
                 metadata={**configuration.model_dump()},
                 max_concurrency=BRAINTRUST_MAX_CONCURRENCY,
                 no_send_logs=no_send_logs,
@@ -51,7 +78,7 @@ def eval(
                 name=BRAINTRUST_PROJECT,
                 data=eval_cases,
                 task=task,
-                scores=[Factuality()],
+                scores=[quality_classifier],
                 metadata={**configuration.model_dump()},
                 max_concurrency=BRAINTRUST_MAX_CONCURRENCY,
                 no_send_logs=no_send_logs,