Extended tracing to all tool calls (#38)

## Pull Request Overview

This PR extends tracing capabilities to all tool calls by introducing a centralized LLM tracing wrapper that ensures consistent observability across the AI chat system.

- Introduces `LLMTracingWrapper.ts` to wrap LLM calls with tracing metadata
- Updates all tool implementations to use the new `callLLMWithTracing` function
- Refactors direct `LLMClient.call()` usage to include proper tracing context

Author: olesho

front_end/panels/ai_chat/BUILD.gn

@@ -50,6 +50,7 @@ devtools_module("ai_chat") {
     "LLM/OpenRouterProvider.ts",
     "LLM/LLMClient.ts",
     "tools/Tools.ts",
+    "tools/LLMTracingWrapper.ts",
     "tools/CritiqueTool.ts",
     "tools/FetcherTool.ts",
     "tools/FinalizeWithCritiqueTool.ts",

front_end/panels/ai_chat/agent_framework/AgentRunner.ts

@@ -12,6 +12,7 @@ import { createLogger } from '../core/Logger.js';
 import { createTracingProvider, getCurrentTracingContext } from '../tracing/TracingConfig.js';
 import type { AgentSession, AgentMessage } from './AgentSessionTypes.js';
 import { AgentErrorHandler } from '../core/AgentErrorHandler.js';
+import { callLLMWithTracing } from '../tools/LLMTracingWrapper.js';
 
 const logger = createLogger('AgentRunner');
 
@@ -1142,18 +1143,29 @@ Format your response as a clear, informative summary that would help a calling a
         content: summaryPrompt
       });
 
-      // Use existing LLM infrastructure
-      const llm = LLMClient.getInstance();
       const provider = AIChatPanel.getProviderForModel(modelName);
 
-      const response = await llm.call({
-        provider,
-        model: modelName,
-        messages: llmMessages,
-        systemPrompt: '', // Empty string instead of undefined
-        // Omit tools parameter entirely to avoid tool_choice conflicts
-        temperature: 0.1 // Lower temperature for more consistent summaries
-      });
+      const response = await callLLMWithTracing(
+        {
+          provider,
+          model: modelName,
+          messages: llmMessages,
+          systemPrompt: '', // Empty string instead of undefined
+          temperature: 0.1,
+          // Omit tools parameter entirely to avoid tool_choice conflicts
+        },
+        {
+          toolName: 'AgentRunner',
+          operationName: 'summarize_agent_progress',
+          context: 'agent_analysis',
+          additionalMetadata: {
+            agentName,
+            completionType,
+            maxIterations,
+            messageCount: messages.length
+          }
+        }
+      );
 
       logger.info(`Generated summary for agent "${agentName}":`, response.text || 'No summary generated.');
       return response.text || 'No summary generated.';

front_end/panels/ai_chat/tools/CritiqueTool.ts

@@ -4,8 +4,8 @@
 
 import { AgentService } from '../core/AgentService.js';
 import { createLogger } from '../core/Logger.js';
-import { LLMClient } from '../LLM/LLMClient.js';
 import { AIChatPanel } from '../ui/AIChatPanel.js';
+import { callLLMWithTracing } from './LLMTracingWrapper.js';
 
 import type { Tool } from './Tools.js';
 
@@ -167,17 +167,26 @@ Return a JSON array of requirement statements. Example format:
 
     try {
       const { model, provider } = AIChatPanel.getNanoModelWithProvider();
-      const llm = LLMClient.getInstance();
 
-      const response = await llm.call({
-        provider,
-        model,
-        messages: [
-          { role: 'user', content: userPrompt }
-        ],
-        systemPrompt,
-        temperature: 0.1,
-      });
+      const response = await callLLMWithTracing(
+        {
+          provider,
+          model,
+          messages: [
+            { role: 'user', content: userPrompt }
+          ],
+          systemPrompt,
+          temperature: 0.1,
+        },
+        {
+          toolName: this.name,
+          operationName: 'extract_requirements',
+          context: 'requirement_analysis',
+          additionalMetadata: {
+            inputLength: userInput.length
+          }
+        }
+      );
 
       if (!response.text) {
         return { success: false, requirements: [], error: 'No response received' };
@@ -260,17 +269,27 @@ ${JSON.stringify(evaluationSchema, null, 2)}`;
 
     try {
       const { model, provider } = AIChatPanel.getNanoModelWithProvider();
-      const llm = LLMClient.getInstance();
 
-      const response = await llm.call({
-        provider,
-        model,
-        messages: [
-          { role: 'user', content: userPrompt }
-        ],
-        systemPrompt,
-        temperature: 0.1,
-      });
+      const response = await callLLMWithTracing(
+        {
+          provider,
+          model,
+          messages: [
+            { role: 'user', content: userPrompt }
+          ],
+          systemPrompt,
+          temperature: 0.1,
+        },
+        {
+          toolName: this.name,
+          operationName: 'evaluate_response',
+          context: 'plan_evaluation',
+          additionalMetadata: {
+            requirementCount: requirements.length,
+            responseLength: finalResponse.length
+          }
+        }
+      );
 
       if (!response.text) {
         return { success: false, error: 'No response received' };
@@ -320,17 +339,27 @@ Be concise, specific, and constructive.`;
 
     try {
       const { model, provider } = AIChatPanel.getNanoModelWithProvider();
-      const llm = LLMClient.getInstance();
 
-      const response = await llm.call({
-        provider,
-        model,
-        messages: [
-          { role: 'user', content: userPrompt }
-        ],
-        systemPrompt,
-        temperature: 0.7,
-      });
+      const response = await callLLMWithTracing(
+        {
+          provider,
+          model,
+          messages: [
+            { role: 'user', content: userPrompt }
+          ],
+          systemPrompt,
+          temperature: 0.7,
+        },
+        {
+          toolName: this.name,
+          operationName: 'generate_feedback',
+          context: 'feedback_generation',
+          additionalMetadata: {
+            satisfiesCriteria: criteria.satisfiesCriteria,
+            missedRequirements: criteria.missedRequirements?.length || 0
+          }
+        }
+      );
 
       return response.text || 'The plan does not meet all requirements, but no specific feedback could be generated.';
     } catch (error: any) {

front_end/panels/ai_chat/tools/FullPageAccessibilityTreeToMarkdownTool.ts

@@ -3,8 +3,8 @@
 // found in the LICENSE file.
 
 import { AgentService } from '../core/AgentService.js';
-import { LLMClient } from '../LLM/LLMClient.js';
 import { AIChatPanel } from '../ui/AIChatPanel.js';
+import { callLLMWithTracing } from './LLMTracingWrapper.js';
 
 import { GetAccessibilityTreeTool, type Tool, type ErrorResult } from './Tools.js';
 
@@ -55,17 +55,27 @@ export class FullPageAccessibilityTreeToMarkdownTool implements Tool<Record<stri
     const prompt = `Accessibility Tree:\n\n\`\`\`\n${accessibilityTreeString}\n\`\`\``;
 
     try {
-      const llm = LLMClient.getInstance();
-      const llmResponse = await llm.call({
-        provider,
-        model,
-        messages: [
-          { role: 'system', content: this.getSystemPrompt() },
-          { role: 'user', content: prompt }
-        ],
-        systemPrompt: this.getSystemPrompt(),
-        temperature: 0.7
-      });
+      const llmResponse = await callLLMWithTracing(
+        {
+          provider,
+          model,
+          messages: [
+            { role: 'system', content: this.getSystemPrompt() },
+            { role: 'user', content: prompt }
+          ],
+          systemPrompt: this.getSystemPrompt(),
+          temperature: 0.7
+        },
+        {
+          toolName: this.name,
+          operationName: 'accessibility_to_markdown',
+          context: 'accessibility_tree_conversion',
+          additionalMetadata: {
+            treeLength: accessibilityTreeString.length
+          }
+        }
+      );
+
       const response = llmResponse.text;
       if (response) {
         return {

front_end/panels/ai_chat/tools/HTMLToMarkdownTool.ts

@@ -7,8 +7,8 @@ import * as Protocol from '../../../generated/protocol.js';
 import * as Utils from '../common/utils.js';
 import { AgentService } from '../core/AgentService.js';
 import { createLogger } from '../core/Logger.js';
-import { LLMClient } from '../LLM/LLMClient.js';
 import { AIChatPanel } from '../ui/AIChatPanel.js';
+import { callLLMWithTracing } from './LLMTracingWrapper.js';
 
 import { waitForPageLoad, type Tool } from './Tools.js';
 
@@ -319,19 +319,28 @@ ${instruction}
   }): Promise<{
     markdownContent: string,
   }> {
-    // Call LLM using the unified client
+    // Call LLM using the unified client with tracing
     const { model, provider } = AIChatPanel.getNanoModelWithProvider();
-    const llm = LLMClient.getInstance();
-    const llmResponse = await llm.call({
-      provider,
-      model,
-      messages: [
-        { role: 'system', content: params.systemPrompt },
-        { role: 'user', content: params.userPrompt }
-      ],
-      systemPrompt: params.systemPrompt,
-      temperature: 0.2 // Lower temperature for more deterministic results
-    });
+    const llmResponse = await callLLMWithTracing(
+      {
+        provider,
+        model,
+        messages: [
+          { role: 'system', content: params.systemPrompt },
+          { role: 'user', content: params.userPrompt }
+        ],
+        systemPrompt: params.systemPrompt,
+        temperature: 0.2 // Lower temperature for more deterministic results
+      },
+      {
+        toolName: this.name,
+        operationName: 'html_to_markdown',
+        context: 'content_extraction',
+        additionalMetadata: {
+          promptLength: params.userPrompt.length
+        }
+      }
+    );
     const response = llmResponse.text;
 
     // Process the response - UnifiedLLMClient returns string directly