more refactor

Author: tysonthomas9

front_end/panels/ai_chat/core/AgentService.ts

@@ -18,7 +18,6 @@ import type { TracingProvider, TracingContext } from '../tracing/TracingProvider
 import { AgentRunnerEventBus } from '../agent_framework/AgentRunnerEventBus.js';
 import { AgentRunner } from '../agent_framework/AgentRunner.js';
 import type { AgentSession, AgentMessage } from '../agent_framework/AgentSessionTypes.js';
-import { AIChatPanel } from '../ui/AIChatPanel.js';
 import type { LLMProvider } from '../LLM/LLMTypes.js';
 
 const logger = createLogger('AgentService');
@@ -166,13 +165,9 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
   /**
    * Initializes the agent with the given API key
    */
-  async initialize(apiKey: string | null, modelName?: string): Promise<void> {
+  async initialize(apiKey: string | null, modelName: string, miniModel: string, nanoModel: string): Promise<void> {
     try {
       this.#apiKey = apiKey;
-
-      if (!modelName) {
-        throw new Error('Model name is required for initialization');
-      }
 
       // Initialize LLM client first
       await this.#initializeLLMClient();
@@ -197,9 +192,7 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
       // Determine selected provider for primary graph execution
       const selectedProvider = (localStorage.getItem('ai_chat_provider') || 'openai') as LLMProvider;
 
-      // Get mini and nano models for tool execution
-      const miniModel = AIChatPanel.getMiniModel();
-      const nanoModel = AIChatPanel.getNanoModel();
+      // Mini and nano models are injected by caller (validated upstream)
 
       // Will throw error if model/provider configuration is invalid
       this.#graph = createAgentGraph(apiKey, modelName, selectedProvider, miniModel, nanoModel);

front_end/panels/ai_chat/evaluation/framework/judges/LLMEvaluator.ts

@@ -9,7 +9,6 @@ import { ErrorHandlingUtils } from '../../utils/ErrorHandlingUtils.js';
 import { PromptTemplates } from '../../utils/PromptTemplates.js';
 import { ResponseParsingUtils } from '../../utils/ResponseParsingUtils.js';
 import type { ScreenshotData, VisionMessage, TextContent, ImageContent } from '../../utils/EvaluationTypes.js';
-import { AIChatPanel } from '../../../ui/AIChatPanel.js';
 
 const logger = createLogger('LLMEvaluator');
 
@@ -21,11 +20,10 @@ export class LLMEvaluator {
   private apiKey: string;
   private defaultModel: string;
 
-  constructor(apiKey: string, defaultModel?: string) {
+  constructor(apiKey: string, defaultModel: string) {
     this.apiKey = apiKey;
-    // Use the provided model, or fall back to saved judge model, or finally to mini model
-    const JUDGE_MODEL_STORAGE_KEY = 'ai_chat_judge_model';
-    this.defaultModel = defaultModel || localStorage.getItem(JUDGE_MODEL_STORAGE_KEY) || AIChatPanel.getMiniModel();
+    // Model must be provided by caller; no fallbacks
+    this.defaultModel = defaultModel;
   }
 
 
@@ -89,8 +87,9 @@ export class LLMEvaluator {
     for (let attempt = 1; attempt <= maxRetries; attempt++) {
       try {
         const llm = LLMClient.getInstance();
+        const provider = await this.#getProviderForModel(model);
         const llmResponse = await llm.call({
-          provider: AIChatPanel.getProviderForModel(model),
+          provider,
           model: model,
           messages: [
             { role: 'system', content: PromptTemplates.buildSystemPrompt({ hasVision: false }) },
@@ -124,6 +123,25 @@ export class LLMEvaluator {
     throw lastError;
   }
 
+  /**
+   * Resolve the provider for a given model using the LLM registry.
+   * Falls back to the currently selected provider if unknown.
+   */
+  async #getProviderForModel(modelId: string): Promise<'openai' | 'litellm' | 'groq' | 'openrouter'> {
+    try {
+      const llm = LLMClient.getInstance();
+      const models = await llm.getAvailableModels();
+      const found = models.find(m => m.id === modelId);
+      if (found) {
+        return found.provider as any;
+      }
+    } catch {
+      // ignore and fall through to fallback
+    }
+    // Fallback: current selection or openai
+    return (localStorage.getItem('ai_chat_provider') || 'openai') as any;
+  }
+
   /**
    * Vision-enhanced evaluation using multimodal LLM
    */
@@ -342,4 +360,4 @@ export class LLMEvaluator {
 
     return evaluations;
   }
-}
+}

front_end/panels/ai_chat/evaluation/remote/EvaluationAgent.ts

@@ -10,7 +10,6 @@ import { createLogger } from '../../core/Logger.js';
 import { createTracingProvider, withTracingContext, isTracingEnabled, getTracingConfig } from '../../tracing/TracingConfig.js';
 import type { TracingProvider, TracingContext } from '../../tracing/TracingProvider.js';
 import type { ChatMessage } from '../../models/ChatTypes.js';
-import { AIChatPanel } from '../../ui/AIChatPanel.js';
 import {
   RegisterMessage,
   ReadyMessage,
@@ -40,6 +39,10 @@ export interface EvaluationAgentOptions {
   clientId: string;
   endpoint: string;
   secretKey?: string;
+  // Explicit models to avoid UI coupling
+  judgeModel: string;
+  miniModel: string;
+  nanoModel: string;
 }
 
 
@@ -56,11 +59,17 @@ export class EvaluationAgent {
   private authResolve: ((value?: void) => void) | null = null;
   private authReject: ((reason?: any) => void) | null = null;
   private tracingProvider: TracingProvider;
+  private judgeModel: string;
+  private miniModel: string;
+  private nanoModel: string;
 
   constructor(options: EvaluationAgentOptions) {
     this.clientId = options.clientId;
     this.endpoint = options.endpoint;
     this.secretKey = options.secretKey;
+    this.judgeModel = options.judgeModel;
+    this.miniModel = options.miniModel;
+    this.nanoModel = options.nanoModel;
     this.tracingProvider = createTracingProvider();
 
     logger.info('EvaluationAgent created with tracing provider', {
@@ -701,21 +710,19 @@ export class EvaluationAgent {
         // Get or create AgentService instance
         const agentService = AgentService.getInstance();
 
-        // Use the current model from localStorage (no override)
-        let modelName = localStorage.getItem('ai_chat_model_selection');
-        if (!modelName) {
-          // Default model
-          modelName = 'gpt-4o';
-        }
+        // Use explicit models from constructor
+        const modelName = this.judgeModel;
+        const miniModel = this.miniModel;
+        const nanoModel = this.nanoModel;
 
         logger.info('Initializing AgentService for chat evaluation', {
           modelName,
           hasApiKey: !!agentService.getApiKey(),
           isInitialized: agentService.isInitialized()
         });
 
-        // Always reinitialize with the current model
-        await agentService.initialize(agentService.getApiKey(), modelName);
+        // Always reinitialize with the current model and explicit mini/nano
+        await agentService.initialize(agentService.getApiKey(), modelName, miniModel, nanoModel);
 
         // Create a child observation for the chat execution
         if (tracingContext) {

front_end/panels/ai_chat/evaluation/runner/EvaluationRunner.ts

@@ -13,21 +13,27 @@ import type { LLMProviderConfig } from '../../LLM/LLMClient.js';
 import { TIMING_CONSTANTS } from '../../core/Constants.js';
 import { createTracingProvider, isTracingEnabled, getTracingConfig } from '../../tracing/TracingConfig.js';
 import type { TracingProvider, TracingContext } from '../../tracing/TracingProvider.js';
-import { AIChatPanel } from '../../ui/AIChatPanel.js';
 
 const logger = createLogger('EvaluationRunner');
 
 /**
  * Example runner for the evaluation framework
  */
+export interface EvaluationRunnerOptions {
+  judgeModel: string;
+  mainModel: string;
+  miniModel: string;
+  nanoModel: string;
+}
+
 export class EvaluationRunner {
   private evaluator: GenericToolEvaluator;
   private llmEvaluator: LLMEvaluator;
   private config: EvaluationConfig;
   private tracingProvider: TracingProvider;
   private sessionId: string;
 
-  constructor(judgeModel?: string) {
+  constructor(options: EvaluationRunnerOptions) {
     // Get API key from AgentService
     const agentService = AgentService.getInstance();
     const apiKey = agentService.getApiKey();
@@ -36,14 +42,9 @@ export class EvaluationRunner {
       throw new Error('API key not configured. Please configure in AI Chat settings.');
     }
 
-    // Use provided judge model or default
-    const evaluationModel = judgeModel || 'gpt-4.1-mini';
-
-    // Get the actual models configured in the UI for tools and agents
-    // TODO: Use a more robust method to get these settings
-    const mainModel = AIChatPanel.instance().getSelectedModel();
-    const miniModel = AIChatPanel.getMiniModel();
-    const nanoModel = AIChatPanel.getNanoModel();
+    // Require explicit models from caller
+    const { judgeModel, mainModel, miniModel, nanoModel } = options;
+    const evaluationModel = judgeModel;
 
     this.config = {
       extractionModel: evaluationModel,

front_end/panels/ai_chat/evaluation/runner/VisionAgentEvaluationRunner.ts

@@ -13,7 +13,6 @@ import { createLogger } from '../../core/Logger.js';
 import { TIMING_CONSTANTS } from '../../core/Constants.js';
 import { createTracingProvider, isTracingEnabled } from '../../tracing/TracingConfig.js';
 import type { TracingProvider, TracingContext } from '../../tracing/TracingProvider.js';
-import { AIChatPanel } from '../../ui/AIChatPanel.js';
 
 const logger = createLogger('VisionAgentEvaluationRunner');
 
@@ -40,6 +39,14 @@ export interface VisionTestCase extends TestCase {
  * Unified agent evaluation runner that supports both standard and vision-based evaluation
  * This replaces AgentEvaluationRunner when vision capabilities are needed
  */
+export interface VisionRunnerOptions {
+  visionEnabled?: boolean;
+  judgeModel: string;
+  mainModel: string;
+  miniModel: string;
+  nanoModel: string;
+}
+
 export class VisionAgentEvaluationRunner {
 
   private llmEvaluator: LLMEvaluator;
@@ -48,7 +55,7 @@ export class VisionAgentEvaluationRunner {
   private globalVisionEnabled: boolean;
   private tracingProvider: TracingProvider;
 
-  constructor(visionEnabled: boolean = false, judgeModel?: string) {
+  constructor(options: VisionRunnerOptions) {
     // Get API key from AgentService
     const agentService = AgentService.getInstance();
     const apiKey = agentService.getApiKey();
@@ -57,13 +64,9 @@ export class VisionAgentEvaluationRunner {
       throw new Error('API key not configured. Please configure in AI Chat settings.');
     }
 
-    // Use provided judge model or default
-    const evaluationModel = judgeModel || 'gpt-4.1-mini';
-
-    // Get the actual models configured in the UI for tools and agents
-    const mainModel = AIChatPanel.instance().getSelectedModel();
-    const miniModel = AIChatPanel.getMiniModel();
-    const nanoModel = AIChatPanel.getNanoModel();
+    // Require explicit models from caller
+    const { judgeModel, mainModel, miniModel, nanoModel } = options;
+    const evaluationModel = judgeModel;
 
     this.config = {
       extractionModel: evaluationModel,
@@ -82,7 +85,7 @@ export class VisionAgentEvaluationRunner {
 
     this.llmEvaluator = new LLMEvaluator(this.config.evaluationApiKey, this.config.evaluationModel);
     this.screenshotTool = new TakeScreenshotTool();
-    this.globalVisionEnabled = visionEnabled;
+    this.globalVisionEnabled = Boolean(options.visionEnabled);
     this.tracingProvider = createTracingProvider();
   }
 
@@ -467,4 +470,4 @@ export class VisionAgentEvaluationRunner {
     this.globalVisionEnabled = enabled;
     logger.info(`Global vision mode: ${enabled ? 'ENABLED' : 'DISABLED'}`);
   }
-}
+}

front_end/panels/ai_chat/ui/AIChatPanel.ts

@@ -1282,7 +1282,10 @@ export class AIChatPanel extends UI.Panel.Panel {
         this.#evaluationAgent = new EvaluationAgent({
           clientId: compositeClientId,
           endpoint: config.endpoint,
-          secretKey: config.secretKey
+          secretKey: config.secretKey,
+          judgeModel: this.#selectedModel,
+          miniModel: this.#miniModel,
+          nanoModel: this.#nanoModel,
         });
 
         await this.#evaluationAgent.connect();
@@ -1351,7 +1354,14 @@ export class AIChatPanel extends UI.Panel.Panel {
 
     // Initialize the agent service
     logger.info('Calling agentService.initialize()...');
-    this.#agentService.initialize(apiKey, this.#selectedModel)
+    const miniForInit = this.#miniModel || this.#selectedModel;
+    const nanoForInit = this.#nanoModel || miniForInit;
+    this.#agentService.initialize(
+        apiKey,
+        this.#selectedModel,
+        miniForInit,
+        nanoForInit,
+      )
       .then(() => {
         logger.info('✅ Agent service initialized successfully');
         this.#setCanSendMessagesState(true, "Agent service initialized successfully");

front_end/panels/ai_chat/ui/EvaluationDialog.ts

@@ -199,13 +199,25 @@ export class EvaluationDialog {
 
     // Initialize evaluation runners
     try {
-      this.#evaluationRunner = new EvaluationRunner(this.#state.judgeModel);
+      // Inject current UI-selected models into the runner to decouple from UI internals
+      this.#evaluationRunner = new EvaluationRunner({
+        judgeModel: this.#state.judgeModel,
+        mainModel: AIChatPanel.instance().getSelectedModel(),
+        miniModel: AIChatPanel.getMiniModel(),
+        nanoModel: AIChatPanel.getNanoModel(),
+      });
     } catch (error) {
       logger.error('Failed to initialize evaluation runner:', error);
     }
 
     try {
-      this.#agentEvaluationRunner = new VisionAgentEvaluationRunner(this.#state.visionEnabled, this.#state.judgeModel);
+      this.#agentEvaluationRunner = new VisionAgentEvaluationRunner({
+        visionEnabled: this.#state.visionEnabled,
+        judgeModel: this.#state.judgeModel,
+        mainModel: AIChatPanel.instance().getSelectedModel(),
+        miniModel: AIChatPanel.getMiniModel(),
+        nanoModel: AIChatPanel.getNanoModel(),
+      });
     } catch (error) {
       logger.error('Failed to initialize agent evaluation runner:', error);
     }
@@ -885,13 +897,24 @@ export class EvaluationDialog {
 
       // Reinitialize evaluation runners with new model
       try {
-        this.#evaluationRunner = new EvaluationRunner(this.#state.judgeModel);
+        this.#evaluationRunner = new EvaluationRunner({
+          judgeModel: this.#state.judgeModel,
+          mainModel: AIChatPanel.instance().getSelectedModel(),
+          miniModel: AIChatPanel.getMiniModel(),
+          nanoModel: AIChatPanel.getNanoModel(),
+        });
       } catch (error) {
         logger.error('Failed to reinitialize evaluation runner:', error);
       }
 
       try {
-        this.#agentEvaluationRunner = new VisionAgentEvaluationRunner(this.#state.visionEnabled, this.#state.judgeModel);
+        this.#agentEvaluationRunner = new VisionAgentEvaluationRunner({
+          visionEnabled: this.#state.visionEnabled,
+          judgeModel: this.#state.judgeModel,
+          mainModel: AIChatPanel.instance().getSelectedModel(),
+          miniModel: AIChatPanel.getMiniModel(),
+          nanoModel: AIChatPanel.getNanoModel(),
+        });
       } catch (error) {
         logger.error('Failed to reinitialize agent evaluation runner:', error);
       }
@@ -1837,4 +1860,4 @@ export class EvaluationDialog {
     }
   }
 
-}
+}