check if element is selector through ax node and extract method

Author: AlinaVarkki

src/McpContext.ts

@@ -260,6 +260,10 @@ export class McpContext implements Context {
     return page.getDefaultNavigationTimeout();
   }
 
+  getAXNodeByUid(uid: string) {
+    return this.#textSnapshot?.idToNode.get(uid);
+  }
+
   async getElementByUid(uid: string): Promise<ElementHandle<Element>> {
     if (!this.#textSnapshot?.idToNode.size) {
       throw new Error(

src/tools/ToolDefinition.ts

@@ -7,6 +7,7 @@
 import type {Dialog, ElementHandle, Page} from 'puppeteer-core';
 import z from 'zod';
 
+import type {TextSnapshotNode} from '../McpContext.js';
 import type {TraceResult} from '../trace-processing/parse.js';
 
 import type {ToolCategories} from './categories.js';
@@ -68,6 +69,7 @@ export type Context = Readonly<{
   closePage(pageIdx: number): Promise<void>;
   setSelectedPageIdx(idx: number): void;
   getElementByUid(uid: string): Promise<ElementHandle<Element>>;
+  getAXNodeByUid(uid: string): TextSnapshotNode | undefined;
   setNetworkConditions(conditions: string | null): void;
   setCpuThrottlingRate(rate: number): void;
   saveTemporaryFile(

src/tools/input.ts

@@ -7,6 +7,8 @@
 import type {ElementHandle} from 'puppeteer-core';
 import z from 'zod';
 
+import type {McpContext} from '../McpContext.js';
+
 import {ToolCategories} from './categories.js';
 import {defineTool} from './ToolDefinition.js';
 
@@ -78,6 +80,40 @@ export const hover = defineTool({
   },
 });
 
+async function fillFormElement(
+  uid: string,
+  value: string,
+  context: McpContext,
+) {
+  const handle = await context.getElementByUid(uid);
+  try {
+    // The AXNode for an option doesn't contain its `value`. We set text content of the option as value.
+    // If the form is a combobox, we need to find the correct option by its text value.
+    // To do that, loop through the children while checking which child's text matches the requested value (requested value is actually the text content).
+    // When the correct option is found, use the element handle to get the real value.
+    const aXNode = context.getAXNodeByUid(uid);
+    if (aXNode && aXNode.role === 'combobox' && aXNode.children) {
+      for (const child of aXNode.children) {
+        if (child.role === 'option' && child.name === value && child.value) {
+          const childHandle = await child.elementHandle();
+          if (childHandle) {
+            const childValueHandle = await childHandle.getProperty('value');
+            const childValue = await childValueHandle.jsonValue();
+            if (childValue) {
+              await handle.asLocator().fill(childValue.toString());
+            }
+            break;
+          }
+        }
+      }
+    } else {
+      await handle.asLocator().fill(value);
+    }
+  } finally {
+    void handle.dispose();
+  }
+}
+
 export const fill = defineTool({
   name: 'fill',
   description: `Type text into a input, text area or select an option from a <select> element.`,
@@ -94,35 +130,15 @@ export const fill = defineTool({
     value: z.string().describe('The value to fill in'),
   },
   handler: async (request, response, context) => {
-    const handle = await context.getElementByUid(request.params.uid);
-    try {
-      await context.waitForEventsAfterAction(async () => {
-        // The AXNode for an option doesn't contain its `value`. We set text content of the option as value.
-        // To fill the form, get the correct option by its text value.
-        const isSelectElement = await handle.evaluate(
-          el => el.tagName === 'SELECT',
-        );
-        if (isSelectElement) {
-          const value = await handle.evaluate((el, text) => {
-            const option = Array.from((el as HTMLSelectElement).options).find(
-              option => option.text === text,
-            );
-            if (option) {
-              return option.value;
-            }
-            return request.params.value;
-          }, request.params.value);
-
-          await handle.asLocator().fill(value);
-        } else {
-          await handle.asLocator().fill(request.params.value);
-        }
-      });
-      response.appendResponseLine(`Successfully filled out the element`);
-      response.setIncludeSnapshot(true);
-    } finally {
-      void handle.dispose();
-    }
+    await context.waitForEventsAfterAction(async () => {
+      await fillFormElement(
+        request.params.uid,
+        request.params.value,
+        context as McpContext,
+      );
+    });
+    response.appendResponseLine(`Successfully filled out the element`);
+    response.setIncludeSnapshot(true);
   },
 });
 
@@ -174,14 +190,13 @@ export const fillForm = defineTool({
   },
   handler: async (request, response, context) => {
     for (const element of request.params.elements) {
-      const handle = await context.getElementByUid(element.uid);
-      try {
-        await context.waitForEventsAfterAction(async () => {
-          await handle.asLocator().fill(element.value);
-        });
-      } finally {
-        void handle.dispose();
-      }
+      await context.waitForEventsAfterAction(async () => {
+        await fillFormElement(
+          element.uid,
+          element.value,
+          context as McpContext,
+        );
+      });
     }
     response.appendResponseLine(`Successfully filled out the form`);
     response.setIncludeSnapshot(true);