fix: add an option value to the snapshot (#362)

Fixes #185 

The standard selector value is not included in the AXNode, so add it
separately.

Author: AlinaVarkki

src/McpContext.ts

@@ -272,6 +272,10 @@ export class McpContext implements Context {
     return page.getDefaultNavigationTimeout();
   }
 
+  getAXNodeByUid(uid: string) {
+    return this.#textSnapshot?.idToNode.get(uid);
+  }
+
   async getElementByUid(uid: string): Promise<ElementHandle<Element>> {
     if (!this.#textSnapshot?.idToNode.size) {
       throw new Error(
@@ -334,6 +338,16 @@ export class McpContext implements Context {
           ? node.children.map(child => assignIds(child))
           : [],
       };
+
+      // The AXNode for an option doesn't contain its `value`.
+      // Therefore, set text content of the option as value.
+      if (node.role === 'option') {
+        const optionText = node.name;
+        if (optionText) {
+          nodeWithId.value = optionText.toString();
+        }
+      }
+
       idToNode.set(nodeWithId.id, nodeWithId);
       return nodeWithId;
     };

src/tools/ToolDefinition.ts

@@ -7,6 +7,7 @@
 import type {Dialog, ElementHandle, Page} from 'puppeteer-core';
 import z from 'zod';
 
+import type {TextSnapshotNode} from '../McpContext.js';
 import type {TraceResult} from '../trace-processing/parse.js';
 
 import type {ToolCategories} from './categories.js';
@@ -68,6 +69,7 @@ export type Context = Readonly<{
   closePage(pageIdx: number): Promise<void>;
   setSelectedPageIdx(idx: number): void;
   getElementByUid(uid: string): Promise<ElementHandle<Element>>;
+  getAXNodeByUid(uid: string): TextSnapshotNode | undefined;
   setNetworkConditions(conditions: string | null): void;
   setCpuThrottlingRate(rate: number): void;
   saveTemporaryFile(

src/tools/input.ts

@@ -7,6 +7,8 @@
 import type {ElementHandle} from 'puppeteer-core';
 import z from 'zod';
 
+import type {McpContext, TextSnapshotNode} from '../McpContext.js';
+
 import {ToolCategories} from './categories.js';
 import {defineTool} from './ToolDefinition.js';
 
@@ -78,6 +80,61 @@ export const hover = defineTool({
   },
 });
 
+// The AXNode for an option doesn't contain its `value`. We set text content of the option as value.
+// If the form is a combobox, we need to find the correct option by its text value.
+// To do that, loop through the children while checking which child's text matches the requested value (requested value is actually the text content).
+// When the correct option is found, use the element handle to get the real value.
+async function selectOption(
+  handle: ElementHandle,
+  aXNode: TextSnapshotNode,
+  value: string,
+) {
+  let optionFound = false;
+  for (const child of aXNode.children) {
+    if (child.role === 'option' && child.name === value && child.value) {
+      optionFound = true;
+      const childHandle = await child.elementHandle();
+      if (childHandle) {
+        try {
+          const childValueHandle = await childHandle.getProperty('value');
+          try {
+            const childValue = await childValueHandle.jsonValue();
+            if (childValue) {
+              await handle.asLocator().fill(childValue.toString());
+            }
+          } finally {
+            void childValueHandle.dispose();
+          }
+          break;
+        } finally {
+          void childHandle.dispose();
+        }
+      }
+    }
+  }
+  if (!optionFound) {
+    throw new Error(`Could not find option with text "${value}"`);
+  }
+}
+
+async function fillFormElement(
+  uid: string,
+  value: string,
+  context: McpContext,
+) {
+  const handle = await context.getElementByUid(uid);
+  try {
+    const aXNode = context.getAXNodeByUid(uid);
+    if (aXNode && aXNode.role === 'combobox') {
+      await selectOption(handle, aXNode, value);
+    } else {
+      await handle.asLocator().fill(value);
+    }
+  } finally {
+    void handle.dispose();
+  }
+}
+
 export const fill = defineTool({
   name: 'fill',
   description: `Type text into a input, text area or select an option from a <select> element.`,
@@ -94,16 +151,15 @@ export const fill = defineTool({
     value: z.string().describe('The value to fill in'),
   },
   handler: async (request, response, context) => {
-    const handle = await context.getElementByUid(request.params.uid);
-    try {
-      await context.waitForEventsAfterAction(async () => {
-        await handle.asLocator().fill(request.params.value);
-      });
-      response.appendResponseLine(`Successfully filled out the element`);
-      response.setIncludeSnapshot(true);
-    } finally {
-      void handle.dispose();
-    }
+    await context.waitForEventsAfterAction(async () => {
+      await fillFormElement(
+        request.params.uid,
+        request.params.value,
+        context as McpContext,
+      );
+    });
+    response.appendResponseLine(`Successfully filled out the element`);
+    response.setIncludeSnapshot(true);
   },
 });
 
@@ -155,14 +211,13 @@ export const fillForm = defineTool({
   },
   handler: async (request, response, context) => {
     for (const element of request.params.elements) {
-      const handle = await context.getElementByUid(element.uid);
-      try {
-        await context.waitForEventsAfterAction(async () => {
-          await handle.asLocator().fill(element.value);
-        });
-      } finally {
-        void handle.dispose();
-      }
+      await context.waitForEventsAfterAction(async () => {
+        await fillFormElement(
+          element.uid,
+          element.value,
+          context as McpContext,
+        );
+      });
     }
     response.appendResponseLine(`Successfully filled out the form`);
     response.setIncludeSnapshot(true);

tests/tools/input.test.ts

@@ -202,6 +202,35 @@ describe('input', () => {
         assert.ok(await page.$('text/test'));
       });
     });
+
+    it('fills out a select by text', async () => {
+      await withBrowser(async (response, context) => {
+        const page = context.getSelectedPage();
+        await page.setContent(
+          `<!DOCTYPE html><select><option value="v1">one</option><option value="v2">two</option></select>`,
+        );
+        await context.createTextSnapshot();
+        await fill.handler(
+          {
+            params: {
+              uid: '1_1',
+              value: 'two',
+            },
+          },
+          response,
+          context,
+        );
+        assert.strictEqual(
+          response.responseLines[0],
+          'Successfully filled out the element',
+        );
+        assert.ok(response.includeSnapshot);
+        const selectedValue = await page.evaluate(
+          () => document.querySelector('select')!.value,
+        );
+        assert.strictEqual(selectedValue, 'v2');
+      });
+    });
   });
 
   describe('drags', () => {