From a728c04466fedaf3bd0e794777a6600daff1c07d Mon Sep 17 00:00:00 2001 From: alinavarkki Date: Tue, 30 Sep 2025 07:51:44 +0200 Subject: [PATCH 01/44] format --- docs/tool-reference.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/tool-reference.md b/docs/tool-reference.md index ad9410a2..0ba03b8e 100644 --- a/docs/tool-reference.md +++ b/docs/tool-reference.md @@ -285,15 +285,16 @@ so returned values have to JSON-serializable. - **args** (array) _(optional)_: An optional list of arguments to pass to the function. - **function** (string) **(required)**: A JavaScript function to run in the currently selected page. - Example without arguments: `() => { +Example without arguments: `() => { return document.title }` or `async () => { return await fetch("example.com") }`. - Example with arguments: `(el) => { +Example with arguments: `(el) => { return el.innerText; }` + --- ### `list_console_messages` From ee14198f27047eac011d1df487b9815bc8546c8a Mon Sep 17 00:00:00 2001 From: alinavarkki Date: Tue, 30 Sep 2025 08:52:08 +0200 Subject: [PATCH 02/44] format --- docs/tool-reference.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/tool-reference.md b/docs/tool-reference.md index 0ba03b8e..ad9410a2 100644 --- a/docs/tool-reference.md +++ b/docs/tool-reference.md @@ -285,16 +285,15 @@ so returned values have to JSON-serializable. - **args** (array) _(optional)_: An optional list of arguments to pass to the function. - **function** (string) **(required)**: A JavaScript function to run in the currently selected page. -Example without arguments: `() => { + Example without arguments: `() => { return document.title }` or `async () => { return await fetch("example.com") }`. -Example with arguments: `(el) => { + Example with arguments: `(el) => { return el.innerText; }` - --- ### `list_console_messages` From 56ab3217b12b9309edfe22a4a9d1472c76a39905 Mon Sep 17 00:00:00 2001 From: alinavarkki Date: Wed, 1 Oct 2025 11:56:17 +0200 Subject: [PATCH 03/44] option handle seperately --- src/McpContext.ts | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/McpContext.ts b/src/McpContext.ts index d1037935..803b4887 100644 --- a/src/McpContext.ts +++ b/src/McpContext.ts @@ -30,6 +30,7 @@ import {WaitForHelper} from './WaitForHelper.js'; export interface TextSnapshotNode extends SerializedAXNode { id: string; children: TextSnapshotNode[]; + value?: string | number; } export interface TextSnapshot { @@ -326,19 +327,36 @@ export class McpContext implements Context { // will be used for the tree serialization and mapping ids back to nodes. let idCounter = 0; const idToNode = new Map(); - const assignIds = (node: SerializedAXNode): TextSnapshotNode => { + const assignIds = async ( + node: SerializedAXNode, + ): Promise => { const nodeWithId: TextSnapshotNode = { ...node, id: `${snapshotId}_${idCounter++}`, - children: node.children - ? node.children.map(child => assignIds(child)) - : [], + children: [], }; + + // The AXNode for an option doesn't contain its value, so add it from the element. + if (node.role === 'option') { + const handle = await node.elementHandle(); + if (handle) { + const valueHandle = await handle.getProperty('value'); + const optionValue = await valueHandle.jsonValue(); + if (optionValue) { + nodeWithId.value = optionValue.toString(); + } + } + } + + nodeWithId.children = node.children + ? await Promise.all(node.children.map(child => assignIds(child))) + : []; + idToNode.set(nodeWithId.id, nodeWithId); return nodeWithId; }; - const rootNodeWithId = assignIds(rootNode); + const rootNodeWithId = await assignIds(rootNode); this.#textSnapshot = { root: rootNodeWithId, snapshotId: String(snapshotId), From b1506edbee2d4b040d64988461250dd115bb0249 Mon Sep 17 00:00:00 2001 From: alinavarkki Date: Wed, 1 Oct 2025 13:52:19 +0200 Subject: [PATCH 04/44] add test --- tests/McpContext.test.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/McpContext.test.ts b/tests/McpContext.test.ts index a054baba..d1ad1958 100644 --- a/tests/McpContext.test.ts +++ b/tests/McpContext.test.ts @@ -33,6 +33,30 @@ describe('McpContext', () => { }); }); + it('should include the value of an option in the text snapshot', async () => { + await withBrowser(async (_response, context) => { + const page = context.getSelectedPage(); + await page.setContent(` + + + `); + await context.createTextSnapshot(); + const snapshot = context.getTextSnapshot(); + assert.ok(snapshot); + + const option1 = snapshot.root.children[0]?.children[0]; + assert.strictEqual(option1?.role, 'option'); + assert.strictEqual(option1?.value, '1'); + + const option2 = snapshot.root.children[0]?.children[1]; + assert.strictEqual(option2?.role, 'option'); + assert.strictEqual(option2?.value, '2'); + }); + }); + it('can store and retrieve performance traces', async () => { await withBrowser(async (_response, context) => { const fakeTrace1 = {} as unknown as TraceResult; From 047c403ee242a52b216da637ee2b0aa7c5d71234 Mon Sep 17 00:00:00 2001 From: alinavarkki Date: Wed, 1 Oct 2025 13:55:14 +0200 Subject: [PATCH 05/44] make value string --- src/McpContext.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/McpContext.ts b/src/McpContext.ts index 803b4887..8e71361a 100644 --- a/src/McpContext.ts +++ b/src/McpContext.ts @@ -30,7 +30,7 @@ import {WaitForHelper} from './WaitForHelper.js'; export interface TextSnapshotNode extends SerializedAXNode { id: string; children: TextSnapshotNode[]; - value?: string | number; + value?: string; } export interface TextSnapshot { From c680b3ce4dd0495f9baa650933ad0a08da3f416d Mon Sep 17 00:00:00 2001 From: alinavarkki Date: Wed, 1 Oct 2025 14:06:04 +0200 Subject: [PATCH 06/44] remove value --- src/McpContext.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/McpContext.ts b/src/McpContext.ts index 8e71361a..af543998 100644 --- a/src/McpContext.ts +++ b/src/McpContext.ts @@ -30,7 +30,6 @@ import {WaitForHelper} from './WaitForHelper.js'; export interface TextSnapshotNode extends SerializedAXNode { id: string; children: TextSnapshotNode[]; - value?: string; } export interface TextSnapshot { From dcced0fd6043e5a2236dada49afe6c853926384a Mon Sep 17 00:00:00 2001 From: alinavarkki Date: Wed, 1 Oct 2025 19:16:08 +0200 Subject: [PATCH 07/44] use option text as value --- src/McpContext.ts | 11 ++++++----- src/tools/input.ts | 21 ++++++++++++++++++++- tests/McpContext.test.ts | 24 ------------------------ tests/tools/input.test.ts | 29 +++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 30 deletions(-) diff --git a/src/McpContext.ts b/src/McpContext.ts index af543998..1b21ebd8 100644 --- a/src/McpContext.ts +++ b/src/McpContext.ts @@ -335,14 +335,15 @@ export class McpContext implements Context { children: [], }; - // The AXNode for an option doesn't contain its value, so add it from the element. + // The AXNode for an option doesn't contain its `value`. + // Therefore, set text content of the option as value. if (node.role === 'option') { const handle = await node.elementHandle(); if (handle) { - const valueHandle = await handle.getProperty('value'); - const optionValue = await valueHandle.jsonValue(); - if (optionValue) { - nodeWithId.value = optionValue.toString(); + const textContentHandle = await handle.getProperty('textContent'); + const optionText = await textContentHandle.jsonValue(); + if (optionText) { + nodeWithId.value = optionText.toString(); } } } diff --git a/src/tools/input.ts b/src/tools/input.ts index eda04e80..95694a72 100644 --- a/src/tools/input.ts +++ b/src/tools/input.ts @@ -97,7 +97,26 @@ export const fill = defineTool({ const handle = await context.getElementByUid(request.params.uid); try { await context.waitForEventsAfterAction(async () => { - await handle.asLocator().fill(request.params.value); + // The AXNode for an option doesn't contain its `value`. We set text content of the option as value. + // To fill the form, get the correct option by its text value. + const isSelectElement = await handle.evaluate( + el => el.tagName === 'SELECT', + ); + if (isSelectElement) { + const value = await handle.evaluate((el, text) => { + const option = Array.from((el as HTMLSelectElement).options).find( + option => option.text === text, + ); + if (option) { + return option.value; + } + return request.params.value; + }, request.params.value); + + await handle.asLocator().fill(value); + } else { + await handle.asLocator().fill(request.params.value); + } }); response.appendResponseLine(`Successfully filled out the element`); response.setIncludeSnapshot(true); diff --git a/tests/McpContext.test.ts b/tests/McpContext.test.ts index d1ad1958..a054baba 100644 --- a/tests/McpContext.test.ts +++ b/tests/McpContext.test.ts @@ -33,30 +33,6 @@ describe('McpContext', () => { }); }); - it('should include the value of an option in the text snapshot', async () => { - await withBrowser(async (_response, context) => { - const page = context.getSelectedPage(); - await page.setContent(` - - - `); - await context.createTextSnapshot(); - const snapshot = context.getTextSnapshot(); - assert.ok(snapshot); - - const option1 = snapshot.root.children[0]?.children[0]; - assert.strictEqual(option1?.role, 'option'); - assert.strictEqual(option1?.value, '1'); - - const option2 = snapshot.root.children[0]?.children[1]; - assert.strictEqual(option2?.role, 'option'); - assert.strictEqual(option2?.value, '2'); - }); - }); - it('can store and retrieve performance traces', async () => { await withBrowser(async (_response, context) => { const fakeTrace1 = {} as unknown as TraceResult; diff --git a/tests/tools/input.test.ts b/tests/tools/input.test.ts index 8329192f..b788c7b5 100644 --- a/tests/tools/input.test.ts +++ b/tests/tools/input.test.ts @@ -202,6 +202,35 @@ describe('input', () => { assert.ok(await page.$('text/test')); }); }); + + it('fills out a select by text', async () => { + await withBrowser(async (response, context) => { + const page = context.getSelectedPage(); + await page.setContent( + ``, + ); + await context.createTextSnapshot(); + await fill.handler( + { + params: { + uid: '1_1', + value: 'two', + }, + }, + response, + context, + ); + assert.strictEqual( + response.responseLines[0], + 'Successfully filled out the element', + ); + assert.ok(response.includeSnapshot); + const selectedValue = await page.evaluate( + () => document.querySelector('select')!.value, + ); + assert.strictEqual(selectedValue, 'v2'); + }); + }); }); describe('drags', () => { From e873cd9b9d2c2b1fe1d22cb4bbd3cbd8a533f0f7 Mon Sep 17 00:00:00 2001 From: alinavarkki Date: Thu, 2 Oct 2025 19:42:13 +0200 Subject: [PATCH 08/44] check if element is selector through ax node and extract method --- src/McpContext.ts | 4 ++ src/tools/ToolDefinition.ts | 2 + src/tools/input.ts | 89 ++++++++++++++++++++++--------------- 3 files changed, 58 insertions(+), 37 deletions(-) diff --git a/src/McpContext.ts b/src/McpContext.ts index 1b21ebd8..5b29ce09 100644 --- a/src/McpContext.ts +++ b/src/McpContext.ts @@ -272,6 +272,10 @@ export class McpContext implements Context { return page.getDefaultNavigationTimeout(); } + getAXNodeByUid(uid: string) { + return this.#textSnapshot?.idToNode.get(uid); + } + async getElementByUid(uid: string): Promise> { if (!this.#textSnapshot?.idToNode.size) { throw new Error( diff --git a/src/tools/ToolDefinition.ts b/src/tools/ToolDefinition.ts index fe2fae7b..56fdb53a 100644 --- a/src/tools/ToolDefinition.ts +++ b/src/tools/ToolDefinition.ts @@ -7,6 +7,7 @@ import type {Dialog, ElementHandle, Page} from 'puppeteer-core'; import z from 'zod'; +import type {TextSnapshotNode} from '../McpContext.js'; import type {TraceResult} from '../trace-processing/parse.js'; import type {ToolCategories} from './categories.js'; @@ -68,6 +69,7 @@ export type Context = Readonly<{ closePage(pageIdx: number): Promise; setSelectedPageIdx(idx: number): void; getElementByUid(uid: string): Promise>; + getAXNodeByUid(uid: string): TextSnapshotNode | undefined; setNetworkConditions(conditions: string | null): void; setCpuThrottlingRate(rate: number): void; saveTemporaryFile( diff --git a/src/tools/input.ts b/src/tools/input.ts index 95694a72..1d595d0f 100644 --- a/src/tools/input.ts +++ b/src/tools/input.ts @@ -7,6 +7,8 @@ import type {ElementHandle} from 'puppeteer-core'; import z from 'zod'; +import type {McpContext} from '../McpContext.js'; + import {ToolCategories} from './categories.js'; import {defineTool} from './ToolDefinition.js'; @@ -78,6 +80,40 @@ export const hover = defineTool({ }, }); +async function fillFormElement( + uid: string, + value: string, + context: McpContext, +) { + const handle = await context.getElementByUid(uid); + try { + // The AXNode for an option doesn't contain its `value`. We set text content of the option as value. + // If the form is a combobox, we need to find the correct option by its text value. + // To do that, loop through the children while checking which child's text matches the requested value (requested value is actually the text content). + // When the correct option is found, use the element handle to get the real value. + const aXNode = context.getAXNodeByUid(uid); + if (aXNode && aXNode.role === 'combobox' && aXNode.children) { + for (const child of aXNode.children) { + if (child.role === 'option' && child.name === value && child.value) { + const childHandle = await child.elementHandle(); + if (childHandle) { + const childValueHandle = await childHandle.getProperty('value'); + const childValue = await childValueHandle.jsonValue(); + if (childValue) { + await handle.asLocator().fill(childValue.toString()); + } + break; + } + } + } + } else { + await handle.asLocator().fill(value); + } + } finally { + void handle.dispose(); + } +} + export const fill = defineTool({ name: 'fill', description: `Type text into a input, text area or select an option from a