feat: read elements from DevTools UI

Author: OrKoN

src/McpContext.ts

@@ -31,13 +31,15 @@ import {WaitForHelper} from './WaitForHelper.js';
 
 export interface TextSnapshotNode extends SerializedAXNode {
   id: string;
+  backendNodeId?: number;
   children: TextSnapshotNode[];
 }
 
 export interface TextSnapshot {
   root: TextSnapshotNode;
   idToNode: Map<string, TextSnapshotNode>;
   snapshotId: string;
+  selectedElementUid?: string;
 }
 
 interface McpContextOptions {
@@ -378,39 +380,72 @@ export class McpContext implements Context {
     return this.#pageToDevToolsPage.get(page);
   }
 
-  async getDevToolsData(): Promise<undefined | {requestId?: number}> {
+  async getDevToolsData(): Promise<
+    undefined | {requestId?: number; uid?: string}
+  > {
     try {
+      this.logger('Getting DevTools UI data')
       const selectedPage = this.getSelectedPage();
       const devtoolsPage = this.getDevToolsPage(selectedPage);
-      if (devtoolsPage) {
-        const cdpRequestId = await devtoolsPage.evaluate(async () => {
+      if (!devtoolsPage) {
+        this.logger('No DevTools page detected');
+        return;
+      }
+      const {cdpRequestId, cdpBackendNodeId} = await devtoolsPage.evaluate(
+        async () => {
           // @ts-expect-error no types
           const UI = await import('/bundled/ui/legacy/legacy.js');
           // @ts-expect-error no types
           const SDK = await import('/bundled/core/sdk/sdk.js');
           const request = UI.Context.Context.instance().flavor(
             SDK.NetworkRequest.NetworkRequest,
           );
-          return request?.requestId();
-        });
+          const node = UI.Context.Context.instance().flavor(
+            SDK.DOMModel.DOMNode,
+          );
+          return {
+            cdpRequestId: request?.requestId(),
+            cdpBackendNodeId: node?.backendNodeId(),
+          };
+        },
+      );
+      const resolveNetworkRequestId = () => {
         if (!cdpRequestId) {
-          this.logger('no context request');
+          this.logger('no network request');
           return;
         }
         const request = this.#networkCollector.find(selectedPage, request => {
           // @ts-expect-error id is internal.
           return request.id === cdpRequestId;
         });
         if (!request) {
-          this.logger('no collected request for ' + cdpRequestId);
+          this.logger('no network request for ' + cdpRequestId);
           return;
         }
-        return {
-          requestId: this.#networkCollector.getIdForResource(request),
-        };
-      } else {
-        this.logger('no devtools page deteched');
-      }
+        return this.#networkCollector.getIdForResource(request);
+      };
+      const resolveNodeUid = (): string | undefined => {
+        if (!cdpBackendNodeId) {
+          this.logger('no cdpBackendNodeId');
+          return;
+        }
+        // TODO: index by backendNodeId instead.
+        const queue = [this.#textSnapshot?.root];
+        while (queue.length) {
+          const current = queue.pop()!;
+          if (current.backendNodeId === cdpBackendNodeId) {
+            return current.id;
+          }
+          for (const child of current.children) {
+            queue.push(child);
+          }
+        }
+        return;
+      };
+      return {
+        requestId: resolveNetworkRequestId(),
+        uid: resolveNodeUid(),
+      };
     } catch (err) {
       this.logger('error getting devtools data', err);
     }
@@ -463,6 +498,10 @@ export class McpContext implements Context {
       snapshotId: String(snapshotId),
       idToNode,
     };
+    const data = await this.getDevToolsData();
+    if (data?.uid) {
+      this.#textSnapshot.selectedElementUid = data.uid;
+    }
   }
 
   getTextSnapshot(): TextSnapshot | null {

src/McpResponse.ts

@@ -15,7 +15,7 @@ import {
   getShortDescriptionForRequest,
   getStatusFromRequest,
 } from './formatters/networkFormatter.js';
-import {formatA11ySnapshot} from './formatters/snapshotFormatter.js';
+import {formatSnapshotNode} from './formatters/snapshotFormatter.js';
 import type {McpContext} from './McpContext.js';
 import type {
   ConsoleMessage,
@@ -184,12 +184,12 @@ export class McpResponse implements Response {
       if (snapshot) {
         if (this.#snapshotParams.filePath) {
           await context.saveFile(
-            new TextEncoder().encode(formatA11ySnapshot(snapshot.root)),
+            new TextEncoder().encode(formatSnapshotNode(snapshot.root, snapshot)),
             this.#snapshotParams.filePath,
           );
           formattedSnapshot = `Saved snapshot to ${this.#snapshotParams.filePath}.`;
         } else {
-          formattedSnapshot = formatA11ySnapshot(snapshot.root);
+          formattedSnapshot = formatSnapshotNode(snapshot.root, snapshot);
         }
       }
     }

src/formatters/snapshotFormatter.ts

@@ -3,19 +3,20 @@
  * Copyright 2025 Google LLC
  * SPDX-License-Identifier: Apache-2.0
  */
-import type {TextSnapshotNode} from '../McpContext.js';
+import type {TextSnapshot, TextSnapshotNode} from '../McpContext.js';
 
-export function formatA11ySnapshot(
-  serializedAXNodeRoot: TextSnapshotNode,
+export function formatSnapshotNode(
+  root: TextSnapshotNode,
+  snapshot?: TextSnapshot,
   depth = 0,
 ): string {
   let result = '';
-  const attributes = getAttributes(serializedAXNodeRoot);
-  const line = ' '.repeat(depth * 2) + attributes.join(' ') + '\n';
+  const attributes = getAttributes(root);
+  const line = ' '.repeat(depth * 2) + attributes.join(' ') + (root.id === snapshot?.selectedElementUid ? ' [selected in DevTools UI]' : '') + '\n';
   result += line;
 
-  for (const child of serializedAXNodeRoot.children) {
-    result += formatA11ySnapshot(child, depth + 1);
+  for (const child of root.children) {
+    result += formatSnapshotNode(child, snapshot, depth + 1);
   }
 
   return result;

src/tools/ToolDefinition.ts

@@ -103,7 +103,7 @@ export type Context = Readonly<{
     text: string;
     timeout?: number | undefined;
   }): Promise<Element>;
-  getDevToolsData(): Promise<undefined | {requestId?: number}>;
+  getDevToolsData(): Promise<undefined | {requestId?: number, uid?: string}>;
 }>;
 
 export function defineTool<Schema extends zod.ZodRawShape>(

tests/formatters/snapshotFormatter.test.ts

@@ -9,12 +9,12 @@ import {describe, it} from 'node:test';
 
 import type {ElementHandle} from 'puppeteer-core';
 
-import {formatA11ySnapshot} from '../../src/formatters/snapshotFormatter.js';
+import {formatSnapshotNode} from '../../src/formatters/snapshotFormatter.js';
 import type {TextSnapshotNode} from '../../src/McpContext.js';
 
 describe('snapshotFormatter', () => {
   it('formats a snapshot with value properties', () => {
-    const snapshot: TextSnapshotNode = {
+    const node: TextSnapshotNode = {
       id: '1_1',
       role: 'textbox',
       name: 'textbox',
@@ -35,7 +35,7 @@ describe('snapshotFormatter', () => {
       },
     };
 
-    const formatted = formatA11ySnapshot(snapshot);
+    const formatted = formatSnapshotNode(node);
     assert.strictEqual(
       formatted,
       `uid=1_1 textbox "textbox" value="value"
@@ -45,7 +45,7 @@ describe('snapshotFormatter', () => {
   });
 
   it('formats a snapshot with boolean properties', () => {
-    const snapshot: TextSnapshotNode = {
+    const node: TextSnapshotNode = {
       id: '1_1',
       role: 'button',
       name: 'button',
@@ -66,7 +66,7 @@ describe('snapshotFormatter', () => {
       },
     };
 
-    const formatted = formatA11ySnapshot(snapshot);
+    const formatted = formatSnapshotNode(node);
     assert.strictEqual(
       formatted,
       `uid=1_1 button "button" disableable disabled
@@ -76,7 +76,7 @@ describe('snapshotFormatter', () => {
   });
 
   it('formats a snapshot with checked properties', () => {
-    const snapshot: TextSnapshotNode = {
+    const node: TextSnapshotNode = {
       id: '1_1',
       role: 'checkbox',
       name: 'checkbox',
@@ -97,7 +97,7 @@ describe('snapshotFormatter', () => {
       },
     };
 
-    const formatted = formatA11ySnapshot(snapshot);
+    const formatted = formatSnapshotNode(node);
     assert.strictEqual(
       formatted,
       `uid=1_1 checkbox "checkbox" checked
@@ -107,7 +107,7 @@ describe('snapshotFormatter', () => {
   });
 
   it('formats a snapshot with multiple different type attributes', () => {
-    const snapshot: TextSnapshotNode = {
+    const node: TextSnapshotNode = {
       id: '1_1',
       role: 'root',
       name: 'root',
@@ -139,7 +139,7 @@ describe('snapshotFormatter', () => {
       },
     };
 
-    const formatted = formatA11ySnapshot(snapshot);
+    const formatted = formatSnapshotNode(node);
     assert.strictEqual(
       formatted,
       `uid=1_1 root "root"