Skip to content

Commit eebcb2d

Browse files
tysonthomas9tysonthomas9
authored
AI Chat Panel with Multi-Agent Framework (#1)
1 parent 9276668 commit eebcb2d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+14792
-2
lines changed

‎.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,3 +54,6 @@ test/perf/.generated
5454
.eslintcache
5555
.stylelintcache
5656
**/tsconfig.tsbuildinfo
57+
58+
# Dependencies
59+
node_modules/

‎.gitmodules

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,4 @@
2525
[submodule "third_party/inspector_protocol"]
2626
path = third_party/inspector_protocol
2727
url = https://chromium.googlesource.com/deps/inspector_protocol
28-
gclient-condition = build_with_chromium == False
28+
gclient-condition = build_with_chromium == False

‎config/gni/devtools_grd_files.gni

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -597,6 +597,41 @@ grd_files_release_sources = [
597597
"front_end/panels/accessibility/accessibility.js",
598598
"front_end/panels/ai_assistance/ai_assistance-meta.js",
599599
"front_end/panels/ai_assistance/ai_assistance.js",
600+
"front_end/panels/ai_chat/ui/AIChatPanel.js",
601+
"front_end/panels/ai_chat/ui/ChatView.js",
602+
"front_end/panels/ai_chat/ui/chatView.css.js",
603+
"front_end/panels/ai_chat/core/AgentService.js",
604+
"front_end/panels/ai_chat/core/State.js",
605+
"front_end/panels/ai_chat/core/Graph.js",
606+
"front_end/panels/ai_chat/core/Types.js",
607+
"front_end/panels/ai_chat/core/ConfigurableGraph.js",
608+
"front_end/panels/ai_chat/core/GraphConfigs.js",
609+
"front_end/panels/ai_chat/core/OpenAIClient.js",
610+
"front_end/panels/ai_chat/core/BaseOrchestratorAgent.js",
611+
"front_end/panels/ai_chat/core/PageInfoManager.js",
612+
"front_end/panels/ai_chat/core/AgentNodes.js",
613+
"front_end/panels/ai_chat/core/ChatOpenAI.js",
614+
"front_end/panels/ai_chat/core/GraphHelpers.js",
615+
"front_end/panels/ai_chat/core/StateGraph.js",
616+
"front_end/panels/ai_chat/tools/Tools.js",
617+
"front_end/panels/ai_chat/tools/CombinedExtractionTool.js",
618+
"front_end/panels/ai_chat/tools/CritiqueTool.js",
619+
"front_end/panels/ai_chat/tools/FetcherTool.js",
620+
"front_end/panels/ai_chat/tools/FinalizeWithCritiqueTool.js",
621+
"front_end/panels/ai_chat/tools/HTMLToMarkdownTool.js",
622+
"front_end/panels/ai_chat/tools/SchemaBasedExtractorTool.js",
623+
"front_end/panels/ai_chat/tools/VisitHistoryManager.js",
624+
"front_end/panels/ai_chat/tools/FullPageAccessibilityTreeToMarkdownTool.js",
625+
"front_end/panels/ai_chat/common/utils.js",
626+
"front_end/panels/ai_chat/common/log.js",
627+
"front_end/panels/ai_chat/common/context.js",
628+
"front_end/panels/ai_chat/common/page.js",
629+
"front_end/panels/ai_chat/ai_chat-meta.js",
630+
"front_end/panels/ai_chat/ai_chat.js",
631+
"front_end/panels/ai_chat/ai_chat_impl.js",
632+
"front_end/panels/ai_chat/agent_framework/AgentRunner.js",
633+
"front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.js",
634+
"front_end/panels/ai_chat/agent_framework/implementation/ConfiguredAgents.js",
600635
"front_end/panels/animation/animation-meta.js",
601636
"front_end/panels/animation/animation.js",
602637
"front_end/panels/application/application-meta.js",

‎front_end/entrypoints/devtools_app/BUILD.gn

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ devtools_entrypoint("entrypoint") {
1010
deps = [
1111
"../../entrypoints/inspector_main:meta",
1212
"../../panels/accessibility:meta",
13+
"../../panels/ai_chat:meta",
1314
"../../panels/animation:meta",
1415
"../../panels/application:meta",
1516
"../../panels/autofill:meta",

‎front_end/entrypoints/devtools_app/devtools_app.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import '../../panels/webauthn/webauthn-meta.js';
2727
import '../../panels/layer_viewer/layer_viewer-meta.js';
2828
import '../../panels/recorder/recorder-meta.js';
2929
import '../../panels/whats_new/whats_new-meta.js';
30+
import '../../panels/ai_chat/ai_chat-meta.js';
3031

3132
import * as Root from '../../core/root/root.js';
3233
import * as Main from '../main/main.js';

‎front_end/models/logs/BUILD.gn

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ devtools_entrypoint("bundle") {
3131
visibility = [
3232
":*",
3333
"../../entrypoints/main/*",
34+
"../../panels/ai_chat/*",
3435
"../../panels/application/*",
3536
"../../panels/console/*",
3637
"../../panels/explain/*",

‎front_end/panels/ai_chat/BUILD.gn

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
# Copyright 2025 The Chromium Authors. All rights reserved.
2+
# Use of this source code is governed by a BSD-style license that can be
3+
# found in the LICENSE file.
4+
5+
import("../../../scripts/build/ninja/devtools_entrypoint.gni")
6+
import("../../../scripts/build/ninja/devtools_module.gni")
7+
import("../../../scripts/build/ninja/generate_css.gni")
8+
import("../../../scripts/build/typescript/typescript.gni")
9+
import("../visibility.gni")
10+
11+
generate_css("css_files") {
12+
sources = [
13+
"ui/chatView.css",
14+
]
15+
}
16+
17+
devtools_module("ai_chat") {
18+
sources = [
19+
"ui/AIChatPanel.ts",
20+
"ui/ChatView.ts",
21+
"ai_chat_impl.ts",
22+
"core/Graph.ts",
23+
"core/State.ts",
24+
"core/Types.ts",
25+
"core/AgentService.ts",
26+
"core/GraphConfigs.ts",
27+
"core/OpenAIClient.ts",
28+
"core/ConfigurableGraph.ts",
29+
"core/BaseOrchestratorAgent.ts",
30+
"core/PageInfoManager.ts",
31+
"core/AgentNodes.ts",
32+
"core/ChatOpenAI.ts",
33+
"core/GraphHelpers.ts",
34+
"core/StateGraph.ts",
35+
"tools/Tools.ts",
36+
"tools/CritiqueTool.ts",
37+
"tools/FetcherTool.ts",
38+
"tools/FinalizeWithCritiqueTool.ts",
39+
"tools/VisitHistoryManager.ts",
40+
"tools/HTMLToMarkdownTool.ts",
41+
"tools/SchemaBasedExtractorTool.ts",
42+
"tools/CombinedExtractionTool.ts",
43+
"tools/FullPageAccessibilityTreeToMarkdownTool.ts",
44+
"agent_framework/ConfigurableAgentTool.ts",
45+
"agent_framework/AgentRunner.ts",
46+
"agent_framework/implementation/ConfiguredAgents.ts",
47+
"common/utils.ts",
48+
"common/log.ts",
49+
"common/context.ts",
50+
"common/page.ts",
51+
]
52+
53+
deps = [
54+
":css_files",
55+
"../../core/common:bundle",
56+
"../../core/i18n:bundle",
57+
"../../core/sdk:bundle",
58+
"../../generated:protocol",
59+
"../../models/logs:bundle",
60+
"../../ui/components/helpers:bundle",
61+
"../../ui/components/markdown_view:bundle",
62+
"../../ui/components/text_editor:bundle",
63+
"../../ui/legacy:bundle",
64+
"../../ui/lit:bundle",
65+
"../../ui/visual_logging:bundle",
66+
]
67+
}
68+
69+
# List of source files also used to determine JS outputs for metadata
70+
_ai_chat_sources = [
71+
"ui/AIChatPanel.ts",
72+
"ui/ChatView.ts",
73+
"ai_chat_impl.ts",
74+
"core/Graph.ts",
75+
"core/State.ts",
76+
"core/Types.ts",
77+
"core/AgentService.ts",
78+
"core/GraphConfigs.ts",
79+
"core/OpenAIClient.ts",
80+
"core/ConfigurableGraph.ts",
81+
"core/BaseOrchestratorAgent.ts",
82+
"core/PageInfoManager.ts",
83+
"core/AgentNodes.ts",
84+
"core/ChatOpenAI.ts",
85+
"core/GraphHelpers.ts",
86+
"core/StateGraph.ts",
87+
"tools/Tools.ts",
88+
"tools/CritiqueTool.ts",
89+
"tools/FetcherTool.ts",
90+
"tools/FinalizeWithCritiqueTool.ts",
91+
"tools/VisitHistoryManager.ts",
92+
"tools/HTMLToMarkdownTool.ts",
93+
"tools/SchemaBasedExtractorTool.ts",
94+
"tools/CombinedExtractionTool.ts",
95+
"tools/FullPageAccessibilityTreeToMarkdownTool.ts",
96+
"agent_framework/ConfigurableAgentTool.ts",
97+
"agent_framework/AgentRunner.ts",
98+
"agent_framework/implementation/ConfiguredAgents.ts",
99+
"common/utils.ts",
100+
"common/log.ts",
101+
"common/context.ts",
102+
"common/page.ts",
103+
]
104+
105+
# Construct the expected JS output paths for the metadata
106+
_ai_chat_js_outputs_for_metadata = []
107+
_ai_chat_module_target_gen_dir = get_label_info(":ai_chat", "target_gen_dir")
108+
109+
foreach(src, _ai_chat_sources) {
110+
_extension = get_path_info(src, "extension")
111+
_relative_file_name = rebase_path(src, ".")
112+
_fileName = get_path_info(_relative_file_name, "dir") + "/" +
113+
get_path_info(_relative_file_name, "name")
114+
115+
# Only consider .ts files that aren't definition files (.d.ts) and .js files
116+
if ((_extension == "ts" &&
117+
get_path_info(get_path_info(src, "name"), "extension") != "d") ||
118+
_extension == "js") {
119+
_ai_chat_js_outputs_for_metadata +=
120+
[ "$_ai_chat_module_target_gen_dir/${_fileName}.js" ]
121+
}
122+
}
123+
124+
# This target generates the metadata needed for the release GRD check
125+
generated_file("ai_chat_release_js_metadata") {
126+
outputs = [ "$target_gen_dir/ai_chat_release_js_metadata-tsconfig.json" ]
127+
contents = "{\"files\":[],\"compilerOptions\":{\"composite\":true}}"
128+
129+
# Make sure the actual JS files are built first
130+
deps = [ ":ai_chat" ]
131+
132+
metadata = {
133+
# These paths should match the format ts_library would produce in debug
134+
grd_files = _ai_chat_js_outputs_for_metadata
135+
}
136+
}
137+
138+
# Metadata target for the CSS file, needed for release GRD check
139+
_css_module_target_gen_dir = get_label_info(":css_files", "target_gen_dir")
140+
_css_outputs_for_metadata = [
141+
"$_css_module_target_gen_dir/ui/chatView.css.js",
142+
]
143+
144+
generated_file("ai_chat_release_css_metadata") {
145+
outputs = [ "$target_gen_dir/ai_chat_release_css_metadata-tsconfig.json" ]
146+
contents = "{\"files\":[],\"compilerOptions\":{\"composite\":true}}"
147+
148+
# Make sure the actual CSS file is built first
149+
deps = [ ":css_files" ]
150+
151+
metadata = {
152+
grd_files = _css_outputs_for_metadata
153+
}
154+
}
155+
156+
devtools_entrypoint("bundle") {
157+
entrypoint = "ai_chat.ts"
158+
159+
deps = [
160+
":ai_chat",
161+
":css_files",
162+
":ai_chat_release_js_metadata", # Add dependency on the metadata target
163+
":ai_chat_release_css_metadata", # Add dependency on the CSS metadata target
164+
]
165+
166+
visibility = [
167+
":*",
168+
"../../entrypoints/*",
169+
]
170+
171+
visibility += devtools_panels_visibility
172+
}
173+
174+
devtools_entrypoint("meta") {
175+
entrypoint = "ai_chat-meta.ts"
176+
177+
deps = [ ":bundle" ]
178+
179+
visibility = [ "../../entrypoints/*" ]
180+
}

‎front_end/panels/ai_chat/Readme.md

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# AI Chat Panel
2+
3+
This panel is a Multi-Agent Framework that allows a user to connect an existing LLM with the chromium browser.
4+
5+
### Setup and Development
6+
7+
1. Build the project and use watch mode
8+
```sh
9+
npm run build -- --watch
10+
```
11+
12+
2. Serve the content of out/Default/gen/front_end on a web server, e.g. via python -m http.server.
13+
14+
```sh
15+
cd out/Default/gen/front_end
16+
17+
python3 -m http.server
18+
```
19+
20+
3. Use the AI Chat panel.
21+
22+
```sh
23+
<path-to-devtools-frontend>/third_party/chrome/chrome-<platform>/chrome --disable-infobars --custom-devtools-frontend=http://localhost:8000/
24+
```
25+
26+
27+
### Agent Architecture
28+
29+
The AI Chat Panel uses the multi-agent framework with the following components:
30+
31+
1. **State Management**: Tracks conversation history, user context, and DevTools state
32+
2. **Tools**: Provides capabilities for DOM inspection, network analysis, and code execution
33+
3. **Workflow**: Defines the agent's reasoning process and decision-making flow
34+
35+
### Implementation Details
36+
37+
The agent implementation is located in `front_end/panels/ai_chat/agent/` and consists of:
38+
39+
- `AgentService.ts`: Main service that interfaces between UI and Graph Workflow
40+
- `State.ts`: Defines the agent's state schema
41+
- `Tools.ts`: Implements DevTools-specific tools
42+
- `Graph.ts`: Defines the agent's workflow and decision-making process
43+
44+
### Usage Example
45+
46+
```typescript
47+
import { AgentService } from './agent/AgentService.js';
48+
49+
// Initialize the agent service
50+
const agentService = new AgentService();
51+
52+
// Send a message to the agent
53+
async function sendMessage(text: string) {
54+
const response = await agentService.sendMessage(text);
55+
// Update UI with response
56+
}
57+
```
58+
59+
## Reference
60+
https://chromium.googlesource.com/devtools/devtools-frontend/+/main/docs/get_the_code.md#Standalone-checkout-Checking-out-source

0 commit comments

Comments
 (0)