Skip to content

feat(amazonq): add image context support #5846

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

feat(amazonq): add image context support #5846

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
92b30dc
feat(q): enable image context
zuoyaofu Jun 20, 2025
79f86f6
feat(q): add image context support
zuoyaofu Jun 21, 2025
9d0fd61
add change log + fix some styles
zuoyaofu Jun 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .changes/next-release/feature-ddbdde68-769a-468f-bb42-e5aece51c729.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"type" : "feature",
"description" : "Add image context support"
}
4 changes: 3 additions & 1 deletion ...tbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/webview/Browser.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import com.google.gson.Gson
import com.intellij.openapi.Disposable
import com.intellij.openapi.project.Project
import com.intellij.openapi.util.Disposer
import com.intellij.ui.jcef.JBCefJSQuery
import org.cef.CefApp
import software.aws.toolkits.jetbrains.services.amazonq.CodeWhispererFeatureConfigService
import software.aws.toolkits.jetbrains.services.amazonq.lsp.flareChat.AwsServerCapabilitiesProvider
Expand Down Expand Up @@ -85,6 +84,9 @@ class Browser(parent: Disposable, private val webUri: URI, val project: Project)
// setup empty state. The message request handlers use this for storing state
// that's persistent between page loads.
jcefBrowser.setProperty("state", "")
jcefBrowser.jbCefClient.addDragHandler({ browser, dragData, mask ->
true // Allow drag operations
}, jcefBrowser.cefBrowser)
// load the web app
jcefBrowser.loadHTML(
getWebviewHTML(
Expand Down
13 changes: 13 additions & 0 deletions ...ommunity/src/software/aws/toolkits/jetbrains/services/amazonq/webview/BrowserConnector.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GET_S
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GetSerializedChatResponse
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.LIST_MCP_SERVERS_REQUEST_METHOD
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.MCP_SERVER_CLICK_REQUEST_METHOD
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIALOG
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIALOG_REQUEST_METHOD
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_SETTINGS
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_WORKSPACE_SETTINGS_KEY
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OpenSettingsNotification
Expand Down Expand Up @@ -485,6 +487,17 @@ class BrowserConnector(
)
}
}
OPEN_FILE_DIALOG -> {
handleChat(AmazonQChatServer.showOpenFileDialog, node)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you walk me through the message flow? does it actually need to transit through flare before requesting the file picker?

.whenComplete { response, _ ->
browser.postChat(
FlareUiMessage(
command = OPEN_FILE_DIALOG_REQUEST_METHOD,
params = response
)
)
}
}
}
}

Expand Down
7 changes: 7 additions & 0 deletions ...s-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQChatServer.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.LIST_
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.LinkClickParams
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ListConversationsParams
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.MCP_SERVER_CLICK_REQUEST_METHOD
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIALOG_REQUEST_METHOD
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.PROMPT_INPUT_OPTIONS_CHANGE
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.PromptInputOptionChangeParams
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.SEND_CHAT_COMMAND_PROMPT
Expand Down Expand Up @@ -222,4 +223,10 @@ object AmazonQChatServer : JsonRpcMethodProvider {
TELEMETRY_EVENT,
Any::class.java
)

val showOpenFileDialog = JsonRpcRequest(
OPEN_FILE_DIALOG_REQUEST_METHOD,
Any::class.java,
LSPAny::class.java
)
}
5 changes: 5 additions & 0 deletions ...mmunity/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQLanguageClient.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GET_S
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GetSerializedChatResult
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OPEN_FILE_DIFF
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OpenFileDiffParams
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.SHOW_OPEN_FILE_DIALOG_REQUEST_METHOD
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.SHOW_SAVE_FILE_DIALOG_REQUEST_METHOD
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowOpenFileDialogParams
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowSaveFileDialogParams
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowSaveFileDialogResult
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.credentials.ConnectionMetadata
Expand All @@ -42,6 +44,9 @@ interface AmazonQLanguageClient : LanguageClient {
@JsonRequest(SHOW_SAVE_FILE_DIALOG_REQUEST_METHOD)
fun showSaveFileDialog(params: ShowSaveFileDialogParams): CompletableFuture<ShowSaveFileDialogResult>

@JsonRequest(SHOW_OPEN_FILE_DIALOG_REQUEST_METHOD)
fun showOpenFileDialog(params: ShowOpenFileDialogParams): CompletableFuture<LSPAny>

@JsonRequest(GET_SERIALIZED_CHAT_REQUEST_METHOD)
fun getSerializedChat(params: LSPAny): CompletableFuture<GetSerializedChatResult>

Expand Down
24 changes: 24 additions & 0 deletions ...ity/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/AmazonQLanguageClientImpl.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import com.intellij.ide.BrowserUtil
import com.intellij.notification.NotificationAction
import com.intellij.notification.NotificationType
import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.fileChooser.FileChooser
import com.intellij.openapi.fileChooser.FileChooserDescriptorFactory
import com.intellij.openapi.fileChooser.FileChooserFactory
import com.intellij.openapi.fileChooser.FileSaverDescriptor
import com.intellij.openapi.fileEditor.FileEditorManager
Expand Down Expand Up @@ -51,6 +53,7 @@ import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.FileP
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GET_SERIALIZED_CHAT_REQUEST_METHOD
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.GetSerializedChatResult
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.OpenFileDiffParams
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowOpenFileDialogParams
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowSaveFileDialogParams
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat.ShowSaveFileDialogResult
import software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.credentials.ConnectionMetadata
Expand Down Expand Up @@ -249,6 +252,27 @@ class AmazonQLanguageClientImpl(private val project: Project) : AmazonQLanguageC
)
}

override fun showOpenFileDialog(params: ShowOpenFileDialogParams): CompletableFuture<LSPAny> {
return CompletableFuture.supplyAsync(
{
val descriptor = if (params.canSelectMany) {
FileChooserDescriptorFactory.createMultipleFilesNoJarsDescriptor().apply {
title = "Select Files"
description = "Choose files to open"
}
} else {
FileChooserDescriptorFactory.createSingleFileNoJarsDescriptor()
}

val chosenFiles = FileChooser.chooseFiles(descriptor, project, null)
val uris = chosenFiles.map { it.url }

mapOf("uris" to uris) as LSPAny
},
ApplicationManager.getApplication()::invokeLater
)
}

override fun getSerializedChat(params: LSPAny): CompletableFuture<GetSerializedChatResult> {
val requestId = UUID.randomUUID().toString()
val result = CompletableFuture<GetSerializedChatResult>()
Expand Down
3 changes: 3 additions & 0 deletions .../software/aws/toolkits/jetbrains/services/amazonq/lsp/model/aws/chat/FlareChatCommands.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,13 @@ const val PROMPT_INPUT_OPTIONS_CHANGE = "aws/chat/promptInputOptionChange"

const val SEND_CHAT_COMMAND_PROMPT = "aws/chat/sendChatPrompt"
const val SHOW_SAVE_FILE_DIALOG_REQUEST_METHOD = "aws/showSaveFileDialog"
const val SHOW_OPEN_FILE_DIALOG_REQUEST_METHOD = "aws/showOpenFileDialog"
const val STOP_CHAT_RESPONSE = "stopChatResponse"
const val SEND_TO_PROMPT = "sendToPrompt"
const val OPEN_FILE_DIALOG = "openFileDialog"
const val TELEMETRY_EVENT = "telemetry/event"

// https://github.com/aws/language-server-runtimes/blob/112feba70219a98a12f13727d67c540205fa9c9f/types/chat.ts#L32
const val LIST_MCP_SERVERS_REQUEST_METHOD = "aws/chat/listMcpServers"
const val MCP_SERVER_CLICK_REQUEST_METHOD = "aws/chat/mcpServerClick"
const val OPEN_FILE_DIALOG_REQUEST_METHOD = "aws/chat/openFileDialog"
13 changes: 13 additions & 0 deletions ...re/aws/toolkits/jetbrains/services/amazonq/lsp/model/aws/chat/showOpenFileDialogParams.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

package software.aws.toolkits.jetbrains.services.amazonq.lsp.model.aws.chat

data class ShowOpenFileDialogParams(
val canSelectFiles: Boolean = false,
val canSelectFolders: Boolean = false,
val canSelectMany: Boolean = false,
val filters: Map<String, List<String>> = emptyMap(),
val defaultUri: String? = null,
val title: String? = null,
Comment on lines +7 to +12
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

only canSelectMany is being used

)
Loading