telemetry(amazonq) add feedback functionality with thumbs up/down (#5336)

Randall-Jiang · web-flow · commit b72f78b945e5 · 2025-02-17T18:10:05.000-08:00
Previously, UTG did not have telemetry to collect user feedback on generated unit tests. This PR introduces telemetry support to address that gap. Additionally, it enables users to submit comments, which will be routed to a private SIM folder.
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -27,7 +27,7 @@ mockitoKotlin = "5.4.0"
 mockk = "1.13.10"
 nimbus-jose-jwt = "9.40"
 node-gradle = "7.0.2"
-telemetryGenerator = "1.0.295"
+telemetryGenerator = "1.0.297"
 testLogger = "4.0.0"
 testRetry = "1.5.10"
 # test-only; platform provides slf4j transitively at runtime
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/CodeTestChatApp.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/CodeTestChatApp.kt
@@ -39,6 +39,9 @@ class CodeTestChatApp(private val scope: CoroutineScope) : AmazonQApp {
             "start-test-gen" to IncomingCodeTestMessage.StartTestGen::class,
             "response-body-link-click" to IncomingCodeTestMessage.ClickedLink::class,
             "button-click" to IncomingCodeTestMessage.ButtonClicked::class,
+            "chat-item-voted" to IncomingCodeTestMessage.ChatItemVoted::class,
+            "chat-item-feedback" to IncomingCodeTestMessage.ChatItemFeedback::class,
+            "button-click" to IncomingCodeTestMessage.ButtonClicked::class,
             "auth-follow-up-was-clicked" to IncomingCodeTestMessage.AuthFollowUpWasClicked::class
         )
 
@@ -80,6 +83,8 @@ class CodeTestChatApp(private val scope: CoroutineScope) : AmazonQApp {
             is IncomingCodeTestMessage.StartTestGen -> inboundAppMessagesHandler.processStartTestGen(message)
             is IncomingCodeTestMessage.ClickedLink -> inboundAppMessagesHandler.processLinkClick(message)
             is IncomingCodeTestMessage.ButtonClicked -> inboundAppMessagesHandler.processButtonClickedMessage(message)
+            is IncomingCodeTestMessage.ChatItemVoted -> inboundAppMessagesHandler.processChatItemVoted(message)
+            is IncomingCodeTestMessage.ChatItemFeedback -> inboundAppMessagesHandler.processChatItemFeedBack(message)
             is IncomingCodeTestMessage.AuthFollowUpWasClicked -> inboundAppMessagesHandler.processAuthFollowUpClick(message)
         }
     }
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/InboundAppMessagesHandler.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/InboundAppMessagesHandler.kt
@@ -22,5 +22,9 @@ interface InboundAppMessagesHandler {
 
     suspend fun processButtonClickedMessage(message: IncomingCodeTestMessage.ButtonClicked)
 
+    suspend fun processChatItemVoted(message: IncomingCodeTestMessage.ChatItemVoted)
+
+    suspend fun processChatItemFeedBack(message: IncomingCodeTestMessage.ChatItemFeedback)
+
     suspend fun processAuthFollowUpClick(message: IncomingCodeTestMessage.AuthFollowUpWasClicked)
 }
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/controller/CodeTestChatController.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/controller/CodeTestChatController.kt
@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 package software.aws.toolkits.jetbrains.services.amazonqCodeTest.controller
-
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.intellij.diff.DiffContentFactory
 import com.intellij.diff.DiffManager
 import com.intellij.diff.DiffManagerEx
@@ -44,8 +44,11 @@ import software.amazon.awssdk.services.codewhispererstreaming.model.TextDocument
 import software.amazon.awssdk.services.codewhispererstreaming.model.UserInputMessage
 import software.amazon.awssdk.services.codewhispererstreaming.model.UserInputMessageContext
 import software.amazon.awssdk.services.codewhispererstreaming.model.UserIntent
+import software.amazon.awssdk.services.toolkittelemetry.model.Sentiment
 import software.aws.toolkits.core.utils.debug
 import software.aws.toolkits.core.utils.getLogger
+import software.aws.toolkits.core.utils.info
+import software.aws.toolkits.core.utils.warn
 import software.aws.toolkits.jetbrains.core.AwsClientManager
 import software.aws.toolkits.jetbrains.core.coroutines.EDT
 import software.aws.toolkits.jetbrains.core.credentials.ToolkitConnectionManager
@@ -81,14 +84,19 @@ import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.TriggerTy
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.v1.ChatSessionV1.Companion.validLanguages
 import software.aws.toolkits.jetbrains.services.cwc.controller.chat.StaticPrompt
 import software.aws.toolkits.jetbrains.services.cwc.controller.chat.StaticTextResponse
+import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.FeedbackComment
 import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.getStartUrl
 import software.aws.toolkits.jetbrains.services.cwc.editor.context.ActiveFileContext
 import software.aws.toolkits.jetbrains.services.cwc.editor.context.ActiveFileContextExtractor
 import software.aws.toolkits.jetbrains.services.cwc.editor.context.ExtractionTriggerType
 import software.aws.toolkits.jetbrains.services.cwc.editor.context.file.FileContext
 import software.aws.toolkits.jetbrains.services.cwc.messages.ChatMessageType
+import software.aws.toolkits.jetbrains.services.telemetry.TelemetryService
+import software.aws.toolkits.jetbrains.utils.notifyError
 import software.aws.toolkits.resources.message
 import software.aws.toolkits.telemetry.AmazonqTelemetry
+import software.aws.toolkits.telemetry.FeatureId
+import software.aws.toolkits.telemetry.InteractionType
 import software.aws.toolkits.telemetry.MetricResult
 import software.aws.toolkits.telemetry.UiTelemetry
 import java.io.File
@@ -403,6 +411,57 @@ class CodeTestChatController(
             .build()
     }
 
+    override suspend fun processChatItemFeedBack(message: IncomingCodeTestMessage.ChatItemFeedback) {
+        LOG.debug { "$FEATURE_NAME: Processing ChatItemFeedBackMessage: ${message.comment}" }
+
+        val session = codeTestChatHelper.getActiveSession()
+
+        val comment = FeedbackComment(
+            conversationId = session.startTestGenerationRequestId,
+            userComment = message.comment.orEmpty(),
+            reason = message.selectedOption,
+            type = "testgen-chat-answer-feedback",
+            messageId = "",
+        )
+
+        try {
+            TelemetryService.getInstance().sendFeedback(
+                sentiment = Sentiment.NEGATIVE,
+                comment = objectMapper.writeValueAsString(comment),
+            )
+            LOG.info { "$FEATURE_NAME answer feedback sent: \"Negative\"" }
+        } catch (e: Throwable) {
+            e.notifyError(message("feedback.submit_failed", e))
+            LOG.warn(e) { "Failed to submit feedback" }
+            return
+        }
+    }
+
+    override suspend fun processChatItemVoted(message: IncomingCodeTestMessage.ChatItemVoted) {
+        LOG.debug { "$FEATURE_NAME: Processing ChatItemVotedMessage: $message" }
+
+        val session = codeTestChatHelper.getActiveSession()
+        when (message.vote) {
+            "upvote" -> {
+                AmazonqTelemetry.feedback(
+                    featureId = FeatureId.AmazonQTest,
+                    interactionType = InteractionType.Upvote,
+                    credentialStartUrl = getStartUrl(project = context.project),
+                    amazonqConversationId = session.startTestGenerationRequestId
+
+                )
+            }
+            "downvote" -> {
+                AmazonqTelemetry.feedback(
+                    featureId = FeatureId.AmazonQTest,
+                    interactionType = InteractionType.Downvote,
+                    credentialStartUrl = getStartUrl(project = context.project),
+                    amazonqConversationId = session.startTestGenerationRequestId
+                )
+            }
+        }
+    }
+
     override suspend fun processNewTabCreatedMessage(message: IncomingCodeTestMessage.NewTabCreated) {
         newTabOpened(message.tabId)
         LOG.debug { "$FEATURE_NAME: New tab created: $message" }
@@ -1312,5 +1371,7 @@ class CodeTestChatController(
 
     companion object {
         private val LOG = getLogger<CodeTestChatController>()
+
+        private val objectMapper = jacksonObjectMapper()
     }
 }
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/messages/CodeTestMessage.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/messages/CodeTestMessage.kt
@@ -95,6 +95,17 @@ sealed interface IncomingCodeTestMessage : CodeTestBaseMessage {
         @JsonProperty("actionID") val actionID: String,
     ) : IncomingCodeTestMessage
 
+    data class ChatItemVoted(
+        @JsonProperty("tabID") val tabId: String,
+        val vote: String,
+    ) : IncomingCodeTestMessage
+
+    data class ChatItemFeedback(
+        @JsonProperty("tabID") val tabId: String,
+        val selectedOption: String,
+        val comment: String?,
+    ) : IncomingCodeTestMessage
+
     data class AuthFollowUpWasClicked(
         @JsonProperty("tabID") val tabId: String,
         val authType: AuthFollowUpType,
diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt
@@ -444,7 +444,7 @@ class TelemetryHelper(private val project: Project, private val sessionStorage:
 
 data class FeedbackComment(
     val conversationId: String,
-    val messageId: String,
+    val messageId: String?,
     val reason: String,
     val userComment: String,
     val type: String = "codewhisperer-chat-answer-feedback",
diff --git a/plugins/amazonq/mynah-ui/src/mynah-ui/ui/apps/codeTestChatConnector.ts b/plugins/amazonq/mynah-ui/src/mynah-ui/ui/apps/codeTestChatConnector.ts
@@ -4,7 +4,7 @@
 import {ExtensionMessage} from "../commands";
 import {ChatPayload, ConnectorProps} from "../connector";
 import {FormButtonIds} from "../forms/constants";
-import {ChatItem, ChatItemAction, ChatItemType, MynahIcons, MynahUIDataModel} from '@aws/mynah-ui-chat'
+import {ChatItem, ChatItemAction, ChatItemType, FeedbackPayload, MynahIcons, MynahUIDataModel} from '@aws/mynah-ui-chat'
 import {CodeReference} from "./amazonqCommonsConnector";
 import {Status} from "@aws/mynah-ui-chat/dist/static";
 import {EmptyMynahUIDataModel} from "@aws/mynah-ui-chat/dist/helper/store";
@@ -13,6 +13,7 @@ import {doesNotMatch} from "node:assert";
 export interface ICodeTestChatConnectorProps {
     sendMessageToExtension: (message: ExtensionMessage) => void
     onChatAnswerReceived?: (tabID: string, message: ChatItem) => void
+    sendFeedback?: (tabId: string, feedbackPayload: FeedbackPayload) => void | undefined
     onUpdateAuthentication: (
         featureDevEnabled: boolean,
         codeTransformEnabled: boolean,
@@ -571,6 +572,24 @@ export class CodeTestChatConnector {
         })
     }
 
+    onChatItemVoted = (tabId: string, messageId: string, vote: string): void | undefined => {
+        this.sendMessageToExtension({
+            tabID: tabId,
+            vote: vote,
+            command: 'chat-item-voted',
+            tabType: 'codetest',
+        })
+    }
+
+    sendFeedback = (tabId: string, feedbackPayload: FeedbackPayload): void | undefined => {
+        this.sendMessageToExtension({
+            command: 'chat-item-feedback',
+            ...feedbackPayload,
+            tabType: 'codetest',
+            tabID: tabId,
+        })
+    }
+
     onResponseBodyLinkClick = (tabID: string, messageId: string, link: string): void => {
         this.sendMessageToExtension({
             command: 'response-body-link-click',
diff --git a/plugins/amazonq/mynah-ui/src/mynah-ui/ui/connector.ts b/plugins/amazonq/mynah-ui/src/mynah-ui/ui/connector.ts
@@ -593,6 +593,9 @@ export class Connector {
             case 'cwc':
                 this.cwChatConnector.onSendFeedback(tabId, feedbackPayload)
                 break
+            case 'codetest':
+                this.codeTestChatConnector.sendFeedback(tabId,feedbackPayload)
+                break
         }
     }
 
@@ -604,6 +607,9 @@ export class Connector {
             case 'featuredev':
                 this.featureDevChatConnector.onChatItemVoted(tabId, messageId, vote)
                 break
+            case 'codetest' :
+                this.codeTestChatConnector.onChatItemVoted(tabId,messageId,vote)
+                break
         }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -39,6 +39,9 @@ class CodeTestChatApp(private val scope: CoroutineScope) : AmazonQApp {`
`39`	`39`	`"start-test-gen" to IncomingCodeTestMessage.StartTestGen::class,`
`40`	`40`	`"response-body-link-click" to IncomingCodeTestMessage.ClickedLink::class,`
`41`	`41`	`"button-click" to IncomingCodeTestMessage.ButtonClicked::class,`
	`42`	`+ "chat-item-voted" to IncomingCodeTestMessage.ChatItemVoted::class,`
	`43`	`+ "chat-item-feedback" to IncomingCodeTestMessage.ChatItemFeedback::class,`
	`44`	`+ "button-click" to IncomingCodeTestMessage.ButtonClicked::class,`
`42`	`45`	`"auth-follow-up-was-clicked" to IncomingCodeTestMessage.AuthFollowUpWasClicked::class`
`43`	`46`	`)`
`44`	`47`
`@@ -80,6 +83,8 @@ class CodeTestChatApp(private val scope: CoroutineScope) : AmazonQApp {`
`80`	`83`	`is IncomingCodeTestMessage.StartTestGen -> inboundAppMessagesHandler.processStartTestGen(message)`
`81`	`84`	`is IncomingCodeTestMessage.ClickedLink -> inboundAppMessagesHandler.processLinkClick(message)`
`82`	`85`	`is IncomingCodeTestMessage.ButtonClicked -> inboundAppMessagesHandler.processButtonClickedMessage(message)`
	`86`	`+ is IncomingCodeTestMessage.ChatItemVoted -> inboundAppMessagesHandler.processChatItemVoted(message)`
	`87`	`+ is IncomingCodeTestMessage.ChatItemFeedback -> inboundAppMessagesHandler.processChatItemFeedBack(message)`
`83`	`88`	`is IncomingCodeTestMessage.AuthFollowUpWasClicked -> inboundAppMessagesHandler.processAuthFollowUpClick(message)`
`84`	`89`	`}`
`85`	`90`	`}`
Original file line number	Diff line number	Diff line change
`@@ -22,5 +22,9 @@ interface InboundAppMessagesHandler {`
`22`	`22`
`23`	`23`	`suspend fun processButtonClickedMessage(message: IncomingCodeTestMessage.ButtonClicked)`
`24`	`24`
	`25`	`+ suspend fun processChatItemVoted(message: IncomingCodeTestMessage.ChatItemVoted)`
	`26`	`+`
	`27`	`+ suspend fun processChatItemFeedBack(message: IncomingCodeTestMessage.ChatItemFeedback)`
	`28`	`+`
`25`	`29`	`suspend fun processAuthFollowUpClick(message: IncomingCodeTestMessage.AuthFollowUpWasClicked)`
`26`	`30`	`}`
Original file line number	Diff line number	Diff line change
`@@ -593,6 +593,9 @@ export class Connector {`
`593`	`593`	`case 'cwc':`
`594`	`594`	`this.cwChatConnector.onSendFeedback(tabId, feedbackPayload)`
`595`	`595`	`break`
	`596`	`+ case 'codetest':`
	`597`	`+ this.codeTestChatConnector.sendFeedback(tabId,feedbackPayload)`
	`598`	`+ break`
`596`	`599`	`}`
`597`	`600`	`}`
`598`	`601`
`@@ -604,6 +607,9 @@ export class Connector {`
`604`	`607`	`case 'featuredev':`
`605`	`608`	`this.featureDevChatConnector.onChatItemVoted(tabId, messageId, vote)`
`606`	`609`	`break`
	`610`	`+ case 'codetest' :`
	`611`	`+ this.codeTestChatConnector.onChatItemVoted(tabId,messageId,vote)`
	`612`	`+ break`
`607`	`613`	`}`
`608`	`614`	`}`
`609`	`615`	`}`