fix: emit telemetry event to RTS when failed or cancelled (#1384)

Author: yueny2020

server/aws-lsp-codewhisperer/src/client/token/bearer-token-service.json

@@ -829,6 +829,10 @@
             "max": 20,
             "min": 0
         },
+        "AgenticChatEventStatus": {
+            "type": "string",
+            "enum": ["SUCCEEDED", "CANCELLED", "FAILED"]
+        },
         "AppStudioState": {
             "type": "structure",
             "required": ["namespace", "propertyName", "propertyContext"],
@@ -1010,6 +1014,9 @@
                 },
                 "hasProjectLevelContext": {
                     "shape": "Boolean"
+                },
+                "result": {
+                    "shape": "AgenticChatEventStatus"
                 }
             }
         },

server/aws-lsp-codewhisperer/src/client/token/codewhispererbearertokenclient.d.ts

@@ -246,6 +246,7 @@ declare namespace CodeWhispererBearerTokenClient {
   export type AdditionalContentEntryInnerContextString = string;
   export type AdditionalContentEntryNameString = string;
   export type AdditionalContentList = AdditionalContentEntry[];
+  export type AgenticChatEventStatus = "SUCCEEDED"|"CANCELLED"|"FAILED"|string;
   export interface AppStudioState {
     /**
      * The namespace of the context. Examples: 'ui.Button', 'ui.Table.DataSource', 'ui.Table.RowActions.Button', 'logic.invokeAWS', 'logic.JavaScript'
@@ -323,6 +324,7 @@ declare namespace CodeWhispererBearerTokenClient {
     responseLength?: Integer;
     numberOfCodeBlocks?: Integer;
     hasProjectLevelContext?: Boolean;
+    result?: AgenticChatEventStatus;
   }
   export type ChatHistory = ChatMessage[];
   export interface ChatInteractWithMessageEvent {

server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts

@@ -427,6 +427,8 @@ export class AgenticChatController implements ChatHandlers {
                     session.pairProgrammingMode,
                     session.getConversationType()
                 )
+                await this.#telemetryController.emitAddMessageMetric(params.tabId, metric.metric, 'Cancelled')
+
                 session.abortRequest()
                 void this.#invalidateAllShellCommands(params.tabId, session)
                 session.rejectAllDeferredToolExecutions(new CancellationError('user'))
@@ -491,7 +493,6 @@ export class AgenticChatController implements ChatHandlers {
                 },
                 params.partialResultToken
             )
-
             if (this.isUserAction(err, token)) {
                 /**
                  * when the session is aborted it generates an error.
@@ -505,7 +506,14 @@ export class AgenticChatController implements ChatHandlers {
                     buttons: [],
                 }
             }
-            return this.#handleRequestError(err, errorMessageId, params.tabId, metric, session.pairProgrammingMode)
+            return this.#handleRequestError(
+                session.conversationId,
+                err,
+                errorMessageId,
+                params.tabId,
+                metric,
+                session.pairProgrammingMode
+            )
         }
     }
 
@@ -1862,7 +1870,7 @@ export class AgenticChatController implements ChatHandlers {
                 cwsprChatFocusFileContextLength: triggerContext.text?.length,
             })
         }
-        await this.#telemetryController.emitAddMessageMetric(params.tabId, metric.metric)
+        await this.#telemetryController.emitAddMessageMetric(params.tabId, metric.metric, 'Succeeded')
 
         this.#telemetryController.updateTriggerInfo(params.tabId, {
             lastMessageTrigger: {
@@ -1882,18 +1890,23 @@ export class AgenticChatController implements ChatHandlers {
     /**
      * Handles errors that occur during the request
      */
-    #handleRequestError(
+    async #handleRequestError(
+        conversationId: string | undefined,
         err: any,
         errorMessageId: string,
         tabId: string,
         metric: Metric<CombinedConversationEvent>,
         agenticCodingMode: boolean
-    ): ChatResult | ResponseError<ChatResult> {
+    ): Promise<ChatResult | ResponseError<ChatResult>> {
         const errorMessage = getErrorMessage(err)
         const requestID = getRequestID(err) ?? ''
         metric.setDimension('cwsprChatResponseCode', getHttpStatusCode(err) ?? 0)
         metric.setDimension('languageServerVersion', this.#features.runtime.serverInfo.version)
 
+        metric.metric.requestIds = [requestID]
+        metric.metric.cwsprChatMessageId = errorMessageId
+        metric.metric.cwsprChatConversationId = conversationId
+        await this.#telemetryController.emitAddMessageMetric(tabId, metric.metric, 'Failed')
         // use custom error message for unactionable errors (user-dependent errors like PromptCharacterLimit)
         if (err.code && err.code in unactionableErrorCodes) {
             const customErrMessage = unactionableErrorCodes[err.code as keyof typeof unactionableErrorCodes]

server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts

@@ -23,6 +23,7 @@ import { AcceptedSuggestionEntry, CodeDiffTracker } from '../../inline-completio
 import { TelemetryService } from '../../../shared/telemetry/telemetryService'
 import { getEndPositionForAcceptedSuggestion, getTelemetryReasonDesc } from '../../../shared/utils'
 import { CodewhispererLanguage } from '../../../shared/languageDetection'
+import { AgenticChatEventStatus } from '../../../client/token/codewhispererbearertokenclient'
 
 export const CONVERSATION_ID_METRIC_KEY = 'cwsprChatConversationId'
 
@@ -239,7 +240,7 @@ export class ChatTelemetryController {
         })
     }
 
-    public emitAddMessageMetric(tabId: string, metric: Partial<CombinedConversationEvent>) {
+    public emitAddMessageMetric(tabId: string, metric: Partial<CombinedConversationEvent>, result?: string) {
         const conversationId = this.getConversationId(tabId)
         // Store the customization value associated with the message
         if (metric.cwsprChatMessageId && metric.codewhispererCustomizationArn) {
@@ -264,6 +265,7 @@ export class ChatTelemetryController {
                 responseLength: metric.cwsprChatResponseLength,
                 numberOfCodeBlocks: metric.cwsprChatResponseCodeSnippetCount,
                 agenticCodingMode: metric.enabled,
+                result: result,
             },
             {
                 chatTriggerInteraction: metric.cwsprChatTriggerInteraction,

server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts

@@ -799,6 +799,7 @@ describe('TelemetryService', () => {
                         responseLength: 3000,
                         numberOfCodeBlocks: 0,
                         hasProjectLevelContext: false,
+                        result: 'SUCCEEDED',
                     },
                 },
             }
@@ -883,28 +884,6 @@ describe('TelemetryService', () => {
             })
         })
 
-        it('should not send ChatAddMessage when conversationId is undefined', () => {
-            telemetryService.emitChatAddMessage(
-                {
-                    messageId: 'message123',
-                    customizationArn: 'cust-123',
-                },
-                {}
-            )
-            sinon.assert.notCalled(codeWhisperServiceStub.sendTelemetryEvent)
-        })
-
-        it('should not send ChatAddMessage when messageId is undefined', () => {
-            telemetryService.emitChatAddMessage(
-                {
-                    conversationId: 'conv123',
-                    customizationArn: 'cust-123',
-                },
-                {}
-            )
-            sinon.assert.notCalled(codeWhisperServiceStub.sendTelemetryEvent)
-        })
-
         it('should not send ChatAddMessage when credentialsType is IAM', () => {
             codeWhisperServiceStub.getCredentialsType.returns('iam')
             telemetryService = new TelemetryService(serviceManagerStub, mockCredentialsProvider, telemetry, logging)

server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts

@@ -19,6 +19,7 @@ import {
     ChatAddMessageEvent,
     UserIntent,
     InlineChatEvent,
+    AgenticChatEventStatus,
 } from '../../client/token/codewhispererbearertokenclient'
 import { getCompletionType, getSsoConnectionType, isAwsError } from '../utils'
 import {
@@ -474,6 +475,7 @@ export class TelemetryService {
             numberOfCodeBlocks?: number
             hasProjectLevelContext?: number
             agenticCodingMode?: boolean
+            result?: string
         },
         additionalParams: Partial<{
             chatTriggerInteraction: string
@@ -498,9 +500,6 @@ export class TelemetryService {
             requestIds?: string[]
         }>
     ) {
-        if (!params.conversationId || !params.messageId) {
-            return
-        }
         const timeBetweenChunks = params.timeBetweenChunks?.slice(0, 100)
         // truncate requestIds if longer than 875 so it does not go over field limit
         const truncatedRequestIds = additionalParams.requestIds?.slice(0, 875)
@@ -541,7 +540,7 @@ export class TelemetryService {
                     cwsprChatFocusFileContextLength: additionalParams.cwsprChatFocusFileContextLength,
                     cwsprChatCodeContextCount: additionalParams.cwsprChatCodeContextCount,
                     cwsprChatCodeContextLength: additionalParams.cwsprChatCodeContextLength,
-                    result: 'Succeeded',
+                    result: params.result ?? 'Succeeded',
                     enabled: params.agenticCodingMode,
                     languageServerVersion: additionalParams.languageServerVersion,
                     requestIds: truncatedRequestIds,
@@ -550,8 +549,9 @@ export class TelemetryService {
         }
 
         const event: ChatAddMessageEvent = {
-            conversationId: params.conversationId,
-            messageId: params.messageId,
+            // Fields conversationId and messageId are required, but failed or cancelled events may not have those values, then just set them as dummy value
+            conversationId: params.conversationId ?? 'DummyConversationId',
+            messageId: params.messageId ?? 'DummyMessageId',
             userIntent: params.userIntent,
             hasCodeSnippet: params.hasCodeSnippet,
             activeEditorTotalCharacters: params.activeEditorTotalCharacters,
@@ -562,6 +562,7 @@ export class TelemetryService {
             responseLength: params.responseLength,
             numberOfCodeBlocks: params.numberOfCodeBlocks,
             hasProjectLevelContext: false,
+            result: params.result?.toUpperCase() ?? 'SUCCEEDED',
         }
         if (params.customizationArn) {
             event.customizationArn = params.customizationArn