telemetry(amazonq): AI code gen % for Q features (#5215)

With the release of many Q features(Inline Suggestion, chat, inline chat, /dev, /test, /doc, /review, /transform), we need to know the % code written by all Q features. This requires calculating and reporting the user written code. The reporting of the code contribution of each Q features was already implemented.

% Code Written by Q = Code Written by Q / ( Code Written by Q + Code Written by User)

Ref: aws/aws-toolkit-vscode#5991

Calculate and report the user written code for each language by listening to document change events while Q is not making changes to the editor.

We add flags to know whether Q is making temporary changes for suggestion rendering or Q suggestion is accepted, by doing so, the document change events are coming from the user.

We ignore certain document changes when their length of new characters exceeds 50. Previous data driven research has shown that user tend to copy a huge file from one place to another, making the user written code count skyrocketing but that is actually some existing code not written by the user.

We plan to first collect data from IDEs and let it run in the background in shadow mode before we finish the service side aggregation, fix possible bugs and eventually present the AI code written % to the customers.

Author: leigaol

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeScan/controller/CodeScanChatHelper.kt

@@ -11,6 +11,8 @@ import software.aws.toolkits.jetbrains.services.amazonqCodeScan.messages.CodeSca
 import software.aws.toolkits.jetbrains.services.amazonqCodeScan.messages.CodeScanChatMessageContent
 import software.aws.toolkits.jetbrains.services.amazonqCodeScan.messages.UpdatePlaceholderMessage
 import software.aws.toolkits.jetbrains.services.amazonqCodeScan.storage.ChatSessionStorage
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
 import software.aws.toolkits.jetbrains.services.cwc.messages.ChatMessageType
 import java.util.UUID
 
@@ -34,7 +36,7 @@ class CodeScanChatHelper(
         clearPreviousItemButtons: Boolean? = false,
     ) {
         if (isInValidSession()) return
-
+        broadcastQEvent(QFeatureEvent.INVOCATION)
         messagePublisher.publish(
             CodeScanChatMessage(
                 tabId = activeCodeScanTabId as String,

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/controller/CodeTestChatController.kt

@@ -71,6 +71,8 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.messages.sendA
 import software.aws.toolkits.jetbrains.services.codewhisperer.credentials.CodeWhispererClientAdaptor
 import software.aws.toolkits.jetbrains.services.codewhisperer.language.CodeWhispererProgrammingLanguage
 import software.aws.toolkits.jetbrains.services.codewhisperer.language.programmingLanguage
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
 import software.aws.toolkits.jetbrains.services.codewhisperer.toolwindow.CodeWhispererCodeReferenceManager
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.isWithin
 import software.aws.toolkits.jetbrains.services.cwc.ChatConstants
@@ -1205,6 +1207,7 @@ class CodeTestChatController(
                 "Processing message: $message " +
                 "tabId: $tabId"
         }
+        broadcastQEvent(QFeatureEvent.INVOCATION)
         when (session.conversationState) {
             ConversationState.WAITING_FOR_BUILD_COMMAND_INPUT -> handleBuildCommandInput(session, message)
             ConversationState.WAITING_FOR_REGENERATE_INPUT -> handleRegenerateInput(session, message)

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqDoc/controller/DocController.kt

@@ -70,6 +70,8 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.Delete
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.NewFileZipInfo
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.SessionStatePhase
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.util.CancellationTokenSource
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
 import software.aws.toolkits.resources.message
 import java.nio.file.Paths
 import java.util.UUID
@@ -718,6 +720,7 @@ class DocController(
                 is PrepareDocGenerationState -> state.filePaths
                 else -> emptyList()
             }
+            broadcastQEvent(QFeatureEvent.INVOCATION)
 
             if (filePaths.isNotEmpty()) {
                 processOpenDiff(

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqFeatureDev/controller/FeatureDevController.kt

@@ -71,6 +71,8 @@ import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.session.Sessio
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.storage.ChatSessionStorage
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.util.InsertAction
 import software.aws.toolkits.jetbrains.services.amazonqFeatureDev.util.getFollowUpOptions
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.content
 import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.FeedbackComment
 import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.getStartUrl
@@ -191,6 +193,7 @@ class FeatureDevController(
         logger.debug { "$FEATURE_NAME: Processing InsertCodeAtCursorPosition: $message" }
 
         withContext(EDT) {
+            broadcastQEvent(QFeatureEvent.STARTS_EDITING)
             val editor: Editor = FileEditorManager.getInstance(context.project).selectedTextEditor ?: return@withContext
 
             val caret: Caret = editor.caretModel.primaryCaret
@@ -202,6 +205,7 @@ class FeatureDevController(
                 }
                 editor.document.insertString(offset, message.code)
             }
+            broadcastQEvent(QFeatureEvent.FINISHES_EDITING)
         }
     }
 
@@ -679,7 +683,7 @@ class FeatureDevController(
             }
 
             session.preloader(message, messenger)
-
+            broadcastQEvent(QFeatureEvent.INVOCATION)
             when (session.sessionState.phase) {
                 SessionStatePhase.CODEGEN -> onCodeGeneration(session, message, tabId)
                 else -> null

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/ChatController.kt

@@ -49,6 +49,8 @@ import software.aws.toolkits.jetbrains.services.amazonq.project.ProjectContextCo
 import software.aws.toolkits.jetbrains.services.amazonq.project.RelevantDocument
 import software.aws.toolkits.jetbrains.services.codewhisperer.settings.CodeWhispererConfigurable
 import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererUserModificationTracker
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
 import software.aws.toolkits.jetbrains.services.cwc.InboundAppMessagesHandler
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.exceptions.ChatApiException
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatRequestData
@@ -215,6 +217,7 @@ class ChatController private constructor(
     }
 
     override suspend fun processInsertCodeAtCursorPosition(message: IncomingCwcMessage.InsertCodeAtCursorPosition) {
+        broadcastQEvent(QFeatureEvent.STARTS_EDITING)
         withContext(EDT) {
             val editor: Editor = FileEditorManager.getInstance(context.project).selectedTextEditor ?: return@withContext
 
@@ -245,6 +248,8 @@ class ChatController private constructor(
             }
         }
         telemetryHelper.recordInteractWithMessage(message)
+
+        broadcastQEvent(QFeatureEvent.FINISHES_EDITING)
     }
 
     override suspend fun processStopResponseMessage(message: IncomingCwcMessage.StopResponse) {
@@ -438,7 +443,7 @@ class ChatController private constructor(
         sessionInfo.history.add(requestData)
         telemetryHelper.recordEnterFocusConversation(tabId)
         telemetryHelper.recordStartConversation(tabId, requestData)
-
+        broadcastQEvent(QFeatureEvent.INVOCATION)
         // Send the request to the API and publish the responses back to the UI.
         // This is launched in a scope attached to the sessionInfo so that the Job can be cancelled on a per-session basis.
         ChatPromptHandler(telemetryHelper).handle(tabId, triggerId, requestData, sessionInfo, shouldAddIndexInProgressMessage)

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/chat/messenger/ChatPromptHandler.kt

@@ -13,6 +13,8 @@ import software.amazon.awssdk.awscore.exception.AwsServiceException
 import software.amazon.awssdk.services.codewhispererstreaming.model.CodeWhispererStreamingException
 import software.aws.toolkits.core.utils.convertMarkdownToHTML
 import software.aws.toolkits.core.utils.extractCodeBlockLanguage
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.exceptions.ChatApiException
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatRequestData
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatResponseEvent
@@ -115,6 +117,8 @@ class ChatPromptHandler(private val telemetryHelper: TelemetryHelper) {
                 )
                 telemetryHelper.recordAddMessage(data, response, responseText.length, statusCode, countTotalNumberOfCodeBlocks(responseText))
                 emit(response)
+
+                broadcastQEvent(QFeatureEvent.INVOCATION)
             }
             .catch { exception ->
                 val statusCode = if (exception is AwsServiceException) exception.statusCode() else 0

plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/inline/InlineChatController.kt

@@ -58,6 +58,8 @@ import software.aws.toolkits.jetbrains.services.amazonq.auth.AuthController
 import software.aws.toolkits.jetbrains.services.amazonq.toolwindow.AMAZON_Q_WINDOW_ID
 import software.aws.toolkits.jetbrains.services.codewhisperer.customization.CodeWhispererModelConfigurator
 import software.aws.toolkits.jetbrains.services.codewhisperer.model.CaretPosition
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatRequestData
 import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.TriggerType
 import software.aws.toolkits.jetbrains.services.cwc.controller.ReferenceLogController
@@ -191,7 +193,6 @@ class InlineChatController(
 
     private fun addPopupListeners(popup: JBPopup, editor: Editor) {
         val popupListener = object : JBPopupListener {
-
             override fun onClosed(event: LightweightWindowEvent) {
                 if (canPopupAbort.get() && event.asPopup().isDisposed) {
                     popupCancelHandler.invoke(editor)
@@ -534,6 +535,7 @@ class InlineChatController(
     private fun insertString(editor: Editor, offset: Int, text: String): RangeMarker {
         lateinit var rangeMarker: RangeMarker
 
+        broadcastQEvent(QFeatureEvent.STARTS_EDITING)
         ApplicationManager.getApplication().invokeAndWait {
             CommandProcessor.getInstance().runUndoTransparentAction {
                 WriteCommandAction.runWriteCommandAction(project) {
@@ -543,18 +545,20 @@ class InlineChatController(
                 highlightCodeWithBackgroundColor(editor, rangeMarker.startOffset, rangeMarker.endOffset, true)
             }
         }
-
+        broadcastQEvent(QFeatureEvent.FINISHES_EDITING)
         return rangeMarker
     }
 
     private fun replaceString(document: Document, start: Int, end: Int, text: String) {
+        broadcastQEvent(QFeatureEvent.STARTS_EDITING)
         ApplicationManager.getApplication().invokeAndWait {
             CommandProcessor.getInstance().runUndoTransparentAction {
                 WriteCommandAction.runWriteCommandAction(project) {
                     document.replaceString(start, end, text)
                 }
             }
         }
+        broadcastQEvent(QFeatureEvent.FINISHES_EDITING)
     }
 
     private fun highlightString(editor: Editor, start: Int, end: Int, isInsert: Boolean) {
@@ -711,6 +715,8 @@ class InlineChatController(
             canPopupAbort.set(true)
             undoChanges()
         }
+
+        broadcastQEvent(QFeatureEvent.FINISHES_EDITING)
         return errorMessage
     }
 

plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/controller/CodeTransformChatController.kt

@@ -111,6 +111,8 @@ import software.aws.toolkits.jetbrains.services.codemodernizer.utils.toVirtualFi
 import software.aws.toolkits.jetbrains.services.codemodernizer.utils.tryGetJdk
 import software.aws.toolkits.jetbrains.services.codemodernizer.utils.unzipFile
 import software.aws.toolkits.jetbrains.services.codemodernizer.utils.validateSctMetadata
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
 import software.aws.toolkits.jetbrains.services.cwc.messages.ChatMessageType
 import software.aws.toolkits.resources.message
 
@@ -136,7 +138,7 @@ class CodeTransformChatController(
         if (objective == "language upgrade" || objective == "sql conversion") {
             telemetry.submitSelection(objective)
         }
-
+        broadcastQEvent(QFeatureEvent.INVOCATION)
         when (objective) {
             "language upgrade" -> this.handleLanguageUpgrade()
             "sql conversion" -> this.handleSQLConversion()

plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/utils/CodeWhispererCodeScanIssueUtils.kt

@@ -38,6 +38,8 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.explorer.CodeWhisp
 import software.aws.toolkits.jetbrains.services.codewhisperer.language.CodeWhispererProgrammingLanguage
 import software.aws.toolkits.jetbrains.services.codewhisperer.language.programmingLanguage
 import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererTelemetryService
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
 import software.aws.toolkits.jetbrains.services.codewhisperer.toolwindow.CodeWhispererCodeReferenceManager
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants.CODE_SCAN_ISSUE_TITLE_MAX_LENGTH
@@ -331,6 +333,7 @@ fun applySuggestedFix(project: Project, issue: CodeWhispererCodeScanIssue) {
     try {
         val manager = CodeWhispererCodeReferenceManager.getInstance(issue.project)
         WriteCommandAction.runWriteCommandAction(issue.project) {
+            broadcastQEvent(QFeatureEvent.STARTS_EDITING)
             val document = FileDocumentManager.getInstance().getDocument(issue.file) ?: return@runWriteCommandAction
 
             val documentContent = document.text
@@ -343,6 +346,7 @@ fun applySuggestedFix(project: Project, issue: CodeWhispererCodeScanIssue) {
                 LOG.debug { "Original content from reference span: $originalContent" }
                 manager.addReferenceLogPanelEntry(reference = reference, null, null, originalContent.split("\n"))
             }
+            broadcastQEvent(QFeatureEvent.FINISHES_EDITING)
         }
         if (issue.suggestedFixes[0].references.isNotEmpty()) {
             manager.toolWindow?.show()

plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt

@@ -134,6 +134,8 @@ interface CodeWhispererClientAdaptor : Disposable {
         acceptedTokenCount: Long,
         totalTokenCount: Long,
         unmodifiedAcceptedTokenCount: Long?,
+        userWrittenCodeCharacterCount: Long?,
+        userWrittenCodeLineCount: Long?,
     ): SendTelemetryEventResponse
 
     fun sendUserModificationTelemetry(
@@ -481,6 +483,8 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
         acceptedTokenCount: Long,
         totalTokenCount: Long,
         unmodifiedAcceptedTokenCount: Long?,
+        userWrittenCodeCharacterCount: Long?,
+        userWrittenCodeLineCount: Long?,
     ): SendTelemetryEventResponse = bearerClient().sendTelemetryEvent { requestBuilder ->
         requestBuilder.telemetryEvent { telemetryEventBuilder ->
             telemetryEventBuilder.codeCoverageEvent {
@@ -490,6 +494,8 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
                 it.totalCharacterCount(totalTokenCount.toInt())
                 it.timestamp(Instant.now())
                 it.unmodifiedAcceptedCharacterCount(unmodifiedAcceptedTokenCount?.toInt())
+                it.userWrittenCodeCharacterCount(userWrittenCodeLineCount?.toInt())
+                it.userWrittenCodeLineCount(userWrittenCodeLineCount?.toInt())
             }
         }
         requestBuilder.optOutPreference(getTelemetryOptOutPreference())

plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/editor/CodeWhispererEditorListener.kt

@@ -15,6 +15,7 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.language.programmi
 import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererInvocationStatus
 import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererInvocationStatusNew
 import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererCodeCoverageTracker
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.UserWrittenCodeTracker
 
 class CodeWhispererEditorListener : EditorFactoryListener {
     override fun editorCreated(event: EditorFactoryEvent) {
@@ -40,6 +41,10 @@ class CodeWhispererEditorListener : EditorFactoryListener {
                         activateTrackerIfNotActive()
                         documentChanged(event)
                     }
+                    UserWrittenCodeTracker.getInstance(project).apply {
+                        activateTrackerIfNotActive()
+                        documentChanged(event)
+                    }
                 }
             },
             editor.disposable

plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/editor/CodeWhispererEditorManager.kt

@@ -15,6 +15,8 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.model.InvocationCo
 import software.aws.toolkits.jetbrains.services.codewhisperer.model.SessionContext
 import software.aws.toolkits.jetbrains.services.codewhisperer.popup.CodeWhispererPopupManager
 import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererTelemetryService
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CaretMovement
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants.PAIRED_BRACKETS
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants.PAIRED_QUOTES
@@ -44,9 +46,12 @@ class CodeWhispererEditorManager {
         val endOffsetToReplace = if (insertEndOffset != -1) insertEndOffset else primaryCaret.offset
 
         WriteCommandAction.runWriteCommandAction(project) {
+            broadcastQEvent(QFeatureEvent.STARTS_EDITING)
             document.replaceString(originalOffset, endOffsetToReplace, reformatted)
             PsiDocumentManager.getInstance(project).commitDocument(document)
             primaryCaret.moveToOffset(endOffset + detail.rightOverlap.length)
+
+            broadcastQEvent(QFeatureEvent.FINISHES_EDITING)
         }
 
         ApplicationManager.getApplication().invokeLater {

plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/editor/CodeWhispererEditorManagerNew.kt

@@ -16,6 +16,8 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.popup.CodeWhispere
 import software.aws.toolkits.jetbrains.services.codewhisperer.popup.CodeWhispererPopupManagerNew
 import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererServiceNew
 import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererTelemetryServiceNew
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.QFeatureEvent
+import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.broadcastQEvent
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CaretMovement
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants.PAIRED_BRACKETS
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants.PAIRED_QUOTES
@@ -51,9 +53,12 @@ class CodeWhispererEditorManagerNew {
         preview.detail.isAccepted = true
 
         WriteCommandAction.runWriteCommandAction(project) {
+            broadcastQEvent(QFeatureEvent.STARTS_EDITING)
             document.replaceString(originalOffset, endOffsetToReplace, reformatted)
             PsiDocumentManager.getInstance(project).commitDocument(document)
             primaryCaret.moveToOffset(endOffset + detail.rightOverlap.length)
+
+            broadcastQEvent(QFeatureEvent.FINISHES_EDITING)
         }
 
         ApplicationManager.getApplication().invokeLater {