Skip to content

Commit 076513a

Browse files
authored
test(amazon): add more debugging information to inline tests (#6644)
## Problem - we have no way to get any information on why an inline test was flaky. It might be because telemetry events are being out of order? - Since fib doesn't have spaces in the contents it's autocompleting on the very first line which may be causing the backend to return no results ## Solution - add more debugging information to understand why there was a failure - add spaces to the test fib function - consider a suggestion shown if the recommendation handler says its visible or if its state has been declared as 'Shown' --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license.
1 parent 82b016f commit 076513a

File tree

1 file changed

+32
-24
lines changed

1 file changed

+32
-24
lines changed

packages/amazonq/test/e2e/inline/inline.test.ts

Lines changed: 32 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import * as vscode from 'vscode'
77
import assert from 'assert'
88
import {
9-
assertTelemetry,
109
closeAllEditors,
1110
getTestWindow,
1211
registerAuthHook,
@@ -15,7 +14,7 @@ import {
1514
toTextEditor,
1615
using,
1716
} from 'aws-core-vscode/test'
18-
import { RecommendationHandler, RecommendationService } from 'aws-core-vscode/codewhisperer'
17+
import { RecommendationHandler, RecommendationService, session } from 'aws-core-vscode/codewhisperer'
1918
import { Commands, globals, sleep, waitUntil } from 'aws-core-vscode/shared'
2019
import { loginToIdC } from '../amazonq/utils/setup'
2120

@@ -50,31 +49,47 @@ describe('Amazon Q Inline', async function () {
5049
const textContents =
5150
contents ??
5251
`function fib() {
53-
54-
52+
53+
5554
}`
5655
await toTextEditor(textContents, fileName, tempFolder, {
5756
selection: new vscode.Range(new vscode.Position(1, 4), new vscode.Position(1, 4)),
5857
})
5958
}
6059

6160
async function waitForRecommendations() {
62-
const ok = await waitUntil(async () => RecommendationHandler.instance.isSuggestionVisible(), waitOptions)
63-
if (!ok) {
64-
assert.fail('Suggestions failed to become visible')
65-
}
66-
}
67-
68-
async function waitForTelemetry() {
6961
const ok = await waitUntil(
7062
async () =>
71-
globals.telemetry.logger.query({
72-
metricName: 'codewhisperer_userTriggerDecision',
73-
}).length > 0,
63+
RecommendationHandler.instance.isSuggestionVisible() || session.getSuggestionState(0) === 'Showed',
7464
waitOptions
7565
)
7666
if (!ok) {
77-
assert.fail('Telemetry failed to be emitted')
67+
assert.fail(
68+
`Suggestions failed to become visible. Suggestion States: ${JSON.stringify(session.suggestionStates)}`
69+
)
70+
}
71+
}
72+
73+
/**
74+
* Waits for a specific telemetry event to be emitted with the expected suggestion state.
75+
* It looks like there might be a potential race condition in codewhisperer causing telemetry
76+
* events to be emitted in different orders
77+
*/
78+
async function waitForTelemetry(metricName: string, suggestionState: string) {
79+
const ok = await waitUntil(async () => {
80+
const events = globals.telemetry.logger.query({
81+
metricName,
82+
})
83+
return events.some((event) => event.codewhispererSuggestionState === suggestionState)
84+
}, waitOptions)
85+
const events = globals.telemetry.logger.query({
86+
metricName,
87+
})
88+
if (!ok) {
89+
assert.fail(`Telemetry failed to be emitted. Current events: ${JSON.stringify(events)}`)
90+
}
91+
if (events.length > 1 && events[events.length - 1].codewhispererSuggestionState !== suggestionState) {
92+
assert.fail(`Telemetry events were emitted in the wrong order. Current events: ${JSON.stringify(events)}`)
7893
}
7994
}
8095

@@ -119,10 +134,7 @@ describe('Amazon Q Inline', async function () {
119134

120135
assert.ok(suggestionAccepted, 'Editor contents should have changed')
121136

122-
await waitForTelemetry()
123-
assertTelemetry('codewhisperer_userTriggerDecision', {
124-
codewhispererSuggestionState: 'Accept',
125-
})
137+
await waitForTelemetry('codewhisperer_userTriggerDecision', 'Accept')
126138
})
127139

128140
it(`${name} invoke reject`, async function () {
@@ -131,11 +143,7 @@ describe('Amazon Q Inline', async function () {
131143

132144
// Contents haven't changed
133145
assert.deepStrictEqual(vscode.window.activeTextEditor?.document.getText(), originalEditorContents)
134-
135-
await waitForTelemetry()
136-
assertTelemetry('codewhisperer_userTriggerDecision', {
137-
codewhispererSuggestionState: 'Reject',
138-
})
146+
await waitForTelemetry('codewhisperer_userTriggerDecision', 'Reject')
139147
})
140148

141149
it(`${name} invoke discard`, async function () {

0 commit comments

Comments
 (0)