Skip to content

test(amazonq): add more tests for /transform #6183

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Dec 9, 2024
12 changes: 11 additions & 1 deletion packages/amazonq/test/e2e/amazonq/framework/messenger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,14 @@ export class Messenger {
this.mynahUIProps.onFollowUpClicked(this.tabID, lastChatItem?.messageId ?? '', option[0])
}

clickCustomFormButton(action: { id: string; text?: string; formItemValues?: Record<string, string> }) {
if (!this.mynahUIProps.onCustomFormAction) {
assert.fail('onCustomFormAction must be defined to use it in the tests')
}

this.mynahUIProps.onCustomFormAction(this.tabID, action)
}

clickFileActionButton(filePath: string, actionName: string) {
if (!this.mynahUIProps.onFileActionClick) {
assert.fail('onFileActionClick must be defined to use it in the tests')
Expand Down Expand Up @@ -173,7 +181,9 @@ export class Messenger {

// Do another check just in case the waitUntil time'd out
if (!event()) {
assert.fail(`Event has not finished loading in: ${this.waitTimeoutInMs} ms`)
assert.fail(
`Event has not finished loading in: ${waitOverrides ? waitOverrides.waitTimeoutInMs : this.waitTimeoutInMs} ms`
)
}
}

Expand Down
270 changes: 270 additions & 0 deletions packages/amazonq/test/e2e/amazonq/transformByQ.test.ts
Copy link
Contributor Author

@dhasani23 dhasani23 Dec 8, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

/runIntegrationTests to trigger them

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

They are also run automatically after merging.

Original file line number Diff line number Diff line change
@@ -0,0 +1,270 @@
/*!
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/

import assert from 'assert'
import { qTestingFramework } from './framework/framework'
import sinon from 'sinon'
import { Messenger } from './framework/messenger'
import { JDKVersion } from 'aws-core-vscode/codewhisperer'
import { GumbyController, startTransformByQ, TabsStorage } from 'aws-core-vscode/amazonqGumby'

describe('Amazon Q Code Transformation', function () {
let framework: qTestingFramework
let tab: Messenger

beforeEach(() => {
framework = new qTestingFramework('gumby', true, [])
tab = framework.createTab()
})

afterEach(() => {
framework.removeTab(tab.tabID)
framework.dispose()
sinon.restore()
})

describe('Quick action availability', () => {
it('Shows /transform when QCT is enabled', async () => {
const command = tab.findCommand('/transform')
if (!command) {
assert.fail('Could not find command')
}

if (command.length > 1) {
assert.fail('Found too many commands with the name /transform')
}
})

it('Does NOT show /transform when QCT is NOT enabled', () => {
framework.dispose()
framework = new qTestingFramework('gumby', false, [])
const tab = framework.createTab()
const command = tab.findCommand('/transform')
if (command.length > 0) {
assert.fail('Found command when it should not have been found')
}
})
})

describe('Starting a transformation from chat', () => {
it('Can click through all user input forms for a Java upgrade', async () => {
sinon.stub(startTransformByQ, 'getValidSQLConversionCandidateProjects').resolves([])
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I stub some validation-related logic in this test file because 1) we already have unit test coverage of those, and 2) these tests are just as useful even with mocked validation functions.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, but when things change in the implementation (e.g. using Flare), stubs/mocks will break easily. Thanks for calling this out in any case.

sinon.stub(GumbyController.prototype, 'validateLanguageUpgradeProjects' as keyof GumbyController).resolves([
{
name: 'qct-sample-java-8-app-main',
path: '/Users/alias/Desktop/qct-sample-java-8-app-main',
JDKVersion: JDKVersion.JDK8,
},
])

tab.addChatMessage({ command: '/transform' })

// wait for /transform to respond with some intro messages and the first user input form
await tab.waitForEvent(() => tab.getChatItems().length > 3, {
waitTimeoutInMs: 5000,
waitIntervalInMs: 1000,
})
const projectForm = tab.getChatItems().pop()
assert.strictEqual(projectForm?.formItems?.[0]?.id ?? undefined, 'GumbyTransformLanguageUpgradeProjectForm')

const projectFormItemValues = {
GumbyTransformLanguageUpgradeProjectForm: '/Users/alias/Desktop/qct-sample-java-8-app-main',
GumbyTransformJdkFromForm: '8',
GumbyTransformJdkToForm: '17',
}
const projectFormValues: Record<string, string> = { ...projectFormItemValues }
// TODO: instead of stubbing, can we create a tab in qTestingFramework with tabType passed in?
// Mynah-UI updates tab type like this: this.tabsStorage.updateTabTypeFromUnknown(affectedTabId, 'gumby')
sinon
.stub(TabsStorage.prototype, 'getTab')
.returns({ id: tab.tabID, status: 'free', type: 'gumby', isSelected: true })
tab.clickCustomFormButton({
id: 'gumbyLanguageUpgradeTransformFormConfirm',
text: 'Confirm',
formItemValues: projectFormValues,
})

// 3 additional chat messages (including message with 2nd form) get sent after 1st form submitted; wait for all of them
await tab.waitForEvent(() => tab.getChatItems().length > 6, {
waitTimeoutInMs: 5000,
waitIntervalInMs: 1000,
})
const skipTestsForm = tab.getChatItems().pop()
assert.strictEqual(skipTestsForm?.formItems?.[0]?.id ?? undefined, 'GumbyTransformSkipTestsForm')

const skipTestsFormItemValues = {
GumbyTransformSkipTestsForm: 'Run unit tests',
}
const skipTestsFormValues: Record<string, string> = { ...skipTestsFormItemValues }
tab.clickCustomFormButton({
id: 'gumbyTransformSkipTestsFormConfirm',
text: 'Confirm',
formItemValues: skipTestsFormValues,
})

// 3 additional chat messages (including message with 3rd form) get sent after 2nd form submitted; wait for all of them
await tab.waitForEvent(() => tab.getChatItems().length > 9, {
waitTimeoutInMs: 5000,
waitIntervalInMs: 1000,
})
const multipleDiffsForm = tab.getChatItems().pop()
assert.strictEqual(
multipleDiffsForm?.formItems?.[0]?.id ?? undefined,
'GumbyTransformOneOrMultipleDiffsForm'
)

const oneOrMultipleDiffsFormItemValues = {
GumbyTransformOneOrMultipleDiffsForm: 'One diff',
}
const oneOrMultipleDiffsFormValues: Record<string, string> = { ...oneOrMultipleDiffsFormItemValues }
tab.clickCustomFormButton({
id: 'gumbyTransformOneOrMultipleDiffsFormConfirm',
text: 'Confirm',
formItemValues: oneOrMultipleDiffsFormValues,
})

// 2 additional chat messages (including message with 4th form) get sent after 3rd form submitted; wait for both of them
await tab.waitForEvent(() => tab.getChatItems().length > 11, {
waitTimeoutInMs: 5000,
waitIntervalInMs: 1000,
})
const jdkPathPrompt = tab.getChatItems().pop()
assert.strictEqual(jdkPathPrompt?.body?.includes('Enter the path to JDK'), true)

// 2 additional chat messages get sent after 4th form submitted; wait for both of them
tab.addChatMessage({ prompt: '/dummy/path/to/jdk8' })
await tab.waitForEvent(() => tab.getChatItems().length > 13, {
waitTimeoutInMs: 5000,
waitIntervalInMs: 1000,
})
const jdkPathResponse = tab.getChatItems().pop()
// this 'Sorry' message is OK - just making sure that the UI components are working correctly
assert.strictEqual(jdkPathResponse?.body?.includes("Sorry, I couldn't locate your Java installation"), true)
})

it('Can provide metadata file for a SQL conversion', async () => {
sinon.stub(startTransformByQ, 'getValidSQLConversionCandidateProjects').resolves([
{
name: 'OracleExample',
path: '/Users/alias/Desktop/OracleExample',
JDKVersion: JDKVersion.JDK17,
},
])
sinon.stub(startTransformByQ, 'getValidLanguageUpgradeCandidateProjects').resolves([])
sinon.stub(GumbyController.prototype, 'validateSQLConversionProjects' as keyof GumbyController).resolves([
{
name: 'OracleExample',
path: '/Users/alias/Desktop/OracleExample',
JDKVersion: JDKVersion.JDK17,
},
])

tab.addChatMessage({ command: '/transform' })

// wait for /transform to respond with some intro messages and the first user input message
await tab.waitForEvent(() => tab.getChatItems().length > 3, {
waitTimeoutInMs: 5000,
waitIntervalInMs: 1000,
})
const selectMetadataMessage = tab.getChatItems().pop()
assert.strictEqual(
selectMetadataMessage?.body?.includes('I can convert the embedded SQL') ?? undefined,
true
)

// verify that we processed the metadata file
const processMetadataFileStub = sinon.stub(
GumbyController.prototype,
'processMetadataFile' as keyof GumbyController
)
tab.clickCustomFormButton({
id: 'gumbySQLConversionMetadataTransformFormConfirm',
text: 'Select metadata file',
})
sinon.assert.calledOnce(processMetadataFileStub)
})

it('Can choose "language upgrade" when eligible for a Java upgrade AND SQL conversion', async () => {
sinon.stub(startTransformByQ, 'getValidSQLConversionCandidateProjects').resolves([
{
name: 'OracleExample',
path: '/Users/alias/Desktop/OracleExample',
JDKVersion: JDKVersion.JDK17,
},
])
sinon.stub(startTransformByQ, 'getValidLanguageUpgradeCandidateProjects').resolves([
{
name: 'qct-sample-java-8-app-main',
path: '/Users/alias/Desktop/qct-sample-java-8-app-main',
JDKVersion: JDKVersion.JDK8,
},
])

tab.addChatMessage({ command: '/transform' })

// wait for /transform to respond with some intro messages and a prompt asking user what they want to do
await tab.waitForEvent(() => tab.getChatItems().length > 2, {
waitTimeoutInMs: 5000,
waitIntervalInMs: 1000,
})
const prompt = tab.getChatItems().pop()
assert.strictEqual(
prompt?.body?.includes('You can enter "language upgrade" or "sql conversion"') ?? undefined,
true
)

// 3 additional chat messages get sent after user enters a choice; wait for all of them
tab.addChatMessage({ prompt: 'language upgrade' })
await tab.waitForEvent(() => tab.getChatItems().length > 5, {
waitTimeoutInMs: 5000,
waitIntervalInMs: 1000,
})
const projectForm = tab.getChatItems().pop()
assert.strictEqual(projectForm?.formItems?.[0]?.id ?? undefined, 'GumbyTransformLanguageUpgradeProjectForm')
})

it('Can choose "sql conversion" when eligible for a Java upgrade AND SQL conversion', async () => {
sinon.stub(startTransformByQ, 'getValidSQLConversionCandidateProjects').resolves([
{
name: 'OracleExample',
path: '/Users/alias/Desktop/OracleExample',
JDKVersion: JDKVersion.JDK17,
},
])
sinon.stub(startTransformByQ, 'getValidLanguageUpgradeCandidateProjects').resolves([
{
name: 'qct-sample-java-8-app-main',
path: '/Users/alias/Desktop/qct-sample-java-8-app-main',
JDKVersion: JDKVersion.JDK8,
},
])

tab.addChatMessage({ command: '/transform' })

// wait for /transform to respond with some intro messages and a prompt asking user what they want to do
await tab.waitForEvent(() => tab.getChatItems().length > 2, {
waitTimeoutInMs: 5000,
waitIntervalInMs: 1000,
})
const prompt = tab.getChatItems().pop()
assert.strictEqual(
prompt?.body?.includes('You can enter "language upgrade" or "sql conversion"') ?? undefined,
true
)

// 3 additional chat messages get sent after user enters a choice; wait for all of them
tab.addChatMessage({ prompt: 'sql conversion' })
await tab.waitForEvent(() => tab.getChatItems().length > 5, {
waitTimeoutInMs: 5000,
waitIntervalInMs: 1000,
})
const selectMetadataMessage = tab.getChatItems().pop()
assert.strictEqual(
selectMetadataMessage?.body?.includes('I can convert the embedded SQL') ?? undefined,
true
)
})
})
})
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,12 @@ import {
} from '../../../shared/telemetry/telemetry'
import { MetadataResult } from '../../../shared/telemetry/telemetryClient'
import { CodeTransformTelemetryState } from '../../telemetry/codeTransformTelemetryState'
import { getAuthType } from '../../../codewhisperer/service/transformByQ/transformApiHandler'
import DependencyVersions from '../../models/dependencies'
import { getStringHash } from '../../../shared/utilities/textUtilities'
import { getVersionData } from '../../../codewhisperer/service/transformByQ/transformMavenHandler'
import AdmZip from 'adm-zip'
import { AuthError } from '../../../auth/sso/server'
import { getAuthType } from '../../../auth/utils'

// These events can be interactions within the chat,
// or elsewhere in the IDE
Expand Down
3 changes: 3 additions & 0 deletions packages/core/src/amazonqGumby/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,7 @@
export { activate } from './activation'
export { default as DependencyVersions } from './models/dependencies'
export { default as MessengerUtils } from './chat/controller/messenger/messengerUtils'
export { GumbyController } from './chat/controller/controller'
export { TabsStorage } from '../amazonq/webview/ui/storages/tabsStorage'
export * as startTransformByQ from '../../src/codewhisperer/commands/startTransformByQ'
Copy link
Contributor Author

@dhasani23 dhasani23 Dec 9, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm unable to run the tests in packages/amazonq/test/e2e/amazonq/transformByQ.test.ts by importing these things from the core module directly where they're defined, as I get an import error when doing that. So instead I have to export them here and then (in the test file, line 11) import them from here.

export * from './errors'
14 changes: 12 additions & 2 deletions packages/core/src/auth/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import { formatError, ToolkitError } from '../shared/errors'
import { asString } from './providers/credentials'
import { TreeNode } from '../shared/treeview/resourceTreeDataProvider'
import { createInputBox } from '../shared/ui/inputPrompter'
import { telemetry } from '../shared/telemetry/telemetry'
import { CredentialSourceId, telemetry } from '../shared/telemetry/telemetry'
import { createCommonButtons, createExitButton, createHelpButton, createRefreshButton } from '../shared/ui/buttons'
import { getIdeProperties, isAmazonQ, isCloud9 } from '../shared/extensionUtilities'
import { addScopes, getDependentAuths } from './secondaryAuth'
Expand All @@ -45,7 +45,7 @@ import { Commands, placeholder } from '../shared/vscode/commands2'
import { Auth } from './auth'
import { validateIsNewSsoUrl, validateSsoUrlFormat } from './sso/validation'
import { getLogger } from '../shared/logger'
import { isValidAmazonQConnection, isValidCodeWhispererCoreConnection } from '../codewhisperer/util/authUtil'
import { AuthUtil, isValidAmazonQConnection, isValidCodeWhispererCoreConnection } from '../codewhisperer/util/authUtil'
import { AuthFormId } from '../login/webview/vue/types'
import { extensionVersion } from '../shared/vscode/env'
import { ExtStartUpSources } from '../shared/telemetry'
Expand Down Expand Up @@ -798,3 +798,13 @@ export function initializeCredentialsProviderManager() {
manager.addProviderFactory(new SharedCredentialsProviderFactory())
manager.addProviders(new Ec2CredentialsProvider(), new EcsCredentialsProvider(), new EnvVarsCredentialsProvider())
}

export async function getAuthType() {
let authType: CredentialSourceId | undefined = undefined
if (AuthUtil.instance.isEnterpriseSsoInUse() && AuthUtil.instance.isConnectionValid()) {
authType = 'iamIdentityCenter'
} else if (AuthUtil.instance.isBuilderIdInUse() && AuthUtil.instance.isConnectionValid()) {
authType = 'awsId'
}
return authType
}
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This above function was previously in transformApiHandler.ts, just moving it here so it's in a shared utils file

Original file line number Diff line number Diff line change
Expand Up @@ -34,38 +34,28 @@ import {
import { sleep } from '../../../shared/utilities/timeoutUtils'
import AdmZip from 'adm-zip'
import globals from '../../../shared/extensionGlobals'
import { CredentialSourceId, telemetry } from '../../../shared/telemetry/telemetry'
import { telemetry } from '../../../shared/telemetry/telemetry'
import { CodeTransformTelemetryState } from '../../../amazonqGumby/telemetry/codeTransformTelemetryState'
import { calculateTotalLatency } from '../../../amazonqGumby/telemetry/codeTransformTelemetry'
import { MetadataResult } from '../../../shared/telemetry/telemetryClient'
import request from '../../../shared/request'
import { JobStoppedError, ZipExceedsSizeLimitError } from '../../../amazonqGumby/errors'
import { writeLogs } from './transformFileHandler'
import { AuthUtil } from '../../util/authUtil'
import { createCodeWhispererChatStreamingClient } from '../../../shared/clients/codewhispererChatClient'
import { downloadExportResultArchive } from '../../../shared/utilities/download'
import { ExportIntent, TransformationDownloadArtifactType } from '@amzn/codewhisperer-streaming'
import fs from '../../../shared/fs/fs'
import { ChatSessionManager } from '../../../amazonqGumby/chat/storages/chatSession'
import { encodeHTML } from '../../../shared/utilities/textUtilities'
import { convertToTimeString } from '../../../shared/datetime'
import { getAuthType } from '../../../auth/utils'

export function getSha256(buffer: Buffer) {
const hasher = crypto.createHash('sha256')
hasher.update(buffer)
return hasher.digest('base64')
}

export async function getAuthType() {
let authType: CredentialSourceId | undefined = undefined
if (AuthUtil.instance.isEnterpriseSsoInUse() && AuthUtil.instance.isConnectionValid()) {
authType = 'iamIdentityCenter'
} else if (AuthUtil.instance.isBuilderIdInUse() && AuthUtil.instance.isConnectionValid()) {
authType = 'awsId'
}
return authType
}

export function throwIfCancelled() {
if (transformByQState.isCancelled()) {
throw new TransformByQStoppedError()
Expand Down
Loading
Loading