diff --git a/.changes/next-release/bugfix-d2a9278c-d71f-463f-b899-f56a587829e8.json b/.changes/next-release/bugfix-d2a9278c-d71f-463f-b899-f56a587829e8.json new file mode 100644 index 00000000000..6356b23fa2c --- /dev/null +++ b/.changes/next-release/bugfix-d2a9278c-d71f-463f-b899-f56a587829e8.json @@ -0,0 +1,4 @@ +{ + "type" : "bugfix", + "description" : "Fix integer overflow when local context index input is larger than 2GB" +} \ No newline at end of file diff --git a/.changes/next-release/bugfix-de10b7d3-8c8a-4968-88cb-ed28ce3a553e.json b/.changes/next-release/bugfix-de10b7d3-8c8a-4968-88cb-ed28ce3a553e.json new file mode 100644 index 00000000000..1c00b0e3f21 --- /dev/null +++ b/.changes/next-release/bugfix-de10b7d3-8c8a-4968-88cb-ed28ce3a553e.json @@ -0,0 +1,4 @@ +{ + "type" : "bugfix", + "description" : "Fix workspace index process quits when hitting a race condition" +} \ No newline at end of file diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/settings/CodeWhispererConfigurable.kt b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/settings/CodeWhispererConfigurable.kt index 6b9d6207cb7..77f901950c7 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/settings/CodeWhispererConfigurable.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/settings/CodeWhispererConfigurable.kt @@ -169,7 +169,7 @@ class CodeWhispererConfigurable(private val project: Project) : row(message("aws.settings.codewhisperer.project_context_index_thread")) { intTextField( - range = IntRange(0, 50) + range = CodeWhispererSettings.CONTEXT_INDEX_THREADS ).bindIntText(codeWhispererSettings::getProjectContextIndexThreadCount, codeWhispererSettings::setProjectContextIndexThreadCount) .apply { connect.subscribe( @@ -186,7 +186,7 @@ class CodeWhispererConfigurable(private val project: Project) : row(message("aws.settings.codewhisperer.project_context_index_max_size")) { intTextField( - range = IntRange(1, 4096) + range = CodeWhispererSettings.CONTEXT_INDEX_SIZE ).bindIntText(codeWhispererSettings::getProjectContextIndexMaxSize, codeWhispererSettings::setProjectContextIndexMaxSize) .apply { connect.subscribe( diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererSettingsTest.kt b/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererSettingsTest.kt index 982266f07bd..84263c62ad2 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererSettingsTest.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererSettingsTest.kt @@ -220,6 +220,41 @@ class CodeWhispererSettingsTest : CodeWhispererTestBase() { assertThat(actual.autoBuildSetting["project1"]).isTrue() } + @Test + fun `context thread count is returned in range`() { + val sut = CodeWhispererSettings.getInstance() + + mapOf( + 1 to 1, + 0 to 0, + -1 to 0, + 123 to 50, + 50 to 50, + 51 to 50, + ).forEach { s, expected -> + sut.setProjectContextIndexThreadCount(s) + assertThat(sut.getProjectContextIndexThreadCount()).isEqualTo(expected) + } + } + + @Test + fun `context index size is returned in range`() { + val sut = CodeWhispererSettings.getInstance() + + mapOf( + 1 to 1, + 0 to 1, + -1 to 1, + 123 to 123, + 2047 to 2047, + 4096 to 4096, + 4097 to 4096, + ).forEach { s, expected -> + sut.setProjectContextIndexMaxSize(s) + assertThat(sut.getProjectContextIndexMaxSize()).isEqualTo(expected) + } + } + @Test fun `toggleMetricOptIn should trigger LSP didChangeConfiguration`() { mockkObject(AmazonQLspService) diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/ProjectContextProvider.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/ProjectContextProvider.kt index 75ab0df4f46..854124a180f 100644 --- a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/ProjectContextProvider.kt +++ b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/ProjectContextProvider.kt @@ -70,7 +70,7 @@ class ProjectContextProvider(val project: Project, private val encoderServer: En data class FileCollectionResult( val files: List, - val fileSize: Int, + val fileSize: Int, // in MB ) // TODO: move to LspMessage.kt @@ -246,59 +246,7 @@ class ProjectContextProvider(val project: Project, private val encoderServer: En } } - private fun willExceedPayloadLimit(currentTotalFileSize: Long, currentFileSize: Long): Boolean { - val maxSize = CodeWhispererSettings.getInstance().getProjectContextIndexMaxSize() - return currentTotalFileSize.let { totalSize -> totalSize > (maxSize * 1024 * 1024 - currentFileSize) } - } - - private fun isBuildOrBin(fileName: String): Boolean { - val regex = Regex("""bin|build|node_modules|venv|\.venv|env|\.idea|\.conda""", RegexOption.IGNORE_CASE) - return regex.find(fileName) != null - } - - fun collectFiles(): FileCollectionResult { - val collectedFiles = mutableListOf() - var currentTotalFileSize = 0L - val allFiles = mutableListOf() - - val projectBaseDirectories = project.getBaseDirectories() - val changeListManager = ChangeListManager.getInstance(project) - - projectBaseDirectories.forEach { - VfsUtilCore.visitChildrenRecursively( - it, - object : VirtualFileVisitor(NO_FOLLOW_SYMLINKS) { - // TODO: refactor this along with /dev & codescan file traversing logic - override fun visitFile(file: VirtualFile): Boolean { - if ((file.isDirectory && isBuildOrBin(file.name)) || - !isWorkspaceSourceContent(file, projectBaseDirectories, changeListManager, additionalGlobalIgnoreRulesForStrictSources) || - (file.isFile && file.length > 10 * 1024 * 1024) - ) { - return false - } - if (file.isFile) { - allFiles.add(file) - return false - } - return true - } - } - ) - } - - for (file in allFiles) { - if (willExceedPayloadLimit(currentTotalFileSize, file.length)) { - break - } - collectedFiles.add(file.path) - currentTotalFileSize += file.length - } - - return FileCollectionResult( - files = collectedFiles.toList(), - fileSize = (currentTotalFileSize / 1024 / 1024).toInt() - ) - } + fun collectFiles(): FileCollectionResult = collectFiles(project.getBaseDirectories(), ChangeListManager.getInstance(project)) private fun queryResultToRelevantDocuments(queryResult: List): List { val documents: MutableList = mutableListOf() @@ -358,5 +306,57 @@ class ProjectContextProvider(val project: Project, private val encoderServer: En companion object { private val logger = getLogger() + private val regex = Regex("""bin|build|node_modules|venv|\.venv|env|\.idea|\.conda""", RegexOption.IGNORE_CASE) + private val mega = (1024 * 1024).toULong() + private val tenMb = 10 * mega.toInt() + + private fun willExceedPayloadLimit(maxSize: ULong, currentTotalFileSize: ULong, currentFileSize: Long) = + currentTotalFileSize.let { totalSize -> totalSize > (maxSize - currentFileSize.toUInt()) } + + private fun isBuildOrBin(fileName: String): Boolean = + regex.find(fileName) != null + + fun collectFiles(projectBaseDirectories: Set, changeListManager: ChangeListManager): FileCollectionResult { + val maxSize = CodeWhispererSettings.getInstance() + .getProjectContextIndexMaxSize().toULong() * mega + val collectedFiles = mutableListOf() + var currentTotalFileSize = 0UL + val allFiles = mutableListOf() + + projectBaseDirectories.forEach { + VfsUtilCore.visitChildrenRecursively( + it, + object : VirtualFileVisitor(NO_FOLLOW_SYMLINKS) { + // TODO: refactor this along with /dev & codescan file traversing logic + override fun visitFile(file: VirtualFile): Boolean { + if ((file.isDirectory && isBuildOrBin(file.name)) || + !isWorkspaceSourceContent(file, projectBaseDirectories, changeListManager, additionalGlobalIgnoreRulesForStrictSources) || + (file.isFile && file.length > tenMb) + ) { + return false + } + if (file.isFile) { + allFiles.add(file) + return false + } + return true + } + } + ) + } + + for (file in allFiles) { + if (willExceedPayloadLimit(maxSize, currentTotalFileSize, file.length)) { + break + } + collectedFiles.add(file.path) + currentTotalFileSize += file.length.toUInt() + } + + return FileCollectionResult( + files = collectedFiles.toList(), + fileSize = (currentTotalFileSize / 1024u / 1024u).toInt() + ) + } } } diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/manifest/ManifestManager.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/manifest/ManifestManager.kt index dca4d460816..484c319db52 100644 --- a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/manifest/ManifestManager.kt +++ b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/manifest/ManifestManager.kt @@ -15,7 +15,7 @@ import software.aws.toolkits.jetbrains.core.getTextFromUrl class ManifestManager { private val cloudFrontUrl = "https://aws-toolkit-language-servers.amazonaws.com/q-context/manifest.json" - val currentVersion = "0.1.46" + val currentVersion = "0.1.49" val currentOs = getOs() private val arch = CpuArch.CURRENT private val mapper = jacksonObjectMapper().apply { configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) } diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/settings/CodeWhispererSettings.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/settings/CodeWhispererSettings.kt index 7f9e443c14f..2c5cac62e57 100644 --- a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/settings/CodeWhispererSettings.kt +++ b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/settings/CodeWhispererSettings.kt @@ -119,7 +119,7 @@ class CodeWhispererSettings : PersistentStateComponent