Make /dev non-blocking UI and reduce ~70% time on large repo (#4415)

The file exclusion logic incorrectly traverses ignored directories which significantly increases collection time on projects with large artifacts. Fix by using a visitor that prunes traversal on ignored directories

Author: andrewyuq

.changes/next-release/bugfix-1c15c377-1231-405a-9047-176e2f6943ce.json

@@ -0,0 +1,4 @@
+{
+  "type" : "bugfix",
+  "description" : "Amazon Q: Fix an issue where /dev usage would cause the UI to freeze and take an unusually long time to complete. (#4269)"
+}

plugins/core/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt

@@ -3,24 +3,38 @@
 
 package software.aws.toolkits.jetbrains.services.amazonq
 
-import com.intellij.openapi.application.runReadAction
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.project.guessProjectDir
 import com.intellij.openapi.vfs.VfsUtil
 import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.openapi.vfs.VirtualFileVisitor
+import com.intellij.openapi.vfs.isFile
+import com.intellij.platform.ide.progress.withBackgroundProgress
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.async
+import kotlinx.coroutines.flow.channelFlow
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withContext
 import org.apache.commons.codec.digest.DigestUtils
-import software.aws.toolkits.core.utils.createTemporaryZipFile
+import software.aws.toolkits.core.utils.outputStream
 import software.aws.toolkits.core.utils.putNextEntry
+import software.aws.toolkits.jetbrains.core.coroutines.EDT
+import software.aws.toolkits.jetbrains.core.coroutines.getCoroutineBgContext
+import software.aws.toolkits.resources.message
 import java.io.File
 import java.io.FileInputStream
+import java.nio.file.Files
+import java.nio.file.Path
 import java.util.Base64
+import java.util.zip.ZipOutputStream
 import kotlin.io.path.Path
 import kotlin.io.path.relativeTo
 
 class FeatureDevSessionContext(val project: Project) {
     // TODO: Need to correct this class location in the modules going further to support both amazonq and codescan.
 
-    private val ignorePatterns = listOf(
+    private val ignorePatterns = setOf(
         "\\.aws-sam",
         "\\.svn",
         "\\.hg/",
@@ -42,47 +56,94 @@ class FeatureDevSessionContext(val project: Project) {
         "/license\\.md$",
         "/License\\.md$",
         "/LICENSE\\.md$",
+        "node_modules/",
+        "build/",
+        "dist/"
     ).map { Regex(it) }
 
     private var _projectRoot = project.guessProjectDir() ?: error("Cannot guess base directory for project ${project.name}")
     private var ignorePatternsWithGitIgnore = emptyList<Regex>()
     private val gitIgnoreFile = File(projectRoot.path, ".gitignore")
 
     init {
-        ignorePatternsWithGitIgnore = ignorePatterns + parseGitIgnore().map { Regex(it) }
+        ignorePatternsWithGitIgnore = (ignorePatterns + parseGitIgnore().map { Regex(it) }).toList()
     }
 
     fun getProjectZip(): ZipCreationResult {
-        val zippedProject = runReadAction { zipFiles(projectRoot) }
+        val zippedProject = runBlocking {
+            withBackgroundProgress(project, message("amazonqFeatureDev.create_plan.background_progress_title")) {
+                zipFiles(projectRoot)
+            }
+        }
         val checkSum256: String = Base64.getEncoder().encodeToString(DigestUtils.sha256(FileInputStream(zippedProject)))
         return ZipCreationResult(zippedProject, checkSum256, zippedProject.length())
     }
 
-    fun ignoreFile(file: File): Boolean = try {
-        ignorePatternsWithGitIgnore.any { p -> p.containsMatchIn(file.path) }
-    } catch (e: Exception) {
-        true
+    private suspend fun ignoreFile(file: File, scope: CoroutineScope): Boolean = with(scope) {
+        val deferredResults = ignorePatternsWithGitIgnore.map { pattern ->
+            async {
+                pattern.containsMatchIn(file.path)
+            }
+        }
+        deferredResults.any { it.await() }
     }
 
-    fun ignoreFile(file: VirtualFile): Boolean = ignoreFile(File(file.path))
+    suspend fun ignoreFile(file: VirtualFile, scope: CoroutineScope): Boolean = ignoreFile(File(file.path), scope)
+
+    suspend fun zipFiles(projectRoot: VirtualFile): File = withContext(getCoroutineBgContext()) {
+        val files = mutableListOf<VirtualFile>()
+        VfsUtil.visitChildrenRecursively(
+            projectRoot,
+            object : VirtualFileVisitor<Unit>() {
+                override fun visitFile(file: VirtualFile): Boolean {
+                    if (file.isFile) {
+                        files.add(file)
+                        return true
+                    }
+                    return runBlocking {
+                        !ignoreFile(file, this)
+                    }
+                }
+            }
+        )
 
-    private fun zipFiles(projectRoot: VirtualFile): File = createTemporaryZipFile {
-        VfsUtil.collectChildrenRecursively(projectRoot).map { virtualFile -> File(virtualFile.path) }.forEach { file ->
-            if (file.isFile() && !ignoreFile(file)) {
+        // Process files in parallel
+        val filesToIncludeFlow = channelFlow {
+            // chunk with some reasonable number because we don't actually need a new job for each file
+            files.chunked(50).forEach { chunk ->
+                launch {
+                    for (file in chunk) {
+                        if (file.isFile && !ignoreFile(file, this)) {
+                            send(file)
+                        }
+                    }
+                }
+            }
+        }
+
+        createTemporaryZipFileAsync { zipOutput ->
+            filesToIncludeFlow.collect { file ->
                 val relativePath = Path(file.path).relativeTo(projectRoot.toNioPath())
-                it.putNextEntry(relativePath.toString(), Path(file.path))
+                zipOutput.putNextEntry(relativePath.toString(), Path(file.path))
             }
         }
     }.toFile()
 
-    private fun parseGitIgnore(): List<String> {
+    private suspend fun createTemporaryZipFileAsync(block: suspend (ZipOutputStream) -> Unit): Path = withContext(EDT) {
+        val file = Files.createTempFile(null, ".zip")
+        ZipOutputStream(file.outputStream()).use { zipOutput -> block(zipOutput) }
+        file
+    }
+
+    private fun parseGitIgnore(): Set<String> {
         if (!gitIgnoreFile.exists()) {
-            return emptyList()
+            return emptySet()
         }
         return gitIgnoreFile.readLines()
             .filterNot { it.isBlank() || it.startsWith("#") }
             .map { it.trim() }
             .map { convertGitIgnorePatternToRegex(it) }
+            .toSet()
     }
 
     // gitignore patterns are not regex, method update needed.

plugins/toolkit/jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/sessionconfig/CodeScanSessionConfig.kt

@@ -10,6 +10,7 @@ import com.intellij.openapi.vfs.VFileProperty
 import com.intellij.openapi.vfs.VfsUtil
 import com.intellij.openapi.vfs.VirtualFile
 import kotlinx.coroutines.TimeoutCancellationException
+import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.time.withTimeout
 import software.aws.toolkits.core.utils.createTemporaryZipFile
 import software.aws.toolkits.core.utils.debug
@@ -140,7 +141,7 @@ class CodeScanSessionConfig(
                 } else {
                     VfsUtil.collectChildrenRecursively(projectRoot).filter {
                         !it.isDirectory && !it.`is`((VFileProperty.SYMLINK)) && (
-                            !featureDevSessionContext.ignoreFile(it)
+                            !featureDevSessionContext.ignoreFile(it, this)
                             )
                     }.fold(0L) { acc, next ->
                         totalSize = acc + next.length
@@ -174,7 +175,7 @@ class CodeScanSessionConfig(
             val current = stack.pop()
 
             if (!current.isDirectory) {
-                if (!featureDevSessionContext.ignoreFile(current)) {
+                if (runBlocking { !featureDevSessionContext.ignoreFile(current, this) }) {
                     if (willExceedPayloadLimit(currentTotalFileSize, current.length)) {
                         break
                     } else {
@@ -190,7 +191,7 @@ class CodeScanSessionConfig(
                 }
             } else {
                 // Directory case: only traverse if not ignored
-                if (!featureDevSessionContext.ignoreFile(current)) {
+                if (runBlocking { !featureDevSessionContext.ignoreFile(current, this) }) {
                     for (child in current.children) {
                         stack.push(child)
                     }

plugins/toolkit/resources/resources/software/aws/toolkits/resources/MessagesBundle.properties

@@ -55,6 +55,7 @@ amazonqFeatureDev.code_generation.updated_code=Code has been updated. Would you
 amazonqFeatureDev.content_length.error_text=The project you have selected for source code is too large to use as context. Please select a different folder to use for this conversation
 amazonqFeatureDev.create_new_plan=What change would you like to discuss?
 amazonqFeatureDev.create_plan=Ok, let me create a plan. This may take a few minutes.
+amazonqFeatureDev.create_plan.background_progress_title=Creating plans ...
 amazonqFeatureDev.error_text=Sorry, we encountered a problem when processing your request.
 amazonqFeatureDev.example_text=You can use /dev to:\n- Add a new feature or logic\n- Write tests\n- Fix a bug in your project\n- Generate a README for a file, folder, or project\n\nTo learn more, visit the [Amazon Q User Guide](https://docs.aws.amazon.com/amazonq/latest/aws-builder-use-ug/getting-started.html)
 amazonqFeatureDev.exception.conversation_not_found=Conversation id must exist before continuing