Implement maxNodeModuleJsDepth, noResolve

internal/compiler/fileloader.go

@@ -64,6 +64,10 @@
 	compilerOptions := opts.Config.CompilerOptions()
 	rootFiles := opts.Config.FileNames()
 	supportedExtensions := tsoptions.GetSupportedExtensions(compilerOptions, nil /*extraFileExtensions*/)
+	var maxNodeModuleJsDepth int
+	if p := opts.Config.CompilerOptions().MaxNodeModuleJsDepth; p != nil {
+		maxNodeModuleJsDepth = *p
+	}
 	loader := fileLoader{
 		opts:               opts,
 		defaultLibraryPath: tspath.GetNormalizedAbsolutePath(opts.Host.DefaultLibraryPath(), opts.Host.GetCurrentDirectory()),
@@ -72,12 +76,11 @@
 			CurrentDirectory:          opts.Host.GetCurrentDirectory(),
 		},
 		parseTasks: &fileLoaderWorker[*parseTask]{
-			wg:          core.NewWorkGroup(singleThreaded),
-			getSubTasks: getSubTasksOfParseTask,
+			wg:       core.NewWorkGroup(singleThreaded),
+			maxDepth: maxNodeModuleJsDepth,
 		},
 		projectReferenceParseTasks: &fileLoaderWorker[*projectReferenceParseTask]{
-			wg:          core.NewWorkGroup(singleThreaded),
-			getSubTasks: getSubTasksOfProjectReferenceParseTask,
+			wg: core.NewWorkGroup(singleThreaded),
 		},
 		rootTasks:           make([]*parseTask, 0, len(rootFiles)+len(libs)),
 		supportedExtensions: core.Flatten(tsoptions.GetSupportedExtensionsWithJsonIfResolveJsonModule(compilerOptions, supportedExtensions)),
@@ -282,30 +285,34 @@
 	return sourceFile
 }
 
-func (p *fileLoader) resolveTripleslashPathReference(moduleName string, containingFile string) string {
+func (p *fileLoader) resolveTripleslashPathReference(moduleName string, containingFile string) resolvedRef {
 	basePath := tspath.GetDirectoryPath(containingFile)
 	referencedFileName := moduleName
 
 	if !tspath.IsRootedDiskPath(moduleName) {
 		referencedFileName = tspath.CombinePaths(basePath, moduleName)
 	}
-	return tspath.NormalizePath(referencedFileName)
+	return resolvedRef{
+		fileName: tspath.NormalizePath(referencedFileName),
+	}
 }
 
 func (p *fileLoader) resolveTypeReferenceDirectives(file *ast.SourceFile, meta *ast.SourceFileMetaData) (
-	toParse []string,
+	toParse []resolvedRef,
 	typeResolutionsInFile module.ModeAwareCache[*module.ResolvedTypeReferenceDirective],
 ) {
 	if len(file.TypeReferenceDirectives) != 0 {
-		toParse = make([]string, 0, len(file.TypeReferenceDirectives))
+		toParse = make([]resolvedRef, 0, len(file.TypeReferenceDirectives))
 		typeResolutionsInFile = make(module.ModeAwareCache[*module.ResolvedTypeReferenceDirective], len(file.TypeReferenceDirectives))
 		for _, ref := range file.TypeReferenceDirectives {
 			redirect := p.projectReferenceFileMapper.getRedirectForResolution(file)
 			resolutionMode := getModeForTypeReferenceDirectiveInFile(ref, file, meta, module.GetCompilerOptionsWithRedirect(p.opts.Config.CompilerOptions(), redirect))
 			resolved := p.resolver.ResolveTypeReferenceDirective(ref.FileName, file.FileName(), resolutionMode, redirect)
 			typeResolutionsInFile[module.ModeAwareCacheKey{Name: ref.FileName, Mode: resolutionMode}] = resolved
 			if resolved.IsResolved() {
-				toParse = append(toParse, resolved.ResolvedFileName)
+				toParse = append(toParse, resolvedRef{
+					fileName: resolved.ResolvedFileName,
+				})
 			}
 		}
 	}
@@ -315,7 +322,7 @@
 const externalHelpersModuleNameText = "tslib" // TODO(jakebailey): dedupe
 
 func (p *fileLoader) resolveImportsAndModuleAugmentations(file *ast.SourceFile, meta *ast.SourceFileMetaData) (
-	toParse []string,
+	toParse []resolvedRef,
 	resolutionsInFile module.ModeAwareCache[*module.ResolvedModule],
 	importHelpersImportSpecifier *ast.Node,
 	jsxRuntimeImportSpecifier_ *jsxRuntimeImportSpecifier,
@@ -353,18 +360,20 @@
 	}
 
 	if len(moduleNames) != 0 {
-		toParse = make([]string, 0, len(moduleNames))
-
-		resolutions := p.resolveModuleNames(moduleNames, file, meta, redirect)
+		toParse = make([]resolvedRef, 0, len(moduleNames))
+		resolutionsInFile = make(module.ModeAwareCache[*module.ResolvedModule], len(moduleNames))
 
-		resolutionsInFile = make(module.ModeAwareCache[*module.ResolvedModule], len(resolutions))
+		for _, entry := range moduleNames {
+			moduleName := entry.Text()
+			if moduleName == "" {
+				continue
+			}
 
-		for _, resolution := range resolutions {
-			resolvedFileName := resolution.resolvedModule.ResolvedFileName
-			// TODO(ercornel): !!!: check if from node modules
+			mode := getModeForUsageLocation(file.FileName(), meta, entry, module.GetCompilerOptionsWithRedirect(p.opts.Config.CompilerOptions(), redirect))
+			resolvedModule := p.resolver.ResolveModuleName(moduleName, file.FileName(), mode, redirect)
+			resolvedFileName := resolvedModule.ResolvedFileName
 
-			mode := getModeForUsageLocation(file.FileName(), meta, resolution.node, optionsForFile)
-			resolutionsInFile[module.ModeAwareCacheKey{Name: resolution.node.Text(), Mode: mode}] = resolution.resolvedModule
+			resolutionsInFile[module.ModeAwareCacheKey{Name: moduleName, Mode: mode}] = resolvedModule
 
 			// add file to program only if:
 			// - resolution was successful
@@ -374,46 +383,37 @@
 
 			// const elideImport = isJSFileFromNodeModules && currentNodeModulesDepth > maxNodeModuleJsDepth;
 
-			// Don't add the file if it has a bad extension (e.g. 'tsx' if we don't have '--allowJs')
-			// This may still end up being an untyped module -- the file won't be included but imports will be allowed.
-			hasAllowedExtension := false
-			if optionsForFile.GetResolveJsonModule() {
-				hasAllowedExtension = tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedTSExtensionsWithJsonFlat)
-			} else if optionsForFile.AllowJs.IsTrue() {
-				hasAllowedExtension = tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedJSExtensionsFlat) || tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedTSExtensionsFlat)
-			} else {
-				hasAllowedExtension = tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedTSExtensionsFlat)
-			}
-			shouldAddFile := resolution.resolvedModule.IsResolved() && hasAllowedExtension
-			// TODO(ercornel): !!!: other checks on whether or not to add the file
-
+			shouldAddFile := resolvedModule.IsResolved() && optionsForFile.NoResolve.IsFalseOrUnknown()
 			if shouldAddFile {
-				// p.findSourceFile(resolvedFileName, FileIncludeReason{Import, 0})
-				toParse = append(toParse, resolvedFileName)
+				// Don't add the file if it has a bad extension (e.g. 'tsx' if we don't have '--allowJs')
+				// This may still end up being an untyped module -- the file won't be included but imports will be allowed.
+				if optionsForFile.GetResolveJsonModule() {
+					shouldAddFile = tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedTSExtensionsWithJsonFlat)
+				} else if optionsForFile.AllowJs.IsTrue() {
+					shouldAddFile = tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedJSExtensionsFlat) || tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedTSExtensionsFlat)
+				} else {
+					shouldAddFile = tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedTSExtensionsFlat)
+				}
 			}
-		}
-	}
-
-	return toParse, resolutionsInFile, importHelpersImportSpecifier, jsxRuntimeImportSpecifier_
-}
 
-func (p *fileLoader) resolveModuleNames(entries []*ast.Node, file *ast.SourceFile, meta *ast.SourceFileMetaData, redirect *tsoptions.ParsedCommandLine) []*resolution {
-	if len(entries) == 0 {
-		return nil
-	}
+			if !shouldAddFile {
+				continue
+			}
 
-	resolvedModules := make([]*resolution, 0, len(entries))
+			isFromNodeModulesSearch := resolvedModule.IsExternalLibraryImport
+			redirect := p.projectReferenceFileMapper.getRedirectForResolution(ast.NewHasFileName(resolvedFileName, p.toPath(resolvedFileName)))
+			// If this is js file source of project reference, dont treat it as js file but as d.ts
+			isJsFile := !tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedTSExtensionsWithJsonFlat) && redirect == nil
+			isJsFileFromNodeModules := isFromNodeModulesSearch && isJsFile && (resolvedFileName == "" || strings.Contains(resolvedFileName, "/node_modules/"))
 
-	for _, entry := range entries {
-		moduleName := entry.Text()
-		if moduleName == "" {
-			continue
+			toParse = append(toParse, resolvedRef{
+				fileName:                resolvedFileName,
+				isJsFileFromNodeModules: isJsFileFromNodeModules,
+			})
 		}
-		resolvedModule := p.resolver.ResolveModuleName(moduleName, file.FileName(), getModeForUsageLocation(file.FileName(), meta, entry, module.GetCompilerOptionsWithRedirect(p.opts.Config.CompilerOptions(), redirect)), redirect)
-		resolvedModules = append(resolvedModules, &resolution{node: entry, resolvedModule: resolvedModule})
 	}
 
-	return resolvedModules
+	return toParse, resolutionsInFile, importHelpersImportSpecifier, jsxRuntimeImportSpecifier_
 }
 
 func (p *fileLoader) createSyntheticImport(text string, file *ast.SourceFile) *ast.Node {

internal/compiler/fileloadertask.go

@@ -1,43 +1,74 @@
 package compiler
 
 import (
+	"math"
+	"sync"
+
 	"github.com/microsoft/typescript-go/internal/collections"
 	"github.com/microsoft/typescript-go/internal/core"
 	"github.com/microsoft/typescript-go/internal/tspath"
 )
 
-type fileLoaderWorkerTask interface {
+type fileLoaderWorkerTask[T any] interface {
 	comparable
 	FileName() string
-	start(loader *fileLoader)
+	isLoaded() bool
+	load(loader *fileLoader)
+	getSubTasks() []T
+	shouldIncreaseDepth() bool
 }
 
-type fileLoaderWorker[K fileLoaderWorkerTask] struct {
+type fileLoaderWorker[K fileLoaderWorkerTask[K]] struct {
 	wg              core.WorkGroup
-	tasksByFileName collections.SyncMap[string, K]
-	getSubTasks     func(t K) []K
+	tasksByFileName collections.SyncMap[string, *queuedTask[K]]
+	maxDepth        int
+}
+
+type queuedTask[K fileLoaderWorkerTask[K]] struct {
+	task        K
+	mu          sync.Mutex
+	lowestDepth int
 }
 
 func (w *fileLoaderWorker[K]) runAndWait(loader *fileLoader, tasks []K) {
-	w.start(loader, tasks)
+	w.start(loader, tasks, 0)
 	w.wg.RunAndWait()
 }
 
-func (w *fileLoaderWorker[K]) start(loader *fileLoader, tasks []K) {
-	if len(tasks) > 0 {
-		for i, task := range tasks {
-			loadedTask, loaded := w.tasksByFileName.LoadOrStore(task.FileName(), task)
-			if loaded {
-				// dedup tasks to ensure correct file order, regardless of which task would be started first
-				tasks[i] = loadedTask
-			} else {
-				w.wg.Queue(func() {
-					task.start(loader)
-					subTasks := w.getSubTasks(task)
-					w.start(loader, subTasks)
-				})
-			}
+func (w *fileLoaderWorker[K]) start(loader *fileLoader, tasks []K, depth int) {
+	for i, task := range tasks {
+		newTask := &queuedTask[K]{task: task, lowestDepth: math.MaxInt}
+		loadedTask, loaded := w.tasksByFileName.LoadOrStore(task.FileName(), newTask)
+		task = loadedTask.task
+		if loaded {
+			tasks[i] = task
+		}
+
+		currentDepth := depth
+		if task.shouldIncreaseDepth() {
+			currentDepth++
+		}
+
+		if currentDepth > w.maxDepth {
+			continue
 		}
+
+		w.wg.Queue(func() {
+			loadedTask.mu.Lock()
+			defer loadedTask.mu.Unlock()
+
+			if !task.isLoaded() {
+				task.load(loader)
+			}
+
+			if currentDepth < loadedTask.lowestDepth {
+				// If we're seeing this task at a lower depth than before,
+				// reprocess its subtasks to ensure they are loaded.
+				loadedTask.lowestDepth = currentDepth
+				subTasks := task.getSubTasks()
+				w.start(loader, subTasks, currentDepth)
+			}
+		})
 	}
 }
 
@@ -49,12 +80,12 @@ func (w *fileLoaderWorker[K]) collectWorker(loader *fileLoader, tasks []K, itera
 	var results []tspath.Path
 	for _, task := range tasks {
 		// ensure we only walk each task once
-		if seen.Has(task) {
+		if !task.isLoaded() || seen.Has(task) {
 			continue
 		}
 		seen.Add(task)
 		var subResults []tspath.Path
-		if subTasks := w.getSubTasks(task); len(subTasks) > 0 {
+		if subTasks := task.getSubTasks(); len(subTasks) > 0 {
 			subResults = w.collectWorker(loader, subTasks, iterate, seen)
 		}
 		iterate(task, subResults)

internal/compiler/parsetask.go

@@ -15,12 +15,14 @@ type parseTask struct {
 	isLib              bool
 	isRedirected       bool
 	subTasks           []*parseTask
+	loaded                bool
 
 	metadata                     *ast.SourceFileMetaData
 	resolutionsInFile            module.ModeAwareCache[*module.ResolvedModule]
 	typeResolutionsInFile        module.ModeAwareCache[*module.ResolvedTypeReferenceDirective]
 	importHelpersImportSpecifier *ast.Node
 	jsxRuntimeImportSpecifier    *jsxRuntimeImportSpecifier
+	isJsFileFromNodeModules      bool
 }
 
 func (t *parseTask) FileName() string {
@@ -31,7 +33,9 @@ func (t *parseTask) Path() tspath.Path {
 	return t.path
 }
 
-func (t *parseTask) start(loader *fileLoader) {
+func (t *parseTask) load(loader *fileLoader) {
+	t.loaded = true
+
 	t.path = loader.toPath(t.normalizedFilePath)
 	redirect := loader.projectReferenceFileMapper.getParseFileRedirect(t)
 	if redirect != "" {
@@ -73,7 +77,7 @@ func (t *parseTask) start(loader *fileLoader) {
 			if !ok {
 				continue
 			}
-			t.addSubTask(tspath.CombinePaths(loader.defaultLibraryPath, name), true)
+			t.addSubTask(resolvedRef{fileName: tspath.CombinePaths(loader.defaultLibraryPath, name)}, true)
 		}
 	}
 
@@ -92,11 +96,25 @@ func (t *parseTask) redirect(loader *fileLoader, fileName string) {
 	t.subTasks = []*parseTask{{normalizedFilePath: tspath.NormalizePath(fileName), isLib: t.isLib}}
 }
 
-func (t *parseTask) addSubTask(fileName string, isLib bool) {
-	normalizedFilePath := tspath.NormalizePath(fileName)
-	t.subTasks = append(t.subTasks, &parseTask{normalizedFilePath: normalizedFilePath, isLib: isLib})
+type resolvedRef struct {
+	fileName                string
+	isJsFileFromNodeModules bool
+}
+
+func (t *parseTask) addSubTask(ref resolvedRef, isLib bool) {
+	normalizedFilePath := tspath.NormalizePath(ref.fileName)
+	subTask := &parseTask{normalizedFilePath: normalizedFilePath, isLib: isLib, isJsFileFromNodeModules: ref.isJsFileFromNodeModules}
+	t.subTasks = append(t.subTasks, subTask)
 }
 
-func getSubTasksOfParseTask(t *parseTask) []*parseTask {
+func (t *parseTask) getSubTasks() []*parseTask {
 	return t.subTasks
 }
+
+func (t *parseTask) shouldIncreaseDepth() bool {
+	return t.isJsFileFromNodeModules
+}
+
+func (t *parseTask) isLoaded() bool {
+	return t.loaded
+}

internal/compiler/program.go

@@ -45,16 +45,6 @@ type Program struct {
 
 	processedFiles
 
-	// The below settings are to track if a .js file should be add to the program if loaded via searching under node_modules.
-	// This works as imported modules are discovered recursively in a depth first manner, specifically:
-	// - For each root file, findSourceFile is called.
-	// - This calls processImportedModules for each module imported in the source file.
-	// - This calls resolveModuleNames, and then calls findSourceFile for each resolved module.
-	// As all these operations happen - and are nested - within the createProgram call, they close over the below variables.
-	// The current resolution depth is tracked by incrementing/decrementing as the depth first search progresses.
-	// maxNodeModuleJsDepth      int
-	currentNodeModulesDepth int
-
 	usesUriStyleNodeCoreModules core.Tristate
 
 	commonSourceDirectory     string
@@ -193,12 +183,6 @@ func NewProgram(opts ProgramOptions) *Program {
 	}
 	p.initCheckerPool()
 
-	// p.maxNodeModuleJsDepth = p.options.MaxNodeModuleJsDepth
-
-	// TODO(ercornel): !!! tracing?
-	// tracing?.push(tracing.Phase.Program, "createProgram", { configFilePath: options.configFilePath, rootDir: options.rootDir }, /*separateBeginAndEnd*/ true);
-	// performance.mark("beforeProgram");
-
 	var libs []string
 
 	if compilerOptions.NoLib != core.TSTrue {
@@ -236,7 +220,6 @@ func (p *Program) UpdateProgram(changedFilePath tspath.Path) (*Program, bool) {
 		nodeModules:                 p.nodeModules,
 		comparePathsOptions:         p.comparePathsOptions,
 		processedFiles:              p.processedFiles,
-		currentNodeModulesDepth:     p.currentNodeModulesDepth,
 		usesUriStyleNodeCoreModules: p.usesUriStyleNodeCoreModules,
 	}
 	result.initCheckerPool()