From 24e52b48a40ea215639befa2b439e16e599382b3 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Thu, 12 Jun 2025 14:08:30 -0700 Subject: [PATCH 1/9] Add configurable concurency options --- internal/compiler/concurrency.go | 55 ++++++++++++++++ internal/compiler/program.go | 65 +++++++++++-------- internal/core/compileroptions.go | 1 + internal/core/tristate.go | 4 ++ internal/diagnostics/diagnostics_generated.go | 2 + .../diagnostics/extraDiagnosticMessages.json | 4 ++ internal/testutil/harnessutil/harnessutil.go | 18 +++-- internal/tsoptions/declscompiler.go | 6 ++ internal/tsoptions/parsinghelpers.go | 2 + 9 files changed, 119 insertions(+), 38 deletions(-) create mode 100644 internal/compiler/concurrency.go diff --git a/internal/compiler/concurrency.go b/internal/compiler/concurrency.go new file mode 100644 index 0000000000..d126564671 --- /dev/null +++ b/internal/compiler/concurrency.go @@ -0,0 +1,55 @@ +package compiler + +import ( + "runtime" + "strconv" + "strings" + + "github.com/microsoft/typescript-go/internal/core" +) + +type concurrency struct { + checkerCount int +} + +func parseConcurrency(options *core.CompilerOptions, numFiles int) concurrency { + if options.SingleThreaded.IsTrue() { + return concurrency{ + checkerCount: 1, + } + } + + checkerCount := 4 + + switch strings.ToLower(options.Concurrency) { + case "default", "auto": + break + case "single", "none": + checkerCount = 1 + case "max": + checkerCount = runtime.GOMAXPROCS(0) + case "half": + checkerCount = max(1, runtime.GOMAXPROCS(0)/2) + case "checker-per-file": + checkerCount = -1 + default: + if v, err := strconv.Atoi(options.Concurrency); err == nil && v > 0 { + checkerCount = v + } + } + + return concurrency{ + checkerCount: checkerCount, + } +} + +func (c concurrency) isSingleThreaded() bool { + return c.checkerCount == 1 +} + +func (c concurrency) getCheckerCount(numFiles int) int { + if c.checkerCount == -1 { + return max(1, numFiles) + } + return max(1, c.checkerCount) +} diff --git a/internal/compiler/program.go b/internal/compiler/program.go index 94c6cad250..a669f9d346 100644 --- a/internal/compiler/program.go +++ b/internal/compiler/program.go @@ -26,7 +26,6 @@ type ProgramOptions struct { Host CompilerHost Config *tsoptions.ParsedCommandLine UseSourceOfProjectReference bool - SingleThreaded core.Tristate CreateCheckerPool func(*Program) CheckerPool TypingsLocation string ProjectName string @@ -39,7 +38,11 @@ func (p *ProgramOptions) canUseProjectReferenceSource() bool { type Program struct { opts ProgramOptions nodeModules map[string]*ast.SourceFile - checkerPool CheckerPool + + concurrency concurrency + concurrencyOnce sync.Once + checkerPool CheckerPool + checkerPoolOnce sync.Once comparePathsOptions tspath.ComparePathsOptions @@ -191,9 +194,6 @@ func NewProgram(opts ProgramOptions) *Program { if p.opts.Host == nil { panic("host required") } - p.initCheckerPool() - - // p.maxNodeModuleJsDepth = p.options.MaxNodeModuleJsDepth // TODO(ercornel): !!! tracing? // tracing?.push(tracing.Phase.Program, "createProgram", { configFilePath: options.configFilePath, rootDir: options.rootDir }, /*separateBeginAndEnd*/ true); @@ -239,7 +239,6 @@ func (p *Program) UpdateProgram(changedFilePath tspath.Path) (*Program, bool) { currentNodeModulesDepth: p.currentNodeModulesDepth, usesUriStyleNodeCoreModules: p.usesUriStyleNodeCoreModules, } - result.initCheckerPool() index := core.FindIndex(result.files, func(file *ast.SourceFile) bool { return file.Path() == newFile.Path() }) result.files = slices.Clone(result.files) result.files[index] = newFile @@ -248,14 +247,6 @@ func (p *Program) UpdateProgram(changedFilePath tspath.Path) (*Program, bool) { return result, true } -func (p *Program) initCheckerPool() { - if p.opts.CreateCheckerPool != nil { - p.checkerPool = p.opts.CreateCheckerPool(p) - } else { - p.checkerPool = newCheckerPool(core.IfElse(p.singleThreaded(), 1, 4), p) - } -} - func canReplaceFileInProgram(file1 *ast.SourceFile, file2 *ast.SourceFile) bool { // TODO(jakebailey): metadata?? return file2 != nil && @@ -299,8 +290,26 @@ func (p *Program) GetConfigFileParsingDiagnostics() []*ast.Diagnostic { return slices.Clip(p.opts.Config.GetConfigFileParsingDiagnostics()) } +func (p *Program) getConcurrency() concurrency { + p.concurrencyOnce.Do(func() { + p.concurrency = parseConcurrency(p.Options(), len(p.files)) + }) + return p.concurrency +} + func (p *Program) singleThreaded() bool { - return p.opts.SingleThreaded.DefaultIfUnknown(p.Options().SingleThreaded).IsTrue() + return p.getConcurrency().isSingleThreaded() +} + +func (p *Program) getCheckerPool() CheckerPool { + p.checkerPoolOnce.Do(func() { + if p.opts.CreateCheckerPool != nil { + p.checkerPool = p.opts.CreateCheckerPool(p) + } else { + p.checkerPool = newCheckerPool(p.getConcurrency().getCheckerCount(len(p.files)), p) + } + }) + return p.checkerPool } func (p *Program) BindSourceFiles() { @@ -317,11 +326,11 @@ func (p *Program) BindSourceFiles() { func (p *Program) CheckSourceFiles(ctx context.Context) { wg := core.NewWorkGroup(p.singleThreaded()) - checkers, done := p.checkerPool.GetAllCheckers(ctx) + checkers, done := p.getCheckerPool().GetAllCheckers(ctx) defer done() for _, checker := range checkers { wg.Queue(func() { - for file := range p.checkerPool.Files(checker) { + for file := range p.getCheckerPool().Files(checker) { checker.CheckSourceFile(ctx, file) } }) @@ -331,11 +340,11 @@ func (p *Program) CheckSourceFiles(ctx context.Context) { // Return the type checker associated with the program. func (p *Program) GetTypeChecker(ctx context.Context) (*checker.Checker, func()) { - return p.checkerPool.GetChecker(ctx) + return p.getCheckerPool().GetChecker(ctx) } func (p *Program) GetTypeCheckers(ctx context.Context) ([]*checker.Checker, func()) { - return p.checkerPool.GetAllCheckers(ctx) + return p.getCheckerPool().GetAllCheckers(ctx) } // Return a checker for the given file. We may have multiple checkers in concurrent scenarios and this @@ -343,7 +352,7 @@ func (p *Program) GetTypeCheckers(ctx context.Context) ([]*checker.Checker, func // types obtained from different checkers, so only non-type data (such as diagnostics or string // representations of types) should be obtained from checkers returned by this method. func (p *Program) GetTypeCheckerForFile(ctx context.Context, file *ast.SourceFile) (*checker.Checker, func()) { - return p.checkerPool.GetCheckerForFile(ctx, file) + return p.getCheckerPool().GetCheckerForFile(ctx, file) } func (p *Program) GetResolvedModule(file ast.HasFileName, moduleReference string, mode core.ResolutionMode) *module.ResolvedModule { @@ -385,7 +394,7 @@ func (p *Program) GetSuggestionDiagnostics(ctx context.Context, sourceFile *ast. func (p *Program) GetGlobalDiagnostics(ctx context.Context) []*ast.Diagnostic { var globalDiagnostics []*ast.Diagnostic - checkers, done := p.checkerPool.GetAllCheckers(ctx) + checkers, done := p.getCheckerPool().GetAllCheckers(ctx) defer done() for _, checker := range checkers { globalDiagnostics = append(globalDiagnostics, checker.GetGlobalDiagnostics()...) @@ -432,11 +441,11 @@ func (p *Program) getSemanticDiagnosticsForFile(ctx context.Context, sourceFile var fileChecker *checker.Checker var done func() if sourceFile != nil { - fileChecker, done = p.checkerPool.GetCheckerForFile(ctx, sourceFile) + fileChecker, done = p.getCheckerPool().GetCheckerForFile(ctx, sourceFile) defer done() } diags := slices.Clip(sourceFile.BindDiagnostics()) - checkers, closeCheckers := p.checkerPool.GetAllCheckers(ctx) + checkers, closeCheckers := p.getCheckerPool().GetAllCheckers(ctx) defer closeCheckers() // Ask for diags from all checkers; checking one file may add diagnostics to other files. @@ -523,13 +532,13 @@ func (p *Program) getSuggestionDiagnosticsForFile(ctx context.Context, sourceFil var fileChecker *checker.Checker var done func() if sourceFile != nil { - fileChecker, done = p.checkerPool.GetCheckerForFile(ctx, sourceFile) + fileChecker, done = p.getCheckerPool().GetCheckerForFile(ctx, sourceFile) defer done() } diags := slices.Clip(sourceFile.BindSuggestionDiagnostics) - checkers, closeCheckers := p.checkerPool.GetAllCheckers(ctx) + checkers, closeCheckers := p.getCheckerPool().GetAllCheckers(ctx) defer closeCheckers() // Ask for diags from all checkers; checking one file may add diagnostics to other files. @@ -640,7 +649,7 @@ func (p *Program) SymbolCount() int { for _, file := range p.files { count += file.SymbolCount } - checkers, done := p.checkerPool.GetAllCheckers(context.Background()) + checkers, done := p.getCheckerPool().GetAllCheckers(context.Background()) defer done() for _, checker := range checkers { count += int(checker.SymbolCount) @@ -650,7 +659,7 @@ func (p *Program) SymbolCount() int { func (p *Program) TypeCount() int { var count int - checkers, done := p.checkerPool.GetAllCheckers(context.Background()) + checkers, done := p.getCheckerPool().GetAllCheckers(context.Background()) defer done() for _, checker := range checkers { count += int(checker.TypeCount) @@ -660,7 +669,7 @@ func (p *Program) TypeCount() int { func (p *Program) InstantiationCount() int { var count int - checkers, done := p.checkerPool.GetAllCheckers(context.Background()) + checkers, done := p.getCheckerPool().GetAllCheckers(context.Background()) defer done() for _, checker := range checkers { count += int(checker.TotalInstantiationCount) diff --git a/internal/core/compileroptions.go b/internal/core/compileroptions.go index e6c7f859e9..5cfeb48f12 100644 --- a/internal/core/compileroptions.go +++ b/internal/core/compileroptions.go @@ -146,6 +146,7 @@ type CompilerOptions struct { PprofDir string `json:"pprofDir,omitzero"` SingleThreaded Tristate `json:"singleThreaded,omitzero"` + Concurrency string `json:"concurrency,omitzero"` Quiet Tristate `json:"quiet,omitzero"` sourceFileAffectingCompilerOptionsOnce sync.Once diff --git a/internal/core/tristate.go b/internal/core/tristate.go index 11b1bc0d36..8ea4d86ba5 100644 --- a/internal/core/tristate.go +++ b/internal/core/tristate.go @@ -13,6 +13,10 @@ const ( TSTrue ) +func (t Tristate) IsUnknown() bool { + return t == TSUnknown +} + func (t Tristate) IsTrue() bool { return t == TSTrue } diff --git a/internal/diagnostics/diagnostics_generated.go b/internal/diagnostics/diagnostics_generated.go index 5fb56a7143..db9c4080cf 100644 --- a/internal/diagnostics/diagnostics_generated.go +++ b/internal/diagnostics/diagnostics_generated.go @@ -4213,3 +4213,5 @@ var Do_not_print_diagnostics = &Message{code: 100000, category: CategoryMessage, var Run_in_single_threaded_mode = &Message{code: 100001, category: CategoryMessage, key: "Run_in_single_threaded_mode_100001", text: "Run in single threaded mode."} var Generate_pprof_CPU_Slashmemory_profiles_to_the_given_directory = &Message{code: 100002, category: CategoryMessage, key: "Generate_pprof_CPU_Slashmemory_profiles_to_the_given_directory_100002", text: "Generate pprof CPU/memory profiles to the given directory."} + +var Specify_how_much_concurrency_will_be_used = &Message{code: 100003, category: CategoryMessage, key: "Specify_how_much_concurrency_will_be_used_100003", text: "Specify how much concurrency will be used."} diff --git a/internal/diagnostics/extraDiagnosticMessages.json b/internal/diagnostics/extraDiagnosticMessages.json index b1de202b28..5c5fc1ab10 100644 --- a/internal/diagnostics/extraDiagnosticMessages.json +++ b/internal/diagnostics/extraDiagnosticMessages.json @@ -10,5 +10,9 @@ "Generate pprof CPU/memory profiles to the given directory.": { "category": "Message", "code": 100002 + }, + "Specify how much concurrency will be used.": { + "category": "Message", + "code": 100003 } } diff --git a/internal/testutil/harnessutil/harnessutil.go b/internal/testutil/harnessutil/harnessutil.go index f816bde694..ebb4b2cce3 100644 --- a/internal/testutil/harnessutil/harnessutil.go +++ b/internal/testutil/harnessutil/harnessutil.go @@ -182,6 +182,11 @@ func CompileFilesEx( compilerOptions.TypeRoots[i] = tspath.GetNormalizedAbsolutePath(typeRoot, currentDirectory) } + if compilerOptions.Concurrency == "" && compilerOptions.SingleThreaded.IsUnknown() && testutil.TestProgramIsSingleThreaded() { + // TODO(jakebailey): replace TestProgramIsSingleThreaded with passed in value + compilerOptions.Concurrency = "1" + } + // Create fake FS for testing testfs := map[string]any{} for _, file := range inputFiles { @@ -841,18 +846,11 @@ func (c *CompilationResult) GetSourceMapRecord() string { } func createProgram(host compiler.CompilerHost, config *tsoptions.ParsedCommandLine) *compiler.Program { - var singleThreaded core.Tristate - if testutil.TestProgramIsSingleThreaded() { - singleThreaded = core.TSTrue - } - programOptions := compiler.ProgramOptions{ - Config: config, - Host: host, - SingleThreaded: singleThreaded, + Config: config, + Host: host, } - program := compiler.NewProgram(programOptions) - return program + return compiler.NewProgram(programOptions) } func EnumerateFiles(folder string, testRegex *regexp.Regexp, recursive bool) ([]string, error) { diff --git a/internal/tsoptions/declscompiler.go b/internal/tsoptions/declscompiler.go index c378ad896a..ab78f96500 100644 --- a/internal/tsoptions/declscompiler.go +++ b/internal/tsoptions/declscompiler.go @@ -225,6 +225,12 @@ var optionsForCompiler = []*CommandLineOption{ Category: diagnostics.Command_line_Options, Description: diagnostics.Run_in_single_threaded_mode, }, + { + Name: "concurrency", + Kind: CommandLineOptionTypeString, + Category: diagnostics.Command_line_Options, + Description: diagnostics.Specify_how_much_concurrency_will_be_used, + }, { Name: "pprofDir", Kind: CommandLineOptionTypeString, diff --git a/internal/tsoptions/parsinghelpers.go b/internal/tsoptions/parsinghelpers.go index e13280b1a5..69b6958888 100644 --- a/internal/tsoptions/parsinghelpers.go +++ b/internal/tsoptions/parsinghelpers.go @@ -430,6 +430,8 @@ func parseCompilerOptions(key string, value any, allOptions *core.CompilerOption allOptions.PprofDir = parseString(value) case "singleThreaded": allOptions.SingleThreaded = parseTristate(value) + case "concurrency": + allOptions.Concurrency = parseString(value) case "quiet": allOptions.Quiet = parseTristate(value) default: From ca9c0554295378a1616f34425cae30a527f2029f Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Thu, 12 Jun 2025 14:18:02 -0700 Subject: [PATCH 2/9] Fix nil --- internal/project/project.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/project/project.go b/internal/project/project.go index 83a53993ac..6b85d022ef 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -341,7 +341,6 @@ func (p *Project) GetLanguageServiceForRequest(ctx context.Context) (*ls.Languag if program == nil { panic("must have gced by other request") } - checkerPool := p.checkerPool snapshot := &snapshot{ project: p, positionEncoding: p.host.PositionEncoding(), @@ -349,7 +348,7 @@ func (p *Project) GetLanguageServiceForRequest(ctx context.Context) (*ls.Languag } languageService := ls.NewLanguageService(ctx, snapshot) cleanup := func() { - if checkerPool.isRequestCheckerInUse(core.GetRequestID(ctx)) { + if p.checkerPool != nil && p.checkerPool.isRequestCheckerInUse(core.GetRequestID(ctx)) { panic(fmt.Errorf("checker for request ID %s not returned to pool at end of request", core.GetRequestID(ctx))) } } From 4675e427454327eb9fdbf1822c739cd1aa2f2259 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Thu, 12 Jun 2025 14:51:53 -0700 Subject: [PATCH 3/9] Test program stuff --- Herebyfile.mjs | 18 ++++- internal/compiler/concurrency.go | 55 -------------- internal/compiler/program.go | 10 +-- internal/core/concurrency.go | 76 ++++++++++++++++++++ internal/testrunner/compiler_runner.go | 3 +- internal/testutil/harnessutil/harnessutil.go | 7 +- internal/testutil/testutil.go | 18 ----- 7 files changed, 102 insertions(+), 85 deletions(-) delete mode 100644 internal/compiler/concurrency.go create mode 100644 internal/core/concurrency.go diff --git a/Herebyfile.mjs b/Herebyfile.mjs index 0ff3044c1c..db50b77511 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -51,6 +51,20 @@ function parseEnvBoolean(name, defaultValue = false) { throw new Error(`Invalid value for ${name}: ${value}`); } +/** + * @param {string} name + * @param {string} defaultValue + * @returns {string} + */ +function parseEnvString(name, defaultValue = "") { + name = "TSGO_HEREBY_" + name.toUpperCase(); + const value = process.env[name]; + if (!value) { + return defaultValue; + } + return value; +} + const { values: rawOptions } = parseArgs({ args: process.argv.slice(2), options: { @@ -65,7 +79,7 @@ const { values: rawOptions } = parseArgs({ race: { type: "boolean", default: parseEnvBoolean("RACE") }, noembed: { type: "boolean", default: parseEnvBoolean("NOEMBED") }, - concurrentTestPrograms: { type: "boolean", default: parseEnvBoolean("CONCURRENT_TEST_PROGRAMS") }, + concurrency: { type: "string", default: parseEnvString("TEST_PROGRAM_CONCURRENCY") }, coverage: { type: "boolean", default: parseEnvBoolean("COVERAGE") }, }, strict: false, @@ -291,7 +305,7 @@ function goTestFlags(taskName) { } const goTestEnv = { - ...(options.concurrentTestPrograms ? { TS_TEST_PROGRAM_SINGLE_THREADED: "false" } : {}), + ...(options.concurrency ? { TSGO_TEST_PROGRAM_CONCURRENCY: "false" } : {}), // Go test caching takes a long time on Windows. // https://github.com/golang/go/issues/72992 ...(process.platform === "win32" ? { GOFLAGS: "-count=1" } : {}), diff --git a/internal/compiler/concurrency.go b/internal/compiler/concurrency.go deleted file mode 100644 index d126564671..0000000000 --- a/internal/compiler/concurrency.go +++ /dev/null @@ -1,55 +0,0 @@ -package compiler - -import ( - "runtime" - "strconv" - "strings" - - "github.com/microsoft/typescript-go/internal/core" -) - -type concurrency struct { - checkerCount int -} - -func parseConcurrency(options *core.CompilerOptions, numFiles int) concurrency { - if options.SingleThreaded.IsTrue() { - return concurrency{ - checkerCount: 1, - } - } - - checkerCount := 4 - - switch strings.ToLower(options.Concurrency) { - case "default", "auto": - break - case "single", "none": - checkerCount = 1 - case "max": - checkerCount = runtime.GOMAXPROCS(0) - case "half": - checkerCount = max(1, runtime.GOMAXPROCS(0)/2) - case "checker-per-file": - checkerCount = -1 - default: - if v, err := strconv.Atoi(options.Concurrency); err == nil && v > 0 { - checkerCount = v - } - } - - return concurrency{ - checkerCount: checkerCount, - } -} - -func (c concurrency) isSingleThreaded() bool { - return c.checkerCount == 1 -} - -func (c concurrency) getCheckerCount(numFiles int) int { - if c.checkerCount == -1 { - return max(1, numFiles) - } - return max(1, c.checkerCount) -} diff --git a/internal/compiler/program.go b/internal/compiler/program.go index a669f9d346..0be42afdd9 100644 --- a/internal/compiler/program.go +++ b/internal/compiler/program.go @@ -39,7 +39,7 @@ type Program struct { opts ProgramOptions nodeModules map[string]*ast.SourceFile - concurrency concurrency + concurrency core.Concurrency concurrencyOnce sync.Once checkerPool CheckerPool checkerPoolOnce sync.Once @@ -290,15 +290,15 @@ func (p *Program) GetConfigFileParsingDiagnostics() []*ast.Diagnostic { return slices.Clip(p.opts.Config.GetConfigFileParsingDiagnostics()) } -func (p *Program) getConcurrency() concurrency { +func (p *Program) getConcurrency() core.Concurrency { p.concurrencyOnce.Do(func() { - p.concurrency = parseConcurrency(p.Options(), len(p.files)) + p.concurrency = core.ParseConcurrency(p.Options()) }) return p.concurrency } func (p *Program) singleThreaded() bool { - return p.getConcurrency().isSingleThreaded() + return p.getConcurrency().SingleThreaded() } func (p *Program) getCheckerPool() CheckerPool { @@ -306,7 +306,7 @@ func (p *Program) getCheckerPool() CheckerPool { if p.opts.CreateCheckerPool != nil { p.checkerPool = p.opts.CreateCheckerPool(p) } else { - p.checkerPool = newCheckerPool(p.getConcurrency().getCheckerCount(len(p.files)), p) + p.checkerPool = newCheckerPool(p.getConcurrency().CheckerCount(len(p.files)), p) } }) return p.checkerPool diff --git a/internal/core/concurrency.go b/internal/core/concurrency.go new file mode 100644 index 0000000000..170773f954 --- /dev/null +++ b/internal/core/concurrency.go @@ -0,0 +1,76 @@ +package core + +import ( + "fmt" + "os" + "runtime" + "strconv" + "strings" + "sync" + + "github.com/microsoft/typescript-go/internal/testutil/race" +) + +type Concurrency struct { + checkerCount int +} + +func ParseConcurrency(options *CompilerOptions) Concurrency { + if options.SingleThreaded.IsTrue() { + return Concurrency{ + checkerCount: 1, + } + } + return parseConcurrency(options.Concurrency) +} + +func parseConcurrency(v string) Concurrency { + checkerCount := 4 + + switch strings.ToLower(v) { + case "default", "auto": + break + case "single", "none": + checkerCount = 1 + case "max": + checkerCount = runtime.GOMAXPROCS(0) + case "half": + checkerCount = max(1, runtime.GOMAXPROCS(0)/2) + case "checker-per-file": + checkerCount = -1 + default: + if v, err := strconv.Atoi(v); err == nil && v > 0 { + checkerCount = v + } + } + + return Concurrency{ + checkerCount: checkerCount, + } +} + +func (c Concurrency) SingleThreaded() bool { + return c.checkerCount == 1 +} + +func (c Concurrency) CheckerCount(numFiles int) int { + if c.checkerCount == -1 { + return max(1, numFiles) + } + return max(1, c.checkerCount) +} + +var testProgramConcurrency = sync.OnceValues(func() (concurrency Concurrency, raw string) { + // Leave Program in SingleThreaded mode unless explicitly configured or in race mode. + v := os.Getenv("TSGO_TEST_PROGRAM_CONCURRENCY") + if v == "" && !race.Enabled { + v = "single" + } + c := parseConcurrency(v) + fmt.Println(c, v) + return c, v +}) + +func TestProgramConcurrency() (concurrency Concurrency, raw string) { + return testProgramConcurrency() +} diff --git a/internal/testrunner/compiler_runner.go b/internal/testrunner/compiler_runner.go index 6770a03eb1..04d6a0647c 100644 --- a/internal/testrunner/compiler_runner.go +++ b/internal/testrunner/compiler_runner.go @@ -335,7 +335,8 @@ var concurrentSkippedErrorBaselines = collections.NewSetFromItems( func (c *compilerTest) verifyDiagnostics(t *testing.T, suiteName string, isSubmodule bool) { t.Run("error", func(t *testing.T) { - if !testutil.TestProgramIsSingleThreaded() && concurrentSkippedErrorBaselines.Has(c.testName) { + concurrency, _ := core.TestProgramConcurrency() + if !concurrency.SingleThreaded() && concurrentSkippedErrorBaselines.Has(c.testName) { t.Skip("Skipping error baseline in concurrent mode") } diff --git a/internal/testutil/harnessutil/harnessutil.go b/internal/testutil/harnessutil/harnessutil.go index ebb4b2cce3..f13ab195d1 100644 --- a/internal/testutil/harnessutil/harnessutil.go +++ b/internal/testutil/harnessutil/harnessutil.go @@ -25,7 +25,6 @@ import ( "github.com/microsoft/typescript-go/internal/repo" "github.com/microsoft/typescript-go/internal/scanner" "github.com/microsoft/typescript-go/internal/sourcemap" - "github.com/microsoft/typescript-go/internal/testutil" "github.com/microsoft/typescript-go/internal/tsoptions" "github.com/microsoft/typescript-go/internal/tspath" "github.com/microsoft/typescript-go/internal/vfs" @@ -182,9 +181,9 @@ func CompileFilesEx( compilerOptions.TypeRoots[i] = tspath.GetNormalizedAbsolutePath(typeRoot, currentDirectory) } - if compilerOptions.Concurrency == "" && compilerOptions.SingleThreaded.IsUnknown() && testutil.TestProgramIsSingleThreaded() { - // TODO(jakebailey): replace TestProgramIsSingleThreaded with passed in value - compilerOptions.Concurrency = "1" + if compilerOptions.Concurrency == "" && compilerOptions.SingleThreaded.IsUnknown() { + _, concurrency := core.TestProgramConcurrency() + compilerOptions.Concurrency = concurrency } // Create fake FS for testing diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 279258f27a..f803dacfe6 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -1,13 +1,9 @@ package testutil import ( - "os" "runtime/debug" - "strconv" - "sync" "testing" - "github.com/microsoft/typescript-go/internal/testutil/race" "gotest.tools/v3/assert" ) @@ -33,17 +29,3 @@ func RecoverAndFail(t *testing.T, msg string) { t.Fatalf("%s:\n%v\n%s", msg, r, string(stack)) } } - -var testProgramIsSingleThreaded = sync.OnceValue(func() bool { - // Leave Program in SingleThreaded mode unless explicitly configured or in race mode. - if v := os.Getenv("TS_TEST_PROGRAM_SINGLE_THREADED"); v != "" { - if b, err := strconv.ParseBool(v); err == nil { - return b - } - } - return !race.Enabled -}) - -func TestProgramIsSingleThreaded() bool { - return testProgramIsSingleThreaded() -} From 1448c31890b714dbe037edd26d5f36ff68da40f6 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Thu, 12 Jun 2025 14:55:00 -0700 Subject: [PATCH 4/9] Remove print --- internal/core/concurrency.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/core/concurrency.go b/internal/core/concurrency.go index 170773f954..d8d7975de4 100644 --- a/internal/core/concurrency.go +++ b/internal/core/concurrency.go @@ -1,7 +1,6 @@ package core import ( - "fmt" "os" "runtime" "strconv" @@ -66,9 +65,7 @@ var testProgramConcurrency = sync.OnceValues(func() (concurrency Concurrency, ra if v == "" && !race.Enabled { v = "single" } - c := parseConcurrency(v) - fmt.Println(c, v) - return c, v + return parseConcurrency(v), v }) func TestProgramConcurrency() (concurrency Concurrency, raw string) { From d341e4e41923de24ee145b1f9bb1735037ab09b8 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Thu, 12 Jun 2025 14:57:17 -0700 Subject: [PATCH 5/9] CI --- .github/workflows/ci.yml | 2 +- internal/core/concurrency.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8aa5b85111..8862610dfc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,7 +97,7 @@ jobs: env: TSGO_HEREBY_RACE: ${{ (matrix.config.race && 'true') || 'false' }} TSGO_HEREBY_NOEMBED: ${{ (matrix.config.noembed && 'true') || 'false' }} - TSGO_HEREBY_CONCURRENT_TEST_PROGRAMS: ${{ (matrix.config.concurrent-test-programs && 'true') || 'false' }} + TSGO_HEREBY_TEST_PROGRAM_CONCURRENCY: ${{ (matrix.config.concurrent-test-programs && 'true') || 'false' }} TSGO_HEREBY_COVERAGE: ${{ (matrix.config.coverage && 'true') || 'false' }} steps: diff --git a/internal/core/concurrency.go b/internal/core/concurrency.go index d8d7975de4..fe7689874a 100644 --- a/internal/core/concurrency.go +++ b/internal/core/concurrency.go @@ -27,9 +27,9 @@ func parseConcurrency(v string) Concurrency { checkerCount := 4 switch strings.ToLower(v) { - case "default", "auto": + case "default", "auto", "true", "yes", "on": break - case "single", "none": + case "single", "none", "false", "no", "off": checkerCount = 1 case "max": checkerCount = runtime.GOMAXPROCS(0) From ff61276667b452e1c05db829f478faf62c173633 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Thu, 12 Jun 2025 15:43:13 -0700 Subject: [PATCH 6/9] Don't make more checkers than files --- internal/core/concurrency.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/core/concurrency.go b/internal/core/concurrency.go index fe7689874a..eab5600365 100644 --- a/internal/core/concurrency.go +++ b/internal/core/concurrency.go @@ -53,10 +53,9 @@ func (c Concurrency) SingleThreaded() bool { } func (c Concurrency) CheckerCount(numFiles int) int { - if c.checkerCount == -1 { - return max(1, numFiles) - } - return max(1, c.checkerCount) + checkerCount := min(c.checkerCount, numFiles) + checkerCount = max(1, checkerCount) + return checkerCount } var testProgramConcurrency = sync.OnceValues(func() (concurrency Concurrency, raw string) { From 99fc50cb7e80e4f9617c873ad09a17d804550351 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Thu, 12 Jun 2025 16:27:37 -0700 Subject: [PATCH 7/9] Testing --- internal/core/concurrency.go | 18 +++++++---- internal/core/concurrency_test.go | 53 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 internal/core/concurrency_test.go diff --git a/internal/core/concurrency.go b/internal/core/concurrency.go index eab5600365..c6997a36fd 100644 --- a/internal/core/concurrency.go +++ b/internal/core/concurrency.go @@ -23,10 +23,10 @@ func ParseConcurrency(options *CompilerOptions) Concurrency { return parseConcurrency(options.Concurrency) } -func parseConcurrency(v string) Concurrency { +func parseConcurrency(s string) Concurrency { checkerCount := 4 - switch strings.ToLower(v) { + switch strings.ToLower(s) { case "default", "auto", "true", "yes", "on": break case "single", "none", "false", "no", "off": @@ -38,8 +38,10 @@ func parseConcurrency(v string) Concurrency { case "checker-per-file": checkerCount = -1 default: - if v, err := strconv.Atoi(v); err == nil && v > 0 { - checkerCount = v + if s != "" { + if v, err := strconv.Atoi(s); err == nil && v > 0 { + checkerCount = v + } } } @@ -53,9 +55,11 @@ func (c Concurrency) SingleThreaded() bool { } func (c Concurrency) CheckerCount(numFiles int) int { - checkerCount := min(c.checkerCount, numFiles) - checkerCount = max(1, checkerCount) - return checkerCount + checkerCount := c.checkerCount + if c.checkerCount == -1 { + return max(1, numFiles) + } + return min(max(1, checkerCount), numFiles) } var testProgramConcurrency = sync.OnceValues(func() (concurrency Concurrency, raw string) { diff --git a/internal/core/concurrency_test.go b/internal/core/concurrency_test.go new file mode 100644 index 0000000000..fb0c9041ad --- /dev/null +++ b/internal/core/concurrency_test.go @@ -0,0 +1,53 @@ +package core + +import ( + "runtime" + "testing" + + "gotest.tools/v3/assert" +) + +func TestConcurrency(t *testing.T) { + tests := []struct { + name string + opts *CompilerOptions + numFiles int + singleThreaded bool + checkerCount int + }{ + {"defaults", &CompilerOptions{}, 100, false, 4}, + {"default", &CompilerOptions{Concurrency: "default"}, 100, false, 4}, + {"auto", &CompilerOptions{Concurrency: "true"}, 100, false, 4}, + {"true", &CompilerOptions{Concurrency: "true"}, 100, false, 4}, + {"yes", &CompilerOptions{Concurrency: "yes"}, 100, false, 4}, + {"on", &CompilerOptions{Concurrency: "on"}, 100, false, 4}, + {"singleThreaded", &CompilerOptions{SingleThreaded: TSTrue}, 100, true, 1}, + {"single", &CompilerOptions{Concurrency: "single"}, 100, true, 1}, + {"none", &CompilerOptions{Concurrency: "none"}, 100, true, 1}, + {"false", &CompilerOptions{Concurrency: "false"}, 100, true, 1}, + {"no", &CompilerOptions{Concurrency: "no"}, 100, true, 1}, + {"off", &CompilerOptions{Concurrency: "off"}, 100, true, 1}, + {"max", &CompilerOptions{Concurrency: "max"}, 1000, false, runtime.GOMAXPROCS(0)}, + {"half", &CompilerOptions{Concurrency: "half"}, 1000, false, runtime.GOMAXPROCS(0) / 2}, + {"checker-per-file", &CompilerOptions{Concurrency: "checker-per-file"}, 100, false, 100}, + {"more than files", &CompilerOptions{Concurrency: "1000"}, 100, false, 100}, + {"10", &CompilerOptions{Concurrency: "10"}, 100, false, 10}, + {"1", &CompilerOptions{Concurrency: "1"}, 100, true, 1}, + {"invalid", &CompilerOptions{Concurrency: "i dunno"}, 100, false, 4}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := ParseConcurrency(tt.opts) + singleThreaded := c.SingleThreaded() + checkerCount := c.CheckerCount(tt.numFiles) + assert.Equal(t, singleThreaded, tt.singleThreaded) + assert.Equal(t, checkerCount, tt.checkerCount) + }) + } + + t.Run("TestProgramConcurrency", func(t *testing.T) { + c, _ := TestProgramConcurrency() + assert.Assert(t, c.CheckerCount(10000) > 0) + }) +} From 9a0905a61c8076d93267ab490d8ebc870b0ad713 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Thu, 12 Jun 2025 16:56:14 -0700 Subject: [PATCH 8/9] lint --- internal/core/concurrency_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/core/concurrency_test.go b/internal/core/concurrency_test.go index fb0c9041ad..6723c0391e 100644 --- a/internal/core/concurrency_test.go +++ b/internal/core/concurrency_test.go @@ -8,6 +8,8 @@ import ( ) func TestConcurrency(t *testing.T) { + t.Parallel() + tests := []struct { name string opts *CompilerOptions @@ -38,6 +40,8 @@ func TestConcurrency(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() + c := ParseConcurrency(tt.opts) singleThreaded := c.SingleThreaded() checkerCount := c.CheckerCount(tt.numFiles) @@ -47,6 +51,8 @@ func TestConcurrency(t *testing.T) { } t.Run("TestProgramConcurrency", func(t *testing.T) { + t.Parallel() + c, _ := TestProgramConcurrency() assert.Assert(t, c.CheckerCount(10000) > 0) }) From e4740ca08d4f8f6f3c9dfc2fe33da756d392e542 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 16 Jun 2025 14:40:36 -0700 Subject: [PATCH 9/9] Fix macos --- internal/core/concurrency_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/core/concurrency_test.go b/internal/core/concurrency_test.go index 6723c0391e..169403cd7e 100644 --- a/internal/core/concurrency_test.go +++ b/internal/core/concurrency_test.go @@ -30,7 +30,7 @@ func TestConcurrency(t *testing.T) { {"no", &CompilerOptions{Concurrency: "no"}, 100, true, 1}, {"off", &CompilerOptions{Concurrency: "off"}, 100, true, 1}, {"max", &CompilerOptions{Concurrency: "max"}, 1000, false, runtime.GOMAXPROCS(0)}, - {"half", &CompilerOptions{Concurrency: "half"}, 1000, false, runtime.GOMAXPROCS(0) / 2}, + {"half", &CompilerOptions{Concurrency: "half"}, 1000, runtime.GOMAXPROCS(0)/2 == 1, runtime.GOMAXPROCS(0) / 2}, {"checker-per-file", &CompilerOptions{Concurrency: "checker-per-file"}, 100, false, 100}, {"more than files", &CompilerOptions{Concurrency: "1000"}, 100, false, 100}, {"10", &CompilerOptions{Concurrency: "10"}, 100, false, 10},