diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8aa5b85111..8862610dfc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,7 +97,7 @@ jobs: env: TSGO_HEREBY_RACE: ${{ (matrix.config.race && 'true') || 'false' }} TSGO_HEREBY_NOEMBED: ${{ (matrix.config.noembed && 'true') || 'false' }} - TSGO_HEREBY_CONCURRENT_TEST_PROGRAMS: ${{ (matrix.config.concurrent-test-programs && 'true') || 'false' }} + TSGO_HEREBY_TEST_PROGRAM_CONCURRENCY: ${{ (matrix.config.concurrent-test-programs && 'true') || 'false' }} TSGO_HEREBY_COVERAGE: ${{ (matrix.config.coverage && 'true') || 'false' }} steps: diff --git a/Herebyfile.mjs b/Herebyfile.mjs index 0ff3044c1c..db50b77511 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -51,6 +51,20 @@ function parseEnvBoolean(name, defaultValue = false) { throw new Error(`Invalid value for ${name}: ${value}`); } +/** + * @param {string} name + * @param {string} defaultValue + * @returns {string} + */ +function parseEnvString(name, defaultValue = "") { + name = "TSGO_HEREBY_" + name.toUpperCase(); + const value = process.env[name]; + if (!value) { + return defaultValue; + } + return value; +} + const { values: rawOptions } = parseArgs({ args: process.argv.slice(2), options: { @@ -65,7 +79,7 @@ const { values: rawOptions } = parseArgs({ race: { type: "boolean", default: parseEnvBoolean("RACE") }, noembed: { type: "boolean", default: parseEnvBoolean("NOEMBED") }, - concurrentTestPrograms: { type: "boolean", default: parseEnvBoolean("CONCURRENT_TEST_PROGRAMS") }, + concurrency: { type: "string", default: parseEnvString("TEST_PROGRAM_CONCURRENCY") }, coverage: { type: "boolean", default: parseEnvBoolean("COVERAGE") }, }, strict: false, @@ -291,7 +305,7 @@ function goTestFlags(taskName) { } const goTestEnv = { - ...(options.concurrentTestPrograms ? { TS_TEST_PROGRAM_SINGLE_THREADED: "false" } : {}), + ...(options.concurrency ? { TSGO_TEST_PROGRAM_CONCURRENCY: "false" } : {}), // Go test caching takes a long time on Windows. // https://github.com/golang/go/issues/72992 ...(process.platform === "win32" ? { GOFLAGS: "-count=1" } : {}), diff --git a/internal/compiler/program.go b/internal/compiler/program.go index 3be053f2d7..6ec3198a5a 100644 --- a/internal/compiler/program.go +++ b/internal/compiler/program.go @@ -26,7 +26,6 @@ type ProgramOptions struct { Host CompilerHost Config *tsoptions.ParsedCommandLine UseSourceOfProjectReference bool - SingleThreaded core.Tristate CreateCheckerPool func(*Program) CheckerPool TypingsLocation string ProjectName string @@ -39,7 +38,11 @@ func (p *ProgramOptions) canUseProjectReferenceSource() bool { type Program struct { opts ProgramOptions nodeModules map[string]*ast.SourceFile - checkerPool CheckerPool + + concurrency core.Concurrency + concurrencyOnce sync.Once + checkerPool CheckerPool + checkerPoolOnce sync.Once comparePathsOptions tspath.ComparePathsOptions @@ -191,9 +194,6 @@ func NewProgram(opts ProgramOptions) *Program { if p.opts.Host == nil { panic("host required") } - p.initCheckerPool() - - // p.maxNodeModuleJsDepth = p.options.MaxNodeModuleJsDepth // TODO(ercornel): !!! tracing? // tracing?.push(tracing.Phase.Program, "createProgram", { configFilePath: options.configFilePath, rootDir: options.rootDir }, /*separateBeginAndEnd*/ true); @@ -239,7 +239,6 @@ func (p *Program) UpdateProgram(changedFilePath tspath.Path) (*Program, bool) { currentNodeModulesDepth: p.currentNodeModulesDepth, usesUriStyleNodeCoreModules: p.usesUriStyleNodeCoreModules, } - result.initCheckerPool() index := core.FindIndex(result.files, func(file *ast.SourceFile) bool { return file.Path() == newFile.Path() }) result.files = slices.Clone(result.files) result.files[index] = newFile @@ -248,14 +247,6 @@ func (p *Program) UpdateProgram(changedFilePath tspath.Path) (*Program, bool) { return result, true } -func (p *Program) initCheckerPool() { - if p.opts.CreateCheckerPool != nil { - p.checkerPool = p.opts.CreateCheckerPool(p) - } else { - p.checkerPool = newCheckerPool(core.IfElse(p.singleThreaded(), 1, 4), p) - } -} - func canReplaceFileInProgram(file1 *ast.SourceFile, file2 *ast.SourceFile) bool { // TODO(jakebailey): metadata?? return file2 != nil && @@ -299,8 +290,26 @@ func (p *Program) GetConfigFileParsingDiagnostics() []*ast.Diagnostic { return slices.Clip(p.opts.Config.GetConfigFileParsingDiagnostics()) } +func (p *Program) getConcurrency() core.Concurrency { + p.concurrencyOnce.Do(func() { + p.concurrency = core.ParseConcurrency(p.Options()) + }) + return p.concurrency +} + func (p *Program) singleThreaded() bool { - return p.opts.SingleThreaded.DefaultIfUnknown(p.Options().SingleThreaded).IsTrue() + return p.getConcurrency().SingleThreaded() +} + +func (p *Program) getCheckerPool() CheckerPool { + p.checkerPoolOnce.Do(func() { + if p.opts.CreateCheckerPool != nil { + p.checkerPool = p.opts.CreateCheckerPool(p) + } else { + p.checkerPool = newCheckerPool(p.getConcurrency().CheckerCount(len(p.files)), p) + } + }) + return p.checkerPool } func (p *Program) BindSourceFiles() { @@ -317,11 +326,11 @@ func (p *Program) BindSourceFiles() { func (p *Program) CheckSourceFiles(ctx context.Context) { wg := core.NewWorkGroup(p.singleThreaded()) - checkers, done := p.checkerPool.GetAllCheckers(ctx) + checkers, done := p.getCheckerPool().GetAllCheckers(ctx) defer done() for _, checker := range checkers { wg.Queue(func() { - for file := range p.checkerPool.Files(checker) { + for file := range p.getCheckerPool().Files(checker) { checker.CheckSourceFile(ctx, file) } }) @@ -331,11 +340,11 @@ func (p *Program) CheckSourceFiles(ctx context.Context) { // Return the type checker associated with the program. func (p *Program) GetTypeChecker(ctx context.Context) (*checker.Checker, func()) { - return p.checkerPool.GetChecker(ctx) + return p.getCheckerPool().GetChecker(ctx) } func (p *Program) GetTypeCheckers(ctx context.Context) ([]*checker.Checker, func()) { - return p.checkerPool.GetAllCheckers(ctx) + return p.getCheckerPool().GetAllCheckers(ctx) } // Return a checker for the given file. We may have multiple checkers in concurrent scenarios and this @@ -343,7 +352,7 @@ func (p *Program) GetTypeCheckers(ctx context.Context) ([]*checker.Checker, func // types obtained from different checkers, so only non-type data (such as diagnostics or string // representations of types) should be obtained from checkers returned by this method. func (p *Program) GetTypeCheckerForFile(ctx context.Context, file *ast.SourceFile) (*checker.Checker, func()) { - return p.checkerPool.GetCheckerForFile(ctx, file) + return p.getCheckerPool().GetCheckerForFile(ctx, file) } func (p *Program) GetResolvedModule(file ast.HasFileName, moduleReference string, mode core.ResolutionMode) *module.ResolvedModule { @@ -385,7 +394,7 @@ func (p *Program) GetSuggestionDiagnostics(ctx context.Context, sourceFile *ast. func (p *Program) GetGlobalDiagnostics(ctx context.Context) []*ast.Diagnostic { var globalDiagnostics []*ast.Diagnostic - checkers, done := p.checkerPool.GetAllCheckers(ctx) + checkers, done := p.getCheckerPool().GetAllCheckers(ctx) defer done() for _, checker := range checkers { globalDiagnostics = append(globalDiagnostics, checker.GetGlobalDiagnostics()...) @@ -432,11 +441,11 @@ func (p *Program) getSemanticDiagnosticsForFile(ctx context.Context, sourceFile var fileChecker *checker.Checker var done func() if sourceFile != nil { - fileChecker, done = p.checkerPool.GetCheckerForFile(ctx, sourceFile) + fileChecker, done = p.getCheckerPool().GetCheckerForFile(ctx, sourceFile) defer done() } diags := slices.Clip(sourceFile.BindDiagnostics()) - checkers, closeCheckers := p.checkerPool.GetAllCheckers(ctx) + checkers, closeCheckers := p.getCheckerPool().GetAllCheckers(ctx) defer closeCheckers() // Ask for diags from all checkers; checking one file may add diagnostics to other files. @@ -525,13 +534,13 @@ func (p *Program) getSuggestionDiagnosticsForFile(ctx context.Context, sourceFil var fileChecker *checker.Checker var done func() if sourceFile != nil { - fileChecker, done = p.checkerPool.GetCheckerForFile(ctx, sourceFile) + fileChecker, done = p.getCheckerPool().GetCheckerForFile(ctx, sourceFile) defer done() } diags := slices.Clip(sourceFile.BindSuggestionDiagnostics) - checkers, closeCheckers := p.checkerPool.GetAllCheckers(ctx) + checkers, closeCheckers := p.getCheckerPool().GetAllCheckers(ctx) defer closeCheckers() // Ask for diags from all checkers; checking one file may add diagnostics to other files. @@ -642,7 +651,7 @@ func (p *Program) SymbolCount() int { for _, file := range p.files { count += file.SymbolCount } - checkers, done := p.checkerPool.GetAllCheckers(context.Background()) + checkers, done := p.getCheckerPool().GetAllCheckers(context.Background()) defer done() for _, checker := range checkers { count += int(checker.SymbolCount) @@ -652,7 +661,7 @@ func (p *Program) SymbolCount() int { func (p *Program) TypeCount() int { var count int - checkers, done := p.checkerPool.GetAllCheckers(context.Background()) + checkers, done := p.getCheckerPool().GetAllCheckers(context.Background()) defer done() for _, checker := range checkers { count += int(checker.TypeCount) @@ -662,7 +671,7 @@ func (p *Program) TypeCount() int { func (p *Program) InstantiationCount() int { var count int - checkers, done := p.checkerPool.GetAllCheckers(context.Background()) + checkers, done := p.getCheckerPool().GetAllCheckers(context.Background()) defer done() for _, checker := range checkers { count += int(checker.TotalInstantiationCount) diff --git a/internal/core/compileroptions.go b/internal/core/compileroptions.go index e6c7f859e9..5cfeb48f12 100644 --- a/internal/core/compileroptions.go +++ b/internal/core/compileroptions.go @@ -146,6 +146,7 @@ type CompilerOptions struct { PprofDir string `json:"pprofDir,omitzero"` SingleThreaded Tristate `json:"singleThreaded,omitzero"` + Concurrency string `json:"concurrency,omitzero"` Quiet Tristate `json:"quiet,omitzero"` sourceFileAffectingCompilerOptionsOnce sync.Once diff --git a/internal/core/concurrency.go b/internal/core/concurrency.go new file mode 100644 index 0000000000..c6997a36fd --- /dev/null +++ b/internal/core/concurrency.go @@ -0,0 +1,76 @@ +package core + +import ( + "os" + "runtime" + "strconv" + "strings" + "sync" + + "github.com/microsoft/typescript-go/internal/testutil/race" +) + +type Concurrency struct { + checkerCount int +} + +func ParseConcurrency(options *CompilerOptions) Concurrency { + if options.SingleThreaded.IsTrue() { + return Concurrency{ + checkerCount: 1, + } + } + return parseConcurrency(options.Concurrency) +} + +func parseConcurrency(s string) Concurrency { + checkerCount := 4 + + switch strings.ToLower(s) { + case "default", "auto", "true", "yes", "on": + break + case "single", "none", "false", "no", "off": + checkerCount = 1 + case "max": + checkerCount = runtime.GOMAXPROCS(0) + case "half": + checkerCount = max(1, runtime.GOMAXPROCS(0)/2) + case "checker-per-file": + checkerCount = -1 + default: + if s != "" { + if v, err := strconv.Atoi(s); err == nil && v > 0 { + checkerCount = v + } + } + } + + return Concurrency{ + checkerCount: checkerCount, + } +} + +func (c Concurrency) SingleThreaded() bool { + return c.checkerCount == 1 +} + +func (c Concurrency) CheckerCount(numFiles int) int { + checkerCount := c.checkerCount + if c.checkerCount == -1 { + return max(1, numFiles) + } + return min(max(1, checkerCount), numFiles) +} + +var testProgramConcurrency = sync.OnceValues(func() (concurrency Concurrency, raw string) { + // Leave Program in SingleThreaded mode unless explicitly configured or in race mode. + v := os.Getenv("TSGO_TEST_PROGRAM_CONCURRENCY") + if v == "" && !race.Enabled { + v = "single" + } + return parseConcurrency(v), v +}) + +func TestProgramConcurrency() (concurrency Concurrency, raw string) { + return testProgramConcurrency() +} diff --git a/internal/core/concurrency_test.go b/internal/core/concurrency_test.go new file mode 100644 index 0000000000..169403cd7e --- /dev/null +++ b/internal/core/concurrency_test.go @@ -0,0 +1,59 @@ +package core + +import ( + "runtime" + "testing" + + "gotest.tools/v3/assert" +) + +func TestConcurrency(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + opts *CompilerOptions + numFiles int + singleThreaded bool + checkerCount int + }{ + {"defaults", &CompilerOptions{}, 100, false, 4}, + {"default", &CompilerOptions{Concurrency: "default"}, 100, false, 4}, + {"auto", &CompilerOptions{Concurrency: "true"}, 100, false, 4}, + {"true", &CompilerOptions{Concurrency: "true"}, 100, false, 4}, + {"yes", &CompilerOptions{Concurrency: "yes"}, 100, false, 4}, + {"on", &CompilerOptions{Concurrency: "on"}, 100, false, 4}, + {"singleThreaded", &CompilerOptions{SingleThreaded: TSTrue}, 100, true, 1}, + {"single", &CompilerOptions{Concurrency: "single"}, 100, true, 1}, + {"none", &CompilerOptions{Concurrency: "none"}, 100, true, 1}, + {"false", &CompilerOptions{Concurrency: "false"}, 100, true, 1}, + {"no", &CompilerOptions{Concurrency: "no"}, 100, true, 1}, + {"off", &CompilerOptions{Concurrency: "off"}, 100, true, 1}, + {"max", &CompilerOptions{Concurrency: "max"}, 1000, false, runtime.GOMAXPROCS(0)}, + {"half", &CompilerOptions{Concurrency: "half"}, 1000, runtime.GOMAXPROCS(0)/2 == 1, runtime.GOMAXPROCS(0) / 2}, + {"checker-per-file", &CompilerOptions{Concurrency: "checker-per-file"}, 100, false, 100}, + {"more than files", &CompilerOptions{Concurrency: "1000"}, 100, false, 100}, + {"10", &CompilerOptions{Concurrency: "10"}, 100, false, 10}, + {"1", &CompilerOptions{Concurrency: "1"}, 100, true, 1}, + {"invalid", &CompilerOptions{Concurrency: "i dunno"}, 100, false, 4}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + c := ParseConcurrency(tt.opts) + singleThreaded := c.SingleThreaded() + checkerCount := c.CheckerCount(tt.numFiles) + assert.Equal(t, singleThreaded, tt.singleThreaded) + assert.Equal(t, checkerCount, tt.checkerCount) + }) + } + + t.Run("TestProgramConcurrency", func(t *testing.T) { + t.Parallel() + + c, _ := TestProgramConcurrency() + assert.Assert(t, c.CheckerCount(10000) > 0) + }) +} diff --git a/internal/core/tristate.go b/internal/core/tristate.go index 11b1bc0d36..8ea4d86ba5 100644 --- a/internal/core/tristate.go +++ b/internal/core/tristate.go @@ -13,6 +13,10 @@ const ( TSTrue ) +func (t Tristate) IsUnknown() bool { + return t == TSUnknown +} + func (t Tristate) IsTrue() bool { return t == TSTrue } diff --git a/internal/diagnostics/diagnostics_generated.go b/internal/diagnostics/diagnostics_generated.go index 5fb56a7143..db9c4080cf 100644 --- a/internal/diagnostics/diagnostics_generated.go +++ b/internal/diagnostics/diagnostics_generated.go @@ -4213,3 +4213,5 @@ var Do_not_print_diagnostics = &Message{code: 100000, category: CategoryMessage, var Run_in_single_threaded_mode = &Message{code: 100001, category: CategoryMessage, key: "Run_in_single_threaded_mode_100001", text: "Run in single threaded mode."} var Generate_pprof_CPU_Slashmemory_profiles_to_the_given_directory = &Message{code: 100002, category: CategoryMessage, key: "Generate_pprof_CPU_Slashmemory_profiles_to_the_given_directory_100002", text: "Generate pprof CPU/memory profiles to the given directory."} + +var Specify_how_much_concurrency_will_be_used = &Message{code: 100003, category: CategoryMessage, key: "Specify_how_much_concurrency_will_be_used_100003", text: "Specify how much concurrency will be used."} diff --git a/internal/diagnostics/extraDiagnosticMessages.json b/internal/diagnostics/extraDiagnosticMessages.json index b1de202b28..5c5fc1ab10 100644 --- a/internal/diagnostics/extraDiagnosticMessages.json +++ b/internal/diagnostics/extraDiagnosticMessages.json @@ -10,5 +10,9 @@ "Generate pprof CPU/memory profiles to the given directory.": { "category": "Message", "code": 100002 + }, + "Specify how much concurrency will be used.": { + "category": "Message", + "code": 100003 } } diff --git a/internal/project/project.go b/internal/project/project.go index 4680e75279..8512f1aa07 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -341,7 +341,6 @@ func (p *Project) GetLanguageServiceForRequest(ctx context.Context) (*ls.Languag if program == nil { panic("must have gced by other request") } - checkerPool := p.checkerPool snapshot := &snapshot{ project: p, positionEncoding: p.host.PositionEncoding(), @@ -349,7 +348,7 @@ func (p *Project) GetLanguageServiceForRequest(ctx context.Context) (*ls.Languag } languageService := ls.NewLanguageService(ctx, snapshot) cleanup := func() { - if checkerPool.isRequestCheckerInUse(core.GetRequestID(ctx)) { + if p.checkerPool != nil && p.checkerPool.isRequestCheckerInUse(core.GetRequestID(ctx)) { panic(fmt.Errorf("checker for request ID %s not returned to pool at end of request", core.GetRequestID(ctx))) } } diff --git a/internal/testrunner/compiler_runner.go b/internal/testrunner/compiler_runner.go index 6770a03eb1..04d6a0647c 100644 --- a/internal/testrunner/compiler_runner.go +++ b/internal/testrunner/compiler_runner.go @@ -335,7 +335,8 @@ var concurrentSkippedErrorBaselines = collections.NewSetFromItems( func (c *compilerTest) verifyDiagnostics(t *testing.T, suiteName string, isSubmodule bool) { t.Run("error", func(t *testing.T) { - if !testutil.TestProgramIsSingleThreaded() && concurrentSkippedErrorBaselines.Has(c.testName) { + concurrency, _ := core.TestProgramConcurrency() + if !concurrency.SingleThreaded() && concurrentSkippedErrorBaselines.Has(c.testName) { t.Skip("Skipping error baseline in concurrent mode") } diff --git a/internal/testutil/harnessutil/harnessutil.go b/internal/testutil/harnessutil/harnessutil.go index f816bde694..f13ab195d1 100644 --- a/internal/testutil/harnessutil/harnessutil.go +++ b/internal/testutil/harnessutil/harnessutil.go @@ -25,7 +25,6 @@ import ( "github.com/microsoft/typescript-go/internal/repo" "github.com/microsoft/typescript-go/internal/scanner" "github.com/microsoft/typescript-go/internal/sourcemap" - "github.com/microsoft/typescript-go/internal/testutil" "github.com/microsoft/typescript-go/internal/tsoptions" "github.com/microsoft/typescript-go/internal/tspath" "github.com/microsoft/typescript-go/internal/vfs" @@ -182,6 +181,11 @@ func CompileFilesEx( compilerOptions.TypeRoots[i] = tspath.GetNormalizedAbsolutePath(typeRoot, currentDirectory) } + if compilerOptions.Concurrency == "" && compilerOptions.SingleThreaded.IsUnknown() { + _, concurrency := core.TestProgramConcurrency() + compilerOptions.Concurrency = concurrency + } + // Create fake FS for testing testfs := map[string]any{} for _, file := range inputFiles { @@ -841,18 +845,11 @@ func (c *CompilationResult) GetSourceMapRecord() string { } func createProgram(host compiler.CompilerHost, config *tsoptions.ParsedCommandLine) *compiler.Program { - var singleThreaded core.Tristate - if testutil.TestProgramIsSingleThreaded() { - singleThreaded = core.TSTrue - } - programOptions := compiler.ProgramOptions{ - Config: config, - Host: host, - SingleThreaded: singleThreaded, + Config: config, + Host: host, } - program := compiler.NewProgram(programOptions) - return program + return compiler.NewProgram(programOptions) } func EnumerateFiles(folder string, testRegex *regexp.Regexp, recursive bool) ([]string, error) { diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 279258f27a..f803dacfe6 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -1,13 +1,9 @@ package testutil import ( - "os" "runtime/debug" - "strconv" - "sync" "testing" - "github.com/microsoft/typescript-go/internal/testutil/race" "gotest.tools/v3/assert" ) @@ -33,17 +29,3 @@ func RecoverAndFail(t *testing.T, msg string) { t.Fatalf("%s:\n%v\n%s", msg, r, string(stack)) } } - -var testProgramIsSingleThreaded = sync.OnceValue(func() bool { - // Leave Program in SingleThreaded mode unless explicitly configured or in race mode. - if v := os.Getenv("TS_TEST_PROGRAM_SINGLE_THREADED"); v != "" { - if b, err := strconv.ParseBool(v); err == nil { - return b - } - } - return !race.Enabled -}) - -func TestProgramIsSingleThreaded() bool { - return testProgramIsSingleThreaded() -} diff --git a/internal/tsoptions/declscompiler.go b/internal/tsoptions/declscompiler.go index c378ad896a..ab78f96500 100644 --- a/internal/tsoptions/declscompiler.go +++ b/internal/tsoptions/declscompiler.go @@ -225,6 +225,12 @@ var optionsForCompiler = []*CommandLineOption{ Category: diagnostics.Command_line_Options, Description: diagnostics.Run_in_single_threaded_mode, }, + { + Name: "concurrency", + Kind: CommandLineOptionTypeString, + Category: diagnostics.Command_line_Options, + Description: diagnostics.Specify_how_much_concurrency_will_be_used, + }, { Name: "pprofDir", Kind: CommandLineOptionTypeString, diff --git a/internal/tsoptions/parsinghelpers.go b/internal/tsoptions/parsinghelpers.go index e13280b1a5..69b6958888 100644 --- a/internal/tsoptions/parsinghelpers.go +++ b/internal/tsoptions/parsinghelpers.go @@ -430,6 +430,8 @@ func parseCompilerOptions(key string, value any, allOptions *core.CompilerOption allOptions.PprofDir = parseString(value) case "singleThreaded": allOptions.SingleThreaded = parseTristate(value) + case "concurrency": + allOptions.Concurrency = parseString(value) case "quiet": allOptions.Quiet = parseTristate(value) default: