From b3b3cd8616122382e9ee7659b64b135cece79dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Gl=C3=BCckert?= Date: Sat, 21 Sep 2024 16:25:55 +0200 Subject: [PATCH 1/7] allow for parallel test execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jannik Glückert --- src/tests.ts | 62 ++++++++++++++++++++++++++++++++++------------------ src/utils.ts | 11 +++++++--- 2 files changed, 49 insertions(+), 24 deletions(-) diff --git a/src/tests.ts b/src/tests.ts index 6942da3..ad364f3 100644 --- a/src/tests.ts +++ b/src/tests.ts @@ -1,3 +1,4 @@ +import * as os from "os"; import * as vscode from "vscode"; import { ExecResult, exec, extensionConfiguration } from "./utils"; import { Tests, DebugEnvironmentConfiguration } from "./types"; @@ -33,32 +34,51 @@ export async function testRunHandler( controller.items.forEach((test) => queue.push(test)); } + // This way the total number of runs shows up from the beginning, + // instead of incrementing as individual runs finish + for (const test of queue) { + run.enqueued(test); + } + const buildDir = workspaceState.get("mesonbuild.buildDir")!; - for (let test of queue) { + const runningTests: Promise[] = []; + const maxRunning = os.cpus().length; + + for (const test of queue) { run.started(test); - let starttime = Date.now(); - try { - await exec( - extensionConfiguration("mesonPath"), - ["test", "-C", buildDir, "--print-errorlog", `"${test.id}"`], - extensionConfiguration("testEnvironment"), - ); - let duration = Date.now() - starttime; - run.passed(test, duration); - } catch (e) { - const execResult = e as ExecResult; - - run.appendOutput(execResult.stdout); - let duration = Date.now() - starttime; - if (execResult.error?.code == 125) { - vscode.window.showErrorMessage("Failed to build tests. Results will not be updated"); - run.errored(test, new vscode.TestMessage(execResult.stderr)); - } else { - run.failed(test, new vscode.TestMessage(execResult.stderr), duration); - } + const runningTest = exec( + extensionConfiguration("mesonPath"), + ["test", "-C", buildDir, "--print-errorlog", `"${test.id}"`], + extensionConfiguration("testEnvironment"), + ) + .then( + (onfulfilled) => { + run.passed(test, onfulfilled.timeMs); + }, + (onrejected) => { + const execResult = onrejected as ExecResult; + + run.appendOutput(execResult.stdout, undefined, test); + if (execResult.error?.code == 125) { + vscode.window.showErrorMessage("Failed to build tests. Results will not be updated"); + run.errored(test, new vscode.TestMessage(execResult.stderr)); + } else { + run.failed(test, new vscode.TestMessage(execResult.stderr), execResult.timeMs); + } + }, + ) + .finally(() => { + runningTests.splice(runningTests.indexOf(runningTest), 1); + }); + + runningTests.push(runningTest); + + if (runningTests.length >= maxRunning) { + await Promise.race(runningTests); } } + await Promise.all(runningTests); run.end(); } diff --git a/src/utils.ts b/src/utils.ts index 18aa4ef..ab831d1 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -19,6 +19,7 @@ import { extensionPath, workspaceState } from "./extension"; export interface ExecResult { stdout: string; stderr: string; + timeMs: number; // Runtime in milliseconds error?: cp.ExecFileException; } @@ -32,11 +33,13 @@ export async function exec( options.env = { ...(options.env ?? process.env), ...extraEnv }; } return new Promise((resolve, reject) => { + const timeStart = Date.now(); cp.execFile(command, args, options, (error, stdout, stderr) => { + const timeMs = Date.now() - timeStart; if (error) { - reject({ error, stdout, stderr }); + reject({ stdout, stderr, timeMs, error }); } else { - resolve({ stdout, stderr }); + resolve({ stdout, stderr, timeMs }); } }); }); @@ -49,8 +52,10 @@ export async function execFeed( stdin: string, ) { return new Promise((resolve) => { + const timeStart = Date.now(); const p = cp.execFile(command, args, options, (error, stdout, stderr) => { - resolve({ stdout, stderr, error: error ? error : undefined }); + const timeMs = Date.now() - timeStart; + resolve({ stdout, stderr, timeMs, error: error ?? undefined }); }); p.stdin?.write(stdin); From 3e5deb27c21faa32cd86f053d0e7a480e32df42a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Gl=C3=BCckert?= Date: Sat, 21 Sep 2024 16:36:42 +0200 Subject: [PATCH 2/7] fix newlines in test output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jannik Glückert --- src/tests.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/tests.ts b/src/tests.ts index ad364f3..bb0bf02 100644 --- a/src/tests.ts +++ b/src/tests.ts @@ -59,7 +59,11 @@ export async function testRunHandler( (onrejected) => { const execResult = onrejected as ExecResult; - run.appendOutput(execResult.stdout, undefined, test); + let stdout = execResult.stdout; + if (os.platform() != "win32") { + stdout = stdout.replace(/\n/g, "\r\n"); + } + run.appendOutput(stdout, undefined, test); if (execResult.error?.code == 125) { vscode.window.showErrorMessage("Failed to build tests. Results will not be updated"); run.errored(test, new vscode.TestMessage(execResult.stderr)); From 3265e88fb29d6d0b3a2db2ecd37b992288fc7df6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Gl=C3=BCckert?= Date: Sat, 21 Sep 2024 17:18:10 +0200 Subject: [PATCH 3/7] support sequential tests in parallel executor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jannik Glückert --- src/tests.ts | 98 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 58 insertions(+), 40 deletions(-) diff --git a/src/tests.ts b/src/tests.ts index bb0bf02..9327d89 100644 --- a/src/tests.ts +++ b/src/tests.ts @@ -26,55 +26,69 @@ export async function testRunHandler( token: vscode.CancellationToken, ) { const run = controller.createTestRun(request, undefined, false); - const queue: vscode.TestItem[] = []; + const parallelTests: vscode.TestItem[] = []; + const sequentialTests: vscode.TestItem[] = []; + const buildDir = workspaceState.get("mesonbuild.buildDir")!; + const mesonTests = await getMesonTests(buildDir); + + // Look up the meson test for a given vscode test, + // put it in the parallel or sequential queue, + // and tell vscode about the enqueued test. + const testAdder = (test: vscode.TestItem) => { + const mesonTest = mesonTests.find((mesonTest) => { + return mesonTest.name == test.id; + })!; + if (mesonTest.is_parallel) { + parallelTests.push(test); + } else { + sequentialTests.push(test); + } + // This way the total number of runs shows up from the beginning, + // instead of incrementing as individual runs finish + run.enqueued(test); + }; if (request.include) { - request.include.forEach((test) => queue.push(test)); + request.include.forEach(testAdder); } else { - controller.items.forEach((test) => queue.push(test)); - } - - // This way the total number of runs shows up from the beginning, - // instead of incrementing as individual runs finish - for (const test of queue) { - run.enqueued(test); + controller.items.forEach(testAdder); } - const buildDir = workspaceState.get("mesonbuild.buildDir")!; - - const runningTests: Promise[] = []; - const maxRunning = os.cpus().length; - - for (const test of queue) { + const dispatchTest = (test: vscode.TestItem) => { run.started(test); - const runningTest = exec( + return exec( extensionConfiguration("mesonPath"), ["test", "-C", buildDir, "--print-errorlog", `"${test.id}"`], extensionConfiguration("testEnvironment"), - ) - .then( - (onfulfilled) => { - run.passed(test, onfulfilled.timeMs); - }, - (onrejected) => { - const execResult = onrejected as ExecResult; - - let stdout = execResult.stdout; - if (os.platform() != "win32") { - stdout = stdout.replace(/\n/g, "\r\n"); - } - run.appendOutput(stdout, undefined, test); - if (execResult.error?.code == 125) { - vscode.window.showErrorMessage("Failed to build tests. Results will not be updated"); - run.errored(test, new vscode.TestMessage(execResult.stderr)); - } else { - run.failed(test, new vscode.TestMessage(execResult.stderr), execResult.timeMs); - } - }, - ) - .finally(() => { - runningTests.splice(runningTests.indexOf(runningTest), 1); - }); + ).then( + (onfulfilled) => { + run.passed(test, onfulfilled.timeMs); + }, + (onrejected) => { + const execResult = onrejected as ExecResult; + + let stdout = execResult.stdout; + if (os.platform() != "win32") { + stdout = stdout.replace(/\n/g, "\r\n"); + } + run.appendOutput(stdout, undefined, test); + if (execResult.error?.code == 125) { + vscode.window.showErrorMessage("Failed to build tests. Results will not be updated"); + run.errored(test, new vscode.TestMessage(execResult.stderr)); + } else { + run.failed(test, new vscode.TestMessage(execResult.stderr), execResult.timeMs); + } + }, + ); + }; + + const runningTests: Promise[] = []; + const maxRunning = os.cpus().length; + + for (const test of parallelTests) { + const runningTest = dispatchTest(test).finally(() => { + runningTests.splice(runningTests.indexOf(runningTest), 1); + }); runningTests.push(runningTest); @@ -84,6 +98,10 @@ export async function testRunHandler( } await Promise.all(runningTests); + for (const test of sequentialTests) { + await dispatchTest(test); + } + run.end(); } From ebccace7e7f5cf04b6e618f5392e12ce5b382bf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Gl=C3=BCckert?= Date: Sat, 21 Sep 2024 18:33:37 +0200 Subject: [PATCH 4/7] add rudimentary test -> test source lookup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jannik Glückert --- src/tests.ts | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/src/tests.ts b/src/tests.ts index 9327d89..d3fb20e 100644 --- a/src/tests.ts +++ b/src/tests.ts @@ -1,12 +1,40 @@ import * as os from "os"; import * as vscode from "vscode"; import { ExecResult, exec, extensionConfiguration } from "./utils"; -import { Tests, DebugEnvironmentConfiguration } from "./types"; +import { Targets, Test, Tests, DebugEnvironmentConfiguration } from "./types"; import { getMesonTests, getMesonTargets } from "./introspection"; import { workspaceState } from "./extension"; +// This is far from complete, but should suffice for the +// "test is made of a single executable is made of a single source file" usecase. +function findSourceOfTest(test: Test, targets: Targets): vscode.Uri | undefined { + const testExe = test.cmd.at(0); + if (!testExe) { + return undefined; + } + + // The meson target such that it is of meson type executable() + // and produces the binary that the test() executes. + const testDependencyTarget = targets.find((target) => { + const depend = test.depends.find((depend) => { + return depend == target.id && target.type == "executable"; + }); + return depend && testExe == target.filename.at(0); + }); + + // The first source file belonging to the target. + const path = testDependencyTarget?.target_sources + ?.find((elem) => { + return elem.sources; + }) + ?.sources?.at(0); + return path ? vscode.Uri.file(path) : undefined; +} + export async function rebuildTests(controller: vscode.TestController) { - let tests = await getMesonTests(workspaceState.get("mesonbuild.buildDir")!); + const buildDir = workspaceState.get("mesonbuild.buildDir")!; + const tests = await getMesonTests(buildDir); + const targets = await getMesonTargets(buildDir); controller.items.forEach((item) => { if (!tests.some((test) => item.id == test.name)) { @@ -15,7 +43,8 @@ export async function rebuildTests(controller: vscode.TestController) { }); for (let testDescr of tests) { - let testItem = controller.createTestItem(testDescr.name, testDescr.name); + const testSourceFile = findSourceOfTest(testDescr, targets); + const testItem = controller.createTestItem(testDescr.name, testDescr.name, testSourceFile); controller.items.add(testItem); } } From 79f6cef5519b00960f520a21dff20a3aa7ac4dab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Gl=C3=BCckert?= Date: Sun, 22 Sep 2024 17:39:00 +0200 Subject: [PATCH 5/7] use full target name in testDebugHandler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this is required when targets exist twice, e.g. due to subprojects Signed-off-by: Jannik Glückert --- src/tests.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tests.ts b/src/tests.ts index d3fb20e..d7edc7f 100644 --- a/src/tests.ts +++ b/src/tests.ts @@ -1,6 +1,6 @@ import * as os from "os"; import * as vscode from "vscode"; -import { ExecResult, exec, extensionConfiguration } from "./utils"; +import { ExecResult, exec, extensionConfiguration, getTargetName } from "./utils"; import { Targets, Test, Tests, DebugEnvironmentConfiguration } from "./types"; import { getMesonTests, getMesonTargets } from "./introspection"; import { workspaceState } from "./extension"; @@ -160,8 +160,8 @@ export async function testDebugHandler( ); let args = ["compile", "-C", buildDir]; - requiredTargets.forEach((target) => { - args.push(target.name); + requiredTargets.forEach(async (target) => { + args.push(await getTargetName(target)); }); try { From 83893b02c84879fd31e62a5bb26a0c8d40c0ae1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Gl=C3=BCckert?= Date: Thu, 31 Oct 2024 20:02:39 +0100 Subject: [PATCH 6/7] make number of test parallelism configurable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jannik Glückert --- package.json | 6 ++++++ src/tests.ts | 12 +++++++++++- src/types.ts | 1 + 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 8515ec9..4e9b5ea 100644 --- a/package.json +++ b/package.json @@ -158,6 +158,12 @@ "default": {}, "description": "Specify the list of additional environment variables used for running tests." }, + "mesonbuild.testJobs": { + "type": "integer", + "default": -1, + "minimum": -1, + "description": "Specify the maximum number of tests executed in parallel. -1 for number of CPUs, 0 for unlimited." + }, "mesonbuild.benchmarkOptions": { "type": "array", "default": [ diff --git a/src/tests.ts b/src/tests.ts index d7edc7f..6a3cbfb 100644 --- a/src/tests.ts +++ b/src/tests.ts @@ -112,7 +112,17 @@ export async function testRunHandler( }; const runningTests: Promise[] = []; - const maxRunning = os.cpus().length; + const maxRunning: number = (() => { + const jobsConfig = extensionConfiguration("testJobs"); + switch (jobsConfig) { + case -1: + return os.cpus().length; + case 0: + return Number.MAX_SAFE_INTEGER; + default: + return jobsConfig; + } + })(); for (const test of parallelTests) { const runningTest = dispatchTest(test).finally(() => { diff --git a/src/types.ts b/src/types.ts index 299dbe6..952609b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -72,6 +72,7 @@ export interface ExtensionConfiguration { setupOptions: string[]; testOptions: string[]; testEnvironment: { [key: string]: string }; + testJobs: number; benchmarkOptions: string[]; buildFolder: string; mesonPath: string; From 93ad6d67ac431754747fb9360efb1c2d93baa3b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Gl=C3=BCckert?= Date: Wed, 16 Apr 2025 22:06:53 +0200 Subject: [PATCH 7/7] gather all tests for rebuild before issuing execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit otherwise multiple simultaneous test runs may trigger rebuilds Signed-off-by: Jannik Glückert --- src/extension.ts | 6 ++-- src/tests.ts | 72 +++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/src/extension.ts b/src/extension.ts index b50511c..af77dec 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -16,7 +16,7 @@ import { } from "./utils"; import { MesonDebugConfigurationProvider, DebuggerType } from "./debug/index"; import { CpptoolsProvider, registerCppToolsProvider } from "./cpptoolsconfigprovider"; -import { testDebugHandler, testRunHandler, rebuildTests } from "./tests"; +import { testDebugHandler, testRunHandler, regenerateTests } from "./tests"; import { activateLinters } from "./linters"; import { activateFormatters } from "./formatters"; import { SettingsKey, TaskQuickPickItem } from "./types"; @@ -136,7 +136,7 @@ export async function activate(ctx: vscode.ExtensionContext) { const changeHandler = async () => { mesonTasks = null; clearCache(); - await rebuildTests(controller); + await regenerateTests(controller); await genEnvFile(buildDir); explorer.refresh(); }; @@ -236,7 +236,7 @@ export async function activate(ctx: vscode.ExtensionContext) { if (!checkMesonIsConfigured(buildDir)) { if (await askConfigureOnOpen()) runFirstTask("reconfigure"); } else { - await rebuildTests(controller); + await regenerateTests(controller); } const server = extensionConfiguration(SettingsKey.languageServer); diff --git a/src/tests.ts b/src/tests.ts index 6a3cbfb..9cb30e8 100644 --- a/src/tests.ts +++ b/src/tests.ts @@ -1,7 +1,7 @@ import * as os from "os"; import * as vscode from "vscode"; import { ExecResult, exec, extensionConfiguration, getTargetName } from "./utils"; -import { Targets, Test, Tests, DebugEnvironmentConfiguration } from "./types"; +import { Target, Targets, Test, Tests, DebugEnvironmentConfiguration } from "./types"; import { getMesonTests, getMesonTargets } from "./introspection"; import { workspaceState } from "./extension"; @@ -31,7 +31,59 @@ function findSourceOfTest(test: Test, targets: Targets): vscode.Uri | undefined return path ? vscode.Uri.file(path) : undefined; } -export async function rebuildTests(controller: vscode.TestController) { +/** + * Ensures that the given test targets are up to date + * @param tests Tests to rebuild + * @param buildDir Meson buildDir + * @returns `ExecResult` of the `meson compile ...` invocation + */ +async function rebuildTests(tests: Test[], buildDir: string): Promise { + // We need to ensure that all test dependencies are built before issuing tests, + // as otherwise each meson test ... invocation will lead to a rebuild on it's own + const dependencies: Set = new Set( + tests.flatMap((test) => { + return test.depends; + }), + ); + + const mesonTargets = await getMesonTargets(buildDir); + const testDependencies: Set = new Set( + mesonTargets.filter((target) => { + return dependencies.has(target.id); + }), + ); + + return exec(extensionConfiguration("mesonPath"), [ + "compile", + "-C", + buildDir, + ...[...testDependencies].map((test) => { + // `test.name` is not guaranteed to be the actual name that meson wants + // format is hash@@realname@type + return `"${/[^@]+@@(.+)@[^@]+/.exec(test.id)![1]}"`; + }), + ]); +} + +/** + * Look up the meson tests that correspond to a given set of vscode TestItems + * @param vsCodeTests TestItems to look up + * @param mesonTests The set of all existing meson tests + * @returns Meson tests corresponding to the TestItems + */ +function vsCodeToMeson(vsCodeTests: readonly vscode.TestItem[], mesonTests: Tests): Test[] { + return vsCodeTests.map((test) => { + return mesonTests.find((mesonTest) => { + return mesonTest.name == test.id; + })!; + }); +} + +/** + * Regenerate the test view in vscode, adding new tests and deleting stale ones + * @param controller VSCode test controller + */ +export async function regenerateTests(controller: vscode.TestController) { const buildDir = workspaceState.get("mesonbuild.buildDir")!; const tests = await getMesonTests(buildDir); const targets = await getMesonTargets(buildDir); @@ -65,9 +117,7 @@ export async function testRunHandler( // put it in the parallel or sequential queue, // and tell vscode about the enqueued test. const testAdder = (test: vscode.TestItem) => { - const mesonTest = mesonTests.find((mesonTest) => { - return mesonTest.name == test.id; - })!; + const mesonTest = vsCodeToMeson([test], mesonTests)[0]; if (mesonTest.is_parallel) { parallelTests.push(test); } else { @@ -83,11 +133,19 @@ export async function testRunHandler( controller.items.forEach(testAdder); } + // we need to ensure that all test dependencies are built before issuing tests, + // as otherwise each meson test ... invocation will lead to a rebuild on it's own + await rebuildTests(vsCodeToMeson(parallelTests.concat(sequentialTests), mesonTests), buildDir).catch((onrejected) => { + const execResult = onrejected as ExecResult; + vscode.window.showErrorMessage("Failed to build tests:\r\n" + execResult.stdout + "\r\n" + execResult.stderr); + run.end(); + }); + const dispatchTest = (test: vscode.TestItem) => { run.started(test); return exec( extensionConfiguration("mesonPath"), - ["test", "-C", buildDir, "--print-errorlog", `"${test.id}"`], + ["test", "-C", buildDir, "--print-errorlog", "--no-rebuild", `"${test.id}"`], extensionConfiguration("testEnvironment"), ).then( (onfulfilled) => { @@ -102,7 +160,7 @@ export async function testRunHandler( } run.appendOutput(stdout, undefined, test); if (execResult.error?.code == 125) { - vscode.window.showErrorMessage("Failed to build tests. Results will not be updated"); + vscode.window.showErrorMessage("Failed to run tests. Results will not be updated"); run.errored(test, new vscode.TestMessage(execResult.stderr)); } else { run.failed(test, new vscode.TestMessage(execResult.stderr), execResult.timeMs);